Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit d7b62727 authored by Matthew Wilcox's avatar Matthew Wilcox
Browse files

radix-tree: Fix __rcu annotations 



Many places were missing __rcu annotations.  A few places needed a few
lines of explanation about why it was safe to not use RCU accessors.
Add a custom CFLAGS setting to the Makefile to ensure that new patches
don't miss RCU annotations.

Signed-off-by: default avatarMatthew Wilcox <mawilcox@microsoft.com>
parent 12320d0f

include/linux/radix-tree.h

+54 −56
Original line number Diff line number Diff line
@@ -222,9 +222,7 @@ static inline unsigned int iter_shift(const struct radix_tree_iter *iter) 


/**

 * radix_tree_deref_slot - dereference a slot

 * @pslot:	pointer to slot, returned by radix_tree_lookup_slot

 * Returns:	item that was stored in that slot with any direct pointer flag

 *		removed.

 * @slot: slot pointer, returned by radix_tree_lookup_slot

 *

 * For use with radix_tree_lookup_slot().  Caller must hold tree at least read

 * locked across slot lookup and dereference. Not required if write lock is
@@ -232,26 +230,27 @@ static inline unsigned int iter_shift(const struct radix_tree_iter *iter) 
 *

 * radix_tree_deref_retry must be used to confirm validity of the pointer if

 * only the read lock is held.

 *

 * Return: entry stored in that slot.

 */

static inline void *radix_tree_deref_slot(void **pslot)

static inline void *radix_tree_deref_slot(void __rcu **slot)

{

	return rcu_dereference(*pslot);

	return rcu_dereference(*slot);

}



/**

 * radix_tree_deref_slot_protected	- dereference a slot without RCU lock but with tree lock held

 * @pslot:	pointer to slot, returned by radix_tree_lookup_slot

 * Returns:	item that was stored in that slot with any direct pointer flag

 *		removed.

 *

 * Similar to radix_tree_deref_slot but only used during migration when a pages

 * mapping is being moved. The caller does not hold the RCU read lock but it

 * must hold the tree lock to prevent parallel updates.

 * radix_tree_deref_slot_protected - dereference a slot with tree lock held

 * @slot: slot pointer, returned by radix_tree_lookup_slot

 *

 * Similar to radix_tree_deref_slot.  The caller does not hold the RCU read

 * lock but it must hold the tree lock to prevent parallel updates.

 *

 * Return: entry stored in that slot.

 */

static inline void *radix_tree_deref_slot_protected(void **pslot,

static inline void *radix_tree_deref_slot_protected(void __rcu **slot,

							spinlock_t *treelock)

{

	return rcu_dereference_protected(*pslot, lockdep_is_held(treelock));

	return rcu_dereference_protected(*slot, lockdep_is_held(treelock));

}



/**
@@ -287,9 +286,9 @@ static inline int radix_tree_exception(void *arg) 
	return unlikely((unsigned long)arg & RADIX_TREE_ENTRY_MASK);

}



int __radix_tree_create(struct radix_tree_root *root, unsigned long index,

int __radix_tree_create(struct radix_tree_root *, unsigned long index,

			unsigned order, struct radix_tree_node **nodep,

			void ***slotp);

			void __rcu ***slotp);

int __radix_tree_insert(struct radix_tree_root *, unsigned long index,

			unsigned order, void *);

static inline int radix_tree_insert(struct radix_tree_root *root,
@@ -298,42 +297,41 @@ static inline int radix_tree_insert(struct radix_tree_root *root, 
	return __radix_tree_insert(root, index, 0, entry);

}

void *__radix_tree_lookup(const struct radix_tree_root *, unsigned long index,

			  struct radix_tree_node **nodep, void ***slotp);

			  struct radix_tree_node **nodep, void __rcu ***slotp);

void *radix_tree_lookup(const struct radix_tree_root *, unsigned long);

void **radix_tree_lookup_slot(const struct radix_tree_root *, unsigned long);

void __rcu **radix_tree_lookup_slot(const struct radix_tree_root *,

					unsigned long index);

typedef void (*radix_tree_update_node_t)(struct radix_tree_node *, void *);

void __radix_tree_replace(struct radix_tree_root *root,

			  struct radix_tree_node *node,

			  void **slot, void *item,

void __radix_tree_replace(struct radix_tree_root *, struct radix_tree_node *,

			  void __rcu **slot, void *entry,

			  radix_tree_update_node_t update_node, void *private);

void radix_tree_iter_replace(struct radix_tree_root *,

		const struct radix_tree_iter *, void **slot, void *item);

void radix_tree_replace_slot(struct radix_tree_root *root,

			     void **slot, void *item);

void __radix_tree_delete_node(struct radix_tree_root *root,

			      struct radix_tree_node *node,

		const struct radix_tree_iter *, void __rcu **slot, void *entry);

void radix_tree_replace_slot(struct radix_tree_root *,

			     void __rcu **slot, void *entry);

void __radix_tree_delete_node(struct radix_tree_root *,

			      struct radix_tree_node *,

			      radix_tree_update_node_t update_node,

			      void *private);

void radix_tree_iter_delete(struct radix_tree_root *,

				struct radix_tree_iter *iter, void **slot);

			struct radix_tree_iter *iter, void __rcu **slot);

void *radix_tree_delete_item(struct radix_tree_root *, unsigned long, void *);

void *radix_tree_delete(struct radix_tree_root *, unsigned long);

void radix_tree_clear_tags(struct radix_tree_root *root,

			   struct radix_tree_node *node,

			   void **slot);

void radix_tree_clear_tags(struct radix_tree_root *, struct radix_tree_node *,

			   void __rcu **slot);

unsigned int radix_tree_gang_lookup(const struct radix_tree_root *,

			void **results, unsigned long first_index,

			unsigned int max_items);

unsigned int radix_tree_gang_lookup_slot(const struct radix_tree_root *,

			void ***results, unsigned long *indices,

			void __rcu ***results, unsigned long *indices,

			unsigned long first_index, unsigned int max_items);

int radix_tree_preload(gfp_t gfp_mask);

int radix_tree_maybe_preload(gfp_t gfp_mask);

int radix_tree_maybe_preload_order(gfp_t gfp_mask, int order);

void radix_tree_init(void);

void *radix_tree_tag_set(struct radix_tree_root *root,

void *radix_tree_tag_set(struct radix_tree_root *,

			unsigned long index, unsigned int tag);

void *radix_tree_tag_clear(struct radix_tree_root *root,

void *radix_tree_tag_clear(struct radix_tree_root *,

			unsigned long index, unsigned int tag);

int radix_tree_tag_get(const struct radix_tree_root *,

			unsigned long index, unsigned int tag);
@@ -341,15 +339,13 @@ void radix_tree_iter_tag_set(struct radix_tree_root *, 
		const struct radix_tree_iter *iter, unsigned int tag);

void radix_tree_iter_tag_clear(struct radix_tree_root *,

		const struct radix_tree_iter *iter, unsigned int tag);

unsigned int

radix_tree_gang_lookup_tag(const struct radix_tree_root *, void **results,

		unsigned long first_index, unsigned int max_items,

		unsigned int tag);

unsigned int

radix_tree_gang_lookup_tag_slot(const struct radix_tree_root *, void ***results,

		unsigned long first_index, unsigned int max_items,

		unsigned int tag);

int radix_tree_tagged(const struct radix_tree_root *root, unsigned int tag);

unsigned int radix_tree_gang_lookup_tag(const struct radix_tree_root *,

		void **results, unsigned long first_index,

		unsigned int max_items, unsigned int tag);

unsigned int radix_tree_gang_lookup_tag_slot(const struct radix_tree_root *,

		void __rcu ***results, unsigned long first_index,

		unsigned int max_items, unsigned int tag);

int radix_tree_tagged(const struct radix_tree_root *, unsigned int tag);



static inline void radix_tree_preload_end(void)

{
@@ -361,7 +357,7 @@ int radix_tree_split(struct radix_tree_root *, unsigned long index, 
			unsigned new_order);

int radix_tree_join(struct radix_tree_root *, unsigned long index,

			unsigned new_order, void *);

void **idr_get_free(struct radix_tree_root *, struct radix_tree_iter *,

void __rcu **idr_get_free(struct radix_tree_root *, struct radix_tree_iter *,

			gfp_t, int end);



enum {
@@ -377,7 +373,7 @@ enum { 
 * @start:	iteration starting index

 * Returns:	NULL

 */

static __always_inline void **

static __always_inline void __rcu **

radix_tree_iter_init(struct radix_tree_iter *iter, unsigned long start)

{

	/*
@@ -406,7 +402,7 @@ radix_tree_iter_init(struct radix_tree_iter *iter, unsigned long start) 
 * Also it fills @iter with data about chunk: position in the tree (index),

 * its end (next_index), and constructs a bit mask for tagged iterating (tags).

 */

void **radix_tree_next_chunk(const struct radix_tree_root *,

void __rcu **radix_tree_next_chunk(const struct radix_tree_root *,

			     struct radix_tree_iter *iter, unsigned flags);



/**
@@ -419,7 +415,8 @@ void **radix_tree_next_chunk(const struct radix_tree_root *, 
 * containing it and updates @iter to describe the entry.  If @index is not

 * present, it returns NULL.

 */

static inline void **radix_tree_iter_lookup(const struct radix_tree_root *root,

static inline void __rcu **

radix_tree_iter_lookup(const struct radix_tree_root *root,

			struct radix_tree_iter *iter, unsigned long index)

{

	radix_tree_iter_init(iter, index);
@@ -436,7 +433,8 @@ static inline void **radix_tree_iter_lookup(const struct radix_tree_root *root, 
 * which is at least @index.  If @index is larger than any present entry, this

 * function returns NULL.  The @iter is updated to describe the entry found.

 */

static inline void **radix_tree_iter_find(const struct radix_tree_root *root,

static inline void __rcu **

radix_tree_iter_find(const struct radix_tree_root *root,

			struct radix_tree_iter *iter, unsigned long index)

{

	radix_tree_iter_init(iter, index);
@@ -453,7 +451,7 @@ static inline void **radix_tree_iter_find(const struct radix_tree_root *root, 
 * and continue the iteration.

 */

static inline __must_check

void **radix_tree_iter_retry(struct radix_tree_iter *iter)

void __rcu **radix_tree_iter_retry(struct radix_tree_iter *iter)

{

	iter->next_index = iter->index;

	iter->tags = 0;
@@ -476,7 +474,7 @@ __radix_tree_iter_add(struct radix_tree_iter *iter, unsigned long slots) 
 * have been invalidated by an insertion or deletion.  Call this function

 * before releasing the lock to continue the iteration from the next index.

 */

void **__must_check radix_tree_iter_resume(void **slot,

void __rcu **__must_check radix_tree_iter_resume(void __rcu **slot,

					struct radix_tree_iter *iter);



/**
@@ -492,11 +490,11 @@ radix_tree_chunk_size(struct radix_tree_iter *iter) 
}



#ifdef CONFIG_RADIX_TREE_MULTIORDER

void ** __radix_tree_next_slot(void **slot, struct radix_tree_iter *iter,

				unsigned flags);

void __rcu **__radix_tree_next_slot(void __rcu **slot,

				struct radix_tree_iter *iter, unsigned flags);

#else

/* Can't happen without sibling entries, but the compiler can't tell that */

static inline void ** __radix_tree_next_slot(void **slot,

static inline void __rcu **__radix_tree_next_slot(void __rcu **slot,

				struct radix_tree_iter *iter, unsigned flags)

{

	return slot;
@@ -522,8 +520,8 @@ static inline void ** __radix_tree_next_slot(void **slot, 
 * b) we are doing non-tagged iteration, and iter->index and iter->next_index

 *    have been set up so that radix_tree_chunk_size() returns 1 or 0.

 */

static __always_inline void **

radix_tree_next_slot(void **slot, struct radix_tree_iter *iter, unsigned flags)

static __always_inline void __rcu **radix_tree_next_slot(void __rcu **slot,

				struct radix_tree_iter *iter, unsigned flags)

{

	if (flags & RADIX_TREE_ITER_TAGGED) {

		iter->tags >>= 1;

lib/Makefile

+2 −0
Original line number Diff line number Diff line
@@ -24,6 +24,8 @@ lib-y := ctype.o string.o vsprintf.o cmdline.o \ 
	 is_single_threaded.o plist.o decompress.o kobject_uevent.o \

	 earlycpio.o seq_buf.o nmi_backtrace.o nodemask.o win_minmax.o



CFLAGS_radix-tree.o += -DCONFIG_SPARSE_RCU_POINTER



lib-$(CONFIG_MMU) += ioremap.o

lib-$(CONFIG_SMP) += cpumask.o

lib-$(CONFIG_HAS_DMA) += dma-noop.o

lib/radix-tree.c

+66 −59
Original line number Diff line number Diff line
@@ -104,7 +104,7 @@ static inline void *node_to_entry(void *ptr) 
static inline

bool is_sibling_entry(const struct radix_tree_node *parent, void *node)

{

	void **ptr = node;

	void __rcu **ptr = node;

	return (parent->slots <= ptr) &&

			(ptr < parent->slots + RADIX_TREE_MAP_SIZE);

}
@@ -116,8 +116,8 @@ bool is_sibling_entry(const struct radix_tree_node *parent, void *node) 
}

#endif



static inline

unsigned long get_slot_offset(const struct radix_tree_node *parent, void **slot)

static inline unsigned long

get_slot_offset(const struct radix_tree_node *parent, void __rcu **slot)

{

	return slot - parent->slots;

}
@@ -126,12 +126,13 @@ static unsigned int radix_tree_descend(const struct radix_tree_node *parent, 
			struct radix_tree_node **nodep, unsigned long index)

{

	unsigned int offset = (index >> parent->shift) & RADIX_TREE_MAP_MASK;

	void **entry = rcu_dereference_raw(parent->slots[offset]);

	void __rcu **entry = rcu_dereference_raw(parent->slots[offset]);



#ifdef CONFIG_RADIX_TREE_MULTIORDER

	if (radix_tree_is_internal_node(entry)) {

		if (is_sibling_entry(parent, entry)) {

			void **sibentry = (void **) entry_to_node(entry);

			void __rcu **sibentry;

			sibentry = (void __rcu **) entry_to_node(entry);

			offset = get_slot_offset(parent, sibentry);

			entry = rcu_dereference_raw(*sibentry);

		}
@@ -618,7 +619,7 @@ static unsigned radix_tree_load_root(const struct radix_tree_root *root, 
static int radix_tree_extend(struct radix_tree_root *root, gfp_t gfp,

				unsigned long index, unsigned int shift)

{

	struct radix_tree_node *slot;

	void *entry;

	unsigned int maxshift;

	int tag;
@@ -627,8 +628,8 @@ static int radix_tree_extend(struct radix_tree_root *root, gfp_t gfp, 
	while (index > shift_maxindex(maxshift))

		maxshift += RADIX_TREE_MAP_SHIFT;



	slot = rcu_dereference_raw(root->rnode);

	if (!slot && (!is_idr(root) || root_tag_get(root, IDR_FREE)))

	entry = rcu_dereference_raw(root->rnode);

	if (!entry && (!is_idr(root) || root_tag_get(root, IDR_FREE)))

		goto out;



	do {
@@ -652,15 +653,19 @@ static int radix_tree_extend(struct radix_tree_root *root, gfp_t gfp, 
		}



		BUG_ON(shift > BITS_PER_LONG);

		if (radix_tree_is_internal_node(slot)) {

			entry_to_node(slot)->parent = node;

		} else if (radix_tree_exceptional_entry(slot)) {

		if (radix_tree_is_internal_node(entry)) {

			entry_to_node(entry)->parent = node;

		} else if (radix_tree_exceptional_entry(entry)) {

			/* Moving an exceptional root->rnode to a node */

			node->exceptional = 1;

		}

		node->slots[0] = slot;

		slot = node_to_entry(node);

		rcu_assign_pointer(root->rnode, slot);

		/*

		 * entry was already in the radix tree, so we do not need

		 * rcu_assign_pointer here

		 */

		node->slots[0] = (void __rcu *)entry;

		entry = node_to_entry(node);

		rcu_assign_pointer(root->rnode, entry);

		shift += RADIX_TREE_MAP_SHIFT;

	} while (shift <= maxshift);

out:
@@ -708,7 +713,7 @@ static inline bool radix_tree_shrink(struct radix_tree_root *root, 
		 * (node->slots[0]), it will be safe to dereference the new

		 * one (root->rnode) as far as dependent read barriers go.

		 */

		root->rnode = child;

		root->rnode = (void __rcu *)child;

		if (is_idr(root) && !tag_get(node, IDR_FREE, 0))

			root_tag_clear(root, IDR_FREE);
@@ -732,7 +737,7 @@ static inline bool radix_tree_shrink(struct radix_tree_root *root, 
		 */

		node->count = 0;

		if (!radix_tree_is_internal_node(child)) {

			node->slots[0] = RADIX_TREE_RETRY;

			node->slots[0] = (void __rcu *)RADIX_TREE_RETRY;

			if (update_node)

				update_node(node, private);

		}
@@ -805,10 +810,10 @@ static bool delete_node(struct radix_tree_root *root, 
 */

int __radix_tree_create(struct radix_tree_root *root, unsigned long index,

			unsigned order, struct radix_tree_node **nodep,

			void ***slotp)

			void __rcu ***slotp)

{

	struct radix_tree_node *node = NULL, *child;

	void **slot = (void **)&root->rnode;

	void __rcu **slot = (void __rcu **)&root->rnode;

	unsigned long maxindex;

	unsigned int shift, offset = 0;

	unsigned long max = index | ((1UL << order) - 1);
@@ -890,8 +895,8 @@ static void radix_tree_free_nodes(struct radix_tree_node *node) 
}



#ifdef CONFIG_RADIX_TREE_MULTIORDER

static inline int insert_entries(struct radix_tree_node *node, void **slot,

				void *item, unsigned order, bool replace)

static inline int insert_entries(struct radix_tree_node *node,

		void __rcu **slot, void *item, unsigned order, bool replace)

{

	struct radix_tree_node *child;

	unsigned i, n, tag, offset, tags = 0;
@@ -953,8 +958,8 @@ static inline int insert_entries(struct radix_tree_node *node, void **slot, 
	return n;

}

#else

static inline int insert_entries(struct radix_tree_node *node, void **slot,

				void *item, unsigned order, bool replace)

static inline int insert_entries(struct radix_tree_node *node,

		void __rcu **slot, void *item, unsigned order, bool replace)

{

	if (*slot)

		return -EEXIST;
@@ -981,7 +986,7 @@ int __radix_tree_insert(struct radix_tree_root *root, unsigned long index, 
			unsigned order, void *item)

{

	struct radix_tree_node *node;

	void **slot;

	void __rcu **slot;

	int error;



	BUG_ON(radix_tree_is_internal_node(item));
@@ -1023,15 +1028,15 @@ EXPORT_SYMBOL(__radix_tree_insert); 
 */

void *__radix_tree_lookup(const struct radix_tree_root *root,

			  unsigned long index, struct radix_tree_node **nodep,

			  void ***slotp)

			  void __rcu ***slotp)

{

	struct radix_tree_node *node, *parent;

	unsigned long maxindex;

	void **slot;

	void __rcu **slot;



 restart:

	parent = NULL;

	slot = (void **)&root->rnode;

	slot = (void __rcu **)&root->rnode;

	radix_tree_load_root(root, &node, &maxindex);

	if (index > maxindex)

		return NULL;
@@ -1066,10 +1071,10 @@ void *__radix_tree_lookup(const struct radix_tree_root *root, 
 *	exclusive from other writers. Any dereference of the slot must be done

 *	using radix_tree_deref_slot.

 */

void **radix_tree_lookup_slot(const struct radix_tree_root *root,

void __rcu **radix_tree_lookup_slot(const struct radix_tree_root *root,

				unsigned long index)

{

	void **slot;

	void __rcu **slot;



	if (!__radix_tree_lookup(root, index, NULL, &slot))

		return NULL;
@@ -1096,7 +1101,7 @@ void *radix_tree_lookup(const struct radix_tree_root *root, unsigned long index) 
EXPORT_SYMBOL(radix_tree_lookup);



static inline void replace_sibling_entries(struct radix_tree_node *node,

				void **slot, int count, int exceptional)

				void __rcu **slot, int count, int exceptional)

{

#ifdef CONFIG_RADIX_TREE_MULTIORDER

	void *ptr = node_to_entry(slot);
@@ -1115,8 +1120,8 @@ static inline void replace_sibling_entries(struct radix_tree_node *node, 
#endif

}



static void replace_slot(void **slot, void *item, struct radix_tree_node *node,

				int count, int exceptional)

static void replace_slot(void __rcu **slot, void *item,

		struct radix_tree_node *node, int count, int exceptional)

{

	if (WARN_ON_ONCE(radix_tree_is_internal_node(item)))

		return;
@@ -1147,7 +1152,7 @@ static bool node_tag_get(const struct radix_tree_root *root, 
 * deleted.

 */

static int calculate_count(struct radix_tree_root *root,

				struct radix_tree_node *node, void **slot,

				struct radix_tree_node *node, void __rcu **slot,

				void *item, void *old)

{

	if (is_idr(root)) {
@@ -1175,7 +1180,7 @@ static int calculate_count(struct radix_tree_root *root, 
 */

void __radix_tree_replace(struct radix_tree_root *root,

			  struct radix_tree_node *node,

			  void **slot, void *item,

			  void __rcu **slot, void *item,

			  radix_tree_update_node_t update_node, void *private)

{

	void *old = rcu_dereference_raw(*slot);
@@ -1188,7 +1193,7 @@ void __radix_tree_replace(struct radix_tree_root *root, 
	 * deleting entries, but that needs accounting against the

	 * node unless the slot is root->rnode.

	 */

	WARN_ON_ONCE(!node && (slot != (void **)&root->rnode) &&

	WARN_ON_ONCE(!node && (slot != (void __rcu **)&root->rnode) &&

			(count || exceptional));

	replace_slot(slot, item, node, count, exceptional);
@@ -1218,7 +1223,7 @@ void __radix_tree_replace(struct radix_tree_root *root, 
 * radix_tree_iter_replace().

 */

void radix_tree_replace_slot(struct radix_tree_root *root,

			     void **slot, void *item)

			     void __rcu **slot, void *item)

{

	__radix_tree_replace(root, NULL, slot, item, NULL, NULL);

}
@@ -1233,7 +1238,8 @@ void radix_tree_replace_slot(struct radix_tree_root *root, 
 * Caller must hold tree write locked across split and replacement.

 */

void radix_tree_iter_replace(struct radix_tree_root *root,

		const struct radix_tree_iter *iter, void **slot, void *item)

				const struct radix_tree_iter *iter,

				void __rcu **slot, void *item)

{

	__radix_tree_replace(root, iter->node, slot, item, NULL, NULL);

}
@@ -1257,7 +1263,7 @@ int radix_tree_join(struct radix_tree_root *root, unsigned long index, 
			unsigned order, void *item)

{

	struct radix_tree_node *node;

	void **slot;

	void __rcu **slot;

	int error;



	BUG_ON(radix_tree_is_internal_node(item));
@@ -1292,7 +1298,7 @@ int radix_tree_split(struct radix_tree_root *root, unsigned long index, 
				unsigned order)

{

	struct radix_tree_node *parent, *node, *child;

	void **slot;

	void __rcu **slot;

	unsigned int offset, end;

	unsigned n, tag, tags = 0;

	gfp_t gfp = root_gfp_mask(root);
@@ -1603,8 +1609,8 @@ static void set_iter_tags(struct radix_tree_iter *iter, 
}



#ifdef CONFIG_RADIX_TREE_MULTIORDER

static void **skip_siblings(struct radix_tree_node **nodep,

			void **slot, struct radix_tree_iter *iter)

static void __rcu **skip_siblings(struct radix_tree_node **nodep,

			void __rcu **slot, struct radix_tree_iter *iter)

{

	void *sib = node_to_entry(slot - 1);
@@ -1621,8 +1627,8 @@ static void **skip_siblings(struct radix_tree_node **nodep, 
	return NULL;

}



void ** __radix_tree_next_slot(void **slot, struct radix_tree_iter *iter,

					unsigned flags)

void __rcu **__radix_tree_next_slot(void __rcu **slot,

				struct radix_tree_iter *iter, unsigned flags)

{

	unsigned tag = flags & RADIX_TREE_ITER_TAG_MASK;

	struct radix_tree_node *node = rcu_dereference_raw(*slot);
@@ -1675,14 +1681,15 @@ void ** __radix_tree_next_slot(void **slot, struct radix_tree_iter *iter, 
}

EXPORT_SYMBOL(__radix_tree_next_slot);

#else

static void **skip_siblings(struct radix_tree_node **nodep,

			void **slot, struct radix_tree_iter *iter)

static void __rcu **skip_siblings(struct radix_tree_node **nodep,

			void __rcu **slot, struct radix_tree_iter *iter)

{

	return slot;

}

#endif



void **radix_tree_iter_resume(void **slot, struct radix_tree_iter *iter)

void __rcu **radix_tree_iter_resume(void __rcu **slot,

					struct radix_tree_iter *iter)

{

	struct radix_tree_node *node;
@@ -1703,7 +1710,7 @@ EXPORT_SYMBOL(radix_tree_iter_resume); 
 * @flags:	RADIX_TREE_ITER_* flags and tag index

 * Returns:	pointer to chunk first slot, or NULL if iteration is over

 */

void **radix_tree_next_chunk(const struct radix_tree_root *root,

void __rcu **radix_tree_next_chunk(const struct radix_tree_root *root,

			     struct radix_tree_iter *iter, unsigned flags)

{

	unsigned tag = flags & RADIX_TREE_ITER_TAG_MASK;
@@ -1740,7 +1747,7 @@ void **radix_tree_next_chunk(const struct radix_tree_root *root, 
		iter->tags = 1;

		iter->node = NULL;

		__set_iter_shift(iter, 0);

		return (void **)&root->rnode;

		return (void __rcu **)&root->rnode;

	}



	do {
@@ -1819,7 +1826,7 @@ radix_tree_gang_lookup(const struct radix_tree_root *root, void **results, 
			unsigned long first_index, unsigned int max_items)

{

	struct radix_tree_iter iter;

	void **slot;

	void __rcu **slot;

	unsigned int ret = 0;



	if (unlikely(!max_items))
@@ -1861,11 +1868,11 @@ EXPORT_SYMBOL(radix_tree_gang_lookup); 
 */

unsigned int

radix_tree_gang_lookup_slot(const struct radix_tree_root *root,

			void ***results, unsigned long *indices,

			void __rcu ***results, unsigned long *indices,

			unsigned long first_index, unsigned int max_items)

{

	struct radix_tree_iter iter;

	void **slot;

	void __rcu **slot;

	unsigned int ret = 0;



	if (unlikely(!max_items))
@@ -1902,7 +1909,7 @@ radix_tree_gang_lookup_tag(const struct radix_tree_root *root, void **results, 
		unsigned int tag)

{

	struct radix_tree_iter iter;

	void **slot;

	void __rcu **slot;

	unsigned int ret = 0;



	if (unlikely(!max_items))
@@ -1939,11 +1946,11 @@ EXPORT_SYMBOL(radix_tree_gang_lookup_tag); 
 */

unsigned int

radix_tree_gang_lookup_tag_slot(const struct radix_tree_root *root,

		void ***results, unsigned long first_index,

		void __rcu ***results, unsigned long first_index,

		unsigned int max_items, unsigned int tag)

{

	struct radix_tree_iter iter;

	void **slot;

	void __rcu **slot;

	unsigned int ret = 0;



	if (unlikely(!max_items))
@@ -1979,7 +1986,7 @@ void __radix_tree_delete_node(struct radix_tree_root *root, 
}



static bool __radix_tree_delete(struct radix_tree_root *root,

				struct radix_tree_node *node, void **slot)

				struct radix_tree_node *node, void __rcu **slot)

{

	void *old = rcu_dereference_raw(*slot);

	int exceptional = radix_tree_exceptional_entry(old) ? -1 : 0;
@@ -2009,7 +2016,7 @@ static bool __radix_tree_delete(struct radix_tree_root *root, 
 * which can access this tree.

 */

void radix_tree_iter_delete(struct radix_tree_root *root,

				struct radix_tree_iter *iter, void **slot)

				struct radix_tree_iter *iter, void __rcu **slot)

{

	if (__radix_tree_delete(root, iter->node, slot))

		iter->index = iter->next_index;
@@ -2030,7 +2037,7 @@ void *radix_tree_delete_item(struct radix_tree_root *root, 
			     unsigned long index, void *item)

{

	struct radix_tree_node *node = NULL;

	void **slot;

	void __rcu **slot;

	void *entry;



	entry = __radix_tree_lookup(root, index, &node, &slot);
@@ -2064,7 +2071,7 @@ EXPORT_SYMBOL(radix_tree_delete); 


void radix_tree_clear_tags(struct radix_tree_root *root,

			   struct radix_tree_node *node,

			   void **slot)

			   void __rcu **slot)

{

	if (node) {

		unsigned int tag, offset = get_slot_offset(node, slot);
@@ -2129,11 +2136,11 @@ int ida_pre_get(struct ida *ida, gfp_t gfp) 
}

EXPORT_SYMBOL(ida_pre_get);



void **idr_get_free(struct radix_tree_root *root,

void __rcu **idr_get_free(struct radix_tree_root *root,

			struct radix_tree_iter *iter, gfp_t gfp, int end)

{

	struct radix_tree_node *node = NULL, *child;

	void **slot = (void **)&root->rnode;

	void __rcu **slot = (void __rcu **)&root->rnode;

	unsigned long maxindex, start = iter->next_index;

	unsigned long max = end > 0 ? end - 1 : INT_MAX;

	unsigned int shift, offset = 0;