Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 64d3e9a9 authored by Matthew Wilcox's avatar Matthew Wilcox
Browse files

xarray: Step through an XArray 



The xas_next and xas_prev functions move the xas index by one position,
and adjust the rest of the iterator state to match it.  This is more
efficient than calling xas_set() as it keeps the iterator at the leaves
of the tree instead of walking the iterator from the root each time.

Signed-off-by: default avatarMatthew Wilcox <willy@infradead.org>
parent 687149fc

include/linux/xarray.h

+67 −0
Original line number Diff line number Diff line
@@ -828,6 +828,12 @@ static inline bool xas_not_node(struct xa_node *node) 
	return ((unsigned long)node & 3) || !node;

}



/* True if the node represents RESTART or an error */

static inline bool xas_frozen(struct xa_node *node)

{

	return (unsigned long)node & 2;

}



/* True if the node represents head-of-tree, RESTART or BOUNDS */

static inline bool xas_top(struct xa_node *node)

{
@@ -1082,4 +1088,65 @@ enum { 
	for (entry = xas_find_marked(xas, max, mark); entry; \

	     entry = xas_next_marked(xas, max, mark))



void *__xas_next(struct xa_state *);

void *__xas_prev(struct xa_state *);



/**

 * xas_prev() - Move iterator to previous index.

 * @xas: XArray operation state.

 *

 * If the @xas was in an error state, it will remain in an error state

 * and this function will return %NULL.  If the @xas has never been walked,

 * it will have the effect of calling xas_load().  Otherwise one will be

 * subtracted from the index and the state will be walked to the correct

 * location in the array for the next operation.

 *

 * If the iterator was referencing index 0, this function wraps

 * around to %ULONG_MAX.

 *

 * Return: The entry at the new index.  This may be %NULL or an internal

 * entry.

 */

static inline void *xas_prev(struct xa_state *xas)

{

	struct xa_node *node = xas->xa_node;



	if (unlikely(xas_not_node(node) || node->shift ||

				xas->xa_offset == 0))

		return __xas_prev(xas);



	xas->xa_index--;

	xas->xa_offset--;

	return xa_entry(xas->xa, node, xas->xa_offset);

}



/**

 * xas_next() - Move state to next index.

 * @xas: XArray operation state.

 *

 * If the @xas was in an error state, it will remain in an error state

 * and this function will return %NULL.  If the @xas has never been walked,

 * it will have the effect of calling xas_load().  Otherwise one will be

 * added to the index and the state will be walked to the correct

 * location in the array for the next operation.

 *

 * If the iterator was referencing index %ULONG_MAX, this function wraps

 * around to 0.

 *

 * Return: The entry at the new index.  This may be %NULL or an internal

 * entry.

 */

static inline void *xas_next(struct xa_state *xas)

{

	struct xa_node *node = xas->xa_node;



	if (unlikely(xas_not_node(node) || node->shift ||

				xas->xa_offset == XA_CHUNK_MASK))

		return __xas_next(xas);



	xas->xa_index++;

	xas->xa_offset++;

	return xa_entry(xas->xa, node, xas->xa_offset);

}



#endif /* _LINUX_XARRAY_H */

lib/test_xarray.c

+115 −0
Original line number Diff line number Diff line
@@ -465,6 +465,120 @@ static noinline void check_find(struct xarray *xa) 
	check_multi_find_2(xa);

}



static noinline void check_move_small(struct xarray *xa, unsigned long idx)

{

	XA_STATE(xas, xa, 0);

	unsigned long i;



	xa_store_index(xa, 0, GFP_KERNEL);

	xa_store_index(xa, idx, GFP_KERNEL);



	rcu_read_lock();

	for (i = 0; i < idx * 4; i++) {

		void *entry = xas_next(&xas);

		if (i <= idx)

			XA_BUG_ON(xa, xas.xa_node == XAS_RESTART);

		XA_BUG_ON(xa, xas.xa_index != i);

		if (i == 0 || i == idx)

			XA_BUG_ON(xa, entry != xa_mk_value(i));

		else

			XA_BUG_ON(xa, entry != NULL);

	}

	xas_next(&xas);

	XA_BUG_ON(xa, xas.xa_index != i);



	do {

		void *entry = xas_prev(&xas);

		i--;

		if (i <= idx)

			XA_BUG_ON(xa, xas.xa_node == XAS_RESTART);

		XA_BUG_ON(xa, xas.xa_index != i);

		if (i == 0 || i == idx)

			XA_BUG_ON(xa, entry != xa_mk_value(i));

		else

			XA_BUG_ON(xa, entry != NULL);

	} while (i > 0);



	xas_set(&xas, ULONG_MAX);

	XA_BUG_ON(xa, xas_next(&xas) != NULL);

	XA_BUG_ON(xa, xas.xa_index != ULONG_MAX);

	XA_BUG_ON(xa, xas_next(&xas) != xa_mk_value(0));

	XA_BUG_ON(xa, xas.xa_index != 0);

	XA_BUG_ON(xa, xas_prev(&xas) != NULL);

	XA_BUG_ON(xa, xas.xa_index != ULONG_MAX);

	rcu_read_unlock();



	xa_erase_index(xa, 0);

	xa_erase_index(xa, idx);

	XA_BUG_ON(xa, !xa_empty(xa));

}



static noinline void check_move(struct xarray *xa)

{

	XA_STATE(xas, xa, (1 << 16) - 1);

	unsigned long i;



	for (i = 0; i < (1 << 16); i++)

		XA_BUG_ON(xa, xa_store_index(xa, i, GFP_KERNEL) != NULL);



	rcu_read_lock();

	do {

		void *entry = xas_prev(&xas);

		i--;

		XA_BUG_ON(xa, entry != xa_mk_value(i));

		XA_BUG_ON(xa, i != xas.xa_index);

	} while (i != 0);



	XA_BUG_ON(xa, xas_prev(&xas) != NULL);

	XA_BUG_ON(xa, xas.xa_index != ULONG_MAX);



	do {

		void *entry = xas_next(&xas);

		XA_BUG_ON(xa, entry != xa_mk_value(i));

		XA_BUG_ON(xa, i != xas.xa_index);

		i++;

	} while (i < (1 << 16));

	rcu_read_unlock();



	for (i = (1 << 8); i < (1 << 15); i++)

		xa_erase_index(xa, i);



	i = xas.xa_index;



	rcu_read_lock();

	do {

		void *entry = xas_prev(&xas);

		i--;

		if ((i < (1 << 8)) || (i >= (1 << 15)))

			XA_BUG_ON(xa, entry != xa_mk_value(i));

		else

			XA_BUG_ON(xa, entry != NULL);

		XA_BUG_ON(xa, i != xas.xa_index);

	} while (i != 0);



	XA_BUG_ON(xa, xas_prev(&xas) != NULL);

	XA_BUG_ON(xa, xas.xa_index != ULONG_MAX);



	do {

		void *entry = xas_next(&xas);

		if ((i < (1 << 8)) || (i >= (1 << 15)))

			XA_BUG_ON(xa, entry != xa_mk_value(i));

		else

			XA_BUG_ON(xa, entry != NULL);

		XA_BUG_ON(xa, i != xas.xa_index);

		i++;

	} while (i < (1 << 16));

	rcu_read_unlock();



	xa_destroy(xa);



	for (i = 0; i < 16; i++)

		check_move_small(xa, 1UL << i);



	for (i = 2; i < 16; i++)

		check_move_small(xa, (1UL << i) - 1);

}



static noinline void check_destroy(struct xarray *xa)

{

	unsigned long index;
@@ -512,6 +626,7 @@ static int xarray_checks(void) 
	check_multi_store(&array);

	check_find(&array);

	check_destroy(&array);

	check_move(&array);



	printk("XArray: %u of %u tests passed\n", tests_passed, tests_run);

	return (tests_run == tests_passed) ? 0 : -EINVAL;

lib/xarray.c

+74 −0
Original line number Diff line number Diff line
@@ -876,6 +876,80 @@ void xas_pause(struct xa_state *xas) 
}

EXPORT_SYMBOL_GPL(xas_pause);



/*

 * __xas_prev() - Find the previous entry in the XArray.

 * @xas: XArray operation state.

 *

 * Helper function for xas_prev() which handles all the complex cases

 * out of line.

 */

void *__xas_prev(struct xa_state *xas)

{

	void *entry;



	if (!xas_frozen(xas->xa_node))

		xas->xa_index--;

	if (xas_not_node(xas->xa_node))

		return xas_load(xas);



	if (xas->xa_offset != get_offset(xas->xa_index, xas->xa_node))

		xas->xa_offset--;



	while (xas->xa_offset == 255) {

		xas->xa_offset = xas->xa_node->offset - 1;

		xas->xa_node = xa_parent(xas->xa, xas->xa_node);

		if (!xas->xa_node)

			return set_bounds(xas);

	}



	for (;;) {

		entry = xa_entry(xas->xa, xas->xa_node, xas->xa_offset);

		if (!xa_is_node(entry))

			return entry;



		xas->xa_node = xa_to_node(entry);

		xas_set_offset(xas);

	}

}

EXPORT_SYMBOL_GPL(__xas_prev);



/*

 * __xas_next() - Find the next entry in the XArray.

 * @xas: XArray operation state.

 *

 * Helper function for xas_next() which handles all the complex cases

 * out of line.

 */

void *__xas_next(struct xa_state *xas)

{

	void *entry;



	if (!xas_frozen(xas->xa_node))

		xas->xa_index++;

	if (xas_not_node(xas->xa_node))

		return xas_load(xas);



	if (xas->xa_offset != get_offset(xas->xa_index, xas->xa_node))

		xas->xa_offset++;



	while (xas->xa_offset == XA_CHUNK_SIZE) {

		xas->xa_offset = xas->xa_node->offset + 1;

		xas->xa_node = xa_parent(xas->xa, xas->xa_node);

		if (!xas->xa_node)

			return set_bounds(xas);

	}



	for (;;) {

		entry = xa_entry(xas->xa, xas->xa_node, xas->xa_offset);

		if (!xa_is_node(entry))

			return entry;



		xas->xa_node = xa_to_node(entry);

		xas_set_offset(xas);

	}

}

EXPORT_SYMBOL_GPL(__xas_next);



/**

 * xas_find() - Find the next present entry in the XArray.

 * @xas: XArray operation state.