Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 0fc9d104 authored by Konstantin Khlebnikov's avatar Konstantin Khlebnikov Committed by Linus Torvalds
Browse files

radix-tree: use iterators in find_get_pages* functions 



Replace radix_tree_gang_lookup_slot() and
radix_tree_gang_lookup_tag_slot() in page-cache lookup functions with
brand-new radix-tree direct iterating.  This avoids the double-scanning
and pointer copying.

Iterator don't stop after nr_pages page-get fails in a row, it continue
lookup till the radix-tree end.  Thus we can safely remove these restart
conditions.

Unfortunately, old implementation didn't forbid nr_pages == 0, this corner
case does not fit into new code, so the patch adds an extra check at the
beginning.

Signed-off-by: default avatarKonstantin Khlebnikov <khlebnikov@openvz.org>
Tested-by: default avatarHugh Dickins <hughd@google.com>
Cc: Christoph Hellwig <hch@lst.de>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent cebbd29e

mm/filemap.c

+38 −48
Original line number Diff line number Diff line
@@ -813,20 +813,19 @@ EXPORT_SYMBOL(find_or_create_page); 
unsigned find_get_pages(struct address_space *mapping, pgoff_t start,

			    unsigned int nr_pages, struct page **pages)

{

	unsigned int i;

	unsigned int ret;

	unsigned int nr_found, nr_skip;

	struct radix_tree_iter iter;

	void **slot;

	unsigned ret = 0;



	if (unlikely(!nr_pages))

		return 0;



	rcu_read_lock();

restart:

	nr_found = radix_tree_gang_lookup_slot(&mapping->page_tree,

				(void ***)pages, NULL, start, nr_pages);

	ret = 0;

	nr_skip = 0;

	for (i = 0; i < nr_found; i++) {

	radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {

		struct page *page;

repeat:

		page = radix_tree_deref_slot((void **)pages[i]);

		page = radix_tree_deref_slot(slot);

		if (unlikely(!page))

			continue;
@@ -837,7 +836,7 @@ unsigned find_get_pages(struct address_space *mapping, pgoff_t start, 
				 * when entry at index 0 moves out of or back

				 * to root: none yet gotten, safe to restart.

				 */

				WARN_ON(start | i);

				WARN_ON(iter.index);

				goto restart;

			}

			/*
@@ -845,7 +844,6 @@ unsigned find_get_pages(struct address_space *mapping, pgoff_t start, 
			 * here as an exceptional entry: so skip over it -

			 * we only reach this from invalidate_mapping_pages().

			 */

			nr_skip++;

			continue;

		}
@@ -853,21 +851,16 @@ unsigned find_get_pages(struct address_space *mapping, pgoff_t start, 
			goto repeat;



		/* Has the page moved? */

		if (unlikely(page != *((void **)pages[i]))) {

		if (unlikely(page != *slot)) {

			page_cache_release(page);

			goto repeat;

		}



		pages[ret] = page;

		ret++;

		if (++ret == nr_pages)

			break;

	}



	/*

	 * If all entries were removed before we could secure them,

	 * try again, because callers stop trying once 0 is returned.

	 */

	if (unlikely(!ret && nr_found > nr_skip))

		goto restart;

	rcu_read_unlock();

	return ret;

}
@@ -887,21 +880,22 @@ unsigned find_get_pages(struct address_space *mapping, pgoff_t start, 
unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index,

			       unsigned int nr_pages, struct page **pages)

{

	unsigned int i;

	unsigned int ret;

	unsigned int nr_found;

	struct radix_tree_iter iter;

	void **slot;

	unsigned int ret = 0;



	if (unlikely(!nr_pages))

		return 0;



	rcu_read_lock();

restart:

	nr_found = radix_tree_gang_lookup_slot(&mapping->page_tree,

				(void ***)pages, NULL, index, nr_pages);

	ret = 0;

	for (i = 0; i < nr_found; i++) {

	radix_tree_for_each_contig(slot, &mapping->page_tree, &iter, index) {

		struct page *page;

repeat:

		page = radix_tree_deref_slot((void **)pages[i]);

		page = radix_tree_deref_slot(slot);

		/* The hole, there no reason to continue */

		if (unlikely(!page))

			continue;

			break;



		if (radix_tree_exception(page)) {

			if (radix_tree_deref_retry(page)) {
@@ -924,7 +918,7 @@ unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index, 
			goto repeat;



		/* Has the page moved? */

		if (unlikely(page != *((void **)pages[i]))) {

		if (unlikely(page != *slot)) {

			page_cache_release(page);

			goto repeat;

		}
@@ -934,14 +928,14 @@ unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index, 
		 * otherwise we can get both false positives and false

		 * negatives, which is just confusing to the caller.

		 */

		if (page->mapping == NULL || page->index != index) {

		if (page->mapping == NULL || page->index != iter.index) {

			page_cache_release(page);

			break;

		}



		pages[ret] = page;

		ret++;

		index++;

		if (++ret == nr_pages)

			break;

	}

	rcu_read_unlock();

	return ret;
@@ -962,19 +956,20 @@ EXPORT_SYMBOL(find_get_pages_contig); 
unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index,

			int tag, unsigned int nr_pages, struct page **pages)

{

	unsigned int i;

	unsigned int ret;

	unsigned int nr_found;

	struct radix_tree_iter iter;

	void **slot;

	unsigned ret = 0;



	if (unlikely(!nr_pages))

		return 0;



	rcu_read_lock();

restart:

	nr_found = radix_tree_gang_lookup_tag_slot(&mapping->page_tree,

				(void ***)pages, *index, nr_pages, tag);

	ret = 0;

	for (i = 0; i < nr_found; i++) {

	radix_tree_for_each_tagged(slot, &mapping->page_tree,

				   &iter, *index, tag) {

		struct page *page;

repeat:

		page = radix_tree_deref_slot((void **)pages[i]);

		page = radix_tree_deref_slot(slot);

		if (unlikely(!page))

			continue;
@@ -998,21 +993,16 @@ unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index, 
			goto repeat;



		/* Has the page moved? */

		if (unlikely(page != *((void **)pages[i]))) {

		if (unlikely(page != *slot)) {

			page_cache_release(page);

			goto repeat;

		}



		pages[ret] = page;

		ret++;

		if (++ret == nr_pages)

			break;

	}



	/*

	 * If all entries were removed before we could secure them,

	 * try again, because callers stop trying once 0 is returned.

	 */

	if (unlikely(!ret && nr_found))

		goto restart;

	rcu_read_unlock();



	if (ret)