Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 341ce06f authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Linus Torvalds
Browse files

page allocator: calculate the alloc_flags for allocation only once 



Factor out the mapping between GFP and alloc_flags only once.  Once
factored out, it only needs to be calculated once but some care must be
taken.

[neilb@suse.de says]
As the test:

-       if (((p->flags & PF_MEMALLOC) || unlikely(test_thread_flag(TIF_MEMDIE)))
-                       && !in_interrupt()) {
-               if (!(gfp_mask & __GFP_NOMEMALLOC)) {

has been replaced with a slightly weaker one:

+       if (alloc_flags & ALLOC_NO_WATERMARKS) {

Without care, this would allow recursion into the allocator via direct
reclaim.  This patch ensures we do not recurse when PF_MEMALLOC is set but
TF_MEMDIE callers are now allowed to directly reclaim where they would
have been prevented in the past.

Signed-off-by: default avatarPeter Zijlstra <a.p.zijlstra@chello.nl>
Acked-by: default avatarPekka Enberg <penberg@cs.helsinki.fi>
Signed-off-by: default avatarMel Gorman <mel@csn.ul.ie>
Cc: Neil Brown <neilb@suse.de>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Cc: Dave Hansen <dave@linux.vnet.ibm.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 3dd28266

mm/page_alloc.c

+50 −44
Original line number Diff line number Diff line
@@ -1574,15 +1574,6 @@ __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, 
	return page;

}



static inline int

is_allocation_high_priority(struct task_struct *p, gfp_t gfp_mask)

{

	if (((p->flags & PF_MEMALLOC) || unlikely(test_thread_flag(TIF_MEMDIE)))

			&& !in_interrupt())

		return 1;

	return 0;

}



/*

 * This is called in the allocator slow-path if the allocation request is of

 * sufficient urgency to ignore watermarks and take other desperate measures
@@ -1618,6 +1609,42 @@ void wake_all_kswapd(unsigned int order, struct zonelist *zonelist, 
		wakeup_kswapd(zone, order);

}



static inline int

gfp_to_alloc_flags(gfp_t gfp_mask)

{

	struct task_struct *p = current;

	int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;

	const gfp_t wait = gfp_mask & __GFP_WAIT;



	/*

	 * The caller may dip into page reserves a bit more if the caller

	 * cannot run direct reclaim, or if the caller has realtime scheduling

	 * policy or is asking for __GFP_HIGH memory.  GFP_ATOMIC requests will

	 * set both ALLOC_HARDER (!wait) and ALLOC_HIGH (__GFP_HIGH).

	 */

	if (gfp_mask & __GFP_HIGH)

		alloc_flags |= ALLOC_HIGH;



	if (!wait) {

		alloc_flags |= ALLOC_HARDER;

		/*

		 * Ignore cpuset if GFP_ATOMIC (!wait) rather than fail alloc.

		 * See also cpuset_zone_allowed() comment in kernel/cpuset.c.

		 */

		alloc_flags &= ~ALLOC_CPUSET;

	} else if (unlikely(rt_task(p)))

		alloc_flags |= ALLOC_HARDER;



	if (likely(!(gfp_mask & __GFP_NOMEMALLOC))) {

		if (!in_interrupt() &&

		    ((p->flags & PF_MEMALLOC) ||

		     unlikely(test_thread_flag(TIF_MEMDIE))))

			alloc_flags |= ALLOC_NO_WATERMARKS;

	}



	return alloc_flags;

}



static inline struct page *

__alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,

	struct zonelist *zonelist, enum zone_type high_zoneidx,
@@ -1648,56 +1675,35 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, 
	 * OK, we're below the kswapd watermark and have kicked background

	 * reclaim. Now things get more complex, so set up alloc_flags according

	 * to how we want to proceed.

	 *

	 * The caller may dip into page reserves a bit more if the caller

	 * cannot run direct reclaim, or if the caller has realtime scheduling

	 * policy or is asking for __GFP_HIGH memory.  GFP_ATOMIC requests will

	 * set both ALLOC_HARDER (!wait) and ALLOC_HIGH (__GFP_HIGH).

	 */

	alloc_flags = ALLOC_WMARK_MIN;

	if ((unlikely(rt_task(p)) && !in_interrupt()) || !wait)

		alloc_flags |= ALLOC_HARDER;

	if (gfp_mask & __GFP_HIGH)

		alloc_flags |= ALLOC_HIGH;

	if (wait)

		alloc_flags |= ALLOC_CPUSET;

	alloc_flags = gfp_to_alloc_flags(gfp_mask);



restart:

	/*

	 * Go through the zonelist again. Let __GFP_HIGH and allocations

	 * coming from realtime tasks go deeper into reserves.

	 *

	 * This is the last chance, in general, before the goto nopage.

	 * Ignore cpuset if GFP_ATOMIC (!wait) rather than fail alloc.

	 * See also cpuset_zone_allowed() comment in kernel/cpuset.c.

	 */

	/* This is the last chance, in general, before the goto nopage. */

	page = get_page_from_freelist(gfp_mask, nodemask, order, zonelist,

						high_zoneidx, alloc_flags,

						preferred_zone,

						migratetype);

			high_zoneidx, alloc_flags & ~ALLOC_NO_WATERMARKS,

			preferred_zone, migratetype);

	if (page)

		goto got_pg;



rebalance:

	/* Allocate without watermarks if the context allows */

	if (is_allocation_high_priority(p, gfp_mask)) {

		/* Do not dip into emergency reserves if specified */

		if (!(gfp_mask & __GFP_NOMEMALLOC)) {

	if (alloc_flags & ALLOC_NO_WATERMARKS) {

		page = __alloc_pages_high_priority(gfp_mask, order,

				zonelist, high_zoneidx, nodemask, preferred_zone,

				migratetype);

				zonelist, high_zoneidx, nodemask,

				preferred_zone, migratetype);

		if (page)

			goto got_pg;

	}



		/* Ensure no recursion into the allocator */

		goto nopage;

	}



	/* Atomic allocations - we can't balance anything */

	if (!wait)

		goto nopage;



	/* Avoid recursion of direct reclaim */

	if (p->flags & PF_MEMALLOC)

		goto nopage;



	/* Try direct reclaim and then allocating */

	page = __alloc_pages_direct_reclaim(gfp_mask, order,

					zonelist, high_zoneidx,