Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 664eedde authored by Mel Gorman's avatar Mel Gorman Committed by Linus Torvalds
Browse files

mm: page_alloc: use jump labels to avoid checking number_of_cpusets 



If cpusets are not in use then we still check a global variable on every
page allocation.  Use jump labels to avoid the overhead.

Signed-off-by: default avatarMel Gorman <mgorman@suse.de>
Reviewed-by: default avatarRik van Riel <riel@redhat.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Jan Kara <jack@suse.cz>
Cc: Michal Hocko <mhocko@suse.cz>
Cc: Hugh Dickins <hughd@google.com>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Theodore Ts'o <tytso@mit.edu>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent ea5e9539

include/linux/cpuset.h

+26 −3
Original line number Diff line number Diff line
@@ -12,10 +12,31 @@ 
#include <linux/cpumask.h>

#include <linux/nodemask.h>

#include <linux/mm.h>

#include <linux/jump_label.h>



#ifdef CONFIG_CPUSETS



extern int number_of_cpusets;	/* How many cpusets are defined in system? */

extern struct static_key cpusets_enabled_key;

static inline bool cpusets_enabled(void)

{

	return static_key_false(&cpusets_enabled_key);

}



static inline int nr_cpusets(void)

{

	/* jump label reference count + the top-level cpuset */

	return static_key_count(&cpusets_enabled_key) + 1;

}



static inline void cpuset_inc(void)

{

	static_key_slow_inc(&cpusets_enabled_key);

}



static inline void cpuset_dec(void)

{

	static_key_slow_dec(&cpusets_enabled_key);

}



extern int cpuset_init(void);

extern void cpuset_init_smp(void);
@@ -32,13 +53,13 @@ extern int __cpuset_node_allowed_hardwall(int node, gfp_t gfp_mask); 


static inline int cpuset_node_allowed_softwall(int node, gfp_t gfp_mask)

{

	return number_of_cpusets <= 1 ||

	return nr_cpusets() <= 1 ||

		__cpuset_node_allowed_softwall(node, gfp_mask);

}



static inline int cpuset_node_allowed_hardwall(int node, gfp_t gfp_mask)

{

	return number_of_cpusets <= 1 ||

	return nr_cpusets() <= 1 ||

		__cpuset_node_allowed_hardwall(node, gfp_mask);

}
@@ -124,6 +145,8 @@ static inline void set_mems_allowed(nodemask_t nodemask) 


#else /* !CONFIG_CPUSETS */



static inline bool cpusets_enabled(void) { return false; }



static inline int cpuset_init(void) { return 0; }

static inline void cpuset_init_smp(void) {}

kernel/cpuset.c

+4 −10
Original line number Diff line number Diff line
@@ -61,12 +61,7 @@ 
#include <linux/cgroup.h>

#include <linux/wait.h>



/*

 * Tracks how many cpusets are currently defined in system.

 * When there is only one cpuset (the root cpuset) we can

 * short circuit some hooks.

 */

int number_of_cpusets __read_mostly;

struct static_key cpusets_enabled_key __read_mostly = STATIC_KEY_INIT_FALSE;



/* See "Frequency meter" comments, below. */
@@ -611,7 +606,7 @@ static int generate_sched_domains(cpumask_var_t **domains, 
		goto done;

	}



	csa = kmalloc(number_of_cpusets * sizeof(cp), GFP_KERNEL);

	csa = kmalloc(nr_cpusets() * sizeof(cp), GFP_KERNEL);

	if (!csa)

		goto done;

	csn = 0;
@@ -1888,7 +1883,7 @@ static int cpuset_css_online(struct cgroup_subsys_state *css) 
	if (is_spread_slab(parent))

		set_bit(CS_SPREAD_SLAB, &cs->flags);



	number_of_cpusets++;

	cpuset_inc();



	if (!test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags))

		goto out_unlock;
@@ -1939,7 +1934,7 @@ static void cpuset_css_offline(struct cgroup_subsys_state *css) 
	if (is_sched_load_balance(cs))

		update_flag(CS_SCHED_LOAD_BALANCE, cs, 0);



	number_of_cpusets--;

	cpuset_dec();

	clear_bit(CS_ONLINE, &cs->flags);



	mutex_unlock(&cpuset_mutex);
@@ -1992,7 +1987,6 @@ int __init cpuset_init(void) 
	if (!alloc_cpumask_var(&cpus_attach, GFP_KERNEL))

		BUG();



	number_of_cpusets = 1;

	return 0;

}

mm/page_alloc.c

+2 −1
Original line number Diff line number Diff line
@@ -1921,7 +1921,8 @@ get_page_from_freelist(gfp_t gfp_mask, nodemask_t *nodemask, unsigned int order, 
		if (IS_ENABLED(CONFIG_NUMA) && zlc_active &&

			!zlc_zone_worth_trying(zonelist, z, allowednodes))

				continue;

		if ((alloc_flags & ALLOC_CPUSET) &&

		if (cpusets_enabled() &&

			(alloc_flags & ALLOC_CPUSET) &&

			!cpuset_zone_allowed_softwall(zone, gfp_mask))

				continue;

		BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);