Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 6c99e9ad authored by Rusty Russell's avatar Rusty Russell Committed by Ingo Molnar
Browse files

sched: convert struct sched_group/sched_domain cpumask_ts to variable bitmaps 



Impact: (future) size reduction for large NR_CPUS.

We move the 'cpumask' member of sched_group to the end, so when we
kmalloc it we can do a minimal allocation: saves space for small
nr_cpu_ids but big CONFIG_NR_CPUS.  Similar trick for 'span' in
sched_domain.

This isn't quite as good as converting to a cpumask_var_t, as some
sched_groups are actually static, but it's safer: we don't have to
figure out where to call alloc_cpumask_var/free_cpumask_var.

Signed-off-by: default avatarRusty Russell <rusty@rustcorp.com.au>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent 758b2cdc

include/linux/sched.h

+7 −4
Original line number Diff line number Diff line
@@ -771,7 +771,6 @@ enum cpu_idle_type { 


struct sched_group {

	struct sched_group *next;	/* Must be a circular list */

	cpumask_t cpumask;



	/*

	 * CPU power of this group, SCHED_LOAD_SCALE being max power for a
@@ -784,11 +783,13 @@ struct sched_group { 
	 * (see include/linux/reciprocal_div.h)

	 */

	u32 reciprocal_cpu_power;



	unsigned long cpumask[];

};



static inline struct cpumask *sched_group_cpus(struct sched_group *sg)

{

	return &sg->cpumask;

	return to_cpumask(sg->cpumask);

}



enum sched_domain_level {
@@ -814,7 +815,6 @@ struct sched_domain { 
	struct sched_domain *parent;	/* top domain must be null terminated */

	struct sched_domain *child;	/* bottom domain must be null terminated */

	struct sched_group *groups;	/* the balancing groups of the domain */

	cpumask_t span;			/* span of all CPUs in this domain */

	unsigned long min_interval;	/* Minimum balance interval ms */

	unsigned long max_interval;	/* Maximum balance interval ms */

	unsigned int busy_factor;	/* less balancing by factor if busy */
@@ -869,11 +869,14 @@ struct sched_domain { 
#ifdef CONFIG_SCHED_DEBUG

	char *name;

#endif



	/* span of all CPUs in this domain */

	unsigned long span[];

};



static inline struct cpumask *sched_domain_span(struct sched_domain *sd)

{

	return &sd->span;

	return to_cpumask(sd->span);

}



extern void partition_sched_domains(int ndoms_new, cpumask_t *doms_new,

kernel/sched.c

+41 −24
Original line number Diff line number Diff line
@@ -7005,19 +7005,34 @@ static void sched_domain_node_span(int node, cpumask_t *span) 


int sched_smt_power_savings = 0, sched_mc_power_savings = 0;



/*

 * The cpus mask in sched_group and sched_domain hangs off the end.

 * FIXME: use cpumask_var_t or dynamic percpu alloc to avoid wasting space

 * for nr_cpu_ids < CONFIG_NR_CPUS.

 */

struct static_sched_group {

	struct sched_group sg;

	DECLARE_BITMAP(cpus, CONFIG_NR_CPUS);

};



struct static_sched_domain {

	struct sched_domain sd;

	DECLARE_BITMAP(span, CONFIG_NR_CPUS);

};



/*

 * SMT sched-domains:

 */

#ifdef CONFIG_SCHED_SMT

static DEFINE_PER_CPU(struct sched_domain, cpu_domains);

static DEFINE_PER_CPU(struct sched_group, sched_group_cpus);

static DEFINE_PER_CPU(struct static_sched_domain, cpu_domains);

static DEFINE_PER_CPU(struct static_sched_group, sched_group_cpus);



static int

cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,

		 cpumask_t *unused)

{

	if (sg)

		*sg = &per_cpu(sched_group_cpus, cpu);

		*sg = &per_cpu(sched_group_cpus, cpu).sg;

	return cpu;

}

#endif /* CONFIG_SCHED_SMT */
@@ -7026,8 +7041,8 @@ cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 
 * multi-core sched-domains:

 */

#ifdef CONFIG_SCHED_MC

static DEFINE_PER_CPU(struct sched_domain, core_domains);

static DEFINE_PER_CPU(struct sched_group, sched_group_core);

static DEFINE_PER_CPU(struct static_sched_domain, core_domains);

static DEFINE_PER_CPU(struct static_sched_group, sched_group_core);

#endif /* CONFIG_SCHED_MC */



#if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT)
@@ -7041,7 +7056,7 @@ cpu_to_core_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 
	cpus_and(*mask, *mask, *cpu_map);

	group = first_cpu(*mask);

	if (sg)

		*sg = &per_cpu(sched_group_core, group);

		*sg = &per_cpu(sched_group_core, group).sg;

	return group;

}

#elif defined(CONFIG_SCHED_MC)
@@ -7050,13 +7065,13 @@ cpu_to_core_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 
		  cpumask_t *unused)

{

	if (sg)

		*sg = &per_cpu(sched_group_core, cpu);

		*sg = &per_cpu(sched_group_core, cpu).sg;

	return cpu;

}

#endif



static DEFINE_PER_CPU(struct sched_domain, phys_domains);

static DEFINE_PER_CPU(struct sched_group, sched_group_phys);

static DEFINE_PER_CPU(struct static_sched_domain, phys_domains);

static DEFINE_PER_CPU(struct static_sched_group, sched_group_phys);



static int

cpu_to_phys_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
@@ -7075,7 +7090,7 @@ cpu_to_phys_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 
	group = cpu;

#endif

	if (sg)

		*sg = &per_cpu(sched_group_phys, group);

		*sg = &per_cpu(sched_group_phys, group).sg;

	return group;

}
@@ -7089,7 +7104,7 @@ static DEFINE_PER_CPU(struct sched_domain, node_domains); 
static struct sched_group ***sched_group_nodes_bycpu;



static DEFINE_PER_CPU(struct sched_domain, allnodes_domains);

static DEFINE_PER_CPU(struct sched_group, sched_group_allnodes);

static DEFINE_PER_CPU(struct static_sched_group, sched_group_allnodes);



static int cpu_to_allnodes_group(int cpu, const cpumask_t *cpu_map,

				 struct sched_group **sg, cpumask_t *nodemask)
@@ -7101,7 +7116,7 @@ static int cpu_to_allnodes_group(int cpu, const cpumask_t *cpu_map, 
	group = first_cpu(*nodemask);



	if (sg)

		*sg = &per_cpu(sched_group_allnodes, group);

		*sg = &per_cpu(sched_group_allnodes, group).sg;

	return group;

}
@@ -7116,7 +7131,7 @@ static void init_numa_sched_groups_power(struct sched_group *group_head) 
		for_each_cpu(j, sched_group_cpus(sg)) {

			struct sched_domain *sd;



			sd = &per_cpu(phys_domains, j);

			sd = &per_cpu(phys_domains, j).sd;

			if (j != cpumask_first(sched_group_cpus(sd->groups))) {

				/*

				 * Only add "power" once for each
@@ -7385,7 +7400,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map, 
#endif



		p = sd;

		sd = &per_cpu(phys_domains, i);

		sd = &per_cpu(phys_domains, i).sd;

		SD_INIT(sd, CPU);

		set_domain_attribute(sd, attr);

		cpumask_copy(sched_domain_span(sd), nodemask);
@@ -7396,7 +7411,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map, 


#ifdef CONFIG_SCHED_MC

		p = sd;

		sd = &per_cpu(core_domains, i);

		sd = &per_cpu(core_domains, i).sd;

		SD_INIT(sd, MC);

		set_domain_attribute(sd, attr);

		*sched_domain_span(sd) = cpu_coregroup_map(i);
@@ -7409,7 +7424,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map, 


#ifdef CONFIG_SCHED_SMT

		p = sd;

		sd = &per_cpu(cpu_domains, i);

		sd = &per_cpu(cpu_domains, i).sd;

		SD_INIT(sd, SIBLING);

		set_domain_attribute(sd, attr);

		cpumask_and(sched_domain_span(sd),
@@ -7485,7 +7500,8 @@ static int __build_sched_domains(const cpumask_t *cpu_map, 
		sched_domain_node_span(i, domainspan);

		cpus_and(*domainspan, *domainspan, *cpu_map);



		sg = kmalloc_node(sizeof(struct sched_group), GFP_KERNEL, i);

		sg = kmalloc_node(sizeof(struct sched_group) + cpumask_size(),

				  GFP_KERNEL, i);

		if (!sg) {

			printk(KERN_WARNING "Can not alloc domain group for "

				"node %d\n", i);
@@ -7518,7 +7534,8 @@ static int __build_sched_domains(const cpumask_t *cpu_map, 
			if (cpus_empty(*tmpmask))

				continue;



			sg = kmalloc_node(sizeof(struct sched_group),

			sg = kmalloc_node(sizeof(struct sched_group) +

					  cpumask_size(),

					  GFP_KERNEL, i);

			if (!sg) {

				printk(KERN_WARNING
@@ -7538,21 +7555,21 @@ static int __build_sched_domains(const cpumask_t *cpu_map, 
	/* Calculate CPU power for physical packages and nodes */

#ifdef CONFIG_SCHED_SMT

	for_each_cpu(i, cpu_map) {

		struct sched_domain *sd = &per_cpu(cpu_domains, i);

		struct sched_domain *sd = &per_cpu(cpu_domains, i).sd;



		init_sched_groups_power(i, sd);

	}

#endif

#ifdef CONFIG_SCHED_MC

	for_each_cpu(i, cpu_map) {

		struct sched_domain *sd = &per_cpu(core_domains, i);

		struct sched_domain *sd = &per_cpu(core_domains, i).sd;



		init_sched_groups_power(i, sd);

	}

#endif



	for_each_cpu(i, cpu_map) {

		struct sched_domain *sd = &per_cpu(phys_domains, i);

		struct sched_domain *sd = &per_cpu(phys_domains, i).sd;



		init_sched_groups_power(i, sd);

	}
@@ -7574,11 +7591,11 @@ static int __build_sched_domains(const cpumask_t *cpu_map, 
	for_each_cpu(i, cpu_map) {

		struct sched_domain *sd;

#ifdef CONFIG_SCHED_SMT

		sd = &per_cpu(cpu_domains, i);

		sd = &per_cpu(cpu_domains, i).sd;

#elif defined(CONFIG_SCHED_MC)

		sd = &per_cpu(core_domains, i);

		sd = &per_cpu(core_domains, i).sd;

#else

		sd = &per_cpu(phys_domains, i);

		sd = &per_cpu(phys_domains, i).sd;

#endif

		cpu_attach_domain(sd, rd, i);

	}