Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 669c55e9 authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar
Browse files

sched: Pre-compute cpumask_weight(sched_domain_span(sd)) 



Dave reported that his large SPARC machines spend lots of time in
hweight64(), try and optimize some of those needless cpumask_weight()
invocations (esp. with the large offstack cpumasks these are very
expensive indeed).

Reported-by: default avatarDavid Miller <davem@davemloft.net>
Signed-off-by: default avatarPeter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent 74f5187a

include/linux/sched.h

+1 −0
Original line number Diff line number Diff line
@@ -960,6 +960,7 @@ struct sched_domain { 
	char *name;

#endif



	unsigned int span_weight;

	/*

	 * Span of all CPUs in this domain.

	 *

kernel/sched.c

+3 −0
Original line number Diff line number Diff line
@@ -6271,6 +6271,9 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) 
	struct rq *rq = cpu_rq(cpu);

	struct sched_domain *tmp;



	for (tmp = sd; tmp; tmp = tmp->parent)

		tmp->span_weight = cpumask_weight(sched_domain_span(tmp));



	/* Remove the sched domains which do not contribute to scheduling. */

	for (tmp = sd; tmp; ) {

		struct sched_domain *parent = tmp->parent;

kernel/sched_fair.c

+5 −7
Original line number Diff line number Diff line
@@ -1508,9 +1508,7 @@ select_task_rq_fair(struct rq *rq, struct task_struct *p, int sd_flag, int wake_ 
		 * Pick the largest domain to update shares over

		 */

		tmp = sd;

		if (affine_sd && (!tmp ||

				  cpumask_weight(sched_domain_span(affine_sd)) >

				  cpumask_weight(sched_domain_span(sd))))

		if (affine_sd && (!tmp || affine_sd->span_weight > sd->span_weight))

			tmp = affine_sd;



		if (tmp) {
@@ -1554,10 +1552,10 @@ select_task_rq_fair(struct rq *rq, struct task_struct *p, int sd_flag, int wake_ 


		/* Now try balancing at a lower domain level of new_cpu */

		cpu = new_cpu;

		weight = cpumask_weight(sched_domain_span(sd));

		weight = sd->span_weight;

		sd = NULL;

		for_each_domain(cpu, tmp) {

			if (weight <= cpumask_weight(sched_domain_span(tmp)))

			if (weight <= tmp->span_weight)

				break;

			if (tmp->flags & sd_flag)

				sd = tmp;
@@ -2243,7 +2241,7 @@ unsigned long __weak arch_scale_freq_power(struct sched_domain *sd, int cpu) 


unsigned long default_scale_smt_power(struct sched_domain *sd, int cpu)

{

	unsigned long weight = cpumask_weight(sched_domain_span(sd));

	unsigned long weight = sd->span_weight;

	unsigned long smt_gain = sd->smt_gain;



	smt_gain /= weight;
@@ -2276,7 +2274,7 @@ unsigned long scale_rt_power(int cpu) 


static void update_cpu_power(struct sched_domain *sd, int cpu)

{

	unsigned long weight = cpumask_weight(sched_domain_span(sd));

	unsigned long weight = sd->span_weight;

	unsigned long power = SCHED_LOAD_SCALE;

	struct sched_group *sdg = sd->groups;