Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 7d1e6a9b authored by Rusty Russell's avatar Rusty Russell Committed by Ingo Molnar
Browse files

sched: convert nohz struct to cpumask_var_t. 



Impact: (future) size reduction for large NR_CPUS.

Dynamically allocating cpumasks (when CONFIG_CPUMASK_OFFSTACK) saves
space for small nr_cpu_ids but big CONFIG_NR_CPUS.  cpumask_var_t
is just a struct cpumask for !CONFIG_CPUMASK_OFFSTACK.

Signed-off-by: default avatarRusty Russell <rusty@rustcorp.com.au>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent c6c4927b

kernel/sched.c

+16 −13
Original line number Original line Diff line number Diff line
@@ -3758,10 +3758,9 @@ static void active_load_balance(struct rq *busiest_rq, int busiest_cpu) 
#ifdef CONFIG_NO_HZ

#ifdef CONFIG_NO_HZ

static struct {

static struct {

	atomic_t load_balancer;

	atomic_t load_balancer;

	cpumask_t cpu_mask;

	cpumask_var_t cpu_mask;

} nohz ____cacheline_aligned = {

} nohz ____cacheline_aligned = {

	.load_balancer = ATOMIC_INIT(-1),

	.load_balancer = ATOMIC_INIT(-1),

	.cpu_mask = CPU_MASK_NONE,

};

};





/*

/*
@@ -3789,7 +3788,7 @@ int select_nohz_load_balancer(int stop_tick) 
	int cpu = smp_processor_id();

	int cpu = smp_processor_id();





	if (stop_tick) {

	if (stop_tick) {

		cpu_set(cpu, nohz.cpu_mask);

		cpumask_set_cpu(cpu, nohz.cpu_mask);

		cpu_rq(cpu)->in_nohz_recently = 1;

		cpu_rq(cpu)->in_nohz_recently = 1;





		/*

		/*
@@ -3803,7 +3802,7 @@ int select_nohz_load_balancer(int stop_tick) 
		}

		}





		/* time for ilb owner also to sleep */

		/* time for ilb owner also to sleep */

		if (cpus_weight(nohz.cpu_mask) == num_online_cpus()) {

		if (cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {

			if (atomic_read(&nohz.load_balancer) == cpu)

			if (atomic_read(&nohz.load_balancer) == cpu)

				atomic_set(&nohz.load_balancer, -1);

				atomic_set(&nohz.load_balancer, -1);

			return 0;

			return 0;
@@ -3816,10 +3815,10 @@ int select_nohz_load_balancer(int stop_tick) 
		} else if (atomic_read(&nohz.load_balancer) == cpu)

		} else if (atomic_read(&nohz.load_balancer) == cpu)

			return 1;

			return 1;

	} else {

	} else {

		if (!cpu_isset(cpu, nohz.cpu_mask))

		if (!cpumask_test_cpu(cpu, nohz.cpu_mask))

			return 0;

			return 0;





		cpu_clear(cpu, nohz.cpu_mask);

		cpumask_clear_cpu(cpu, nohz.cpu_mask);





		if (atomic_read(&nohz.load_balancer) == cpu)

		if (atomic_read(&nohz.load_balancer) == cpu)

			if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)

			if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
@@ -3930,12 +3929,13 @@ static void run_rebalance_domains(struct softirq_action *h) 
	 */

	 */

	if (this_rq->idle_at_tick &&

	if (this_rq->idle_at_tick &&

	    atomic_read(&nohz.load_balancer) == this_cpu) {

	    atomic_read(&nohz.load_balancer) == this_cpu) {

		cpumask_t cpus = nohz.cpu_mask;

		struct rq *rq;

		struct rq *rq;

		int balance_cpu;

		int balance_cpu;





		cpu_clear(this_cpu, cpus);

		for_each_cpu(balance_cpu, nohz.cpu_mask) {

		for_each_cpu(balance_cpu, &cpus) {

			if (balance_cpu == this_cpu)

				continue;



			/*

			/*

			 * If this cpu gets work to do, stop the load balancing

			 * If this cpu gets work to do, stop the load balancing

			 * work being done for other cpus. Next load

			 * work being done for other cpus. Next load
@@ -3973,7 +3973,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu) 
		rq->in_nohz_recently = 0;

		rq->in_nohz_recently = 0;





		if (atomic_read(&nohz.load_balancer) == cpu) {

		if (atomic_read(&nohz.load_balancer) == cpu) {

			cpu_clear(cpu, nohz.cpu_mask);

			cpumask_clear_cpu(cpu, nohz.cpu_mask);

			atomic_set(&nohz.load_balancer, -1);

			atomic_set(&nohz.load_balancer, -1);

		}

		}
@@ -3986,7 +3986,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu) 
			 * TBD: Traverse the sched domains and nominate

			 * TBD: Traverse the sched domains and nominate

			 * the nearest cpu in the nohz.cpu_mask.

			 * the nearest cpu in the nohz.cpu_mask.

			 */

			 */

			int ilb = first_cpu(nohz.cpu_mask);

			int ilb = cpumask_first(nohz.cpu_mask);





			if (ilb < nr_cpu_ids)

			if (ilb < nr_cpu_ids)

				resched_cpu(ilb);

				resched_cpu(ilb);
@@ -3998,7 +3998,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu) 
	 * cpus with ticks stopped, is it time for that to stop?

	 * cpus with ticks stopped, is it time for that to stop?

	 */

	 */

	if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) == cpu &&

	if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) == cpu &&

	    cpus_weight(nohz.cpu_mask) == num_online_cpus()) {

	    cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {

		resched_cpu(cpu);

		resched_cpu(cpu);

		return;

		return;

	}

	}
@@ -4008,7 +4008,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu) 
	 * someone else, then no need raise the SCHED_SOFTIRQ

	 * someone else, then no need raise the SCHED_SOFTIRQ

	 */

	 */

	if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) != cpu &&

	if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) != cpu &&

	    cpu_isset(cpu, nohz.cpu_mask))

	    cpumask_test_cpu(cpu, nohz.cpu_mask))

		return;

		return;

#endif

#endif

	if (time_after_eq(jiffies, rq->next_balance))

	if (time_after_eq(jiffies, rq->next_balance))
@@ -8309,6 +8309,9 @@ void __init sched_init(void) 




	/* Allocate the nohz_cpu_mask if CONFIG_CPUMASK_OFFSTACK */

	/* Allocate the nohz_cpu_mask if CONFIG_CPUMASK_OFFSTACK */

	alloc_bootmem_cpumask_var(&nohz_cpu_mask);

	alloc_bootmem_cpumask_var(&nohz_cpu_mask);

#ifdef CONFIG_NO_HZ

	alloc_bootmem_cpumask_var(&nohz.cpu_mask);

#endif





	scheduler_running = 1;

	scheduler_running = 1;

}

}