Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit b7031a02 authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar
Browse files

sched/fair: Add NOHZ_STATS_KICK 



Split the NOHZ idle balancer into doing two separate actions:

 - update blocked load statistic

 - actually load-balance

Since the latter requires the former, ensure this happens. For now
always tag both bits at the same time.

Prepares for a future where we can toggle only the STATS bit.

Suggested-by: default avatarVincent Guittot <vincent.guittot@linaro.org>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent a22e47a4

kernel/sched/core.c

+2 −2
Original line number Diff line number Diff line
@@ -583,7 +583,7 @@ static inline bool got_nohz_idle_kick(void) 
{

	int cpu = smp_processor_id();



	if (!(atomic_read(nohz_flags(cpu)) & NOHZ_BALANCE_KICK))

	if (!(atomic_read(nohz_flags(cpu)) & NOHZ_KICK_MASK))

		return false;



	if (idle_cpu(cpu) && !need_resched())
@@ -593,7 +593,7 @@ static inline bool got_nohz_idle_kick(void) 
	 * We can't run Idle Load Balance on this CPU for this time so we

	 * cancel it and clear NOHZ_BALANCE_KICK

	 */

	atomic_andnot(NOHZ_BALANCE_KICK, nohz_flags(cpu));

	atomic_andnot(NOHZ_KICK_MASK, nohz_flags(cpu));

	return false;

}

kernel/sched/fair.c

+35 −17
Original line number Diff line number Diff line
@@ -9082,8 +9082,8 @@ static void nohz_balancer_kick(void) 
	if (ilb_cpu >= nr_cpu_ids)

		return;



	flags = atomic_fetch_or(NOHZ_BALANCE_KICK, nohz_flags(ilb_cpu));

	if (flags & NOHZ_BALANCE_KICK)

	flags = atomic_fetch_or(NOHZ_KICK_MASK, nohz_flags(ilb_cpu));

	if (flags & NOHZ_KICK_MASK)

		return;

	/*

	 * Use smp_send_reschedule() instead of resched_cpu().
@@ -9202,8 +9202,6 @@ static void rebalance_domains(struct rq *rq, enum cpu_idle_type idle) 
	int need_serialize, need_decay = 0;

	u64 max_cost = 0;



	update_blocked_averages(cpu);



	rcu_read_lock();

	for_each_domain(cpu, sd) {

		/*
@@ -9298,20 +9296,27 @@ static void rebalance_domains(struct rq *rq, enum cpu_idle_type idle) 
 * In CONFIG_NO_HZ_COMMON case, the idle balance kickee will do the

 * rebalancing for all the CPUs for whom scheduler ticks are stopped.

 */

static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)

static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)

{

	int this_cpu = this_rq->cpu;

	struct rq *rq;

	int balance_cpu;

	/* Earliest time when we have to do rebalance again */

	unsigned long next_balance = jiffies + 60*HZ;

	int update_next_balance = 0;

	int this_cpu = this_rq->cpu;

	unsigned int flags;

	int balance_cpu;

	struct rq *rq;



	if (!(atomic_read(nohz_flags(this_cpu)) & NOHZ_BALANCE_KICK))

		return;

	if (!(atomic_read(nohz_flags(this_cpu)) & NOHZ_KICK_MASK))

		return false;



	if (idle != CPU_IDLE) {

		atomic_andnot(NOHZ_KICK_MASK, nohz_flags(this_cpu));

		return false;

	}



	flags = atomic_fetch_andnot(NOHZ_KICK_MASK, nohz_flags(this_cpu));



	if (idle != CPU_IDLE)

		goto end;

	SCHED_WARN_ON((flags & NOHZ_KICK_MASK) == NOHZ_BALANCE_KICK);



	for_each_cpu(balance_cpu, nohz.idle_cpus_mask) {

		if (balance_cpu == this_cpu || !idle_cpu(balance_cpu))
@@ -9339,6 +9344,8 @@ static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) 
			cpu_load_update_idle(rq);

			rq_unlock_irq(rq, &rf);



			update_blocked_averages(rq->cpu);

			if (flags & NOHZ_BALANCE_KICK)

				rebalance_domains(rq, CPU_IDLE);

		}
@@ -9348,6 +9355,10 @@ static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) 
		}

	}



	update_blocked_averages(this_cpu);

	if (flags & NOHZ_BALANCE_KICK)

		rebalance_domains(this_rq, CPU_IDLE);



	/*

	 * next_balance will be updated only when there is a need.

	 * When the CPU is attached to null domain for ex, it will not be
@@ -9355,8 +9366,8 @@ static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) 
	 */

	if (likely(update_next_balance))

		nohz.next_balance = next_balance;

end:

	atomic_andnot(NOHZ_BALANCE_KICK, nohz_flags(this_cpu));



	return true;

}



/*
@@ -9443,7 +9454,10 @@ static inline bool nohz_kick_needed(struct rq *rq) 
	return kick;

}

#else

static void nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) { }

static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)

{

	return false;

}

#endif



/*
@@ -9464,7 +9478,11 @@ static __latent_entropy void run_rebalance_domains(struct softirq_action *h) 
	 * load balance only within the local sched_domain hierarchy

	 * and abort nohz_idle_balance altogether if we pull some load.

	 */

	nohz_idle_balance(this_rq, idle);

	if (nohz_idle_balance(this_rq, idle))

		return;



	/* normal load balance */

	update_blocked_averages(this_rq->cpu);

	rebalance_domains(this_rq, idle);

}

kernel/sched/sched.h

+4 −0
Original line number Diff line number Diff line
@@ -2036,9 +2036,13 @@ extern void cfs_bandwidth_usage_dec(void); 
#ifdef CONFIG_NO_HZ_COMMON

#define NOHZ_TICK_STOPPED_BIT	0

#define NOHZ_BALANCE_KICK_BIT	1

#define NOHZ_STATS_KICK_BIT	2



#define NOHZ_TICK_STOPPED	BIT(NOHZ_TICK_STOPPED_BIT)

#define NOHZ_BALANCE_KICK	BIT(NOHZ_BALANCE_KICK_BIT)

#define NOHZ_STATS_KICK		BIT(NOHZ_STATS_KICK_BIT)



#define NOHZ_KICK_MASK	(NOHZ_BALANCE_KICK | NOHZ_STATS_KICK)



#define nohz_flags(cpu)	(&cpu_rq(cpu)->nohz_flags)