Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c2ea72fd authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge branch 'sched-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull scheduler fixes from Ingo Molnar:
 "A migrate_tasks() locking fix, and a late-coming nohz change plus a
  nohz debug check"

* 'sched-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  sched: 'Annotate' migrate_tasks()
  nohz: Assert existing housekeepers when nohz full enabled
  nohz: Affine unpinned timers to housekeepers
parents a706797f 5473e0cc

include/linux/tick.h

+9 −0
Original line number Diff line number Diff line
@@ -147,11 +147,20 @@ static inline void tick_nohz_full_add_cpus_to(struct cpumask *mask) 
		cpumask_or(mask, mask, tick_nohz_full_mask);

}



static inline int housekeeping_any_cpu(void)

{

	return cpumask_any_and(housekeeping_mask, cpu_online_mask);

}



extern void tick_nohz_full_kick(void);

extern void tick_nohz_full_kick_cpu(int cpu);

extern void tick_nohz_full_kick_all(void);

extern void __tick_nohz_task_switch(void);

#else

static inline int housekeeping_any_cpu(void)

{

	return smp_processor_id();

}

static inline bool tick_nohz_full_enabled(void) { return false; }

static inline bool tick_nohz_full_cpu(int cpu) { return false; }

static inline void tick_nohz_full_add_cpus_to(struct cpumask *mask) { }

kernel/sched/core.c

+31 −5
Original line number Diff line number Diff line
@@ -621,18 +621,21 @@ int get_nohz_timer_target(void) 
	int i, cpu = smp_processor_id();

	struct sched_domain *sd;



	if (!idle_cpu(cpu))

	if (!idle_cpu(cpu) && is_housekeeping_cpu(cpu))

		return cpu;



	rcu_read_lock();

	for_each_domain(cpu, sd) {

		for_each_cpu(i, sched_domain_span(sd)) {

			if (!idle_cpu(i)) {

			if (!idle_cpu(i) && is_housekeeping_cpu(cpu)) {

				cpu = i;

				goto unlock;

			}

		}

	}



	if (!is_housekeeping_cpu(cpu))

		cpu = housekeeping_any_cpu();

unlock:

	rcu_read_unlock();

	return cpu;
@@ -5178,24 +5181,47 @@ static void migrate_tasks(struct rq *dead_rq) 
			break;



		/*

		 * Ensure rq->lock covers the entire task selection

		 * until the migration.

		 * pick_next_task assumes pinned rq->lock.

		 */

		lockdep_pin_lock(&rq->lock);

		next = pick_next_task(rq, &fake_task);

		BUG_ON(!next);

		next->sched_class->put_prev_task(rq, next);



		/*

		 * Rules for changing task_struct::cpus_allowed are holding

		 * both pi_lock and rq->lock, such that holding either

		 * stabilizes the mask.

		 *

		 * Drop rq->lock is not quite as disastrous as it usually is

		 * because !cpu_active at this point, which means load-balance

		 * will not interfere. Also, stop-machine.

		 */

		lockdep_unpin_lock(&rq->lock);

		raw_spin_unlock(&rq->lock);

		raw_spin_lock(&next->pi_lock);

		raw_spin_lock(&rq->lock);



		/*

		 * Since we're inside stop-machine, _nothing_ should have

		 * changed the task, WARN if weird stuff happened, because in

		 * that case the above rq->lock drop is a fail too.

		 */

		if (WARN_ON(task_rq(next) != rq || !task_on_rq_queued(next))) {

			raw_spin_unlock(&next->pi_lock);

			continue;

		}



		/* Find suitable destination for @next, with force if needed. */

		dest_cpu = select_fallback_rq(dead_rq->cpu, next);



		lockdep_unpin_lock(&rq->lock);

		rq = __migrate_task(rq, next, dest_cpu);

		if (rq != dead_rq) {

			raw_spin_unlock(&rq->lock);

			rq = dead_rq;

			raw_spin_lock(&rq->lock);

		}

		raw_spin_unlock(&next->pi_lock);

	}



	rq->stop = stop;

kernel/time/tick-sched.c

+11 −4
Original line number Diff line number Diff line
@@ -298,8 +298,9 @@ static int tick_nohz_cpu_down_callback(struct notifier_block *nfb, 
	switch (action & ~CPU_TASKS_FROZEN) {

	case CPU_DOWN_PREPARE:

		/*

		 * If we handle the timekeeping duty for full dynticks CPUs,

		 * we can't safely shutdown that CPU.

		 * The boot CPU handles housekeeping duty (unbound timers,

		 * workqueues, timekeeping, ...) on behalf of full dynticks

		 * CPUs. It must remain online when nohz full is enabled.

		 */

		if (tick_nohz_full_running && tick_do_timer_cpu == cpu)

			return NOTIFY_BAD;
@@ -370,6 +371,12 @@ void __init tick_nohz_init(void) 
	cpu_notifier(tick_nohz_cpu_down_callback, 0);

	pr_info("NO_HZ: Full dynticks CPUs: %*pbl.\n",

		cpumask_pr_args(tick_nohz_full_mask));



	/*

	 * We need at least one CPU to handle housekeeping work such

	 * as timekeeping, unbound timers, workqueues, ...

	 */

	WARN_ON_ONCE(cpumask_empty(housekeeping_mask));

}

#endif