Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit cf4b0b2c authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge branch 'timers-fixes-for-linus' of... 

Merge branch 'timers-fixes-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip

* 'timers-fixes-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip

:
  hrtimer: prevent migration of per CPU hrtimers
  hrtimer: mark migration state
  hrtimer: fix migration of CB_IRQSAFE_NO_SOFTIRQ hrtimers
  hrtimer: migrate pending list on cpu offline

Acked-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
Acked-by: default avatarBenjamin Herrenschmidt <benh@kernel.crashing.org>
Tested-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
parents 94aca1da ccc7dadf

include/linux/hrtimer.h

+14 −4
Original line number Diff line number Diff line
@@ -47,14 +47,22 @@ enum hrtimer_restart { 
 *	HRTIMER_CB_IRQSAFE:		Callback may run in hardirq context

 *	HRTIMER_CB_IRQSAFE_NO_RESTART:	Callback may run in hardirq context and

 *					does not restart the timer

 *	HRTIMER_CB_IRQSAFE_NO_SOFTIRQ:	Callback must run in hardirq context

 *					Special mode for tick emultation

 *	HRTIMER_CB_IRQSAFE_PERCPU:	Callback must run in hardirq context

 *					Special mode for tick emulation and

 *					scheduler timer. Such timers are per

 *					cpu and not allowed to be migrated on

 *					cpu unplug.

 *	HRTIMER_CB_IRQSAFE_UNLOCKED:	Callback should run in hardirq context

 *					with timer->base lock unlocked

 *					used for timers which call wakeup to

 *					avoid lock order problems with rq->lock

 */

enum hrtimer_cb_mode {

	HRTIMER_CB_SOFTIRQ,

	HRTIMER_CB_IRQSAFE,

	HRTIMER_CB_IRQSAFE_NO_RESTART,

	HRTIMER_CB_IRQSAFE_NO_SOFTIRQ,

	HRTIMER_CB_IRQSAFE_PERCPU,

	HRTIMER_CB_IRQSAFE_UNLOCKED,

};



/*
@@ -67,9 +75,10 @@ enum hrtimer_cb_mode { 
 * 0x02		callback function running

 * 0x04		callback pending (high resolution mode)

 *

 * Special case:

 * Special cases:

 * 0x03		callback function running and enqueued

 *		(was requeued on another CPU)

 * 0x09		timer was migrated on CPU hotunplug

 * The "callback function running and enqueued" status is only possible on

 * SMP. It happens for example when a posix timer expired and the callback

 * queued a signal. Between dropping the lock which protects the posix timer
@@ -87,6 +96,7 @@ enum hrtimer_cb_mode { 
#define HRTIMER_STATE_ENQUEUED	0x01

#define HRTIMER_STATE_CALLBACK	0x02

#define HRTIMER_STATE_PENDING	0x04

#define HRTIMER_STATE_MIGRATE	0x08



/**

 * struct hrtimer - the basic hrtimer structure

kernel/hrtimer.c

+85 −10
Original line number Diff line number Diff line
@@ -672,13 +672,14 @@ static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, 
			 */

			BUG_ON(timer->function(timer) != HRTIMER_NORESTART);

			return 1;

		case HRTIMER_CB_IRQSAFE_NO_SOFTIRQ:

		case HRTIMER_CB_IRQSAFE_PERCPU:

		case HRTIMER_CB_IRQSAFE_UNLOCKED:

			/*

			 * This is solely for the sched tick emulation with

			 * dynamic tick support to ensure that we do not

			 * restart the tick right on the edge and end up with

			 * the tick timer in the softirq ! The calling site

			 * takes care of this.

			 * takes care of this. Also used for hrtimer sleeper !

			 */

			debug_hrtimer_deactivate(timer);

			return 1;
@@ -1245,7 +1246,8 @@ static void __run_hrtimer(struct hrtimer *timer) 
	timer_stats_account_hrtimer(timer);



	fn = timer->function;

	if (timer->cb_mode == HRTIMER_CB_IRQSAFE_NO_SOFTIRQ) {

	if (timer->cb_mode == HRTIMER_CB_IRQSAFE_PERCPU ||

	    timer->cb_mode == HRTIMER_CB_IRQSAFE_UNLOCKED) {

		/*

		 * Used for scheduler timers, avoid lock inversion with

		 * rq->lock and tasklist_lock.
@@ -1452,7 +1454,7 @@ void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task) 
	sl->timer.function = hrtimer_wakeup;

	sl->task = task;

#ifdef CONFIG_HIGH_RES_TIMERS

	sl->timer.cb_mode = HRTIMER_CB_IRQSAFE_NO_SOFTIRQ;

	sl->timer.cb_mode = HRTIMER_CB_IRQSAFE_UNLOCKED;

#endif

}
@@ -1591,29 +1593,95 @@ static void __cpuinit init_hrtimers_cpu(int cpu) 


#ifdef CONFIG_HOTPLUG_CPU



static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base,

				struct hrtimer_clock_base *new_base)

static int migrate_hrtimer_list(struct hrtimer_clock_base *old_base,

				struct hrtimer_clock_base *new_base, int dcpu)

{

	struct hrtimer *timer;

	struct rb_node *node;

	int raise = 0;



	while ((node = rb_first(&old_base->active))) {

		timer = rb_entry(node, struct hrtimer, node);

		BUG_ON(hrtimer_callback_running(timer));

		debug_hrtimer_deactivate(timer);

		__remove_hrtimer(timer, old_base, HRTIMER_STATE_INACTIVE, 0);



		/*

		 * Should not happen. Per CPU timers should be

		 * canceled _before_ the migration code is called

		 */

		if (timer->cb_mode == HRTIMER_CB_IRQSAFE_PERCPU) {

			__remove_hrtimer(timer, old_base,

					 HRTIMER_STATE_INACTIVE, 0);

			WARN(1, "hrtimer (%p %p)active but cpu %d dead\n",

			     timer, timer->function, dcpu);

			continue;

		}



		/*

		 * Mark it as STATE_MIGRATE not INACTIVE otherwise the

		 * timer could be seen as !active and just vanish away

		 * under us on another CPU

		 */

		__remove_hrtimer(timer, old_base, HRTIMER_STATE_MIGRATE, 0);

		timer->base = new_base;

		/*

		 * Enqueue the timer. Allow reprogramming of the event device

		 */

		enqueue_hrtimer(timer, new_base, 1);



#ifdef CONFIG_HIGH_RES_TIMERS

		/*

		 * Happens with high res enabled when the timer was

		 * already expired and the callback mode is

		 * HRTIMER_CB_IRQSAFE_UNLOCKED (hrtimer_sleeper). The

		 * enqueue code does not move them to the soft irq

		 * pending list for performance/latency reasons, but

		 * in the migration state, we need to do that

		 * otherwise we end up with a stale timer.

		 */

		if (timer->state == HRTIMER_STATE_MIGRATE) {

			timer->state = HRTIMER_STATE_PENDING;

			list_add_tail(&timer->cb_entry,

				      &new_base->cpu_base->cb_pending);

			raise = 1;

		}

#endif

		/* Clear the migration state bit */

		timer->state &= ~HRTIMER_STATE_MIGRATE;

	}

	return raise;

}



#ifdef CONFIG_HIGH_RES_TIMERS

static int migrate_hrtimer_pending(struct hrtimer_cpu_base *old_base,

				   struct hrtimer_cpu_base *new_base)

{

	struct hrtimer *timer;

	int raise = 0;



	while (!list_empty(&old_base->cb_pending)) {

		timer = list_entry(old_base->cb_pending.next,

				   struct hrtimer, cb_entry);



		__remove_hrtimer(timer, timer->base, HRTIMER_STATE_PENDING, 0);

		timer->base = &new_base->clock_base[timer->base->index];

		list_add_tail(&timer->cb_entry, &new_base->cb_pending);

		raise = 1;

	}

	return raise;

}

#else

static int migrate_hrtimer_pending(struct hrtimer_cpu_base *old_base,

				   struct hrtimer_cpu_base *new_base)

{

	return 0;

}

#endif



static void migrate_hrtimers(int cpu)

{

	struct hrtimer_cpu_base *old_base, *new_base;

	int i;

	int i, raise = 0;



	BUG_ON(cpu_online(cpu));

	old_base = &per_cpu(hrtimer_bases, cpu);
@@ -1626,14 +1694,21 @@ static void migrate_hrtimers(int cpu) 
	spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING);



	for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {

		migrate_hrtimer_list(&old_base->clock_base[i],

				     &new_base->clock_base[i]);

		if (migrate_hrtimer_list(&old_base->clock_base[i],

					 &new_base->clock_base[i], cpu))

			raise = 1;

	}



	if (migrate_hrtimer_pending(old_base, new_base))

		raise = 1;



	spin_unlock(&old_base->lock);

	spin_unlock(&new_base->lock);

	local_irq_enable();

	put_cpu_var(hrtimer_bases);



	if (raise)

		hrtimer_raise_softirq();

}

#endif /* CONFIG_HOTPLUG_CPU */

kernel/sched.c

+2 −2
Original line number Diff line number Diff line
@@ -201,7 +201,7 @@ void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime) 
	hrtimer_init(&rt_b->rt_period_timer,

			CLOCK_MONOTONIC, HRTIMER_MODE_REL);

	rt_b->rt_period_timer.function = sched_rt_period_timer;

	rt_b->rt_period_timer.cb_mode = HRTIMER_CB_IRQSAFE_NO_SOFTIRQ;

	rt_b->rt_period_timer.cb_mode = HRTIMER_CB_IRQSAFE_UNLOCKED;

}



static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
@@ -1119,7 +1119,7 @@ static void init_rq_hrtick(struct rq *rq) 


	hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);

	rq->hrtick_timer.function = hrtick;

	rq->hrtick_timer.cb_mode = HRTIMER_CB_IRQSAFE_NO_SOFTIRQ;

	rq->hrtick_timer.cb_mode = HRTIMER_CB_IRQSAFE_PERCPU;

}

#else

static inline void hrtick_clear(struct rq *rq)

kernel/time/tick-sched.c

+1 −1
Original line number Diff line number Diff line
@@ -625,7 +625,7 @@ void tick_setup_sched_timer(void) 
	 */

	hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);

	ts->sched_timer.function = tick_sched_timer;

	ts->sched_timer.cb_mode = HRTIMER_CB_IRQSAFE_NO_SOFTIRQ;

	ts->sched_timer.cb_mode = HRTIMER_CB_IRQSAFE_PERCPU;



	/* Get the next period (per cpu) */

	ts->sched_timer.expires = tick_init_jiffy_update();

kernel/trace/trace_sysprof.c

+1 −1
Original line number Diff line number Diff line
@@ -202,7 +202,7 @@ static void start_stack_timer(int cpu) 


	hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);

	hrtimer->function = stack_trace_timer_fn;

	hrtimer->cb_mode = HRTIMER_CB_IRQSAFE_NO_SOFTIRQ;

	hrtimer->cb_mode = HRTIMER_CB_IRQSAFE_PERCPU;



	hrtimer_start(hrtimer, ns_to_ktime(sample_period), HRTIMER_MODE_REL);

}