Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 58088ad0 authored by Paul Turner's avatar Paul Turner Committed by Ingo Molnar
Browse files

sched: Add a timer to handle CFS bandwidth refresh 



This patch adds a per-task_group timer which handles the refresh of the global
CFS bandwidth pool.

Since the RT pool is using a similar timer there's some small refactoring to
share this support.

Signed-off-by: default avatarPaul Turner <pjt@google.com>
Reviewed-by: default avatarHidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
Signed-off-by: default avatarPeter Zijlstra <a.p.zijlstra@chello.nl>
Link: http://lkml.kernel.org/r/20110721184757.277271273@google.com


Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent ec12cb7f

kernel/sched.c

+86 −21
Original line number Diff line number Diff line
@@ -196,33 +196,36 @@ static inline int rt_bandwidth_enabled(void) 
	return sysctl_sched_rt_runtime >= 0;

}



static void start_rt_bandwidth(struct rt_bandwidth *rt_b)

static void start_bandwidth_timer(struct hrtimer *period_timer, ktime_t period)

{

	ktime_t now;



	if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF)

		return;



	if (hrtimer_active(&rt_b->rt_period_timer))

		return;



	raw_spin_lock(&rt_b->rt_runtime_lock);

	for (;;) {

	unsigned long delta;

		ktime_t soft, hard;

	ktime_t soft, hard, now;



		if (hrtimer_active(&rt_b->rt_period_timer))

	for (;;) {

		if (hrtimer_active(period_timer))

			break;



		now = hrtimer_cb_get_time(&rt_b->rt_period_timer);

		hrtimer_forward(&rt_b->rt_period_timer, now, rt_b->rt_period);

		now = hrtimer_cb_get_time(period_timer);

		hrtimer_forward(period_timer, now, period);



		soft = hrtimer_get_softexpires(&rt_b->rt_period_timer);

		hard = hrtimer_get_expires(&rt_b->rt_period_timer);

		soft = hrtimer_get_softexpires(period_timer);

		hard = hrtimer_get_expires(period_timer);

		delta = ktime_to_ns(ktime_sub(hard, soft));

		__hrtimer_start_range_ns(&rt_b->rt_period_timer, soft, delta,

		__hrtimer_start_range_ns(period_timer, soft, delta,

					 HRTIMER_MODE_ABS_PINNED, 0);

	}

}



static void start_rt_bandwidth(struct rt_bandwidth *rt_b)

{

	if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF)

		return;



	if (hrtimer_active(&rt_b->rt_period_timer))

		return;



	raw_spin_lock(&rt_b->rt_runtime_lock);

	start_bandwidth_timer(&rt_b->rt_period_timer, rt_b->rt_period);

	raw_spin_unlock(&rt_b->rt_runtime_lock);

}
@@ -253,6 +256,9 @@ struct cfs_bandwidth { 
	ktime_t period;

	u64 quota, runtime;

	s64 hierarchal_quota;



	int idle, timer_active;

	struct hrtimer period_timer;

#endif

};
@@ -403,6 +409,28 @@ static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) 
}



static inline u64 default_cfs_period(void);

static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun);



static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer)

{

	struct cfs_bandwidth *cfs_b =

		container_of(timer, struct cfs_bandwidth, period_timer);

	ktime_t now;

	int overrun;

	int idle = 0;



	for (;;) {

		now = hrtimer_cb_get_time(timer);

		overrun = hrtimer_forward(timer, now, cfs_b->period);



		if (!overrun)

			break;



		idle = do_sched_cfs_period_timer(cfs_b, overrun);

	}



	return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;

}



static void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)

{
@@ -410,6 +438,9 @@ static void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) 
	cfs_b->runtime = 0;

	cfs_b->quota = RUNTIME_INF;

	cfs_b->period = ns_to_ktime(default_cfs_period());



	hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);

	cfs_b->period_timer.function = sched_cfs_period_timer;

}



static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq)
@@ -417,8 +448,34 @@ static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) 
	cfs_rq->runtime_enabled = 0;

}



/* requires cfs_b->lock, may release to reprogram timer */

static void __start_cfs_bandwidth(struct cfs_bandwidth *cfs_b)

{

	/*

	 * The timer may be active because we're trying to set a new bandwidth

	 * period or because we're racing with the tear-down path

	 * (timer_active==0 becomes visible before the hrtimer call-back

	 * terminates).  In either case we ensure that it's re-programmed

	 */

	while (unlikely(hrtimer_active(&cfs_b->period_timer))) {

		raw_spin_unlock(&cfs_b->lock);

		/* ensure cfs_b->lock is available while we wait */

		hrtimer_cancel(&cfs_b->period_timer);



		raw_spin_lock(&cfs_b->lock);

		/* if someone else restarted the timer then we're done */

		if (cfs_b->timer_active)

			return;

	}



	cfs_b->timer_active = 1;

	start_bandwidth_timer(&cfs_b->period_timer, cfs_b->period);

}



static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b)

{}

{

	hrtimer_cancel(&cfs_b->period_timer);

}

#else

static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}

static void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
@@ -9078,7 +9135,7 @@ static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime); 


static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)

{

	int i, ret = 0;

	int i, ret = 0, runtime_enabled;

	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg);



	if (tg == &root_task_group)
@@ -9105,10 +9162,18 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota) 
	if (ret)

		goto out_unlock;



	runtime_enabled = quota != RUNTIME_INF;

	raw_spin_lock_irq(&cfs_b->lock);

	cfs_b->period = ns_to_ktime(period);

	cfs_b->quota = quota;

	cfs_b->runtime = quota;



	/* restart the period timer (if active) to handle new period expiry */

	if (runtime_enabled && cfs_b->timer_active) {

		/* force a reprogram */

		cfs_b->timer_active = 0;

		__start_cfs_bandwidth(cfs_b);

	}

	raw_spin_unlock_irq(&cfs_b->lock);



	for_each_possible_cpu(i) {
@@ -9116,7 +9181,7 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota) 
		struct rq *rq = rq_of(cfs_rq);



		raw_spin_lock_irq(&rq->lock);

		cfs_rq->runtime_enabled = quota != RUNTIME_INF;

		cfs_rq->runtime_enabled = runtime_enabled;

		cfs_rq->runtime_remaining = 0;

		raw_spin_unlock_irq(&rq->lock);

	}

kernel/sched_fair.c

+37 −3
Original line number Diff line number Diff line
@@ -1284,9 +1284,16 @@ static void assign_cfs_rq_runtime(struct cfs_rq *cfs_rq) 
	raw_spin_lock(&cfs_b->lock);

	if (cfs_b->quota == RUNTIME_INF)

		amount = min_amount;

	else if (cfs_b->runtime > 0) {

	else {

		/* ensure bandwidth timer remains active under consumption */

		if (!cfs_b->timer_active)

			__start_cfs_bandwidth(cfs_b);



		if (cfs_b->runtime > 0) {

			amount = min(cfs_b->runtime, min_amount);

			cfs_b->runtime -= amount;

			cfs_b->idle = 0;

		}

	}

	raw_spin_unlock(&cfs_b->lock);
@@ -1315,6 +1322,33 @@ static __always_inline void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, 
	__account_cfs_rq_runtime(cfs_rq, delta_exec);

}



/*

 * Responsible for refilling a task_group's bandwidth and unthrottling its

 * cfs_rqs as appropriate. If there has been no activity within the last

 * period the timer is deactivated until scheduling resumes; cfs_b->idle is

 * used to track this state.

 */

static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun)

{

	int idle = 1;



	raw_spin_lock(&cfs_b->lock);

	/* no need to continue the timer with no bandwidth constraint */

	if (cfs_b->quota == RUNTIME_INF)

		goto out_unlock;



	idle = cfs_b->idle;

	cfs_b->runtime = cfs_b->quota;



	/* mark as potentially idle for the upcoming period */

	cfs_b->idle = 1;

out_unlock:

	if (idle)

		cfs_b->timer_active = 0;

	raw_spin_unlock(&cfs_b->lock);



	return idle;

}

#else

static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq,

				     unsigned long delta_exec) {}