Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit ee9ddb5f authored by Steve Muckle's avatar Steve Muckle
Browse files

sched: add sync wakeup recognition in select_best_cpu 



If a wakeup is a sync wakeup, we need to discount the currently
running task's load from the waker's CPU as we calculate the best
CPU for the waking task to land on.

Change-Id: I00c5df626d17868323d60fb90b4513c0dd314825
Signed-off-by: default avatarSteve Muckle <smuckle@codeaurora.org>
parent 6e778f0c

kernel/sched/fair.c

+72 −25
Original line number Diff line number Diff line
@@ -1487,10 +1487,34 @@ static inline u64 cpu_load(int cpu) 
	return scale_load_to_cpu(rq->cumulative_runnable_avg, cpu);

}



static inline u64 cpu_load_sync(int cpu, int sync)

{

	struct rq *rq = cpu_rq(cpu);

	u64 load;



	load = rq->cumulative_runnable_avg;



	/*

	 * If load is being checked in a sync wakeup environment,

	 * we may want to discount the load of the currently running

	 * task.

	 */

	if (sync && cpu == smp_processor_id()) {

		if (load > rq->curr->ravg.demand)

			load -= rq->curr->ravg.demand;

		else

			load = 0;

	}



	return scale_load_to_cpu(load, cpu);

}



static int

spill_threshold_crossed(struct task_struct *p, struct rq *rq, int cpu)

spill_threshold_crossed(struct task_struct *p, struct rq *rq, int cpu,

			int sync)

{

	u64 total_load = cpu_load(cpu) + scale_load_to_cpu(task_load(p), cpu);

	u64 total_load = cpu_load_sync(cpu, sync) +

		scale_load_to_cpu(task_load(p), cpu);



	if (total_load > sched_spill_load ||

	    (rq->nr_running + 1) > sysctl_sched_spill_nr_run)
@@ -1508,6 +1532,26 @@ int mostly_idle_cpu(int cpu) 
		&& !sched_cpu_high_irqload(cpu);

}



static int mostly_idle_cpu_sync(int cpu, int sync)

{

	struct rq *rq = cpu_rq(cpu);

	u64 load = cpu_load_sync(cpu, sync);

	int nr_running;



	nr_running = rq->nr_running;



	/*

	 * Sync wakeups mean that the waker task will go to sleep

	 * soon so we should discount its load from this test.

	 */

	if (sync && cpu == smp_processor_id())

		nr_running--;



	return load <= rq->mostly_idle_load &&

		nr_running <= rq->mostly_idle_nr_run &&

		!sched_cpu_high_irqload(cpu);

}



static int boost_refcount;

static DEFINE_SPINLOCK(boost_lock);

static DEFINE_MUTEX(boost_mutex);
@@ -1622,15 +1666,15 @@ static int task_will_fit(struct task_struct *p, int cpu) 
	return 0;

}



static int eligible_cpu(struct task_struct *p, int cpu)

static int eligible_cpu(struct task_struct *p, int cpu, int sync)

{

	struct rq *rq = cpu_rq(cpu);



	if (mostly_idle_cpu(cpu))

	if (mostly_idle_cpu_sync(cpu, sync))

		return 1;



	if (rq->capacity != max_capacity)

		return !spill_threshold_crossed(p, rq, cpu);

		return !spill_threshold_crossed(p, rq, cpu, sync);



	return 0;

}
@@ -1706,7 +1750,7 @@ static unsigned int power_cost(struct task_struct *p, int cpu) 
	return power_cost_at_freq(cpu, task_freq);

}



static int best_small_task_cpu(struct task_struct *p)

static int best_small_task_cpu(struct task_struct *p, int sync)

{

	int best_busy_cpu = -1, best_fallback_cpu = -1;

	int min_cost_cpu = -1, min_cstate_cpu = -1;
@@ -1728,8 +1772,8 @@ static int best_small_task_cpu(struct task_struct *p) 
	for_each_cpu(i, &search_cpus) {



		trace_sched_cpu_load(cpu_rq(i), idle_cpu(i),

				     mostly_idle_cpu(i), sched_irqload(i),

				     power_cost(p, i));

				     mostly_idle_cpu_sync(i, sync),

				     sched_irqload(i), power_cost(p, i));



		cpu_cost = power_cost(p, i);

		if (cpu_cost < min_cost) {
@@ -1743,7 +1787,8 @@ static int best_small_task_cpu(struct task_struct *p) 
	/* Optimization to steer task towards the minimum power

	   cost CPU. The tradeoff is that we may have to check

	   the same information again in pass 2 */

	if (!cpu_rq(min_cost_cpu)->cstate && mostly_idle_cpu(min_cost_cpu))

	if (!cpu_rq(min_cost_cpu)->cstate &&

	    mostly_idle_cpu_sync(min_cost_cpu, sync))

		return min_cost_cpu;



	for_each_cpu(i, &search_cpus) {
@@ -1766,11 +1811,11 @@ static int best_small_task_cpu(struct task_struct *p) 
			continue;

		}



		if (mostly_idle_cpu(i))

		if (mostly_idle_cpu_sync(i, sync))

			return i;



		load = cpu_load(i);

		if (!spill_threshold_crossed(p, rq, i)) {

		load = cpu_load_sync(i, sync);

		if (!spill_threshold_crossed(p, rq, i, sync)) {

			if (load < min_busy_load) {

				min_busy_load = load;

				best_busy_cpu = i;
@@ -1857,7 +1902,8 @@ static int select_packing_target(struct task_struct *p, int best_cpu) 




/* return cheapest cpu that can fit this task */

static int select_best_cpu(struct task_struct *p, int target, int reason)

static int select_best_cpu(struct task_struct *p, int target, int reason,

			   int sync)

{

	int i, best_cpu = -1, fallback_idle_cpu = -1, min_cstate_cpu = -1;

	int prev_cpu = task_cpu(p);
@@ -1872,7 +1918,7 @@ static int select_best_cpu(struct task_struct *p, int target, int reason) 
	trace_sched_task_load(p, small_task, boost, reason);



	if (small_task && !boost) {

		best_cpu = best_small_task_cpu(p);

		best_cpu = best_small_task_cpu(p, sync);

		goto done;

	}
@@ -1880,8 +1926,8 @@ static int select_best_cpu(struct task_struct *p, int target, int reason) 
	for_each_cpu_and(i, tsk_cpus_allowed(p), cpu_online_mask) {



		trace_sched_cpu_load(cpu_rq(i), idle_cpu(i),

				     mostly_idle_cpu(i), sched_irqload(i),

				     power_cost(p, i));

				     mostly_idle_cpu_sync(i, sync),

				     sched_irqload(i), power_cost(p, i));



		if (skip_cpu(p, i, reason))

			continue;
@@ -1892,8 +1938,8 @@ static int select_best_cpu(struct task_struct *p, int target, int reason) 
		 * where the task will fit.

		 */

		if (!task_will_fit(p, i)) {

			if (mostly_idle_cpu(i)) {

				load = cpu_load(i);

			if (mostly_idle_cpu_sync(i, sync)) {

				load = cpu_load_sync(i, sync);

				if (load < min_fallback_load) {

					min_fallback_load = load;

					fallback_idle_cpu = i;
@@ -1902,7 +1948,7 @@ static int select_best_cpu(struct task_struct *p, int target, int reason) 
			continue;

		}



		if (!eligible_cpu(p, i))

		if (!eligible_cpu(p, i, sync))

			continue;



		/*
@@ -1911,7 +1957,7 @@ static int select_best_cpu(struct task_struct *p, int target, int reason) 
		 * spill.

		 */



		load = cpu_load(i);

		load = cpu_load_sync(i, sync);

		cpu_cost = power_cost(p, i);

		cstate = cpu_rq(i)->cstate;
@@ -1979,7 +2025,7 @@ static int select_best_cpu(struct task_struct *p, int target, int reason) 
	}



	if (min_cstate_cpu >= 0 && (prefer_idle ||

			!(best_cpu >= 0 && mostly_idle_cpu(best_cpu))))

		!(best_cpu >= 0 && mostly_idle_cpu_sync(best_cpu, sync))))

		best_cpu = min_cstate_cpu;

done:

	if (best_cpu < 0) {
@@ -2325,7 +2371,7 @@ void check_for_migration(struct rq *rq, struct task_struct *p) 
		return;



	raw_spin_lock(&migration_lock);

	new_cpu = select_best_cpu(p, cpu, reason);

	new_cpu = select_best_cpu(p, cpu, reason, 0);



	if (new_cpu != cpu) {

		active_balance = kick_active_balance(rq, p, new_cpu);
@@ -2354,7 +2400,8 @@ static inline int task_will_fit(struct task_struct *p, int cpu) 
	return 1;

}



static inline int select_best_cpu(struct task_struct *p, int target, int reason)

static inline int select_best_cpu(struct task_struct *p, int target,

				  int reason, int sync)

{

	return 0;

}
@@ -2370,7 +2417,7 @@ static inline int power_cost(struct task_struct *p, int cpu) 
}



static inline int

spill_threshold_crossed(struct task_struct *p, struct rq *rq, int cpu)

spill_threshold_crossed(struct task_struct *p, struct rq *rq, int cpu, int sync)

{

	return 0;

}
@@ -4701,7 +4748,7 @@ select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flags) 
		return prev_cpu;



	if (sched_enable_hmp)

		return select_best_cpu(p, prev_cpu, 0);

		return select_best_cpu(p, prev_cpu, 0, sync);



	if (sd_flag & SD_BALANCE_WAKE) {

		if (cpumask_test_cpu(cpu, tsk_cpus_allowed(p)))