Merge "sched: add sync wakeup recognition in select_best_cpu" (9c3c63a0) · Commits · e / devices / android_kernel_sony_msm8994

kernel/sched/fair.c

+72 −25

Original line number	Diff line number	Diff line
		@@ -1487,10 +1487,34 @@ static inline u64 cpu_load(int cpu)
		return scale_load_to_cpu(rq->cumulative_runnable_avg, cpu);
		}

		static inline u64 cpu_load_sync(int cpu, int sync)
		{
		struct rq *rq = cpu_rq(cpu);
		u64 load;

		load = rq->cumulative_runnable_avg;

		/*
		* If load is being checked in a sync wakeup environment,
		* we may want to discount the load of the currently running
		* task.
		*/
		if (sync && cpu == smp_processor_id()) {
		if (load > rq->curr->ravg.demand)
		load -= rq->curr->ravg.demand;
		else
		load = 0;
		}

		return scale_load_to_cpu(load, cpu);
		}

		static int
		spill_threshold_crossed(struct task_struct p, struct rq rq, int cpu)
		spill_threshold_crossed(struct task_struct p, struct rq rq, int cpu,
		int sync)
		{
		u64 total_load = cpu_load(cpu) + scale_load_to_cpu(task_load(p), cpu);
		u64 total_load = cpu_load_sync(cpu, sync) +
		scale_load_to_cpu(task_load(p), cpu);

		if (total_load > sched_spill_load \|\|
		(rq->nr_running + 1) > sysctl_sched_spill_nr_run)
		@@ -1508,6 +1532,26 @@ int mostly_idle_cpu(int cpu)
		&& !sched_cpu_high_irqload(cpu);
		}

		static int mostly_idle_cpu_sync(int cpu, int sync)
		{
		struct rq *rq = cpu_rq(cpu);
		u64 load = cpu_load_sync(cpu, sync);
		int nr_running;

		nr_running = rq->nr_running;

		/*
		* Sync wakeups mean that the waker task will go to sleep
		* soon so we should discount its load from this test.
		*/
		if (sync && cpu == smp_processor_id())
		nr_running--;

		return load <= rq->mostly_idle_load &&
		nr_running <= rq->mostly_idle_nr_run &&
		!sched_cpu_high_irqload(cpu);
		}

		static int boost_refcount;
		static DEFINE_SPINLOCK(boost_lock);
		static DEFINE_MUTEX(boost_mutex);
		@@ -1622,15 +1666,15 @@ static int task_will_fit(struct task_struct *p, int cpu)
		return 0;
		}

		static int eligible_cpu(struct task_struct *p, int cpu)
		static int eligible_cpu(struct task_struct *p, int cpu, int sync)
		{
		struct rq *rq = cpu_rq(cpu);

		if (mostly_idle_cpu(cpu))
		if (mostly_idle_cpu_sync(cpu, sync))
		return 1;

		if (rq->capacity != max_capacity)
		return !spill_threshold_crossed(p, rq, cpu);
		return !spill_threshold_crossed(p, rq, cpu, sync);

		return 0;
		}
		@@ -1706,7 +1750,7 @@ static unsigned int power_cost(struct task_struct *p, int cpu)
		return power_cost_at_freq(cpu, task_freq);
		}

		static int best_small_task_cpu(struct task_struct *p)
		static int best_small_task_cpu(struct task_struct *p, int sync)
		{
		int best_busy_cpu = -1, best_fallback_cpu = -1;
		int min_cost_cpu = -1, min_cstate_cpu = -1;
		@@ -1728,8 +1772,8 @@ static int best_small_task_cpu(struct task_struct *p)
		for_each_cpu(i, &search_cpus) {

		trace_sched_cpu_load(cpu_rq(i), idle_cpu(i),
		mostly_idle_cpu(i), sched_irqload(i),
		power_cost(p, i));
		mostly_idle_cpu_sync(i, sync),
		sched_irqload(i), power_cost(p, i));

		cpu_cost = power_cost(p, i);
		if (cpu_cost < min_cost) {
		@@ -1743,7 +1787,8 @@ static int best_small_task_cpu(struct task_struct *p)
		/* Optimization to steer task towards the minimum power
		cost CPU. The tradeoff is that we may have to check
		the same information again in pass 2 */
		if (!cpu_rq(min_cost_cpu)->cstate && mostly_idle_cpu(min_cost_cpu))
		if (!cpu_rq(min_cost_cpu)->cstate &&
		mostly_idle_cpu_sync(min_cost_cpu, sync))
		return min_cost_cpu;

		for_each_cpu(i, &search_cpus) {
		@@ -1766,11 +1811,11 @@ static int best_small_task_cpu(struct task_struct *p)
		continue;
		}

		if (mostly_idle_cpu(i))
		if (mostly_idle_cpu_sync(i, sync))
		return i;

		load = cpu_load(i);
		if (!spill_threshold_crossed(p, rq, i)) {
		load = cpu_load_sync(i, sync);
		if (!spill_threshold_crossed(p, rq, i, sync)) {
		if (load < min_busy_load) {
		min_busy_load = load;
		best_busy_cpu = i;
		@@ -1857,7 +1902,8 @@ static int select_packing_target(struct task_struct *p, int best_cpu)


		/* return cheapest cpu that can fit this task */
		static int select_best_cpu(struct task_struct *p, int target, int reason)
		static int select_best_cpu(struct task_struct *p, int target, int reason,
		int sync)
		{
		int i, best_cpu = -1, fallback_idle_cpu = -1, min_cstate_cpu = -1;
		int prev_cpu = task_cpu(p);
		@@ -1872,7 +1918,7 @@ static int select_best_cpu(struct task_struct *p, int target, int reason)
		trace_sched_task_load(p, small_task, boost, reason);

		if (small_task && !boost) {
		best_cpu = best_small_task_cpu(p);
		best_cpu = best_small_task_cpu(p, sync);
		goto done;
		}

		@@ -1880,8 +1926,8 @@ static int select_best_cpu(struct task_struct *p, int target, int reason)
		for_each_cpu_and(i, tsk_cpus_allowed(p), cpu_online_mask) {

		trace_sched_cpu_load(cpu_rq(i), idle_cpu(i),
		mostly_idle_cpu(i), sched_irqload(i),
		power_cost(p, i));
		mostly_idle_cpu_sync(i, sync),
		sched_irqload(i), power_cost(p, i));

		if (skip_cpu(p, i, reason))
		continue;
		@@ -1892,8 +1938,8 @@ static int select_best_cpu(struct task_struct *p, int target, int reason)
		* where the task will fit.
		*/
		if (!task_will_fit(p, i)) {
		if (mostly_idle_cpu(i)) {
		load = cpu_load(i);
		if (mostly_idle_cpu_sync(i, sync)) {
		load = cpu_load_sync(i, sync);
		if (load < min_fallback_load) {
		min_fallback_load = load;
		fallback_idle_cpu = i;
		@@ -1902,7 +1948,7 @@ static int select_best_cpu(struct task_struct *p, int target, int reason)
		continue;
		}

		if (!eligible_cpu(p, i))
		if (!eligible_cpu(p, i, sync))
		continue;

		/*
		@@ -1911,7 +1957,7 @@ static int select_best_cpu(struct task_struct *p, int target, int reason)
		* spill.
		*/

		load = cpu_load(i);
		load = cpu_load_sync(i, sync);
		cpu_cost = power_cost(p, i);
		cstate = cpu_rq(i)->cstate;

		@@ -1979,7 +2025,7 @@ static int select_best_cpu(struct task_struct *p, int target, int reason)
		}

		if (min_cstate_cpu >= 0 && (prefer_idle \|\|
		!(best_cpu >= 0 && mostly_idle_cpu(best_cpu))))
		!(best_cpu >= 0 && mostly_idle_cpu_sync(best_cpu, sync))))
		best_cpu = min_cstate_cpu;
		done:
		if (best_cpu < 0) {
		@@ -2325,7 +2371,7 @@ void check_for_migration(struct rq rq, struct task_struct p)
		return;

		raw_spin_lock(&migration_lock);
		new_cpu = select_best_cpu(p, cpu, reason);
		new_cpu = select_best_cpu(p, cpu, reason, 0);

		if (new_cpu != cpu) {
		active_balance = kick_active_balance(rq, p, new_cpu);
		@@ -2354,7 +2400,8 @@ static inline int task_will_fit(struct task_struct *p, int cpu)
		return 1;
		}

		static inline int select_best_cpu(struct task_struct *p, int target, int reason)
		static inline int select_best_cpu(struct task_struct *p, int target,
		int reason, int sync)
		{
		return 0;
		}
		@@ -2370,7 +2417,7 @@ static inline int power_cost(struct task_struct *p, int cpu)
		}

		static inline int
		spill_threshold_crossed(struct task_struct p, struct rq rq, int cpu)
		spill_threshold_crossed(struct task_struct p, struct rq rq, int cpu, int sync)
		{
		return 0;
		}
		@@ -4701,7 +4748,7 @@ select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flags)
		return prev_cpu;

		if (sched_enable_hmp)
		return select_best_cpu(p, prev_cpu, 0);
		return select_best_cpu(p, prev_cpu, 0, sync);

		if (sd_flag & SD_BALANCE_WAKE) {
		if (cpumask_test_cpu(cpu, tsk_cpus_allowed(p)))