Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 87df0beb authored by Srivatsa Vaddagiri's avatar Srivatsa Vaddagiri Committed by Steve Muckle
Browse files

sched: support legacy mode better 



It should be possible to bypass all HMP scheduler changes at runtime
by setting sysctl_sched_enable_hmp_task_placement and
sysctl_sched_enable_power_aware to 0.  Fix various code paths to honor
this requirement.

Change-Id: I74254e68582b3f9f1b84661baf7dae14f981c025
Signed-off-by: default avatarSrivatsa Vaddagiri <vatsa@codeaurora.org>
parent 1ef9206d

include/linux/sched.h

+1 −1
Original line number Diff line number Diff line
@@ -1981,7 +1981,7 @@ extern unsigned long long 
task_sched_runtime(struct task_struct *task);



/* sched_exec is called by processes performing an exec */

#if defined(CONFIG_SMP) && !defined(CONFIG_SCHED_HMP)

#if defined(CONFIG_SMP)

extern void sched_exec(void);

#else

#define sched_exec()   {}

kernel/sched/core.c

+16 −3
Original line number Diff line number Diff line
@@ -1076,6 +1076,9 @@ int rq_freq_margin(struct rq *rq) 
	int margin;

	u64 demand;



	if (!sysctl_sched_enable_hmp_task_placement)

		return INT_MAX;



	demand = scale_load_to_cpu(rq->prev_runnable_sum, rq->cpu);

	demand *= 128;

	demand = div64_u64(demand, max_task_load());
@@ -1331,6 +1334,9 @@ static void init_cpu_efficiency(void) 
	int i, efficiency;

	unsigned int max = 0, min = UINT_MAX;



	if (!sysctl_sched_enable_hmp_task_placement)

		return;



	for_each_possible_cpu(i) {

		efficiency = arch_get_cpu_efficiency(i);

		cpu_rq(i)->efficiency = efficiency;
@@ -1371,7 +1377,7 @@ static inline void set_window_start(struct rq *rq) 
	int cpu = cpu_of(rq);

	struct rq *sync_rq = cpu_rq(sync_cpu);



	if (likely(rq->window_start))

	if (rq->window_start || !sysctl_sched_enable_hmp_task_placement)

		return;



	if (cpu == sync_cpu) {
@@ -1655,6 +1661,9 @@ static int register_sched_callback(void) 
{

	int ret;



	if (!sysctl_sched_enable_hmp_task_placement)

		return 0;



	ret = cpufreq_register_notifier(&notifier_policy_block,

						CPUFREQ_POLICY_NOTIFIER);
@@ -1824,7 +1833,8 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu) 


		atomic_notifier_call_chain(&task_migration_notifier, 0, &tmn);



		if (p->on_rq || p->state == TASK_WAKING)

		if (sysctl_sched_enable_hmp_task_placement &&

		    (p->on_rq || p->state == TASK_WAKING))

			fixup_busy_time(p, new_cpu);

	}
@@ -3511,7 +3521,7 @@ static void update_cpu_load_active(struct rq *this_rq) 
	calc_load_account_active(this_rq);

}



#if defined(CONFIG_SMP) && !defined(CONFIG_SCHED_HMP)

#if defined(CONFIG_SMP)



/*

 * sched_exec - execve() is a valuable balancing opportunity, because at
@@ -3523,6 +3533,9 @@ void sched_exec(void) 
	unsigned long flags;

	int dest_cpu;



	if (sysctl_sched_enable_hmp_task_placement)

		return;



	raw_spin_lock_irqsave(&p->pi_lock, flags);

	dest_cpu = p->sched_class->select_task_rq(p, SD_BALANCE_EXEC, 0);

	if (dest_cpu == smp_processor_id())

kernel/sched/fair.c

+30 −16
Original line number Diff line number Diff line
@@ -1453,6 +1453,9 @@ int sched_set_boost(int enable) 
	unsigned long flags;

	int ret = 0;



	if (!sysctl_sched_enable_hmp_task_placement)

		return -EINVAL;



	spin_lock_irqsave(&boost_lock, flags);



	if (enable == 1) {
@@ -1758,6 +1761,9 @@ done: 


void inc_nr_big_small_task(struct rq *rq, struct task_struct *p)

{

	if (!sysctl_sched_enable_hmp_task_placement)

		return;



	if (is_big_task(p))

		rq->nr_big_tasks++;

	else if (is_small_task(p))
@@ -1766,6 +1772,9 @@ void inc_nr_big_small_task(struct rq *rq, struct task_struct *p) 


void dec_nr_big_small_task(struct rq *rq, struct task_struct *p)

{

	if (!sysctl_sched_enable_hmp_task_placement)

		return;



	if (is_big_task(p))

		rq->nr_big_tasks--;

	else if (is_small_task(p))
@@ -1831,7 +1840,7 @@ int sched_hmp_proc_update_handler(struct ctl_table *table, int write, 
	unsigned int old_val = *data;



	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);

	if (ret || !write)

	if (ret || !write || !sysctl_sched_enable_hmp_task_placement)

		return ret;



	if ((sysctl_sched_downmigrate_pct > sysctl_sched_upmigrate_pct) ||
@@ -1953,7 +1962,8 @@ static inline int migration_needed(struct rq *rq, struct task_struct *p) 
{

	int nice = TASK_NICE(p);



	if (is_small_task(p) || p->state != TASK_RUNNING)

	if (is_small_task(p) || p->state != TASK_RUNNING ||

			!sysctl_sched_enable_hmp_task_placement)

		return 0;



	/* Todo: cgroup-based control? */
@@ -2556,6 +2566,9 @@ add_to_scaled_stat(int cpu, struct sched_avg *sa, u64 delta) 
	u64 scaled_delta;

	int sf;



	if (!sysctl_sched_enable_hmp_task_placement)

		return;



	if (unlikely(cur_freq > max_possible_freq ||

		     (cur_freq == max_freq &&

		      max_freq < cpu_max_possible_freq)))
@@ -2570,6 +2583,9 @@ add_to_scaled_stat(int cpu, struct sched_avg *sa, u64 delta) 


static inline void decay_scaled_stat(struct sched_avg *sa, u64 periods)

{

	if (!sysctl_sched_enable_hmp_task_placement)

		return;



	sa->runnable_avg_sum_scaled =

		decay_load(sa->runnable_avg_sum_scaled,

			   periods);
@@ -6057,11 +6073,7 @@ ret: 
	return NULL;

}



/*

 * find_busiest_queue - find the busiest runqueue among the cpus in group.

 */

#ifdef CONFIG_SCHED_HMP

static struct rq *find_busiest_queue(struct lb_env *env,

static struct rq *find_busiest_queue_hmp(struct lb_env *env,

				     struct sched_group *group)

{

	struct rq *busiest = NULL, *rq;
@@ -6082,7 +6094,10 @@ static struct rq *find_busiest_queue(struct lb_env *env, 


	return busiest;

}

#else /* CONFIG_SCHED_HMP */



/*

 * find_busiest_queue - find the busiest runqueue among the cpus in group.

 */

static struct rq *find_busiest_queue(struct lb_env *env,

				     struct sched_group *group)

{
@@ -6090,6 +6105,9 @@ static struct rq *find_busiest_queue(struct lb_env *env, 
	unsigned long max_load = 0;

	int i;



	if (sysctl_sched_enable_hmp_task_placement)

		return find_busiest_queue_hmp(env, group);



	for_each_cpu(i, sched_group_cpus(group)) {

		unsigned long power = power_of(i);

		unsigned long capacity = DIV_ROUND_CLOSEST(power,
@@ -6128,7 +6146,6 @@ static struct rq *find_busiest_queue(struct lb_env *env, 


	return busiest;

}

#endif /* CONFIG_SCHED_HMP */



/*

 * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but
@@ -6891,9 +6908,7 @@ end: 
	clear_bit(NOHZ_BALANCE_KICK, nohz_flags(this_cpu));

}



#ifdef CONFIG_SCHED_HMP



static inline int _nohz_kick_needed(struct rq *rq, int cpu, int *type)

static inline int _nohz_kick_needed_hmp(struct rq *rq, int cpu, int *type)

{

	struct sched_domain *sd;

	int i;
@@ -6928,12 +6943,13 @@ static inline int _nohz_kick_needed(struct rq *rq, int cpu, int *type) 
	return 0;

}



#else /* CONFIG_SCHED_HMP */



static inline int _nohz_kick_needed(struct rq *rq, int cpu, int *type)

{

	unsigned long now = jiffies;



	if (sysctl_sched_enable_hmp_task_placement)

		return _nohz_kick_needed_hmp(rq, cpu, type);



	/*

	 * None are in tickless mode and hence no need for NOHZ idle load

	 * balancing.
@@ -6947,8 +6963,6 @@ static inline int _nohz_kick_needed(struct rq *rq, int cpu, int *type) 
	return (rq->nr_running >= 2);

}



#endif /* CONFIG_SCHED_HMP */



/*

 * Current heuristic for kicking the idle load balancer in the presence

 * of an idle cpu is the system.

kernel/sched/rt.c

+9 −20
Original line number Diff line number Diff line
@@ -1258,32 +1258,22 @@ static void yield_task_rt(struct rq *rq) 
#ifdef CONFIG_SMP

static int find_lowest_rq(struct task_struct *task);



/* TODO: Move this to a power aware config feature. There's

 * no strict dependency between SCHED_HMP and this. Its just

 * a different algorithm optimizing for power

 */

#ifdef CONFIG_SCHED_HMP

static int

select_task_rq_rt(struct task_struct *p, int sd_flag, int flags)

select_task_rq_rt_hmp(struct task_struct *p, int sd_flag, int flags)

{

	int cpu, target;



	cpu = task_cpu(p);



	if (p->nr_cpus_allowed == 1)

		goto out;



	rcu_read_lock();

	target = find_lowest_rq(p);

	if (target != -1)

		cpu = target;

	rcu_read_unlock();



out:

	return cpu;

}



#else /* CONFIG_SCHED_HMP */

static int

select_task_rq_rt(struct task_struct *p, int sd_flag, int flags)

{
@@ -1296,6 +1286,9 @@ select_task_rq_rt(struct task_struct *p, int sd_flag, int flags) 
	if (p->nr_cpus_allowed == 1)

		goto out;



	if (sysctl_sched_enable_hmp_task_placement)

		return select_task_rq_rt_hmp(p, sd_flag, flags);



	/* For anything but wake ups, just return the task_cpu */

	if (sd_flag != SD_BALANCE_WAKE && sd_flag != SD_BALANCE_FORK)

		goto out;
@@ -1341,7 +1334,6 @@ select_task_rq_rt(struct task_struct *p, int sd_flag, int flags) 
out:

	return cpu;

}

#endif /* CONFIG_SCHED_HMP */



static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)

{
@@ -1521,12 +1513,7 @@ next_idx: 


static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask);



/* TODO: Move this to a power aware config feature. There's

 * no strict dependency between SCHED_HMP and this. Its just

 * a different algorithm optimizing for power

 */

#ifdef CONFIG_SCHED_HMP

static int find_lowest_rq(struct task_struct *task)

static int find_lowest_rq_hmp(struct task_struct *task)

{

	struct cpumask *lowest_mask = __get_cpu_var(local_cpu_mask);

	int cpu_cost, min_cost = INT_MAX;
@@ -1566,7 +1553,7 @@ static int find_lowest_rq(struct task_struct *task) 
	}

	return best_cpu;

}

#else /* CONFIG_SCHED_HMP */



static int find_lowest_rq(struct task_struct *task)

{

	struct sched_domain *sd;
@@ -1574,6 +1561,9 @@ static int find_lowest_rq(struct task_struct *task) 
	int this_cpu = smp_processor_id();

	int cpu      = task_cpu(task);



	if (sysctl_sched_enable_hmp_task_placement)

		return find_lowest_rq_hmp(task);



	/* Make sure the mask is initialized first */

	if (unlikely(!lowest_mask))

		return -1;
@@ -1640,7 +1630,6 @@ static int find_lowest_rq(struct task_struct *task) 
		return cpu;

	return -1;

}

#endif /* CONFIG_SCHED_HMP */



/* Will lock the rq it finds */

static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq)

kernel/sched/sched.h

+4 −0
Original line number Diff line number Diff line
@@ -753,6 +753,8 @@ static inline unsigned long capacity_scale_cpu_freq(int cpu) 


#ifdef CONFIG_SCHED_HMP



extern unsigned int sysctl_sched_enable_hmp_task_placement;



int mostly_idle_cpu(int cpu);

extern void check_for_migration(struct rq *rq, struct task_struct *p);

extern void pre_big_small_task_count_change(void);
@@ -764,6 +766,8 @@ extern unsigned int power_cost_at_freq(int cpu, unsigned int freq); 


#else /* CONFIG_SCHED_HMP */



#define sysctl_sched_enable_hmp_task_placement 0



static inline void check_for_migration(struct rq *rq, struct task_struct *p) { }

static inline void pre_big_small_task_count_change(void) { }

static inline void post_big_small_task_count_change(void) { }