Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit fcc018af authored by qctecmdr's avatar qctecmdr Committed by Gerrit - the friendly Code Review server
Browse files

Merge "sched/walt: Improve the scheduler"

parents 1176200a a6f81a0b

kernel/sched/core.c

+0 −2
Original line number Diff line number Diff line
@@ -6319,7 +6319,6 @@ int sched_cpu_activate(unsigned int cpu) 
	rq_unlock_irqrestore(rq, &rf);



	update_max_interval();

	walt_update_min_max_capacity();



	return 0;

}
@@ -6359,7 +6358,6 @@ int sched_cpu_deactivate(unsigned int cpu) 
		return ret;

	}

	sched_domains_numa_masks_clear(cpu);

	walt_update_min_max_capacity();

	return 0;

}

kernel/sched/debug.c

+0 −4
Original line number Diff line number Diff line
@@ -674,8 +674,6 @@ do { \ 
	P(cpu_capacity);

#endif

#ifdef CONFIG_SCHED_WALT

	P(cluster->load_scale_factor);

	P(cluster->capacity);

	P(cluster->max_possible_capacity);

	P(cluster->efficiency);

	P(cluster->cur_freq);
@@ -765,8 +763,6 @@ static void sched_debug_header(struct seq_file *m) 
	P(sysctl_sched_features);

#ifdef CONFIG_SCHED_WALT

	P(sched_init_task_load_windows);

	P(min_capacity);

	P(max_capacity);

	P(sched_ravg_window);

	P(sched_load_granule);

#endif

kernel/sched/fair.c

+1 −1
Original line number Diff line number Diff line
@@ -8625,7 +8625,7 @@ static int detach_tasks(struct lb_env *env) 
	if (!same_cluster(env->dst_cpu, env->src_cpu))

		env->flags |= LBF_IGNORE_PREFERRED_CLUSTER_TASKS;



	if (cpu_capacity(env->dst_cpu) < cpu_capacity(env->src_cpu))

	if (capacity_orig_of(env->dst_cpu) < capacity_orig_of(env->src_cpu))

		env->flags |= LBF_IGNORE_BIG_TASKS;



redo:

kernel/sched/sched.h

+0 −153
Original line number Diff line number Diff line
@@ -130,9 +130,7 @@ struct sched_cluster { 
	int max_power_cost;

	int min_power_cost;

	int max_possible_capacity;

	int capacity;

	int efficiency; /* Differentiate cpus with different IPC capability */

	int load_scale_factor;

	unsigned int exec_scale_factor;

	/*

	 * max_freq = user maximum
@@ -2729,12 +2727,6 @@ enum sched_boost_policy { 
	SCHED_BOOST_ON_ALL,

};



/*

 * Returns the rq capacity of any rq in a group. This does not play

 * well with groups where rq capacity can change independently.

 */

#define group_rq_capacity(group) cpu_capacity(group_first_cpu(group))



#ifdef CONFIG_SCHED_WALT



static inline int cluster_first_cpu(struct sched_cluster *cluster)
@@ -2764,8 +2756,6 @@ extern unsigned int max_possible_freq; 
extern unsigned int min_max_freq;

extern unsigned int max_possible_efficiency;

extern unsigned int min_possible_efficiency;

extern unsigned int max_capacity;

extern unsigned int min_capacity;

extern unsigned int max_possible_capacity;

extern unsigned int min_max_possible_capacity;

extern unsigned int max_power_cost;
@@ -2793,21 +2783,11 @@ static inline int asym_cap_siblings(int cpu1, int cpu2) 
		cpumask_test_cpu(cpu2, &asym_cap_sibling_cpus));

}



static inline int cpu_capacity(int cpu)

{

	return cpu_rq(cpu)->cluster->capacity;

}



static inline int cpu_max_possible_capacity(int cpu)

{

	return cpu_rq(cpu)->cluster->max_possible_capacity;

}



static inline int cpu_load_scale_factor(int cpu)

{

	return cpu_rq(cpu)->cluster->load_scale_factor;

}



static inline unsigned int cluster_max_freq(struct sched_cluster *cluster)

{

	/*
@@ -2828,65 +2808,6 @@ static inline unsigned int cpu_max_possible_freq(int cpu) 
	return cpu_rq(cpu)->cluster->max_possible_freq;

}



/* Keep track of max/min capacity possible across CPUs "currently" */

static inline void __update_min_max_capacity(void)

{

	int i;

	int max_cap = 0, min_cap = INT_MAX;



	for_each_possible_cpu(i) {

		if (!cpu_active(i))

			continue;



		max_cap = max(max_cap, cpu_capacity(i));

		min_cap = min(min_cap, cpu_capacity(i));

	}



	max_capacity = max_cap;

	min_capacity = min_cap;

}



/*

 * Return load_scale_factor of a cpu in reference to "most" efficient cpu, so

 * that "most" efficient cpu gets a load_scale_factor of 1

 */

static inline unsigned long

load_scale_cpu_efficiency(struct sched_cluster *cluster)

{

	return DIV_ROUND_UP(1024 * max_possible_efficiency,

			    cluster->efficiency);

}



/*

 * Return load_scale_factor of a cpu in reference to cpu with best max_freq

 * (max_possible_freq), so that one with best max_freq gets a load_scale_factor

 * of 1.

 */

static inline unsigned long load_scale_cpu_freq(struct sched_cluster *cluster)

{

	return DIV_ROUND_UP(1024 * max_possible_freq,

			   cluster_max_freq(cluster));

}



static inline int compute_load_scale_factor(struct sched_cluster *cluster)

{

	int load_scale = 1024;



	/*

	 * load_scale_factor accounts for the fact that task load

	 * is in reference to "best" performing cpu. Task's load will need to be

	 * scaled (up) by a factor to determine suitability to be placed on a

	 * (little) cpu.

	 */

	load_scale *= load_scale_cpu_efficiency(cluster);

	load_scale >>= 10;



	load_scale *= load_scale_cpu_freq(cluster);

	load_scale >>= 10;



	return load_scale;

}



static inline bool hmp_capable(void)

{

	return max_possible_capacity != min_max_possible_capacity;
@@ -2902,55 +2823,6 @@ static inline bool is_min_capacity_cpu(int cpu) 
	return cpu_max_possible_capacity(cpu) == min_max_possible_capacity;

}



/*

 * 'load' is in reference to "best cpu" at its best frequency.

 * Scale that in reference to a given cpu, accounting for how bad it is

 * in reference to "best cpu".

 */

static inline u64 scale_load_to_cpu(u64 task_load, int cpu)

{

	u64 lsf = cpu_load_scale_factor(cpu);



	if (lsf != 1024) {

		task_load *= lsf;

		task_load /= 1024;

	}



	return task_load;

}



/*

 * Return 'capacity' of a cpu in reference to "least" efficient cpu, such that

 * least efficient cpu gets capacity of 1024

 */

static unsigned long

capacity_scale_cpu_efficiency(struct sched_cluster *cluster)

{

	return (1024 * cluster->efficiency) / min_possible_efficiency;

}



/*

 * Return 'capacity' of a cpu in reference to cpu with lowest max_freq

 * (min_max_freq), such that one with lowest max_freq gets capacity of 1024.

 */

static unsigned long capacity_scale_cpu_freq(struct sched_cluster *cluster)

{

	return (1024 * cluster_max_freq(cluster)) / min_max_freq;

}



static inline int compute_capacity(struct sched_cluster *cluster)

{

	int capacity = 1024;



	capacity *= capacity_scale_cpu_efficiency(cluster);

	capacity >>= 10;



	capacity *= capacity_scale_cpu_freq(cluster);

	capacity >>= 10;



	return capacity;

}



static inline unsigned int task_load(struct task_struct *p)

{

	return p->ravg.demand;
@@ -2961,12 +2833,6 @@ static inline unsigned int task_pl(struct task_struct *p) 
	return p->ravg.pred_demand;

}



#define pct_to_real(tunable)	\

		(div64_u64((u64)tunable * (u64)max_task_load(), 100))



#define real_to_pct(tunable)	\

		(div64_u64((u64)tunable * (u64)100, (u64)max_task_load()))



static inline bool task_in_related_thread_group(struct task_struct *p)

{

	return !!(rcu_access_pointer(p->grp) != NULL);
@@ -3087,8 +2953,6 @@ static inline void walt_fixup_cum_window_demand(struct rq *rq, s64 scaled_delta) 
		rq->cum_window_demand_scaled = 0;

}



extern void update_cpu_cluster_capacity(const cpumask_t *cpus);



extern unsigned long thermal_cap(int cpu);



extern void clear_walt_request(int cpu);
@@ -3132,8 +2996,6 @@ static inline enum sched_boost_policy task_boost_policy(struct task_struct *p) 
	return policy;

}



extern void walt_update_min_max_capacity(void);



static inline bool is_min_capacity_cluster(struct sched_cluster *cluster)

{

	return is_min_capacity_cpu(cluster_first_cpu(cluster));
@@ -3200,18 +3062,6 @@ static inline struct sched_cluster *rq_cluster(struct rq *rq) 


static inline int asym_cap_siblings(int cpu1, int cpu2) { return 0; }



static inline u64 scale_load_to_cpu(u64 load, int cpu)

{

	return load;

}



#ifdef CONFIG_SMP

static inline int cpu_capacity(int cpu)

{

	return SCHED_CAPACITY_SCALE;

}

#endif



static inline void set_preferred_cluster(struct related_thread_group *grp) { }



static inline bool task_in_related_thread_group(struct task_struct *p)
@@ -3247,8 +3097,6 @@ static inline int alloc_related_thread_groups(void) { return 0; } 
static inline void walt_fixup_cum_window_demand(struct rq *rq,

						s64 scaled_delta) { }



static inline void update_cpu_cluster_capacity(const cpumask_t *cpus) { }



#ifdef CONFIG_SMP

static inline unsigned long thermal_cap(int cpu)

{
@@ -3285,7 +3133,6 @@ static inline unsigned int power_cost(int cpu, u64 demand) 
#endif



static inline void note_task_waking(struct task_struct *p, u64 wallclock) { }

static inline void walt_update_min_max_capacity(void) { }

#endif	/* CONFIG_SCHED_WALT */



struct sched_avg_stats {

kernel/sched/walt.c

+21 −51
Original line number Diff line number Diff line
@@ -149,9 +149,6 @@ unsigned int max_possible_freq = 1; 
 * capacity (cpu_power) of cpus.

 */

unsigned int min_max_freq = 1;



unsigned int max_capacity = 1024; /* max(rq->capacity) */

unsigned int min_capacity = 1024; /* min(rq->capacity) */

unsigned int max_possible_capacity = 1024; /* max(rq->max_possible_capacity) */

unsigned int

min_max_possible_capacity = 1024; /* min(rq->max_possible_capacity) */
@@ -2074,8 +2071,11 @@ void mark_task_starting(struct task_struct *p) 
}



#define pct_to_min_scaled(tunable) \

		div64_u64(((u64)sched_ravg_window * tunable) << 10, \

			   (u64)sched_cluster[0]->load_scale_factor * 100)

		div64_u64(((u64)sched_ravg_window * tunable *	\

			  cluster_max_freq(sched_cluster[0]) *	\

			  sched_cluster[0]->efficiency),	\

			  ((u64)max_possible_freq *		\

			  max_possible_efficiency * 100))



static inline void walt_update_group_thresholds(void)

{
@@ -2085,6 +2085,19 @@ static inline void walt_update_group_thresholds(void) 
			pct_to_min_scaled(sysctl_sched_group_downmigrate_pct);

}



static void walt_cpus_capacity_changed(const cpumask_t *cpus)

{

	unsigned long flags;



	acquire_rq_locks_irqsave(cpu_possible_mask, &flags);



	if (cpumask_intersects(cpus, &sched_cluster[0]->cpus))

		walt_update_group_thresholds();



	release_rq_locks_irqrestore(cpu_possible_mask, &flags);

}





static cpumask_t all_cluster_cpus = CPU_MASK_NONE;

DECLARE_BITMAP(all_cluster_ids, NR_CPUS);

struct sched_cluster *sched_cluster[NR_CPUS];
@@ -2121,10 +2134,8 @@ static struct sched_cluster *alloc_new_cluster(const struct cpumask *cpus) 
	INIT_LIST_HEAD(&cluster->list);

	cluster->max_power_cost		=	1;

	cluster->min_power_cost		=	1;

	cluster->capacity		=	1024;

	cluster->max_possible_capacity	=	1024;

	cluster->efficiency		=	1;

	cluster->load_scale_factor	=	1024;

	cluster->cur_freq		=	1;

	cluster->max_freq		=	1;

	cluster->max_mitigated_freq	=	UINT_MAX;
@@ -2164,7 +2175,7 @@ static int compute_max_possible_capacity(struct sched_cluster *cluster) 
{

	int capacity = 1024;



	capacity *= capacity_scale_cpu_efficiency(cluster);

	capacity *= (1024 * cluster->efficiency) / min_possible_efficiency;

	capacity >>= 10;



	capacity *= (1024 * cluster->max_possible_freq) / min_max_freq;
@@ -2173,15 +2184,6 @@ static int compute_max_possible_capacity(struct sched_cluster *cluster) 
	return capacity;

}



void walt_update_min_max_capacity(void)

{

	unsigned long flags;



	acquire_rq_locks_irqsave(cpu_possible_mask, &flags);

	__update_min_max_capacity();

	release_rq_locks_irqrestore(cpu_possible_mask, &flags);

}



unsigned int max_power_cost = 1;



static int
@@ -2249,10 +2251,8 @@ static void update_all_clusters_stats(void) 
	for_each_sched_cluster(cluster) {

		u64 mpc;



		cluster->capacity = compute_capacity(cluster);

		mpc = cluster->max_possible_capacity =

			compute_max_possible_capacity(cluster);

		cluster->load_scale_factor = compute_load_scale_factor(cluster);



		cluster->exec_scale_factor =

			DIV_ROUND_UP(cluster->efficiency * 1024,
@@ -2269,7 +2269,6 @@ static void update_all_clusters_stats(void) 
	min_max_possible_capacity = lowest_mpc;

	walt_update_group_thresholds();



	__update_min_max_capacity();

	release_rq_locks_irqrestore(cpu_possible_mask, &flags);

}
@@ -2314,10 +2313,8 @@ struct sched_cluster init_cluster = { 
	.id			=	0,

	.max_power_cost		=	1,

	.min_power_cost		=	1,

	.capacity		=	1024,

	.max_possible_capacity	=	1024,

	.efficiency		=	1,

	.load_scale_factor	=	1024,

	.cur_freq		=	1,

	.max_freq		=	1,

	.max_mitigated_freq	=	UINT_MAX,
@@ -2349,8 +2346,6 @@ static int cpufreq_notifier_policy(struct notifier_block *nb, 
	if (val != CPUFREQ_NOTIFY)

		return 0;



	walt_update_min_max_capacity();



	max_possible_freq = max(max_possible_freq, policy->cpuinfo.max_freq);

	if (min_max_freq == 1)

		min_max_freq = UINT_MAX;
@@ -2391,7 +2386,7 @@ static int cpufreq_notifier_policy(struct notifier_block *nb, 
	}



	if (update_capacity)

		update_cpu_cluster_capacity(policy->related_cpus);

		walt_cpus_capacity_changed(policy->related_cpus);



	return 0;

}
@@ -2868,31 +2863,6 @@ int sync_cgroup_colocation(struct task_struct *p, bool insert) 
}

#endif



void update_cpu_cluster_capacity(const cpumask_t *cpus)

{

	int i;

	struct sched_cluster *cluster;

	struct cpumask cpumask;

	unsigned long flags;



	cpumask_copy(&cpumask, cpus);

	acquire_rq_locks_irqsave(cpu_possible_mask, &flags);



	for_each_cpu(i, &cpumask) {

		cluster = cpu_rq(i)->cluster;

		cpumask_andnot(&cpumask, &cpumask, &cluster->cpus);



		cluster->capacity = compute_capacity(cluster);

		cluster->load_scale_factor = compute_load_scale_factor(cluster);

	}



	__update_min_max_capacity();

	if (cpumask_intersects(cpus, &sched_cluster[0]->cpus))

		walt_update_group_thresholds();



	release_rq_locks_irqrestore(cpu_possible_mask, &flags);

}



static unsigned long max_cap[NR_CPUS];

static unsigned long thermal_cap_cpu[NR_CPUS];
@@ -2958,7 +2928,7 @@ void sched_update_cpu_freq_min_max(const cpumask_t *cpus, u32 fmin, u32 fmax) 
	spin_unlock_irqrestore(&cpu_freq_min_max_lock, flags);



	if (update_capacity)

		update_cpu_cluster_capacity(cpus);

		walt_cpus_capacity_changed(cpus);

}



void note_task_waking(struct task_struct *p, u64 wallclock)