Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit d0ba188e authored by Vikram Mulukutla's avatar Vikram Mulukutla Committed by Joonwoo Park
Browse files

sched: Update rq->cpu_capacity_orig on thermal (LMH) throttling 



capacity_orig_of is supposed to return the maximum capacity
of the a CPU based on its FMAX. Given that LMH may limit
the FMAX, let's make sure that cpu_capacity_orig is also
updated when that happens.

Change-Id: I82fbd8587d7b4b82b50fb80d6c9dc78344837bf5
Signed-off-by: default avatarVikram Mulukutla <markivx@codeaurora.org>
parent e71a8452

include/linux/sched.h

+9 −3
Original line number Diff line number Diff line
@@ -2681,9 +2681,6 @@ static inline int sched_update_freq_max_load(const cpumask_t *cpumask) 
	return 0;

}



static inline void sched_update_cpu_freq_min_max(const cpumask_t *cpus,

					u32 fmin, u32 fmax) { }



static inline void

sched_set_cpu_cstate(int cpu, int cstate, int wakeup_energy, int wakeup_latency)

{
@@ -2702,6 +2699,8 @@ extern int sched_set_group_id(struct task_struct *p, unsigned int group_id); 
extern unsigned int sched_get_group_id(struct task_struct *p);

extern int sched_set_init_task_load(struct task_struct *p, int init_load_pct);

extern u32 sched_get_init_task_load(struct task_struct *p);

extern void sched_update_cpu_freq_min_max(const cpumask_t *cpus, u32 fmin,

					  u32 fmax);

#else

static inline int

register_cpu_cycle_counter_cb(struct cpu_cycle_counter_cb *cb)
@@ -2711,6 +2710,13 @@ register_cpu_cycle_counter_cb(struct cpu_cycle_counter_cb *cb) 
static inline void sched_set_io_is_busy(int val) {};

#endif /* CONFIG_SCHED_WALT */



#ifndef CONFIG_SCHED_WALT

#ifndef CONFIG_SCHED_HMP

static inline void sched_update_cpu_freq_min_max(const cpumask_t *cpus,

					u32 fmin, u32 fmax) { }

#endif /* CONFIG_SCHED_HMP */

#endif /* CONFIG_SCHED_WALT */



#ifdef CONFIG_NO_HZ_COMMON

void calc_load_enter_idle(void);

void calc_load_exit_idle(void);

kernel/sched/fair.c

+2 −0
Original line number Diff line number Diff line
@@ -8129,6 +8129,8 @@ static void update_cpu_capacity(struct sched_domain *sd, int cpu) 
	int max_cap_cpu;

	unsigned long flags;



	capacity = min(capacity, thermal_cap(cpu));



	cpu_rq(cpu)->cpu_capacity_orig = capacity;



	mcc = &cpu_rq(cpu)->rd->max_cpu_capacity;

kernel/sched/sched.h

+13 −5
Original line number Diff line number Diff line
@@ -2644,6 +2644,10 @@ inc_cum_window_demand(struct rq *rq, struct task_struct *p, s64 delta) 
	rq->cum_window_demand += delta;

}



extern void update_cpu_cluster_capacity(const cpumask_t *cpus);



extern unsigned long thermal_cap(int cpu);



#else	/* CONFIG_SCHED_WALT */



struct hmp_sched_stats;
@@ -2766,6 +2770,15 @@ dec_cum_window_demand(struct rq *rq, struct task_struct *p) { } 
static inline void

inc_cum_window_demand(struct rq *rq, struct task_struct *p, s64 delta) { }



static inline void update_cpu_cluster_capacity(const cpumask_t *cpus) { }



#ifdef CONFIG_SMP

static inline unsigned long thermal_cap(int cpu)

{

	return cpu_rq(cpu)->cpu_capacity_orig;

}

#endif



#endif	/* CONFIG_SCHED_WALT */



#ifdef CONFIG_SCHED_HMP
@@ -2818,8 +2831,6 @@ dec_nr_big_task(struct hmp_sched_stats *stats, struct task_struct *p) 
	BUG_ON(stats->nr_big_tasks < 0);

}



extern void update_cpu_cluster_capacity(const cpumask_t *cpus);



static inline bool is_short_burst_task(struct task_struct *p)

{

	return p->ravg.avg_burst < sysctl_sched_short_burst &&
@@ -2897,7 +2908,4 @@ inc_nr_big_task(struct hmp_sched_stats *stats, struct task_struct *p) { } 
static inline void

dec_nr_big_task(struct hmp_sched_stats *stats, struct task_struct *p) { }



static inline void

update_cpu_cluster_capacity(const cpumask_t *cpus) { }



#endif /* CONFIG_SCHED_HMP */

kernel/sched/walt.c

+93 −0
Original line number Diff line number Diff line
@@ -2122,6 +2122,8 @@ void init_clusters(void) 
	INIT_LIST_HEAD(&cluster_head);

}



static unsigned long cpu_max_table_freq[NR_CPUS];



static int cpufreq_notifier_policy(struct notifier_block *nb,

		unsigned long val, void *data)

{
@@ -2147,6 +2149,9 @@ static int cpufreq_notifier_policy(struct notifier_block *nb, 
	BUG_ON(!min_max_freq);

	BUG_ON(!policy->max);



	for_each_cpu(i, &policy_cluster)

		cpu_max_table_freq[i] = policy->cpuinfo.max_freq;



	for_each_cpu(i, &policy_cluster) {

		cluster = cpu_rq(i)->cluster;

		cpumask_andnot(&policy_cluster, &policy_cluster,
@@ -2631,6 +2636,94 @@ int sync_cgroup_colocation(struct task_struct *p, bool insert) 
}

#endif



void update_cpu_cluster_capacity(const cpumask_t *cpus)

{

	int i;

	struct sched_cluster *cluster;

	struct cpumask cpumask;

	unsigned long flags;



	cpumask_copy(&cpumask, cpus);

	acquire_rq_locks_irqsave(cpu_possible_mask, &flags);



	for_each_cpu(i, &cpumask) {

		cluster = cpu_rq(i)->cluster;

		cpumask_andnot(&cpumask, &cpumask, &cluster->cpus);



		cluster->capacity = compute_capacity(cluster);

		cluster->load_scale_factor = compute_load_scale_factor(cluster);

	}



	__update_min_max_capacity();



	release_rq_locks_irqrestore(cpu_possible_mask, &flags);

}



static unsigned long max_cap[NR_CPUS];

static unsigned long thermal_cap_cpu[NR_CPUS];



unsigned long thermal_cap(int cpu)

{

	return thermal_cap_cpu[cpu] ?: cpu_rq(cpu)->cpu_capacity_orig;

}



unsigned long do_thermal_cap(int cpu, unsigned long thermal_max_freq)

{

	struct sched_domain *sd;

	struct sched_group *sg;

	struct rq *rq = cpu_rq(cpu);

	int nr_cap_states;



	if (!max_cap[cpu]) {

		rcu_read_lock();

		sd = rcu_dereference(per_cpu(sd_ea, cpu));

		if (!sd || !sd->groups || !sd->groups->sge ||

		    !sd->groups->sge->cap_states) {

			rcu_read_unlock();

			return rq->cpu_capacity_orig;

		}

		sg = sd->groups;

		nr_cap_states = sg->sge->nr_cap_states;

		max_cap[cpu] = sg->sge->cap_states[nr_cap_states - 1].cap;

		rcu_read_unlock();

	}



	if (cpu_max_table_freq[cpu] &&

	    unlikely(thermal_max_freq && thermal_max_freq

		!= cpu_max_table_freq[cpu])) {

		return div64_ul(thermal_max_freq * max_cap[cpu],

				cpu_max_table_freq[cpu]);

	} else {

		return rq->cpu_capacity_orig;

	}

}



static DEFINE_SPINLOCK(cpu_freq_min_max_lock);

void sched_update_cpu_freq_min_max(const cpumask_t *cpus, u32 fmin, u32 fmax)

{

	struct cpumask cpumask;

	struct sched_cluster *cluster;

	int i, update_capacity = 0;

	unsigned long flags;



	spin_lock_irqsave(&cpu_freq_min_max_lock, flags);

	cpumask_copy(&cpumask, cpus);



	for_each_cpu(i, &cpumask)

		thermal_cap_cpu[i] = do_thermal_cap(i, fmax);



	for_each_cpu(i, &cpumask) {

		cluster = cpu_rq(i)->cluster;

		cpumask_andnot(&cpumask, &cpumask, &cluster->cpus);

		update_capacity += (cluster->max_mitigated_freq != fmax);

		cluster->max_mitigated_freq = fmax;

	}

	spin_unlock_irqrestore(&cpu_freq_min_max_lock, flags);



	if (update_capacity)

		update_cpu_cluster_capacity(cpus);

}



/*

 * Task's cpu usage is accounted in:

 *	rq->curr/prev_runnable_sum,  when its ->grp is NULL