Merge "cpufreq: schedutil: Fix race condition in computing hispeed_util" (5245fb4a) · Commits · e / devices / android_kernel_fairphone_FP3

include/linux/sched.h

+1 −0

Original line number	Diff line number	Diff line
		@@ -3905,6 +3905,7 @@ static inline unsigned long rlimit_max(unsigned int limit)
		#define SCHED_CPUFREQ_DL (1U << 1)
		#define SCHED_CPUFREQ_IOWAIT (1U << 2)
		#define SCHED_CPUFREQ_INTERCLUSTER_MIG (1U << 3)
		#define SCHED_CPUFREQ_WALT (1U << 4)

		#define SCHED_CPUFREQ_RT_DL (SCHED_CPUFREQ_RT \| SCHED_CPUFREQ_DL)

kernel/sched/cpufreq_schedutil.c

+70 −9

Original line number	Diff line number	Diff line
		@@ -36,6 +36,10 @@ struct sugov_policy {
		raw_spinlock_t update_lock; /* For shared policies */
		u64 last_freq_update_time;
		s64 freq_update_delay_ns;
		u64 last_ws;
		u64 curr_cycles;
		u64 last_cyc_update_time;
		unsigned long avg_cap;
		unsigned int next_freq;
		unsigned int cached_raw_freq;
		unsigned long hispeed_util;
		@@ -199,19 +203,63 @@ static void sugov_iowait_boost(struct sugov_cpu sg_cpu, unsigned long util,
		sg_cpu->iowait_boost >>= 1;
		}

		static unsigned long freq_to_util(struct sugov_policy *sg_policy,
		unsigned int freq)
		{
		return mult_frac(sg_policy->max, freq,
		sg_policy->policy->cpuinfo.max_freq);
		}

		#define KHZ 1000
		static void sugov_track_cycles(struct sugov_policy *sg_policy,
		unsigned int prev_freq,
		u64 upto)
		{
		u64 delta_ns, cycles;
		/* Track cycles in current window */
		delta_ns = upto - sg_policy->last_cyc_update_time;
		cycles = (prev_freq * delta_ns) / (NSEC_PER_SEC / KHZ);
		sg_policy->curr_cycles += cycles;
		sg_policy->last_cyc_update_time = upto;
		}

		static void sugov_calc_avg_cap(struct sugov_policy *sg_policy, u64 curr_ws,
		unsigned int prev_freq)
		{
		u64 last_ws = sg_policy->last_ws;
		unsigned int avg_freq;

		WARN_ON(curr_ws < last_ws);
		if (curr_ws <= last_ws)
		return;

		/* If we skipped some windows */
		if (curr_ws > (last_ws + sched_ravg_window)) {
		avg_freq = prev_freq;
		/* Reset tracking history */
		sg_policy->last_cyc_update_time = curr_ws;
		} else {
		sugov_track_cycles(sg_policy, prev_freq, curr_ws);
		avg_freq = sg_policy->curr_cycles;
		avg_freq /= sched_ravg_window / (NSEC_PER_SEC / KHZ);
		}
		sg_policy->avg_cap = freq_to_util(sg_policy, avg_freq);
		sg_policy->curr_cycles = 0;
		sg_policy->last_ws = curr_ws;
		}

		#define NL_RATIO 75
		#define HISPEED_LOAD 90
		static void sugov_walt_adjust(struct sugov_cpu sg_cpu, unsigned long util,
		unsigned long *max)
		{
		struct sugov_policy *sg_policy = sg_cpu->sg_policy;
		unsigned long cap_cur = capacity_curr_of(sg_cpu->cpu);
		bool is_migration = sg_cpu->flags & SCHED_CPUFREQ_INTERCLUSTER_MIG;
		unsigned long nl = sg_cpu->walt_load.nl;
		unsigned long cpu_util = sg_cpu->util;
		bool is_hiload;

		is_hiload = (cpu_util >= mult_frac(cap_cur,
		is_hiload = (cpu_util >= mult_frac(sg_policy->avg_cap,
		HISPEED_LOAD,
		100));

		@@ -247,6 +295,8 @@ static void sugov_update_single(struct update_util_data *hook, u64 time,
		} else {
		sugov_get_util(&util, &max, sg_cpu->cpu);
		sugov_iowait_boost(sg_cpu, &util, &max);
		sugov_calc_avg_cap(sg_policy, sg_cpu->walt_load.ws,
		sg_policy->policy->cur);
		sugov_walt_adjust(sg_cpu, &util, &max);
		next_f = get_next_freq(sg_policy, util, max);
		}
		@@ -322,12 +372,11 @@ static void sugov_update_shared(struct update_util_data *hook, u64 time,
		raw_spin_lock(&sg_policy->update_lock);

		if (sg_policy->max != max) {
		hs_util = mult_frac(max,
		sg_policy->tunables->hispeed_freq,
		sg_policy->policy->cpuinfo.max_freq);
		sg_policy->max = max;
		hs_util = freq_to_util(sg_policy,
		sg_policy->tunables->hispeed_freq);
		hs_util = mult_frac(hs_util, TARGET_LOAD, 100);
		sg_policy->hispeed_util = hs_util;
		sg_policy->max = max;
		}

		sg_cpu->util = util;
		@@ -337,6 +386,9 @@ static void sugov_update_shared(struct update_util_data *hook, u64 time,
		sugov_set_iowait_boost(sg_cpu, time, flags);
		sg_cpu->last_update = time;

		sugov_calc_avg_cap(sg_policy, sg_cpu->walt_load.ws,
		sg_policy->policy->cur);

		trace_sugov_util_update(sg_cpu->cpu, sg_cpu->util, max,
		sg_cpu->walt_load.nl,
		sg_cpu->walt_load.pl, flags);
		@@ -354,6 +406,10 @@ static void sugov_work(struct kthread_work *work)
		struct sugov_policy *sg_policy = container_of(work, struct sugov_policy, work);

		mutex_lock(&sg_policy->work_lock);
		raw_spin_lock(&sg_policy->update_lock);
		sugov_track_cycles(sg_policy, sg_policy->policy->cur,
		sched_ktime_clock());
		raw_spin_unlock(&sg_policy->update_lock);
		__cpufreq_driver_target(sg_policy->policy, sg_policy->next_freq,
		CPUFREQ_RELATION_L);
		mutex_unlock(&sg_policy->work_lock);
		@@ -438,11 +494,12 @@ static ssize_t hispeed_freq_store(struct gov_attr_set *attr_set,

		tunables->hispeed_freq = val;
		list_for_each_entry(sg_policy, &attr_set->policy_list, tunables_hook) {
		hs_util = mult_frac(sg_policy->max,
		sg_policy->tunables->hispeed_freq,
		sg_policy->policy->cpuinfo.max_freq);
		raw_spin_lock(&sg_policy->update_lock);
		hs_util = freq_to_util(sg_policy,
		sg_policy->tunables->hispeed_freq);
		hs_util = mult_frac(hs_util, TARGET_LOAD, 100);
		sg_policy->hispeed_util = hs_util;
		raw_spin_unlock(&sg_policy->update_lock);
		}

		return count;
		@@ -725,6 +782,10 @@ static void sugov_limits(struct cpufreq_policy *policy)

		if (!policy->fast_switch_enabled) {
		mutex_lock(&sg_policy->work_lock);
		raw_spin_lock(&sg_policy->update_lock);
		sugov_track_cycles(sg_policy, sg_policy->policy->cur,
		sched_ktime_clock());
		raw_spin_unlock(&sg_policy->update_lock);
		cpufreq_policy_apply_limits(policy);
		mutex_unlock(&sg_policy->work_lock);
		}

kernel/sched/sched.h

+15 −3

Original line number	Diff line number	Diff line
		@@ -1779,6 +1779,7 @@ struct sched_walt_cpu_load {
		unsigned long prev_window_util;
		unsigned long nl;
		unsigned long pl;
		u64 ws;
		};

		static inline unsigned long cpu_util_cum(int cpu, int delta)
		@@ -1828,6 +1829,7 @@ cpu_util_freq(int cpu, struct sched_walt_cpu_load *walt_load)
		walt_load->prev_window_util = util;
		walt_load->nl = nl;
		walt_load->pl = 0;
		walt_load->ws = rq->window_start;
		}
		}
		#endif
		@@ -2207,6 +2209,15 @@ static inline u64 irq_time_read(int cpu)
		}
		#endif /* CONFIG_IRQ_TIME_ACCOUNTING */

		#ifdef CONFIG_SCHED_WALT
		u64 sched_ktime_clock(void);
		#else /* CONFIG_SCHED_WALT */
		static inline u64 sched_ktime_clock(void)
		{
		return 0;
		}
		#endif /* CONFIG_SCHED_WALT */

		#ifdef CONFIG_CPU_FREQ
		DECLARE_PER_CPU(struct update_util_data *, cpufreq_update_util_data);

		@@ -2239,8 +2250,10 @@ static inline void cpufreq_update_util(struct rq *rq, unsigned int flags)
		#ifdef CONFIG_SCHED_WALT
		/*
		* Skip if we've already reported, but not if this is an inter-cluster
		* migration
		* migration. Also only allow WALT update sites.
		*/
		if (!(flags & SCHED_CPUFREQ_WALT))
		return;
		if (!sched_disable_window_stats &&
		(rq->load_reported_window == rq->window_start) &&
		!(flags & SCHED_CPUFREQ_INTERCLUSTER_MIG))
		@@ -2251,7 +2264,7 @@ static inline void cpufreq_update_util(struct rq *rq, unsigned int flags)
		data = rcu_dereference_sched(*per_cpu_ptr(&cpufreq_update_util_data,
		cpu_of(rq)));
		if (data)
		data->func(data, sched_clock(), flags);
		data->func(data, sched_ktime_clock(), flags);
		}

		static inline void cpufreq_update_this_cpu(struct rq *rq, unsigned int flags)
		@@ -2336,7 +2349,6 @@ extern unsigned int __read_mostly sched_downmigrate;
		extern unsigned int __read_mostly sysctl_sched_spill_nr_run;
		extern unsigned int __read_mostly sched_load_granule;

		extern u64 sched_ktime_clock(void);
		extern int register_cpu_cycle_counter_cb(struct cpu_cycle_counter_cb *cb);
		extern void reset_cpu_hmp_stats(int cpu, int reset_cra);
		extern int update_preferred_cluster(struct related_thread_group *grp,

kernel/sched/walt.c

+5 −3

Original line number	Diff line number	Diff line
		@@ -871,8 +871,10 @@ void fixup_busy_time(struct task_struct *p, int new_cpu)
		migrate_top_tasks(p, src_rq, dest_rq);

		if (!same_freq_domain(new_cpu, task_cpu(p))) {
		cpufreq_update_util(dest_rq, SCHED_CPUFREQ_INTERCLUSTER_MIG);
		cpufreq_update_util(src_rq, SCHED_CPUFREQ_INTERCLUSTER_MIG);
		cpufreq_update_util(dest_rq, SCHED_CPUFREQ_INTERCLUSTER_MIG \|
		SCHED_CPUFREQ_WALT);
		cpufreq_update_util(src_rq, SCHED_CPUFREQ_INTERCLUSTER_MIG \|
		SCHED_CPUFREQ_WALT);
		}

		if (p == src_rq->ed_task) {
		@@ -3040,7 +3042,7 @@ void walt_irq_work(struct irq_work *irq_work)

		for_each_sched_cluster(cluster)
		for_each_cpu(cpu, &cluster->cpus)
		cpufreq_update_util(cpu_rq(cpu), 0);
		cpufreq_update_util(cpu_rq(cpu), SCHED_CPUFREQ_WALT);

		for_each_cpu(cpu, cpu_possible_mask)
		raw_spin_unlock(&cpu_rq(cpu)->lock);

kernel/sched/walt.h

+0 −7

Original line number	Diff line number	Diff line
		@@ -194,8 +194,6 @@ static inline int exiting_task(struct task_struct *p)
		return (p->ravg.sum_history[0] == EXITING_TASK_MARKER);
		}

		extern u64 sched_ktime_clock(void);

		static inline struct sched_cluster *cpu_cluster(int cpu)
		{
		return cpu_rq(cpu)->cluster;
		@@ -335,11 +333,6 @@ static inline void mark_task_starting(struct task_struct *p) { }
		static inline void set_window_start(struct rq *rq) { }
		static inline int sched_cpu_high_irqload(int cpu) { return 0; }

		static inline u64 sched_ktime_clock(void)
		{
		return 0;
		}

		static inline void sched_account_irqstart(int cpu, struct task_struct *curr,
		u64 wallclock)
		{