Merge "cpufreq: schedutil: Keep track of average policy capacity" (c13a462a) · Commits · e / devices / android_kernel_fairphone_FP4

kernel/sched/cpufreq_schedutil.c

+238 −5

Original line number	Diff line number	Diff line
		@@ -20,13 +20,22 @@ struct sugov_tunables {
		struct gov_attr_set attr_set;
		unsigned int up_rate_limit_us;
		unsigned int down_rate_limit_us;
		unsigned int hispeed_load;
		unsigned int hispeed_freq;
		bool pl;
		};

		struct sugov_policy {
		struct cpufreq_policy *policy;

		u64 last_ws;
		u64 curr_cycles;
		u64 last_cyc_update_time;
		unsigned long avg_cap;
		struct sugov_tunables *tunables;
		struct list_head tunables_hook;
		unsigned long hispeed_util;
		unsigned long max;

		raw_spinlock_t update_lock; /* For shared policies */
		u64 last_freq_update_time;
		@@ -57,6 +66,11 @@ struct sugov_cpu {
		unsigned int iowait_boost_max;
		u64 last_update;

		struct sched_walt_cpu_load walt_load;

		unsigned long util;
		unsigned int flags;

		unsigned long bw_dl;
		unsigned long max;

		@@ -67,6 +81,7 @@ struct sugov_cpu {
		};

		static DEFINE_PER_CPU(struct sugov_cpu, sugov_cpu);
		static unsigned int stale_ns;

		/********************** Governor internals *********************/

		@@ -166,6 +181,7 @@ static void sugov_deferred_update(struct sugov_policy *sg_policy, u64 time,
		irq_work_queue(&sg_policy->irq_work);
		}

		#define TARGET_LOAD 80
		/**
		* get_next_freq - Compute a new frequency for a given cpufreq policy.
		* @sg_policy: schedutil policy object to compute the new frequency for.
		@@ -311,13 +327,21 @@ unsigned long schedutil_freq_util(int cpu, unsigned long util,
		static unsigned long sugov_get_util(struct sugov_cpu *sg_cpu)
		{
		struct rq *rq = cpu_rq(sg_cpu->cpu);
		unsigned long util = boosted_cpu_util(sg_cpu->cpu, cpu_util_rt(rq));
		unsigned long max = arch_scale_cpu_capacity(NULL, sg_cpu->cpu);
		#ifdef CONFIG_SCHED_WALT

		sg_cpu->max = max;
		sg_cpu->bw_dl = cpu_bw_dl(rq);

		return cpu_util_freq(sg_cpu->cpu, &sg_cpu->walt_load);
		#else
		unsigned long util = boosted_cpu_util(sg_cpu->cpu, cpu_util_rt(rq));

		sg_cpu->max = max;
		sg_cpu->bw_dl = cpu_bw_dl(rq);

		return schedutil_freq_util(sg_cpu->cpu, util, max, FREQUENCY_UTIL);
		#endif
		}

		/**
		@@ -466,6 +490,77 @@ static bool sugov_cpu_is_busy(struct sugov_cpu *sg_cpu)
		static inline bool sugov_cpu_is_busy(struct sugov_cpu *sg_cpu) { return false; }
		#endif /* CONFIG_NO_HZ_COMMON */

		static unsigned long freq_to_util(struct sugov_policy *sg_policy,
		unsigned int freq)
		{
		return mult_frac(sg_policy->max, freq,
		sg_policy->policy->cpuinfo.max_freq);
		}

		#define KHZ 1000
		static void sugov_track_cycles(struct sugov_policy *sg_policy,
		unsigned int prev_freq,
		u64 upto)
		{
		u64 delta_ns, cycles;
		/* Track cycles in current window */
		delta_ns = upto - sg_policy->last_cyc_update_time;
		cycles = (prev_freq * delta_ns) / (NSEC_PER_SEC / KHZ);
		sg_policy->curr_cycles += cycles;
		sg_policy->last_cyc_update_time = upto;
		}

		static void sugov_calc_avg_cap(struct sugov_policy *sg_policy, u64 curr_ws,
		unsigned int prev_freq)
		{
		u64 last_ws = sg_policy->last_ws;
		unsigned int avg_freq;

		WARN_ON(curr_ws < last_ws);
		if (curr_ws <= last_ws)
		return;

		/* If we skipped some windows */
		if (curr_ws > (last_ws + sched_ravg_window)) {
		avg_freq = prev_freq;
		/* Reset tracking history */
		sg_policy->last_cyc_update_time = curr_ws;
		} else {
		sugov_track_cycles(sg_policy, prev_freq, curr_ws);
		avg_freq = sg_policy->curr_cycles;
		avg_freq /= sched_ravg_window / (NSEC_PER_SEC / KHZ);
		}
		sg_policy->avg_cap = freq_to_util(sg_policy, avg_freq);
		sg_policy->curr_cycles = 0;
		sg_policy->last_ws = curr_ws;
		}

		#define NL_RATIO 75
		#define DEFAULT_HISPEED_LOAD 90
		static void sugov_walt_adjust(struct sugov_cpu sg_cpu, unsigned long util,
		unsigned long *max)
		{
		struct sugov_policy *sg_policy = sg_cpu->sg_policy;
		unsigned long cap_cur = capacity_curr_of(sg_cpu->cpu);
		bool is_migration = sg_cpu->flags & SCHED_CPUFREQ_INTERCLUSTER_MIG;
		unsigned long nl = sg_cpu->walt_load.nl;
		unsigned long cpu_util = sg_cpu->util;
		bool is_hiload;

		is_hiload = (cpu_util >= mult_frac(cap_cur,
		sg_policy->tunables->hispeed_load,
		100));

		if (is_hiload && !is_migration)
		util = max(util, sg_policy->hispeed_util);

		if (is_hiload && nl >= mult_frac(cpu_util, NL_RATIO, 100))
		util = max;

		if (sg_policy->tunables->pl)
		util = max(util, sg_cpu->walt_load.pl);
		}

		/*
		* Make sugov_should_update_freq() ignore the rate limit when DL
		* has increased the utilization.
		@@ -481,7 +576,7 @@ static void sugov_update_single(struct update_util_data *hook, u64 time,
		{
		struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util);
		struct sugov_policy *sg_policy = sg_cpu->sg_policy;
		unsigned long util, max;
		unsigned long util, max, hs_util;
		unsigned int next_f;
		bool busy;

		@@ -495,9 +590,22 @@ static void sugov_update_single(struct update_util_data *hook, u64 time,

		busy = sugov_cpu_is_busy(sg_cpu);

		util = sugov_get_util(sg_cpu);
		sg_cpu->util = util = sugov_get_util(sg_cpu);
		max = sg_cpu->max;
		sg_cpu->flags = flags;

		if (sg_policy->max != max) {
		sg_policy->max = max;
		hs_util = freq_to_util(sg_policy,
		sg_policy->tunables->hispeed_freq);
		hs_util = mult_frac(hs_util, TARGET_LOAD, 100);
		sg_policy->hispeed_util = hs_util;
		}

		sugov_iowait_apply(sg_cpu, time, &util, &max);
		sugov_calc_avg_cap(sg_policy, sg_cpu->walt_load.ws,
		sg_policy->policy->cur);
		sugov_walt_adjust(sg_cpu, &util, &max);
		next_f = get_next_freq(sg_policy, util, max);
		/*
		* Do not reduce the frequency if the CPU has not been idle
		@@ -528,21 +636,46 @@ static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, u64 time)
		{
		struct sugov_policy *sg_policy = sg_cpu->sg_policy;
		struct cpufreq_policy *policy = sg_policy->policy;
		u64 last_freq_update_time = sg_policy->last_freq_update_time;
		unsigned long util = 0, max = 1;
		unsigned int j;

		for_each_cpu(j, policy->cpus) {
		struct sugov_cpu *j_sg_cpu = &per_cpu(sugov_cpu, j);
		unsigned long j_util, j_max;
		s64 delta_ns;

		j_util = sugov_get_util(j_sg_cpu);
		/*
		* If the CPU utilization was last updated before the previous
		* frequency update and the time elapsed between the last update
		* of the CPU utilization and the last frequency update is long
		* enough, don't take the CPU into account as it probably is
		* idle now (and clear iowait_boost for it).
		*/
		delta_ns = last_freq_update_time - j_sg_cpu->last_update;
		if (delta_ns > stale_ns) {
		sugov_iowait_reset(j_sg_cpu, last_freq_update_time,
		false);
		continue;
		}

		/*
		* If the util value for all CPUs in a policy is 0, just using >
		* will result in a max value of 1. WALT stats can later update
		* the aggregated util value, causing get_next_freq() to compute
		* freq = max_freq * 1.25 * (util / max) for nonzero util,
		* leading to spurious jumps to fmax.
		*/
		j_util = j_sg_cpu->util;
		j_max = j_sg_cpu->max;
		sugov_iowait_apply(j_sg_cpu, time, &j_util, &j_max);

		if (j_util * max > j_max * util) {
		if (j_util * max >= j_max * util) {
		util = j_util;
		max = j_max;
		}

		sugov_walt_adjust(j_sg_cpu, &util, &max);
		}

		return get_next_freq(sg_policy, util, max);
		@@ -553,13 +686,26 @@ sugov_update_shared(struct update_util_data *hook, u64 time, unsigned int flags)
		{
		struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util);
		struct sugov_policy *sg_policy = sg_cpu->sg_policy;
		unsigned long hs_util;
		unsigned int next_f;

		sg_cpu->util = sugov_get_util(sg_cpu);
		sg_cpu->flags = flags;
		raw_spin_lock(&sg_policy->update_lock);

		if (sg_policy->max != sg_cpu->max) {
		sg_policy->max = sg_cpu->max;
		hs_util = freq_to_util(sg_policy,
		sg_policy->tunables->hispeed_freq);
		hs_util = mult_frac(hs_util, TARGET_LOAD, 100);
		sg_policy->hispeed_util = hs_util;
		}

		sugov_iowait_boost(sg_cpu, time, flags);
		sg_cpu->last_update = time;

		sugov_calc_avg_cap(sg_policy, sg_cpu->walt_load.ws,
		sg_policy->policy->cur);
		ignore_dl_rate_limit(sg_cpu, sg_policy);

		if (sugov_should_update_freq(sg_policy, time)) {
		@@ -593,6 +739,8 @@ static void sugov_work(struct kthread_work *work)
		raw_spin_lock_irqsave(&sg_policy->update_lock, flags);
		freq = sg_policy->next_freq;
		sg_policy->work_in_progress = false;
		sugov_track_cycles(sg_policy, sg_policy->policy->cur,
		ktime_get_ns());
		raw_spin_unlock_irqrestore(&sg_policy->update_lock, flags);

		mutex_lock(&sg_policy->work_lock);
		@@ -686,9 +834,86 @@ static ssize_t down_rate_limit_us_store(struct gov_attr_set *attr_set,
		static struct governor_attr up_rate_limit_us = __ATTR_RW(up_rate_limit_us);
		static struct governor_attr down_rate_limit_us = __ATTR_RW(down_rate_limit_us);

		static ssize_t hispeed_load_show(struct gov_attr_set attr_set, char buf)
		{
		struct sugov_tunables *tunables = to_sugov_tunables(attr_set);

		return scnprintf(buf, PAGE_SIZE, "%u\n", tunables->hispeed_load);
		}

		static ssize_t hispeed_load_store(struct gov_attr_set *attr_set,
		const char *buf, size_t count)
		{
		struct sugov_tunables *tunables = to_sugov_tunables(attr_set);

		if (kstrtouint(buf, 10, &tunables->hispeed_load))
		return -EINVAL;

		tunables->hispeed_load = min(100U, tunables->hispeed_load);

		return count;
		}

		static ssize_t hispeed_freq_show(struct gov_attr_set attr_set, char buf)
		{
		struct sugov_tunables *tunables = to_sugov_tunables(attr_set);

		return scnprintf(buf, PAGE_SIZE, "%u\n", tunables->hispeed_freq);
		}

		static ssize_t hispeed_freq_store(struct gov_attr_set *attr_set,
		const char *buf, size_t count)
		{
		struct sugov_tunables *tunables = to_sugov_tunables(attr_set);
		unsigned int val;
		struct sugov_policy *sg_policy;
		unsigned long hs_util;
		unsigned long flags;

		if (kstrtouint(buf, 10, &val))
		return -EINVAL;

		tunables->hispeed_freq = val;
		list_for_each_entry(sg_policy, &attr_set->policy_list, tunables_hook) {
		raw_spin_lock_irqsave(&sg_policy->update_lock, flags);
		hs_util = freq_to_util(sg_policy,
		sg_policy->tunables->hispeed_freq);
		hs_util = mult_frac(hs_util, TARGET_LOAD, 100);
		sg_policy->hispeed_util = hs_util;
		raw_spin_unlock_irqrestore(&sg_policy->update_lock, flags);
		}

		return count;
		}

		static ssize_t pl_show(struct gov_attr_set attr_set, char buf)
		{
		struct sugov_tunables *tunables = to_sugov_tunables(attr_set);

		return scnprintf(buf, PAGE_SIZE, "%u\n", tunables->pl);
		}

		static ssize_t pl_store(struct gov_attr_set attr_set, const char buf,
		size_t count)
		{
		struct sugov_tunables *tunables = to_sugov_tunables(attr_set);

		if (kstrtobool(buf, &tunables->pl))
		return -EINVAL;

		return count;
		}

		static struct governor_attr hispeed_load = __ATTR_RW(hispeed_load);
		static struct governor_attr hispeed_freq = __ATTR_RW(hispeed_freq);
		static struct governor_attr pl = __ATTR_RW(pl);

		static struct attribute *sugov_attributes[] = {
		&up_rate_limit_us.attr,
		&down_rate_limit_us.attr,
		&hispeed_load.attr,
		&hispeed_freq.attr,
		&pl.attr,
		NULL
		};

		@@ -846,9 +1071,12 @@ static int sugov_init(struct cpufreq_policy *policy)

		tunables->up_rate_limit_us = cpufreq_policy_transition_delay_us(policy);
		tunables->down_rate_limit_us = cpufreq_policy_transition_delay_us(policy);
		tunables->hispeed_load = DEFAULT_HISPEED_LOAD;
		tunables->hispeed_freq = 0;

		policy->governor_data = sg_policy;
		sg_policy->tunables = tunables;
		stale_ns = sched_ravg_window + (sched_ravg_window >> 3);

		ret = kobject_init_and_add(&tunables->attr_set.kobj, &sugov_tunables_ktype,
		get_governor_parent_kobj(policy), "%s",
		@@ -953,9 +1181,14 @@ static void sugov_stop(struct cpufreq_policy *policy)
		static void sugov_limits(struct cpufreq_policy *policy)
		{
		struct sugov_policy *sg_policy = policy->governor_data;
		unsigned long flags;

		if (!policy->fast_switch_enabled) {
		mutex_lock(&sg_policy->work_lock);
		raw_spin_lock_irqsave(&sg_policy->update_lock, flags);
		sugov_track_cycles(sg_policy, sg_policy->policy->cur,
		ktime_get_ns());
		raw_spin_unlock_irqrestore(&sg_policy->update_lock, flags);
		cpufreq_policy_apply_limits(policy);
		mutex_unlock(&sg_policy->work_lock);
		}