Merge "cpufreq: schedutil: Avoid processing certain notifications" (40b522e7) · Commits · e / devices / android_kernel_oneplus_sm8150

drivers/cpufreq/qcom-cpufreq.c

+17 −1

Original line number	Diff line number	Diff line
		@@ -42,6 +42,8 @@ struct cpufreq_suspend_t {
		};

		static DEFINE_PER_CPU(struct cpufreq_suspend_t, suspend_data);
		static DEFINE_PER_CPU(int, cached_resolve_idx);
		static DEFINE_PER_CPU(unsigned int, cached_resolve_freq);

		static int set_cpu_freq(struct cpufreq_policy *policy, unsigned int new_freq,
		unsigned int index)
		@@ -74,6 +76,7 @@ static int msm_cpufreq_target(struct cpufreq_policy *policy,
		int ret = 0;
		int index;
		struct cpufreq_frequency_table *table;
		int first_cpu = cpumask_first(policy->related_cpus);

		mutex_lock(&per_cpu(suspend_data, policy->cpu).suspend_mutex);

		@@ -88,6 +91,11 @@ static int msm_cpufreq_target(struct cpufreq_policy *policy,
		}

		table = policy->freq_table;
		if (per_cpu(cached_resolve_freq, first_cpu) == target_freq)
		index = per_cpu(cached_resolve_idx, first_cpu);
		else
		index = cpufreq_frequency_table_target(policy, target_freq,
		relation);

		pr_debug("CPU[%d] target %d relation %d (%d-%d) selected %d\n",
		policy->cpu, target_freq, relation,
		@@ -104,10 +112,17 @@ static unsigned int msm_cpufreq_resolve_freq(struct cpufreq_policy *policy,
		unsigned int target_freq)
		{
		int index;
		int first_cpu = cpumask_first(policy->related_cpus);
		unsigned int freq;

		index = cpufreq_frequency_table_target(policy, target_freq,
		CPUFREQ_RELATION_L);
		return policy->freq_table[index].frequency;
		freq = policy->freq_table[index].frequency;

		per_cpu(cached_resolve_idx, first_cpu) = index;
		per_cpu(cached_resolve_freq, first_cpu) = freq;

		return freq;
		}

		static int msm_cpufreq_verify(struct cpufreq_policy *policy)
		@@ -461,6 +476,7 @@ static int __init msm_cpufreq_register(void)
		for_each_possible_cpu(cpu) {
		mutex_init(&(per_cpu(suspend_data, cpu).suspend_mutex));
		per_cpu(suspend_data, cpu).device_suspended = 0;
		per_cpu(cached_resolve_freq, cpu) = UINT_MAX;
		}

		rc = platform_driver_register(&msm_cpufreq_plat_driver);

include/trace/events/power.h

+54 −0

Original line number	Diff line number	Diff line
		@@ -590,6 +590,60 @@ DEFINE_EVENT(dev_pm_qos_request, dev_pm_qos_remove_request,

		TP_ARGS(name, type, new_value)
		);

		TRACE_EVENT(sugov_util_update,
		TP_PROTO(int cpu,
		unsigned long util, unsigned long avg_cap,
		unsigned long max_cap, unsigned long nl, unsigned long pl,
		unsigned int flags),
		TP_ARGS(cpu, util, avg_cap, max_cap, nl, pl, flags),
		TP_STRUCT__entry(
		__field( int, cpu)
		__field( unsigned long, util)
		__field( unsigned long, avg_cap)
		__field( unsigned long, max_cap)
		__field( unsigned long, nl)
		__field( unsigned long, pl)
		__field( unsigned int, flags)
		),
		TP_fast_assign(
		__entry->cpu = cpu;
		__entry->util = util;
		__entry->avg_cap = avg_cap;
		__entry->max_cap = max_cap;
		__entry->nl = nl;
		__entry->pl = pl;
		__entry->flags = flags;
		),
		TP_printk("cpu=%d util=%lu avg_cap=%lu max_cap=%lu nl=%lu pl=%lu flags=0x%x",
		__entry->cpu, __entry->util, __entry->avg_cap,
		__entry->max_cap, __entry->nl,
		__entry->pl, __entry->flags)
		);

		TRACE_EVENT(sugov_next_freq,
		TP_PROTO(unsigned int cpu, unsigned long util, unsigned long max,
		unsigned int freq),
		TP_ARGS(cpu, util, max, freq),
		TP_STRUCT__entry(
		__field( unsigned int, cpu)
		__field( unsigned long, util)
		__field( unsigned long, max)
		__field( unsigned int, freq)
		),
		TP_fast_assign(
		__entry->cpu = cpu;
		__entry->util = util;
		__entry->max = max;
		__entry->freq = freq;
		),
		TP_printk("cpu=%u util=%lu max=%lu freq=%u",
		__entry->cpu,
		__entry->util,
		__entry->max,
		__entry->freq)
		);

		#endif /* _TRACE_POWER_H */

		/* This part must be outside protection */

kernel/sched/cpufreq_schedutil.c

+283 −9

Original line number	Diff line number	Diff line
		@@ -24,6 +24,9 @@
		struct sugov_tunables {
		struct gov_attr_set attr_set;
		unsigned int rate_limit_us;
		unsigned int hispeed_load;
		unsigned int hispeed_freq;
		bool pl;
		};

		struct sugov_policy {
		@@ -35,8 +38,14 @@ struct sugov_policy {
		raw_spinlock_t update_lock; /* For shared policies */
		u64 last_freq_update_time;
		s64 freq_update_delay_ns;
		u64 last_ws;
		u64 curr_cycles;
		u64 last_cyc_update_time;
		unsigned long avg_cap;
		unsigned int next_freq;
		unsigned int cached_raw_freq;
		unsigned long hispeed_util;
		unsigned long max;

		/* The next fields are only needed if fast switch cannot be used. */
		struct irq_work irq_work;
		@@ -59,6 +68,8 @@ struct sugov_cpu {
		unsigned int iowait_boost_max;
		u64 last_update;

		struct sched_walt_cpu_load walt_load;

		/* The fields below are only needed when sharing a policy. */
		unsigned long util;
		unsigned long max;
		@@ -71,6 +82,8 @@ struct sugov_cpu {
		};

		static DEFINE_PER_CPU(struct sugov_cpu, sugov_cpu);
		static unsigned int stale_ns;
		static DEFINE_PER_CPU(struct sugov_tunables *, cached_tunables);

		/********************** Governor internals *********************/

		@@ -98,9 +111,6 @@ static bool sugov_should_update_freq(struct sugov_policy *sg_policy, u64 time)
		!cpufreq_can_do_remote_dvfs(sg_policy->policy))
		return false;

		if (sg_policy->work_in_progress)
		return false;

		if (unlikely(sg_policy->need_freq_update)) {
		sg_policy->need_freq_update = false;
		/*
		@@ -139,6 +149,7 @@ static void sugov_update_commit(struct sugov_policy *sg_policy, u64 time,
		}
		}

		#define TARGET_LOAD 80
		/**
		* get_next_freq - Compute a new frequency for a given cpufreq policy.
		* @sg_policy: schedutil policy object to compute the new frequency for.
		@@ -169,6 +180,7 @@ static unsigned int get_next_freq(struct sugov_policy *sg_policy,
		policy->cpuinfo.max_freq : policy->cur;

		freq = (freq + (freq >> 2)) * util / max;
		trace_sugov_next_freq(policy->cpu, util, max, freq);

		if (freq == sg_policy->cached_raw_freq && sg_policy->next_freq != UINT_MAX)
		return sg_policy->next_freq;
		@@ -180,11 +192,14 @@ static void sugov_get_util(unsigned long util, unsigned long max, int cpu)
		{
		struct rq *rq = cpu_rq(cpu);
		unsigned long cfs_max;
		struct sugov_cpu *loadcpu = &per_cpu(sugov_cpu, cpu);

		cfs_max = arch_scale_cpu_capacity(NULL, cpu);

		*util = min(rq->cfs.avg.util_avg, cfs_max);
		*max = cfs_max;

		*util = cpu_util_freq(cpu, &loadcpu->walt_load);
		}

		static void sugov_set_iowait_boost(struct sugov_cpu *sg_cpu, u64 time,
		@@ -254,16 +269,90 @@ static bool sugov_cpu_is_busy(struct sugov_cpu *sg_cpu)
		static inline bool sugov_cpu_is_busy(struct sugov_cpu *sg_cpu) { return false; }
		#endif /* CONFIG_NO_HZ_COMMON */

		static unsigned long freq_to_util(struct sugov_policy *sg_policy,
		unsigned int freq)
		{
		return mult_frac(sg_policy->max, freq,
		sg_policy->policy->cpuinfo.max_freq);
		}

		#define KHZ 1000
		static void sugov_track_cycles(struct sugov_policy *sg_policy,
		unsigned int prev_freq,
		u64 upto)
		{
		u64 delta_ns, cycles;
		/* Track cycles in current window */
		delta_ns = upto - sg_policy->last_cyc_update_time;
		cycles = (prev_freq * delta_ns) / (NSEC_PER_SEC / KHZ);
		sg_policy->curr_cycles += cycles;
		sg_policy->last_cyc_update_time = upto;
		}

		static void sugov_calc_avg_cap(struct sugov_policy *sg_policy, u64 curr_ws,
		unsigned int prev_freq)
		{
		u64 last_ws = sg_policy->last_ws;
		unsigned int avg_freq;

		WARN_ON(curr_ws < last_ws);
		if (curr_ws <= last_ws)
		return;

		/* If we skipped some windows */
		if (curr_ws > (last_ws + sched_ravg_window)) {
		avg_freq = prev_freq;
		/* Reset tracking history */
		sg_policy->last_cyc_update_time = curr_ws;
		} else {
		sugov_track_cycles(sg_policy, prev_freq, curr_ws);
		avg_freq = sg_policy->curr_cycles;
		avg_freq /= sched_ravg_window / (NSEC_PER_SEC / KHZ);
		}
		sg_policy->avg_cap = freq_to_util(sg_policy, avg_freq);
		sg_policy->curr_cycles = 0;
		sg_policy->last_ws = curr_ws;
		}

		#define NL_RATIO 75
		#define DEFAULT_HISPEED_LOAD 90
		static void sugov_walt_adjust(struct sugov_cpu sg_cpu, unsigned long util,
		unsigned long *max)
		{
		struct sugov_policy *sg_policy = sg_cpu->sg_policy;
		bool is_migration = sg_cpu->flags & SCHED_CPUFREQ_INTERCLUSTER_MIG;
		unsigned long nl = sg_cpu->walt_load.nl;
		unsigned long cpu_util = sg_cpu->util;
		bool is_hiload;

		is_hiload = (cpu_util >= mult_frac(sg_policy->avg_cap,
		sg_policy->tunables->hispeed_load,
		100));

		if (is_hiload && !is_migration)
		util = max(util, sg_policy->hispeed_util);

		if (is_hiload && nl >= mult_frac(cpu_util, NL_RATIO, 100))
		util = max;

		if (sg_policy->tunables->pl)
		util = max(util, sg_cpu->walt_load.pl);
		}

		static void sugov_update_single(struct update_util_data *hook, u64 time,
		unsigned int flags)
		{
		struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util);
		struct sugov_policy *sg_policy = sg_cpu->sg_policy;
		struct cpufreq_policy *policy = sg_policy->policy;
		unsigned long util, max;
		unsigned long util, max, hs_util;
		unsigned int next_f;
		bool busy;

		if (!sg_policy->tunables->pl && flags & SCHED_CPUFREQ_PL)
		return;

		flags &= ~SCHED_CPUFREQ_RT_DL;
		sugov_set_iowait_boost(sg_cpu, time, flags);
		sg_cpu->last_update = time;

		@@ -272,11 +361,32 @@ static void sugov_update_single(struct update_util_data *hook, u64 time,

		busy = sugov_cpu_is_busy(sg_cpu);

		raw_spin_lock(&sg_policy->update_lock);

		if (flags & SCHED_CPUFREQ_RT_DL) {
		next_f = policy->cpuinfo.max_freq;
		} else {
		sugov_get_util(&util, &max, sg_cpu->cpu);
		if (sg_policy->max != max) {
		sg_policy->max = max;
		hs_util = freq_to_util(sg_policy,
		sg_policy->tunables->hispeed_freq);
		hs_util = mult_frac(hs_util, TARGET_LOAD, 100);
		sg_policy->hispeed_util = hs_util;
		}

		sg_cpu->util = util;
		sg_cpu->max = max;
		sg_cpu->flags = flags;

		trace_sugov_util_update(sg_cpu->cpu, sg_cpu->util,
		sg_policy->avg_cap, max, sg_cpu->walt_load.nl,
		sg_cpu->walt_load.pl, flags);

		sugov_iowait_boost(sg_cpu, &util, &max);
		sugov_calc_avg_cap(sg_policy, sg_cpu->walt_load.ws,
		sg_policy->policy->cur);
		sugov_walt_adjust(sg_cpu, &util, &max);
		next_f = get_next_freq(sg_policy, util, max);
		/*
		* Do not reduce the frequency if the CPU has not been idle
		@@ -290,12 +400,14 @@ static void sugov_update_single(struct update_util_data *hook, u64 time,
		}
		}
		sugov_update_commit(sg_policy, time, next_f);
		raw_spin_unlock(&sg_policy->update_lock);
		}

		static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, u64 time)
		{
		struct sugov_policy *sg_policy = sg_cpu->sg_policy;
		struct cpufreq_policy *policy = sg_policy->policy;
		u64 last_freq_update_time = sg_policy->last_freq_update_time;
		unsigned long util = 0, max = 1;
		unsigned int j;

		@@ -311,8 +423,8 @@ static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, u64 time)
		* enough, don't take the CPU into account as it probably is
		* idle now (and clear iowait_boost for it).
		*/
		delta_ns = time - j_sg_cpu->last_update;
		if (delta_ns > TICK_NSEC) {
		delta_ns = last_freq_update_time - j_sg_cpu->last_update;
		if (delta_ns > stale_ns) {
		j_sg_cpu->iowait_boost = 0;
		j_sg_cpu->iowait_boost_pending = false;
		continue;
		@@ -320,14 +432,22 @@ static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, u64 time)
		if (j_sg_cpu->flags & SCHED_CPUFREQ_RT_DL)
		return policy->cpuinfo.max_freq;

		/*
		* If the util value for all CPUs in a policy is 0, just using >
		* will result in a max value of 1. WALT stats can later update
		* the aggregated util value, causing get_next_freq() to compute
		* freq = max_freq * 1.25 * (util / max) for nonzero util,
		* leading to spurious jumps to fmax.
		*/
		j_util = j_sg_cpu->util;
		j_max = j_sg_cpu->max;
		if (j_util * max > j_max * util) {
		if (j_util * max >= j_max * util) {
		util = j_util;
		max = j_max;
		}

		sugov_iowait_boost(j_sg_cpu, &util, &max);
		sugov_walt_adjust(j_sg_cpu, &util, &max);
		}

		return get_next_freq(sg_policy, util, max);
		@@ -338,13 +458,26 @@ static void sugov_update_shared(struct update_util_data *hook, u64 time,
		{
		struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util);
		struct sugov_policy *sg_policy = sg_cpu->sg_policy;
		unsigned long util, max;
		unsigned long util, max, hs_util;
		unsigned int next_f;

		if (!sg_policy->tunables->pl && flags & SCHED_CPUFREQ_PL)
		return;

		sugov_get_util(&util, &max, sg_cpu->cpu);

		flags &= ~SCHED_CPUFREQ_RT_DL;

		raw_spin_lock(&sg_policy->update_lock);

		if (sg_policy->max != max) {
		sg_policy->max = max;
		hs_util = freq_to_util(sg_policy,
		sg_policy->tunables->hispeed_freq);
		hs_util = mult_frac(hs_util, TARGET_LOAD, 100);
		sg_policy->hispeed_util = hs_util;
		}

		sg_cpu->util = util;
		sg_cpu->max = max;
		sg_cpu->flags = flags;
		@@ -352,6 +485,13 @@ static void sugov_update_shared(struct update_util_data *hook, u64 time,
		sugov_set_iowait_boost(sg_cpu, time, flags);
		sg_cpu->last_update = time;

		sugov_calc_avg_cap(sg_policy, sg_cpu->walt_load.ws,
		sg_policy->policy->cur);

		trace_sugov_util_update(sg_cpu->cpu, sg_cpu->util, sg_policy->avg_cap,
		max, sg_cpu->walt_load.nl,
		sg_cpu->walt_load.pl, flags);

		if (sugov_should_update_freq(sg_policy, time)) {
		if (flags & SCHED_CPUFREQ_RT_DL)
		next_f = sg_policy->policy->cpuinfo.max_freq;
		@@ -367,8 +507,13 @@ static void sugov_update_shared(struct update_util_data *hook, u64 time,
		static void sugov_work(struct kthread_work *work)
		{
		struct sugov_policy *sg_policy = container_of(work, struct sugov_policy, work);
		unsigned long flags;

		mutex_lock(&sg_policy->work_lock);
		raw_spin_lock_irqsave(&sg_policy->update_lock, flags);
		sugov_track_cycles(sg_policy, sg_policy->policy->cur,
		ktime_get_ns());
		raw_spin_unlock_irqrestore(&sg_policy->update_lock, flags);
		__cpufreq_driver_target(sg_policy->policy, sg_policy->next_freq,
		CPUFREQ_RELATION_L);
		mutex_unlock(&sg_policy->work_lock);
		@@ -433,10 +578,86 @@ static ssize_t rate_limit_us_store(struct gov_attr_set attr_set, const char bu
		return count;
		}

		static ssize_t hispeed_load_show(struct gov_attr_set attr_set, char buf)
		{
		struct sugov_tunables *tunables = to_sugov_tunables(attr_set);

		return scnprintf(buf, PAGE_SIZE, "%u\n", tunables->hispeed_load);
		}

		static ssize_t hispeed_load_store(struct gov_attr_set *attr_set,
		const char *buf, size_t count)
		{
		struct sugov_tunables *tunables = to_sugov_tunables(attr_set);

		if (kstrtouint(buf, 10, &tunables->hispeed_load))
		return -EINVAL;

		tunables->hispeed_load = min(100U, tunables->hispeed_load);

		return count;
		}

		static ssize_t hispeed_freq_show(struct gov_attr_set attr_set, char buf)
		{
		struct sugov_tunables *tunables = to_sugov_tunables(attr_set);

		return scnprintf(buf, PAGE_SIZE, "%u\n", tunables->hispeed_freq);
		}

		static ssize_t hispeed_freq_store(struct gov_attr_set *attr_set,
		const char *buf, size_t count)
		{
		struct sugov_tunables *tunables = to_sugov_tunables(attr_set);
		unsigned int val;
		struct sugov_policy *sg_policy;
		unsigned long hs_util;
		unsigned long flags;

		if (kstrtouint(buf, 10, &val))
		return -EINVAL;

		tunables->hispeed_freq = val;
		list_for_each_entry(sg_policy, &attr_set->policy_list, tunables_hook) {
		raw_spin_lock_irqsave(&sg_policy->update_lock, flags);
		hs_util = freq_to_util(sg_policy,
		sg_policy->tunables->hispeed_freq);
		hs_util = mult_frac(hs_util, TARGET_LOAD, 100);
		sg_policy->hispeed_util = hs_util;
		raw_spin_unlock_irqrestore(&sg_policy->update_lock, flags);
		}

		return count;
		}

		static ssize_t pl_show(struct gov_attr_set attr_set, char buf)
		{
		struct sugov_tunables *tunables = to_sugov_tunables(attr_set);

		return scnprintf(buf, PAGE_SIZE, "%u\n", tunables->pl);
		}

		static ssize_t pl_store(struct gov_attr_set attr_set, const char buf,
		size_t count)
		{
		struct sugov_tunables *tunables = to_sugov_tunables(attr_set);

		if (kstrtobool(buf, &tunables->pl))
		return -EINVAL;

		return count;
		}

		static struct governor_attr rate_limit_us = __ATTR_RW(rate_limit_us);
		static struct governor_attr hispeed_load = __ATTR_RW(hispeed_load);
		static struct governor_attr hispeed_freq = __ATTR_RW(hispeed_freq);
		static struct governor_attr pl = __ATTR_RW(pl);

		static struct attribute *sugov_attributes[] = {
		&rate_limit_us.attr,
		&hispeed_load.attr,
		&hispeed_freq.attr,
		&pl.attr,
		NULL
		};

		@@ -533,6 +754,30 @@ static struct sugov_tunables sugov_tunables_alloc(struct sugov_policy sg_polic
		return tunables;
		}

		static void sugov_tunables_save(struct cpufreq_policy *policy,
		struct sugov_tunables *tunables)
		{
		int cpu;
		struct sugov_tunables *cached = per_cpu(cached_tunables, policy->cpu);

		if (!have_governor_per_policy())
		return;

		if (!cached) {
		cached = kzalloc(sizeof(*tunables), GFP_KERNEL);
		if (!cached)
		return;

		for_each_cpu(cpu, policy->related_cpus)
		per_cpu(cached_tunables, cpu) = cached;
		}

		cached->pl = tunables->pl;
		cached->hispeed_load = tunables->hispeed_load;
		cached->hispeed_freq = tunables->hispeed_freq;
		cached->rate_limit_us = tunables->rate_limit_us;
		}

		static void sugov_tunables_free(struct sugov_tunables *tunables)
		{
		if (!have_governor_per_policy())
		@@ -541,6 +786,23 @@ static void sugov_tunables_free(struct sugov_tunables *tunables)
		kfree(tunables);
		}

		static void sugov_tunables_restore(struct cpufreq_policy *policy)
		{
		struct sugov_policy *sg_policy = policy->governor_data;
		struct sugov_tunables *tunables = sg_policy->tunables;
		struct sugov_tunables *cached = per_cpu(cached_tunables, policy->cpu);

		if (!cached)
		return;

		tunables->pl = cached->pl;
		tunables->hispeed_load = cached->hispeed_load;
		tunables->hispeed_freq = cached->hispeed_freq;
		tunables->rate_limit_us = cached->rate_limit_us;
		sg_policy->freq_update_delay_ns =
		tunables->rate_limit_us * NSEC_PER_USEC;
		}

		static int sugov_init(struct cpufreq_policy *policy)
		{
		struct sugov_policy *sg_policy;
		@@ -584,9 +846,14 @@ static int sugov_init(struct cpufreq_policy *policy)
		}

		tunables->rate_limit_us = cpufreq_policy_transition_delay_us(policy);
		tunables->hispeed_load = DEFAULT_HISPEED_LOAD;
		tunables->hispeed_freq = 0;

		policy->governor_data = sg_policy;
		sg_policy->tunables = tunables;
		stale_ns = sched_ravg_window + (sched_ravg_window >> 3);

		sugov_tunables_restore(policy);

		ret = kobject_init_and_add(&tunables->attr_set.kobj, &sugov_tunables_ktype,
		get_governor_parent_kobj(policy), "%s",
		@@ -627,8 +894,10 @@ static void sugov_exit(struct cpufreq_policy *policy)

		count = gov_attr_set_put(&tunables->attr_set, &sg_policy->tunables_hook);
		policy->governor_data = NULL;
		if (!count)
		if (!count) {
		sugov_tunables_save(policy, tunables);
		sugov_tunables_free(tunables);
		}

		mutex_unlock(&global_tunables_lock);

		@@ -689,9 +958,14 @@ static void sugov_stop(struct cpufreq_policy *policy)
		static void sugov_limits(struct cpufreq_policy *policy)
		{
		struct sugov_policy *sg_policy = policy->governor_data;
		unsigned long flags;

		if (!policy->fast_switch_enabled) {
		mutex_lock(&sg_policy->work_lock);
		raw_spin_lock_irqsave(&sg_policy->update_lock, flags);
		sugov_track_cycles(sg_policy, sg_policy->policy->cur,
		ktime_get_ns());
		raw_spin_unlock_irqrestore(&sg_policy->update_lock, flags);
		cpufreq_policy_apply_limits(policy);
		mutex_unlock(&sg_policy->work_lock);
		}