Merge "sched: walt: select sysctl_sched_is_big_little dynamically" (d0e67aec) · Commits · e / devices / android_kernel_fairphone_FP3

kernel/sched/energy.c

+3 −0

Original line number	Diff line number	Diff line
		@@ -28,6 +28,8 @@
		#include <linux/pm_opp.h>
		#include <linux/platform_device.h>

		#include "sched.h"

		struct sched_group_energy *sge_array[NR_CPUS][NR_SD_LEVELS];

		static void free_resources(void)
		@@ -269,6 +271,7 @@ static int sched_energy_probe(struct platform_device *pdev)

		kfree(max_frequencies);

		walt_sched_energy_populated_callback();
		dev_info(&pdev->dev, "Sched-energy-costs capacity updated\n");
		return 0;

kernel/sched/fair.c

+154 −263

Original line number	Diff line number	Diff line
		@@ -6746,107 +6746,6 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target)
		return target;
		}

		static inline int find_best_target(struct task_struct *p, bool boosted, bool prefer_idle)
		{
		int iter_cpu;
		int target_cpu = -1;
		int target_util = 0;
		int backup_capacity = 0;
		int best_idle_cpu = -1;
		int best_idle_cstate = INT_MAX;
		int backup_cpu = -1;
		unsigned long task_util_boosted, new_util;

		task_util_boosted = boosted_task_util(p);
		for (iter_cpu = 0; iter_cpu < NR_CPUS; iter_cpu++) {
		int cur_capacity;
		struct rq *rq;
		int idle_idx;

		/*
		* Iterate from higher cpus for boosted tasks.
		*/
		int i = boosted ? NR_CPUS-iter_cpu-1 : iter_cpu;

		if (!cpu_online(i) \|\| !cpumask_test_cpu(i, tsk_cpus_allowed(p)))
		continue;

		/*
		* p's blocked utilization is still accounted for on prev_cpu
		* so prev_cpu will receive a negative bias due to the double
		* accounting. However, the blocked utilization may be zero.
		*/
		new_util = cpu_util(i) + task_util_boosted;

		/*
		* Ensure minimum capacity to grant the required boost.
		* The target CPU can be already at a capacity level higher
		* than the one required to boost the task.
		*/
		if (new_util > capacity_orig_of(i))
		continue;

		#ifdef CONFIG_SCHED_WALT
		if (sched_cpu_high_irqload(i))
		continue;
		#endif
		/*
		* Unconditionally favoring tasks that prefer idle cpus to
		* improve latency.
		*/
		if (idle_cpu(i) && prefer_idle) {
		if (best_idle_cpu < 0)
		best_idle_cpu = i;
		continue;
		}

		cur_capacity = capacity_curr_of(i);
		rq = cpu_rq(i);
		idle_idx = idle_get_state_idx(rq);

		if (new_util < cur_capacity) {
		if (cpu_rq(i)->nr_running) {
		if(prefer_idle) {
		// Find a target cpu with lowest
		// utilization.
		if (target_util == 0 \|\|
		target_util < new_util) {
		target_cpu = i;
		target_util = new_util;
		}
		} else {
		// Find a target cpu with highest
		// utilization.
		if (target_util == 0 \|\|
		target_util > new_util) {
		target_cpu = i;
		target_util = new_util;
		}
		}
		} else if (!prefer_idle) {
		if (best_idle_cpu < 0 \|\|
		(sysctl_sched_cstate_aware &&
		best_idle_cstate > idle_idx)) {
		best_idle_cstate = idle_idx;
		best_idle_cpu = i;
		}
		}
		} else if (backup_capacity == 0 \|\|
		backup_capacity > cur_capacity) {
		// Find a backup cpu with least capacity.
		backup_capacity = cur_capacity;
		backup_cpu = i;
		}
		}

		if (prefer_idle && best_idle_cpu >= 0)
		target_cpu = best_idle_cpu;
		else if (target_cpu < 0)
		target_cpu = best_idle_cpu >= 0 ? best_idle_cpu : backup_cpu;

		return target_cpu;
		}

		/*
		* Should task be woken to any available idle cpu?
		*
		@@ -6923,9 +6822,9 @@ is_packing_eligible(struct task_struct *p, unsigned long task_util,
		static int energy_aware_wake_cpu(struct task_struct *p, int target, int sync)
		{
		struct sched_domain *sd;
		struct sched_group sg, sg_target;
		struct sched_group sg, sg_target, *start_sg;
		int target_max_cap = INT_MAX;
		int target_cpu, targeted_cpus = 0;
		int target_cpu = -1, targeted_cpus = 0;
		unsigned long task_util_boosted = 0, curr_util = 0;
		long new_util, new_util_cum;
		int i;
		@@ -6977,9 +6876,8 @@ static int energy_aware_wake_cpu(struct task_struct *p, int target, int sync)
		return cpu;
		}

		if (sysctl_sched_is_big_little) {
		task_util_boosted = boosted_task_util(p);

		if (sysctl_sched_is_big_little) {
		/*
		* Find group with sufficient capacity. We only get here if no cpu is
		* overutilized. We may end up overutilizing a cpu by adding the task,
		@@ -7032,9 +6930,10 @@ static int energy_aware_wake_cpu(struct task_struct *p, int target, int sync)
		target_max_cap = capacity_of(max_cap_cpu);
		}
		} while (sg = sg->next, sg != sd->groups);
		}

		target_cpu = -1;

		start_sg = sg_target;
		next_sg:
		cpumask_copy(&search_cpus, tsk_cpus_allowed(p));
		cpumask_and(&search_cpus, &search_cpus,
		sched_group_cpus(sg_target));
		@@ -7190,6 +7089,18 @@ static int energy_aware_wake_cpu(struct task_struct *p, int target, int sync)
		goto retry;
		}

		/*
		* If we don't find a CPU that fits this task without
		* increasing OPP, expand the search to the other
		* groups on a SMP system.
		*/
		if (!sysctl_sched_is_big_little && target_cpu == -1) {
		if (sg_target->next != start_sg) {
		sg_target = sg_target->next;
		goto next_sg;
		}
		}

		if (target_cpu == -1 \|\|
		(target_cpu != min_util_cpu && !safe_to_pack &&
		!is_packing_eligible(p, task_util_boosted, sg_target,
		@@ -7203,26 +7114,6 @@ static int energy_aware_wake_cpu(struct task_struct *p, int target, int sync)
		else
		target_cpu = task_cpu(p);
		}
		} else {
		/*
		* Find a cpu with sufficient capacity
		*/
		#ifdef CONFIG_CGROUP_SCHEDTUNE
		bool boosted = schedtune_task_boost(p) > 0;
		bool prefer_idle = schedtune_prefer_idle(p) > 0;
		#else
		bool boosted = 0;
		bool prefer_idle = 0;
		#endif
		int tmp_target = find_best_target(p, boosted, prefer_idle);

		target_cpu = task_cpu(p);
		if (tmp_target >= 0) {
		target_cpu = tmp_target;
		if ((boosted \|\| prefer_idle) && idle_cpu(target_cpu))
		return target_cpu;
		}
		}

		if (target_cpu != task_cpu(p) && !avoid_prev_cpu &&
		!cpu_isolated(task_cpu(p))) {

kernel/sched/sched.h

+16 −3

Original line number	Diff line number	Diff line
		@@ -11,6 +11,7 @@
		#include <linux/irq_work.h>
		#include <linux/tick.h>
		#include <linux/slab.h>
		#include <linux/sched_energy.h>

		#include "cpupri.h"
		#include "cpudeadline.h"
		@@ -2708,11 +2709,21 @@ extern void sched_boost_parse_dt(void);
		extern void clear_ed_task(struct task_struct p, struct rq rq);
		extern bool early_detection_notify(struct rq *rq, u64 wallclock);

		static inline unsigned int power_cost(int cpu, u64 demand)
		static inline unsigned int power_cost(int cpu, bool max)
		{
		struct sched_group_energy *sge = sge_array[cpu][SD_LEVEL1];

		if (!sge \|\| !sge->nr_cap_states)
		return cpu_max_possible_capacity(cpu);

		if (max)
		return sge->cap_states[sge->nr_cap_states - 1].power;
		else
		return sge->cap_states[0].power;
		}

		extern void walt_sched_energy_populated_callback(void);

		#else /* CONFIG_SCHED_WALT */

		struct walt_sched_stats;
		@@ -2829,11 +2840,13 @@ static inline bool early_detection_notify(struct rq *rq, u64 wallclock)
		return 0;
		}

		static inline unsigned int power_cost(int cpu, u64 demand)
		static inline unsigned int power_cost(int cpu, bool max)
		{
		return SCHED_CAPACITY_SCALE;
		}

		static inline void walt_sched_energy_populated_callback(void) { }

		#endif /* CONFIG_SCHED_WALT */

		static inline bool energy_aware(void)

kernel/sched/walt.c

+84 −5

Original line number	Diff line number	Diff line
		@@ -2145,7 +2145,7 @@ compare_clusters(void priv, struct list_head a, struct list_head *b)
		return ret;
		}

		void sort_clusters(void)
		static void sort_clusters(void)
		{
		struct sched_cluster *cluster;
		struct list_head new_head;
		@@ -2155,9 +2155,9 @@ void sort_clusters(void)

		for_each_sched_cluster(cluster) {
		cluster->max_power_cost = power_cost(cluster_first_cpu(cluster),
		max_task_load());
		true);
		cluster->min_power_cost = power_cost(cluster_first_cpu(cluster),
		0);
		false);

		if (cluster->max_power_cost > tmp_max)
		tmp_max = cluster->max_power_cost;
		@@ -2176,6 +2176,37 @@ void sort_clusters(void)
		move_list(&cluster_head, &new_head, false);
		}

		void walt_sched_energy_populated_callback(void)
		{
		struct sched_cluster *cluster;
		int prev_max = 0;

		mutex_lock(&cluster_lock);

		if (num_clusters == 1) {
		sysctl_sched_is_big_little = 0;
		mutex_unlock(&cluster_lock);
		return;
		}

		sort_clusters();

		for_each_sched_cluster(cluster) {
		if (cluster->min_power_cost > prev_max) {
		prev_max = cluster->max_power_cost;
		continue;
		}
		/*
		* We assume no overlap in the power curves of
		* clusters on a big.LITTLE system.
		*/
		sysctl_sched_is_big_little = 0;
		break;
		}

		mutex_unlock(&cluster_lock);
		}

		static void update_all_clusters_stats(void)
		{
		struct sched_cluster *cluster;
		@@ -2337,10 +2368,58 @@ static struct notifier_block notifier_policy_block = {
		.notifier_call = cpufreq_notifier_policy
		};

		static int cpufreq_notifier_trans(struct notifier_block *nb,
		unsigned long val, void *data)
		{
		struct cpufreq_freqs freq = (struct cpufreq_freqs )data;
		unsigned int cpu = freq->cpu, new_freq = freq->new;
		unsigned long flags;
		struct sched_cluster *cluster;
		struct cpumask policy_cpus = cpu_rq(cpu)->freq_domain_cpumask;
		int i, j;

		if (val != CPUFREQ_POSTCHANGE)
		return NOTIFY_DONE;

		if (cpu_cur_freq(cpu) == new_freq)
		return NOTIFY_OK;

		for_each_cpu(i, &policy_cpus) {
		cluster = cpu_rq(i)->cluster;

		if (!use_cycle_counter) {
		for_each_cpu(j, &cluster->cpus) {
		struct rq *rq = cpu_rq(j);

		raw_spin_lock_irqsave(&rq->lock, flags);
		update_task_ravg(rq->curr, rq, TASK_UPDATE,
		ktime_get_ns(), 0);
		raw_spin_unlock_irqrestore(&rq->lock, flags);
		}
		}

		cluster->cur_freq = new_freq;
		cpumask_andnot(&policy_cpus, &policy_cpus, &cluster->cpus);
		}

		return NOTIFY_OK;
		}

		static struct notifier_block notifier_trans_block = {
		.notifier_call = cpufreq_notifier_trans
		};

		static int register_walt_callback(void)
		{
		return cpufreq_register_notifier(&notifier_policy_block,
		int ret;

		ret = cpufreq_register_notifier(&notifier_policy_block,
		CPUFREQ_POLICY_NOTIFIER);
		if (!ret)
		ret = cpufreq_register_notifier(&notifier_trans_block,
		CPUFREQ_TRANSITION_NOTIFIER);

		return ret;
		}
		/*
		* cpufreq callbacks can be registered at core_initcall or later time.

kernel/sched/walt.h

+0 −2

Original line number	Diff line number	Diff line
		@@ -281,8 +281,6 @@ static inline int same_cluster(int src_cpu, int dst_cpu)
		return cpu_rq(src_cpu)->cluster == cpu_rq(dst_cpu)->cluster;
		}

		void sort_clusters(void);

		void walt_irq_work(struct irq_work *irq_work);

		void walt_sched_init(struct rq *rq);