Merge "sched: Improve the scheduler" (52831360) · Commits · e / devices / android_kernel_xiaomi_sm6125

include/linux/sched/sysctl.h

+2 −0

Original line number	Diff line number	Diff line
		@@ -40,6 +40,8 @@ extern unsigned int sysctl_sched_cpu_high_irqload;
		extern unsigned int sysctl_sched_boost;
		extern unsigned int sysctl_sched_group_upmigrate_pct;
		extern unsigned int sysctl_sched_group_downmigrate_pct;
		extern unsigned int sysctl_sched_conservative_pl;
		extern unsigned int sysctl_sched_many_wakeup_threshold;
		extern unsigned int sysctl_sched_walt_rotate_big_tasks;
		extern unsigned int sysctl_sched_min_task_util_for_boost;
		extern unsigned int sysctl_sched_min_task_util_for_colocation;

kernel/sched/cpufreq_schedutil.c

+15 −2

Original line number	Diff line number	Diff line
		@@ -161,6 +161,15 @@ static inline bool use_pelt(void)
		#endif
		}

		static inline bool conservative_pl(void)
		{
		#ifdef CONFIG_SCHED_WALT
		return sysctl_sched_conservative_pl;
		#else
		return false;
		#endif
		}

		static unsigned long freq_to_util(struct sugov_policy *sg_policy,
		unsigned int freq)
		{
		@@ -379,6 +388,7 @@ static void sugov_walt_adjust(struct sugov_cpu sg_cpu, unsigned long util,
		unsigned long nl = sg_cpu->walt_load.nl;
		unsigned long cpu_util = sg_cpu->util;
		bool is_hiload;
		unsigned long pl = sg_cpu->walt_load.pl;

		if (unlikely(!sysctl_sched_use_walt_cpu_util))
		return;
		@@ -393,8 +403,11 @@ static void sugov_walt_adjust(struct sugov_cpu sg_cpu, unsigned long util,
		if (is_hiload && nl >= mult_frac(cpu_util, NL_RATIO, 100))
		util = max;

		if (sg_policy->tunables->pl)
		util = max(util, sg_cpu->walt_load.pl);
		if (sg_policy->tunables->pl) {
		if (conservative_pl())
		pl = mult_frac(pl, TARGET_LOAD, 100);
		util = max(util, pl);
		}
		}

		static void sugov_update_single(struct update_util_data *hook, u64 time,

kernel/sched/fair.c

+33 −6

Original line number	Diff line number	Diff line
		@@ -5827,7 +5827,7 @@ static unsigned long __cpu_norm_util(unsigned long util, unsigned long capacity)
		}

		static inline bool
		bias_to_waker_cpu(struct task_struct p, int cpu, struct cpumask rtg_target)
		bias_to_this_cpu(struct task_struct p, int cpu, struct cpumask rtg_target)
		{
		bool base_test = cpumask_test_cpu(cpu, &p->cpus_allowed) &&
		cpu_active(cpu) && task_fits_max(p, cpu);
		@@ -7374,6 +7374,7 @@ struct find_best_target_env {
		int placement_boost;
		bool need_idle;
		int fastpath;
		int skip_cpu;
		};

		static bool is_packing_eligible(struct task_struct *p, int target_cpu,
		@@ -7447,6 +7448,7 @@ enum fastpaths {
		NONE = 0,
		SYNC_WAKEUP,
		PREV_CPU_FASTPATH,
		MANY_WAKEUP,
		};

		static inline int find_best_target(struct task_struct p, int backup_cpu,
		@@ -7547,6 +7549,9 @@ static inline int find_best_target(struct task_struct p, int backup_cpu,
		if (sched_cpu_high_irqload(i))
		continue;

		if (fbt_env->skip_cpu == i)
		continue;

		/*
		* p's blocked utilization is still accounted for on prev_cpu
		* so prev_cpu will receive a negative bias due to the double
		@@ -8105,11 +8110,22 @@ static inline struct cpumask find_rtg_target(struct task_struct p)

		return rtg_target;
		}

		static inline bool is_many_wakeup(int sibling_count_hint)
		{
		return sibling_count_hint >= sysctl_sched_many_wakeup_threshold;
		}

		#else
		static inline struct cpumask find_rtg_target(struct task_struct p)
		{
		return NULL;
		}

		static inline bool is_many_wakeup(int sibling_count_hint)
		{
		return false;
		}
		#endif

		/*
		@@ -8120,7 +8136,7 @@ static inline struct cpumask find_rtg_target(struct task_struct p)
		static int find_energy_efficient_cpu(struct sched_domain *sd,
		struct task_struct *p,
		int cpu, int prev_cpu,
		int sync)
		int sync, int sibling_count_hint)
		{
		int use_fbt = sched_feat(FIND_BEST_TARGET);
		int cpu_iter, eas_cpu_idx = EAS_CPU_NXT;
		@@ -8144,12 +8160,19 @@ static int find_energy_efficient_cpu(struct sched_domain *sd,
		sync = 0;

		if (sysctl_sched_sync_hint_enable && sync &&
		bias_to_waker_cpu(p, cpu, rtg_target)) {
		bias_to_this_cpu(p, cpu, rtg_target)) {
		target_cpu = cpu;
		fbt_env.fastpath = SYNC_WAKEUP;
		goto out;
		}

		if (is_many_wakeup(sibling_count_hint) && prev_cpu != cpu &&
		bias_to_this_cpu(p, prev_cpu, rtg_target)) {
		target_cpu = prev_cpu;
		fbt_env.fastpath = MANY_WAKEUP;
		goto out;
		}

		/* prepopulate energy diff environment */
		eenv = get_eenv(p, prev_cpu);
		if (eenv->max_cpu_count < 2)
		@@ -8203,6 +8226,8 @@ static int find_energy_efficient_cpu(struct sched_domain *sd,
		fbt_env.rtg_target = rtg_target;
		fbt_env.placement_boost = placement_boost;
		fbt_env.need_idle = need_idle;
		fbt_env.skip_cpu = is_many_wakeup(sibling_count_hint) ?
		cpu : -1;

		/* Find a cpu with sufficient capacity */
		target_cpu = find_best_target(p, &eenv->cpu[EAS_CPU_BKP].cpu_id,
		@@ -8343,7 +8368,8 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_f
		if (energy_aware()) {
		rcu_read_lock();
		new_cpu = find_energy_efficient_cpu(energy_sd, p,
		cpu, prev_cpu, sync);
		cpu, prev_cpu, sync,
		sibling_count_hint);
		rcu_read_unlock();
		return new_cpu;
		}
		@@ -8417,7 +8443,8 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_f

		} else {
		if (energy_sd)
		new_cpu = find_energy_efficient_cpu(energy_sd, p, cpu, prev_cpu, sync);
		new_cpu = find_energy_efficient_cpu(energy_sd, p, cpu,
		prev_cpu, sync, sibling_count_hint);

		/* if we did an energy-aware placement and had no choices available
		* then fall back to the default find_idlest_cpu choice
		@@ -13055,7 +13082,7 @@ void check_for_migration(struct rq rq, struct task_struct p)

		raw_spin_lock(&migration_lock);
		rcu_read_lock();
		new_cpu = find_energy_efficient_cpu(sd, p, cpu, prev_cpu, 0);
		new_cpu = find_energy_efficient_cpu(sd, p, cpu, prev_cpu, 0, 1);
		rcu_read_unlock();
		if ((new_cpu != prev_cpu) && (capacity_orig_of(new_cpu) >
		capacity_orig_of(prev_cpu))) {

kernel/sched/sched.h

+0 −1

Original line number	Diff line number	Diff line
		@@ -2123,7 +2123,6 @@ cpu_util_freq_walt(int cpu, struct sched_walt_cpu_load *walt_load)

		nl = div64_u64(nl * (100 + boost),
		walt_cpu_util_freq_divisor);
		pl = div64_u64(pl * (100 + boost), 100);

		walt_load->prev_window_util = util;
		walt_load->nl = nl;

kernel/sched/walt.c

+3 −0

Original line number	Diff line number	Diff line
		@@ -922,6 +922,9 @@ void set_window_start(struct rq *rq)
		unsigned int max_possible_efficiency = 1;
		unsigned int min_possible_efficiency = UINT_MAX;

		unsigned int sysctl_sched_conservative_pl;
		unsigned int sysctl_sched_many_wakeup_threshold = 1000;

		#define INC_STEP 8
		#define DEC_STEP 2
		#define CONSISTENT_THRES 16