Merge "ANDROID:sched/fair: prefer energy efficient CPUs for !prefer_idle tasks" (c6abd655) · Commits · e / devices / android_kernel_oneplus_sm8150

include/linux/sched.h

+0 −1

Original line number	Diff line number	Diff line
		@@ -745,7 +745,6 @@ struct task_struct {
		struct sched_entity se;
		struct sched_rt_entity rt;
		u64 last_sleep_ts;
		u64 last_cpu_selected_ts;
		#ifdef CONFIG_SCHED_WALT
		struct ravg ravg;
		/*

include/trace/events/sched.h

+1 −1

Original line number	Diff line number	Diff line
		@@ -1300,7 +1300,7 @@ TRACE_EVENT(sched_task_util,
		__field(bool, sync )
		__field(bool, need_idle )
		__field(int, fastpath )
		__field(bool, placement_boost )
		__field(int, placement_boost )
		__field(int, rtg_cpu )
		__field(u64, latency )
		),

kernel/sched/core.c

+0 −1

Original line number	Diff line number	Diff line
		@@ -2295,7 +2295,6 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
		p->se.nr_migrations = 0;
		p->se.vruntime = 0;
		p->last_sleep_ts = 0;
		p->last_cpu_selected_ts = 0;

		INIT_LIST_HEAD(&p->se.group_node);

kernel/sched/fair.c

+56 −73

Original line number	Diff line number	Diff line
		@@ -7005,7 +7005,8 @@ static inline bool task_fits_max(struct task_struct *p, int cpu)
		if (capacity == max_capacity)
		return true;

		if (task_boost_on_big_eligible(p) && is_min_capacity_cpu(cpu))
		if (task_boost_policy(p) == SCHED_BOOST_ON_BIG
		&& is_min_capacity_cpu(cpu))
		return false;

		return task_fits_capacity(p, capacity, cpu);
		@@ -7013,8 +7014,9 @@ static inline bool task_fits_max(struct task_struct *p, int cpu)

		struct find_best_target_env {
		struct cpumask *rtg_target;
		bool placement_boost;
		int placement_boost;
		bool need_idle;
		int fastpath;
		};

		static bool is_packing_eligible(struct task_struct *p, int target_cpu,
		@@ -7024,7 +7026,7 @@ static bool is_packing_eligible(struct task_struct *p, int target_cpu,
		{
		unsigned long tutil, estimated_capacity;

		if (fbt_env->placement_boost \|\| fbt_env->need_idle)
		if (task_placement_boost_enabled(p) \|\| fbt_env->need_idle)
		return false;

		if (best_idle_cstate == -1)
		@@ -7080,6 +7082,12 @@ static int start_cpu(struct task_struct *p, bool boosted,
		return start_cpu;
		}

		enum fastpaths {
		NONE = 0,
		SYNC_WAKEUP,
		PREV_CPU_FASTPATH,
		};

		static inline int find_best_target(struct task_struct p, int backup_cpu,
		bool boosted, bool prefer_idle,
		struct find_best_target_env *fbt_env)
		@@ -7116,6 +7124,26 @@ static inline int find_best_target(struct task_struct p, int backup_cpu,
		if (!sd)
		return -1;

		/* fast path for prev_cpu */
		if ((capacity_orig_of(prev_cpu) == capacity_orig_of(cpu)) &&
		!cpu_isolated(prev_cpu) && cpu_online(prev_cpu) &&
		idle_cpu(prev_cpu)) {

		if (idle_get_state_idx(cpu_rq(prev_cpu)) <= 1) {
		/*
		* Since target_cpu and backup_cpu are both -1s the
		* caller will choose prev_cpu and importantly skip
		* energy evaluation
		*/
		target_cpu = -1;

		fbt_env->fastpath = PREV_CPU_FASTPATH;
		trace_sched_find_best_target(p, prefer_idle, min_util,
		cpu, -1, -1, -1, target_cpu, -1);
		goto out;
		}
		}

		/* Scan CPUs in all SDs */
		sg = sd->groups;
		do {
		@@ -7259,10 +7287,10 @@ static inline int find_best_target(struct task_struct p, int backup_cpu,
		}

		/*
		* Favor CPUs with smaller capacity for Non latency
		* sensitive tasks.
		* Skip processing placement further if we are visiting
		* cpus with lower capacity than start cpu
		*/
		if (capacity_orig > target_capacity)
		if (capacity_orig < capacity_orig_of(cpu))
		continue;

		/*
		@@ -7298,7 +7326,8 @@ static inline int find_best_target(struct task_struct p, int backup_cpu,
		* IOW, prefer a deep IDLE LITTLE CPU vs a
		* shallow idle big CPU.
		*/
		if (sysctl_sched_cstate_aware &&
		if (capacity_orig >= target_capacity &&
		sysctl_sched_cstate_aware &&
		best_idle_cstate < idle_idx)
		continue;

		@@ -7308,7 +7337,6 @@ static inline int find_best_target(struct task_struct p, int backup_cpu,
		new_util_cuml > best_idle_cuml_util)))
		continue;

		/* Keep track of best idle CPU */
		target_capacity = capacity_orig;
		best_idle_cstate = idle_idx;
		best_idle_cuml_util = new_util_cuml;
		@@ -7345,7 +7373,8 @@ static inline int find_best_target(struct task_struct p, int backup_cpu,
		active_cpus_count++;

		/* Favor CPUs with maximum spare capacity */
		if ((capacity_orig - new_util) < target_max_spare_cap)
		if (capacity_orig >= target_capacity &&
		(capacity_orig - new_util) < target_max_spare_cap)
		continue;

		target_max_spare_cap = capacity_orig - new_util;
		@@ -7355,20 +7384,16 @@ static inline int find_best_target(struct task_struct p, int backup_cpu,
		}

		/*
		* We start with group where the task should be placed. When
		* placement boost is active reset the target_capacity to keep
		* traversing the other higher clusters. Don't reset it if we
		* are already at the highest cluster.
		*/
		if (fbt_env->placement_boost &&
		!is_max_capacity_cpu(group_first_cpu(sg)))
		target_capacity = ULONG_MAX;

		/*
		* if we have found a target cpu within a group, don't bother
		* checking other groups
		* If we've found a cpu, but the boost is ON_ALL we continue
		* visiting other clusters. If the boost is ON_BIG we visit
		* next cluster if they are higher in capacity. If we are
		* not in any kind of boost, we break.
		*/
		if (target_capacity != ULONG_MAX)
		if ((target_cpu != -1 \|\| best_idle_cpu != -1) &&
		(fbt_env->placement_boost == SCHED_BOOST_NONE \|\|
		(fbt_env->placement_boost == SCHED_BOOST_ON_BIG &&
		(capacity_orig_of(group_first_cpu(sg)) >
		capacity_orig_of(group_first_cpu(sg->next))))))
		break;

		} while (sg = sg->next, sg != sd->groups);
		@@ -7437,7 +7462,7 @@ static inline int find_best_target(struct task_struct p, int backup_cpu,
		target_cpu = *backup_cpu;
		*backup_cpu = -1;
		}

		out:
		return target_cpu;
		}

		@@ -7605,39 +7630,6 @@ static inline int wake_to_idle(struct task_struct *p)
		(p->flags & PF_WAKE_UP_IDLE);
		}

		#define SCHED_SELECT_PREV_CPU_NSEC 2000000
		#define SCHED_FORCE_CPU_SELECTION_NSEC 20000000

		static inline bool
		bias_to_prev_cpu(struct task_struct p, struct cpumask rtg_target)
		{
		int prev_cpu = task_cpu(p);
		#ifdef CONFIG_SCHED_WALT
		u64 ms = p->ravg.mark_start;
		#else
		u64 ms = sched_clock();
		#endif

		if (cpu_isolated(prev_cpu) \|\| !idle_cpu(prev_cpu))
		return false;

		if (!ms)
		return false;

		if (ms - p->last_cpu_selected_ts >= SCHED_SELECT_PREV_CPU_NSEC) {
		p->last_cpu_selected_ts = ms;
		return false;
		}

		if (ms - p->last_sleep_ts >= SCHED_SELECT_PREV_CPU_NSEC)
		return false;

		if (rtg_target && !cpumask_test_cpu(prev_cpu, rtg_target))
		return false;

		return true;
		}

		#ifdef CONFIG_SCHED_WALT
		static inline struct cpumask find_rtg_target(struct task_struct p)
		{
		@@ -7668,12 +7660,6 @@ static inline struct cpumask find_rtg_target(struct task_struct p)
		}
		#endif

		enum fastpaths {
		NONE = 0,
		SYNC_WAKEUP,
		PREV_CPU_BIAS,
		};

		/*
		* Needs to be called inside rcu_read_lock critical section.
		* sd is a pointer to the sched domain we wish to use for an
		@@ -7691,11 +7677,12 @@ static int find_energy_efficient_cpu(struct sched_domain *sd,
		struct cpumask *rtg_target = find_rtg_target(p);
		struct find_best_target_env fbt_env;
		bool need_idle = wake_to_idle(p);
		bool placement_boost = task_placement_boost_enabled(p);
		int placement_boost = task_boost_policy(p);
		u64 start_t = 0;
		int fastpath = 0;
		int next_cpu = -1, backup_cpu = -1;

		fbt_env.fastpath = 0;

		if (trace_sched_task_util_enabled())
		start_t = sched_clock();

		@@ -7705,7 +7692,7 @@ static int find_energy_efficient_cpu(struct sched_domain *sd,
		if (sysctl_sched_sync_hint_enable && sync &&
		bias_to_waker_cpu(p, cpu, rtg_target)) {
		energy_cpu = cpu;
		fastpath = SYNC_WAKEUP;
		fbt_env.fastpath = SYNC_WAKEUP;
		goto out;
		}

		@@ -7755,11 +7742,6 @@ static int find_energy_efficient_cpu(struct sched_domain *sd,
		prefer_idle = sched_feat(EAS_PREFER_IDLE) ?
		(schedtune_prefer_idle(p) > 0) : 0;

		if (bias_to_prev_cpu(p, rtg_target)) {
		fastpath = PREV_CPU_BIAS;
		goto out;
		}

		eenv->max_cpu_count = EAS_CPU_BKP + 1;

		fbt_env.rtg_target = rtg_target;
		@@ -7796,8 +7778,9 @@ static int find_energy_efficient_cpu(struct sched_domain *sd,
		p->state == TASK_WAKING)
		delta = task_util(p);
		#endif
		if (use_fbt && (fbt_env.placement_boost \|\| fbt_env.need_idle \|\|
		(rtg_target && !cpumask_test_cpu(prev_cpu, rtg_target)) \|\|
		if (use_fbt && (task_placement_boost_enabled(p) \|\| fbt_env.need_idle \|\|
		(rtg_target && (!cpumask_test_cpu(prev_cpu, rtg_target) \|\|
		cpumask_test_cpu(next_cpu, rtg_target))) \|\|
		__cpu_overutilized(prev_cpu, delta) \|\|
		!task_fits_max(p, prev_cpu) \|\| cpu_isolated(prev_cpu))) {
		energy_cpu = eenv->cpu[EAS_CPU_NXT].cpu_id;
		@@ -7810,7 +7793,7 @@ static int find_energy_efficient_cpu(struct sched_domain *sd,

		out:
		trace_sched_task_util(p, next_cpu, backup_cpu, energy_cpu, sync,
		need_idle, fastpath, placement_boost,
		need_idle, fbt_env.fastpath, placement_boost,
		rtg_target ? cpumask_first(rtg_target) : -1, start_t);
		return energy_cpu;
		}

kernel/sched/sched.h

+13 −12

Original line number	Diff line number	Diff line
		@@ -2902,12 +2902,13 @@ static inline bool task_placement_boost_enabled(struct task_struct *p)
		return false;
		}

		static inline bool task_boost_on_big_eligible(struct task_struct *p)
		{
		bool boost_on_big = task_sched_boost(p) &&
		sched_boost_policy() == SCHED_BOOST_ON_BIG;

		if (boost_on_big) {
		static inline enum sched_boost_policy task_boost_policy(struct task_struct *p)
		{
		enum sched_boost_policy policy = task_sched_boost(p) ?
		sched_boost_policy() :
		SCHED_BOOST_NONE;
		if (policy == SCHED_BOOST_ON_BIG) {
		/*
		* Filter out tasks less than min task util threshold
		* under conservative boost.
		@@ -2915,10 +2916,10 @@ static inline bool task_boost_on_big_eligible(struct task_struct *p)
		if (sysctl_sched_boost == CONSERVATIVE_BOOST &&
		task_util(p) <=
		sysctl_sched_min_task_util_for_boost_colocation)
		boost_on_big = false;
		policy = SCHED_BOOST_NONE;
		}

		return boost_on_big;
		return policy;
		}

		#else /* CONFIG_SCHED_WALT */
		@@ -2937,11 +2938,6 @@ static inline bool task_placement_boost_enabled(struct task_struct *p)
		return false;
		}

		static inline bool task_boost_on_big_eligible(struct task_struct *p)
		{
		return false;
		}

		static inline void check_for_migration(struct rq rq, struct task_struct p) { }

		static inline int sched_boost(void)
		@@ -2949,6 +2945,11 @@ static inline int sched_boost(void)
		return 0;
		}

		static inline enum sched_boost_policy task_boost_policy(struct task_struct *p)
		{
		return SCHED_BOOST_NONE;
		}

		static inline bool
		task_in_cum_window_demand(struct rq rq, struct task_struct p)
		{