Merge "sched/walt: drop preferred_cluster from rtg" (0f7beccd) · Commits · e / devices / android_kernel_fairphone_FP4

include/trace/events/sched.h

+14 −8

Original line number	Original line	Diff line number	Diff line
	@@ -1209,11 +1209,13 @@ TRACE_EVENT(sched_task_util,

	TP_PROTO(struct task_struct *p, int best_energy_cpu,		TP_PROTO(struct task_struct *p, int best_energy_cpu,
	bool sync, bool need_idle, int fastpath,		bool sync, bool need_idle, int fastpath,
	bool placement_boost, int rtg_cpu, u64 start_t,		bool placement_boost, u64 start_t,
	bool stune_boosted),		bool stune_boosted, bool is_rtg, bool rtg_skip_min,
			int start_cpu),

	TP_ARGS(p, best_energy_cpu, sync, need_idle, fastpath,		TP_ARGS(p, best_energy_cpu, sync, need_idle, fastpath,
	placement_boost, rtg_cpu, start_t, stune_boosted),		placement_boost, start_t, stune_boosted, is_rtg, rtg_skip_min,
			start_cpu),

	TP_STRUCT__entry(		TP_STRUCT__entry(
	__field(int, pid)		__field(int, pid)
	@@ -1229,6 +1231,8 @@ TRACE_EVENT(sched_task_util,
	__field(u64, latency)		__field(u64, latency)
	__field(bool, stune_boosted)		__field(bool, stune_boosted)
	__field(bool, is_rtg)		__field(bool, is_rtg)
			__field(bool, rtg_skip_min)
			__field(int, start_cpu)
	),		),

	TP_fast_assign(		TP_fast_assign(
	@@ -1241,17 +1245,19 @@ TRACE_EVENT(sched_task_util,
	__entry->need_idle = need_idle;		__entry->need_idle = need_idle;
	__entry->fastpath = fastpath;		__entry->fastpath = fastpath;
	__entry->placement_boost = placement_boost;		__entry->placement_boost = placement_boost;
	__entry->rtg_cpu = rtg_cpu;
	__entry->latency = (sched_clock() - start_t);		__entry->latency = (sched_clock() - start_t);
	__entry->stune_boosted = stune_boosted;		__entry->stune_boosted = stune_boosted;
	__entry->is_rtg = task_in_related_thread_group(p);		__entry->is_rtg = is_rtg;
			__entry->rtg_skip_min = rtg_skip_min;
			__entry->start_cpu = start_cpu;
	),		),

	TP_printk("pid=%d comm=%s util=%lu prev_cpu=%d best_energy_cpu=%d sync=%d need_idle=%d fastpath=%d placement_boost=%d rtg_cpu=%d latency=%llu stune_boosted=%d is_rtg=%d",		TP_printk("pid=%d comm=%s util=%lu prev_cpu=%d best_energy_cpu=%d sync=%d need_idle=%d fastpath=%d placement_boost=%d latency=%llu stune_boosted=%d is_rtg=%d rtg_skip_min=%d start_cpu=%d",
	__entry->pid, __entry->comm, __entry->util, __entry->prev_cpu,		__entry->pid, __entry->comm, __entry->util, __entry->prev_cpu,
	__entry->best_energy_cpu, __entry->sync, __entry->need_idle,		__entry->best_energy_cpu, __entry->sync, __entry->need_idle,
	__entry->fastpath, __entry->placement_boost, __entry->rtg_cpu,		__entry->fastpath, __entry->placement_boost,
	__entry->latency, __entry->stune_boosted, __entry->is_rtg)		__entry->latency, __entry->stune_boosted,
			__entry->is_rtg, __entry->rtg_skip_min, __entry->start_cpu)
	)		)

	/*		/*

include/trace/events/walt.h

+7 −11

Original line number	Original line	Diff line number	Diff line
	@@ -364,23 +364,19 @@ TRACE_EVENT(sched_set_preferred_cluster,

	TP_STRUCT__entry(		TP_STRUCT__entry(
	__field(int, id)		__field(int, id)
	__field(u64, demand)		__field(u64, total_demand)
	__field(int, cluster_first_cpu)		__field(bool, skip_min)
	__array(char, comm, TASK_COMM_LEN)
	__field(pid_t, pid)
	__field(unsigned int, task_demand)
	),		),

	TP_fast_assign(		TP_fast_assign(
	__entry->id = grp->id;		__entry->id = grp->id;
	__entry->demand = total_demand;		__entry->total_demand = total_demand;
	__entry->cluster_first_cpu = grp->preferred_cluster ?		__entry->skip_min = grp->skip_min;
	cluster_first_cpu(grp->preferred_cluster) : -1;
	),		),

	TP_printk("group_id %d total_demand %llu preferred_cluster_first_cpu %d",		TP_printk("group_id %d total_demand %llu skip_min %d",
	__entry->id, __entry->demand,		__entry->id, __entry->total_demand,
	__entry->cluster_first_cpu)		__entry->skip_min)
	);		);

	TRACE_EVENT(sched_migration_update_sum,		TRACE_EVENT(sched_migration_update_sum,

kernel/sched/fair.c

+64 −63

Original line number	Original line	Diff line number	Diff line
	@@ -27,6 +27,7 @@
	#include "walt.h"		#include "walt.h"

	#ifdef CONFIG_SMP		#ifdef CONFIG_SMP
			static inline bool get_rtg_status(struct task_struct *p);
	static inline bool task_fits_max(struct task_struct *p, int cpu);		static inline bool task_fits_max(struct task_struct *p, int cpu);
	#endif /* CONFIG_SMP */		#endif /* CONFIG_SMP */

	@@ -3810,13 +3811,14 @@ util_est_dequeue(struct cfs_rq cfs_rq, struct task_struct p, bool task_sleep)
	}		}

	static inline bool		static inline bool
	bias_to_waker_cpu(struct task_struct p, int cpu, struct cpumask rtg_target)		bias_to_waker_cpu(struct task_struct *p, int cpu, int start_cpu)
	{		{
	bool base_test = cpumask_test_cpu(cpu, &p->cpus_allowed) &&		bool base_test = cpumask_test_cpu(cpu, &p->cpus_allowed) &&
	cpu_active(cpu) && task_fits_max(p, cpu);		cpu_active(cpu);
	bool rtg_test = rtg_target && cpumask_test_cpu(cpu, rtg_target);		bool start_cap_test = (capacity_orig_of(cpu) >=
			capacity_orig_of(start_cpu));

	return base_test && (!rtg_target \|\| rtg_test);		return base_test && start_cap_test;
	}		}

	static inline bool task_fits_capacity(struct task_struct *p,		static inline bool task_fits_capacity(struct task_struct *p,
	@@ -3849,11 +3851,23 @@ static inline bool task_fits_max(struct task_struct *p, int cpu)
	return task_fits_capacity(p, capacity, cpu);		return task_fits_capacity(p, capacity, cpu);
	}		}

			static inline bool task_demand_fits(struct task_struct *p, int cpu)
			{
			unsigned long capacity = capacity_orig_of(cpu);
			unsigned long max_capacity = cpu_rq(cpu)->rd->max_cpu_capacity.val;

			if (capacity == max_capacity)
			return true;

			return task_fits_capacity(p, capacity, cpu);
			}

	struct find_best_target_env {		struct find_best_target_env {
	struct cpumask *rtg_target;		bool is_rtg;
	int placement_boost;		int placement_boost;
	bool need_idle;		bool need_idle;
	int fastpath;		int fastpath;
			int start_cpu;
	};		};

	static inline void adjust_cpus_for_packing(struct task_struct *p,		static inline void adjust_cpus_for_packing(struct task_struct *p,
	@@ -3891,7 +3905,7 @@ static inline void adjust_cpus_for_packing(struct task_struct *p,
	return;		return;
	}		}

	if (fbt_env->rtg_target)		if (fbt_env->is_rtg)
	*best_idle_cpu = -1;		*best_idle_cpu = -1;
	}		}

	@@ -6695,36 +6709,35 @@ unsigned long capacity_curr_of(int cpu)
	return cap_scale(max_cap, scale_freq);		return cap_scale(max_cap, scale_freq);
	}		}

	static int get_start_cpu(struct task_struct *p, bool boosted,		static int get_start_cpu(struct task_struct *p)
	struct cpumask *rtg_target)
	{		{
	struct root_domain *rd = cpu_rq(smp_processor_id())->rd;		struct root_domain *rd = cpu_rq(smp_processor_id())->rd;
	int start_cpu = -1;		int start_cpu = rd->min_cap_orig_cpu;
			bool boosted = schedtune_task_boost(p) > 0 \|\|
			task_boost_policy(p) == SCHED_BOOST_ON_BIG;
			bool task_skip_min = get_rtg_status(p) &&
			(task_util(p) > sched_task_filter_util);

	if (boosted) {		/*
	if (rd->mid_cap_orig_cpu != -1 &&		* note about min/mid/max_cap_orig_cpu - either all of them will be -ve
	task_fits_max(p, rd->mid_cap_orig_cpu))		* or just mid will be -1, there never be any other combinations of -1s
	return rd->mid_cap_orig_cpu;		* beyond these
	return rd->max_cap_orig_cpu;		*/
			if (task_skip_min \|\| boosted) {
			start_cpu = rd->mid_cap_orig_cpu == -1 ?
			rd->max_cap_orig_cpu : rd->mid_cap_orig_cpu;
	}		}
			if (start_cpu == -1 \|\| start_cpu == rd->max_cap_orig_cpu)
			return start_cpu;

	/* A task always fits on its rtg_target */		if (start_cpu == rd->min_cap_orig_cpu &&
	if (rtg_target) {		!task_demand_fits(p, start_cpu)) {
	int rtg_target_cpu = cpumask_first_and(rtg_target,		start_cpu = rd->mid_cap_orig_cpu == -1 ?
	cpu_online_mask);		rd->max_cap_orig_cpu : rd->mid_cap_orig_cpu;

	if (rtg_target_cpu < nr_cpu_ids)
	return rtg_target_cpu;
	}		}

	/* Where the task should land based on its demand */		if (start_cpu == rd->mid_cap_orig_cpu &&
	if (rd->min_cap_orig_cpu != -1		!task_demand_fits(p, start_cpu))
	&& task_fits_max(p, rd->min_cap_orig_cpu))
	start_cpu = rd->min_cap_orig_cpu;
	else if (rd->mid_cap_orig_cpu != -1
	&& task_fits_max(p, rd->mid_cap_orig_cpu))
	start_cpu = rd->mid_cap_orig_cpu;
	else
	start_cpu = rd->max_cap_orig_cpu;		start_cpu = rd->max_cap_orig_cpu;

	return start_cpu;		return start_cpu;
	@@ -6777,10 +6790,7 @@ static void find_best_target(struct sched_domain sd, cpumask_t cpus,
	target_capacity = 0;		target_capacity = 0;

	/* Find start CPU based on boost value */		/* Find start CPU based on boost value */
	start_cpu = get_start_cpu(p, boosted, fbt_env->rtg_target);		start_cpu = fbt_env->start_cpu;
	if (start_cpu < 0)
	goto out;

	/* Find SD for the start CPU */		/* Find SD for the start CPU */
	start_sd = rcu_dereference(per_cpu(sd_asym_cpucapacity, start_cpu));		start_sd = rcu_dereference(per_cpu(sd_asym_cpucapacity, start_cpu));
	if (!start_sd)		if (!start_sd)
	@@ -7404,38 +7414,25 @@ static inline int wake_to_idle(struct task_struct *p)
	}		}

	#ifdef CONFIG_SCHED_WALT		#ifdef CONFIG_SCHED_WALT
	static inline bool is_task_util_above_min_thresh(struct task_struct *p)		static inline bool get_rtg_status(struct task_struct *p)
	{
	return task_util(p) > sched_task_filter_util;
	}

	static inline struct cpumask find_rtg_target(struct task_struct p)
	{		{
	struct related_thread_group *grp;		struct related_thread_group *grp;
	struct cpumask *rtg_target;		bool ret = false;

	rcu_read_lock();		rcu_read_lock();

	grp = task_related_thread_group(p);		grp = task_related_thread_group(p);
	if (grp && grp->preferred_cluster && is_task_util_above_min_thresh(p)) {		if (grp)
	rtg_target = &grp->preferred_cluster->cpus;		ret = grp->skip_min;

	if (!task_fits_max(p, cpumask_first(rtg_target)))
	rtg_target = NULL;
	else if (cpumask_subset(rtg_target, &asym_cap_sibling_cpus))
	rtg_target = &asym_cap_sibling_cpus;
	} else {
	rtg_target = NULL;
	}

	rcu_read_unlock();		rcu_read_unlock();

	return rtg_target;		return ret;
	}		}
	#else		#else
	static inline struct cpumask find_rtg_target(struct task_struct p)		static inline bool get_rtg_status(struct task_struct *p)
	{		{
	return NULL;		return false;
	}		}
	#endif		#endif

	@@ -7509,13 +7506,19 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, int sy
	struct perf_domain *pd;		struct perf_domain *pd;
	struct sched_domain *sd;		struct sched_domain *sd;
	cpumask_t *candidates;		cpumask_t *candidates;
	struct cpumask *rtg_target = find_rtg_target(p);		bool is_rtg;
	struct find_best_target_env fbt_env;		struct find_best_target_env fbt_env;
	bool need_idle = wake_to_idle(p);		bool need_idle = wake_to_idle(p);
	int placement_boost = task_boost_policy(p);		int placement_boost = task_boost_policy(p);
	u64 start_t = 0;		u64 start_t = 0;
	int delta = 0;		int delta = 0;
	int boosted = (schedtune_task_boost(p) > 0);		int boosted = (schedtune_task_boost(p) > 0);
			int start_cpu = get_start_cpu(p);

			if (start_cpu < 0)
			goto eas_not_ready;

			is_rtg = task_in_related_thread_group(p);

	fbt_env.fastpath = 0;		fbt_env.fastpath = 0;

	@@ -7526,7 +7529,7 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, int sy
	sync = 0;		sync = 0;

	if (sysctl_sched_sync_hint_enable && sync &&		if (sysctl_sched_sync_hint_enable && sync &&
	bias_to_waker_cpu(p, cpu, rtg_target)) {		bias_to_waker_cpu(p, cpu, start_cpu)) {
	best_energy_cpu = cpu;		best_energy_cpu = cpu;
	fbt_env.fastpath = SYNC_WAKEUP;		fbt_env.fastpath = SYNC_WAKEUP;
	goto sync_wakeup;		goto sync_wakeup;
	@@ -7556,9 +7559,10 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, int sy
	cpumask_clear(candidates);		cpumask_clear(candidates);

	if (sched_feat(FIND_BEST_TARGET)) {		if (sched_feat(FIND_BEST_TARGET)) {
	fbt_env.rtg_target = rtg_target;		fbt_env.is_rtg = is_rtg;
	fbt_env.placement_boost = placement_boost;		fbt_env.placement_boost = placement_boost;
	fbt_env.need_idle = need_idle;		fbt_env.need_idle = need_idle;
			fbt_env.start_cpu = start_cpu;

	find_best_target(NULL, candidates, p, &fbt_env);		find_best_target(NULL, candidates, p, &fbt_env);
	} else {		} else {
	@@ -7583,9 +7587,7 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, int sy
	delta = task_util(p);		delta = task_util(p);
	#endif		#endif
	if (task_placement_boost_enabled(p) \|\| need_idle \|\| boosted \|\|		if (task_placement_boost_enabled(p) \|\| need_idle \|\| boosted \|\|
	(rtg_target && (!cpumask_test_cpu(prev_cpu, rtg_target) \|\|		is_rtg \|\| __cpu_overutilized(prev_cpu, delta) \|\|
	cpumask_test_cpu(cpu, rtg_target))) \|\|
	__cpu_overutilized(prev_cpu, delta) \|\|
	!task_fits_max(p, prev_cpu) \|\| cpu_isolated(prev_cpu)) {		!task_fits_max(p, prev_cpu) \|\| cpu_isolated(prev_cpu)) {
	best_energy_cpu = cpu;		best_energy_cpu = cpu;
	goto unlock;		goto unlock;
	@@ -7619,9 +7621,8 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, int sy

	sync_wakeup:		sync_wakeup:
	trace_sched_task_util(p, best_energy_cpu, sync,		trace_sched_task_util(p, best_energy_cpu, sync,
	need_idle, fbt_env.fastpath, placement_boost,		need_idle, fbt_env.fastpath, placement_boost, start_t,
	rtg_target ? cpumask_first(rtg_target) : -1, start_t,		boosted, is_rtg, get_rtg_status(p), start_cpu);
	boosted);

	/*		/*
	* Pick the best CPU if prev_cpu cannot be used, or if it saves at		* Pick the best CPU if prev_cpu cannot be used, or if it saves at
	@@ -7637,7 +7638,7 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu, int sy

	fail:		fail:
	rcu_read_unlock();		rcu_read_unlock();
			eas_not_ready:
	return -1;		return -1;
	}		}

kernel/sched/sched.h

+1 −1

Original line number	Original line	Diff line number	Diff line
	@@ -2739,7 +2739,7 @@ struct related_thread_group {
	raw_spinlock_t lock;		raw_spinlock_t lock;
	struct list_head tasks;		struct list_head tasks;
	struct list_head list;		struct list_head list;
	struct sched_cluster *preferred_cluster;		bool skip_min;
	struct rcu_head rcu;		struct rcu_head rcu;
	u64 last_update;		u64 last_update;
	u64 downmigrate_ts;		u64 downmigrate_ts;

kernel/sched/walt.c

+10 −13

Original line number	Original line	Diff line number	Diff line
	@@ -2508,18 +2508,19 @@ void update_best_cluster(struct related_thread_group *grp,
	u64 demand, bool boost)		u64 demand, bool boost)
	{		{
	if (boost) {		if (boost) {
	grp->preferred_cluster = sched_cluster[1];		grp->skip_min = true;
	return;		return;
	}		}

	if (grp->preferred_cluster == sched_cluster[0]) {		if (!grp->skip_min) {
	if (demand >= sched_group_upmigrate)		if (demand >= sched_group_upmigrate) {
	grp->preferred_cluster = sched_cluster[1];		grp->skip_min = true;
			}
	return;		return;
	}		}
	if (demand < sched_group_downmigrate) {		if (demand < sched_group_downmigrate) {
	if (!sysctl_sched_coloc_downmigrate_ns) {		if (!sysctl_sched_coloc_downmigrate_ns) {
	grp->preferred_cluster = sched_cluster[0];		grp->skip_min = false;
	return;		return;
	}		}
	if (!grp->downmigrate_ts) {		if (!grp->downmigrate_ts) {
	@@ -2528,8 +2529,8 @@ void update_best_cluster(struct related_thread_group *grp,
	}		}
	if (grp->last_update - grp->downmigrate_ts >		if (grp->last_update - grp->downmigrate_ts >
	sysctl_sched_coloc_downmigrate_ns) {		sysctl_sched_coloc_downmigrate_ns) {
	grp->preferred_cluster = sched_cluster[0];
	grp->downmigrate_ts = 0;		grp->downmigrate_ts = 0;
			grp->skip_min = false;
	}		}
	} else if (grp->downmigrate_ts)		} else if (grp->downmigrate_ts)
	grp->downmigrate_ts = 0;		grp->downmigrate_ts = 0;
	@@ -2544,7 +2545,7 @@ int preferred_cluster(struct sched_cluster cluster, struct task_struct p)

	grp = task_related_thread_group(p);		grp = task_related_thread_group(p);
	if (grp)		if (grp)
	rc = ((grp->preferred_cluster == cluster) \|\|		rc = (sched_cluster[(int)grp->skip_min] == cluster \|\|
	cpumask_subset(&cluster->cpus, &asym_cap_sibling_cpus));		cpumask_subset(&cluster->cpus, &asym_cap_sibling_cpus));

	rcu_read_unlock();		rcu_read_unlock();
	@@ -2562,7 +2563,7 @@ static void _set_preferred_cluster(struct related_thread_group *grp)
	return;		return;

	if (!hmp_capable()) {		if (!hmp_capable()) {
	grp->preferred_cluster = sched_cluster[0];		grp->skip_min = false;
	return;		return;
	}		}

	@@ -3056,11 +3057,7 @@ static bool is_rtgb_active(void)
	return false;		return false;

	grp = lookup_related_thread_group(DEFAULT_CGROUP_COLOC_ID);		grp = lookup_related_thread_group(DEFAULT_CGROUP_COLOC_ID);
	if (!grp \|\| !grp->preferred_cluster \|\|		return grp && grp->skip_min;
	is_min_capacity_cluster(grp->preferred_cluster))
	return false;

	return true;
	}		}

	/*		/*