Merge "sched: Improve the scheduler" (5e528878) · Commits · e / devices / android_kernel_oneplus_sm7250

kernel/sched/fair.c

+9 −1

Original line number	Diff line number	Diff line
		@@ -7021,9 +7021,14 @@ static void find_best_target(struct sched_domain sd, cpumask_t cpus,
		* visiting other clusters. If the boost is ON_BIG we visit
		* next cluster if they are higher in capacity. If we are
		* not in any kind of boost, we break.
		*
		* And always visit higher capacity group, if solo cpu group
		* is not in idle.
		*/
		if (!prefer_idle && !boosted &&
		(target_cpu != -1 \|\| best_idle_cpu != -1) &&
		((target_cpu != -1 && (sg->group_weight > 1 \|\|
		!next_group_higher_cap)) \|\|
		best_idle_cpu != -1) &&
		(fbt_env->placement_boost == SCHED_BOOST_NONE \|\|
		!is_full_throttle_boost() \|\|
		(fbt_env->placement_boost == SCHED_BOOST_ON_BIG &&
		@@ -7342,8 +7347,11 @@ static inline struct cpumask find_rtg_target(struct task_struct p)
		grp = task_related_thread_group(p);
		if (grp && grp->preferred_cluster && is_task_util_above_min_thresh(p)) {
		rtg_target = &grp->preferred_cluster->cpus;

		if (!task_fits_max(p, cpumask_first(rtg_target)))
		rtg_target = NULL;
		else if (cpumask_subset(rtg_target, &asym_cap_sibling_cpus))
		rtg_target = &asym_cap_sibling_cpus;
		} else {
		rtg_target = NULL;
		}

kernel/sched/sched.h

+8 −0

Original line number	Diff line number	Diff line
		@@ -145,6 +145,8 @@ struct sched_cluster {
		u64 aggr_grp_load;
		u64 coloc_boost_load;
		};

		extern cpumask_t asym_cap_sibling_cpus;
		#endif /* CONFIG_SCHED_WALT */

		/* task_struct::on_rq states: */
		@@ -2730,6 +2732,12 @@ extern void add_new_task_to_grp(struct task_struct *new);
		#define RESTRAINED_BOOST_DISABLE -3
		#define MAX_NUM_BOOST_TYPE (RESTRAINED_BOOST+1)

		static inline int asym_cap_siblings(int cpu1, int cpu2)
		{
		return (cpumask_test_cpu(cpu1, &asym_cap_sibling_cpus) &&
		cpumask_test_cpu(cpu2, &asym_cap_sibling_cpus));
		}

		static inline int cpu_capacity(int cpu)
		{
		return cpu_rq(cpu)->cluster->capacity;

kernel/sched/walt.c

+27 −4

Original line number	Diff line number	Diff line
		@@ -2085,6 +2085,7 @@ struct sched_cluster *sched_cluster[NR_CPUS];
		int num_clusters;

		struct list_head cluster_head;
		cpumask_t asym_cap_sibling_cpus = CPU_MASK_NONE;

		static void
		insert_cluster(struct sched_cluster cluster, struct list_head head)
		@@ -2269,6 +2270,7 @@ void update_cluster_topology(void)
		{
		struct cpumask cpus = *cpu_possible_mask;
		const struct cpumask *cluster_cpus;
		struct sched_cluster *cluster;
		struct list_head new_head;
		int i;

		@@ -2289,6 +2291,15 @@ void update_cluster_topology(void)
		*/
		move_list(&cluster_head, &new_head, false);
		update_all_clusters_stats();

		for_each_sched_cluster(cluster) {
		if (cpumask_weight(&cluster->cpus) == 1)
		cpumask_or(&asym_cap_sibling_cpus,
		&asym_cap_sibling_cpus, &cluster->cpus);
		}

		if (cpumask_weight(&asym_cap_sibling_cpus) == 1)
		cpumask_clear(&asym_cap_sibling_cpus);
		}

		struct sched_cluster init_cluster = {
		@@ -2529,7 +2540,8 @@ static struct sched_cluster best_cluster(struct related_thread_group grp,
		struct sched_cluster *last_best_cluster = sched_cluster[0];

		for_each_sched_cluster(cluster) {
		if (cpumask_weight(&cluster->cpus) <= 1)

		if (cluster == sched_cluster[MAX_NR_CLUSTERS - 1])
		continue;

		last_best_cluster = cluster;
		@@ -2549,7 +2561,8 @@ int preferred_cluster(struct sched_cluster cluster, struct task_struct p)

		grp = task_related_thread_group(p);
		if (grp)
		rc = (grp->preferred_cluster == cluster);
		rc = ((grp->preferred_cluster == cluster) \|\|
		cpumask_subset(&cluster->cpus, &asym_cap_sibling_cpus));

		rcu_read_unlock();
		return rc;
		@@ -3148,7 +3161,7 @@ void walt_irq_work(struct irq_work *irq_work)
		int cpu;
		u64 wc;
		bool is_migration = false;
		u64 total_grp_load = 0;
		u64 total_grp_load = 0, min_cluster_grp_load = 0;
		int level = 0;

		/* Am I the window rollover work or the migration work? */
		@@ -3184,11 +3197,21 @@ void walt_irq_work(struct irq_work *irq_work)
		total_grp_load += aggr_grp_load;
		cluster->coloc_boost_load = 0;

		if (is_min_capacity_cluster(cluster))
		min_cluster_grp_load = aggr_grp_load;
		raw_spin_unlock(&cluster->load_lock);
		}

		if (total_grp_load)
		if (total_grp_load) {
		if (cpumask_weight(&asym_cap_sibling_cpus)) {
		u64 big_grp_load =
		total_grp_load - min_cluster_grp_load;

		for_each_cpu(cpu, &asym_cap_sibling_cpus)
		cpu_cluster(cpu)->aggr_grp_load = big_grp_load;
		}
		walt_update_coloc_boost_load();
		}

		for_each_sched_cluster(cluster) {
		cpumask_t cluster_online_cpus;

kernel/sched/walt.h

+4 −0

Original line number	Diff line number	Diff line
		@@ -10,6 +10,8 @@

		#include <linux/sched/sysctl.h>

		#define MAX_NR_CLUSTERS 3

		#define WINDOW_STATS_RECENT 0
		#define WINDOW_STATS_MAX 1
		#define WINDOW_STATS_MAX_RECENT_AVG 2
		@@ -273,6 +275,8 @@ static inline void assign_cluster_ids(struct list_head *head)
		cluster->id = pos;
		sched_cluster[pos++] = cluster;
		}

		WARN_ON(pos > MAX_NR_CLUSTERS);
		}

		static inline int same_cluster(int src_cpu, int dst_cpu)