Merge "sched: Improve the scheduler" (f0ac9e95) · Commits · e / devices / android_kernel_oneplus_sm7250

kernel/sched/boost.c

+161 −96

Original line number	Diff line number	Diff line
		@@ -16,11 +16,12 @@
		* boost is responsible for disabling it as well.
		*/

		unsigned int sysctl_sched_boost;
		static enum sched_boost_policy boost_policy;
		unsigned int sysctl_sched_boost; /* To/from userspace */
		unsigned int sched_boost_type; /* currently activated sched boost */
		enum sched_boost_policy boost_policy;

		static enum sched_boost_policy boost_policy_dt = SCHED_BOOST_NONE;
		static DEFINE_MUTEX(boost_mutex);
		static int boost_refcount[MAX_NUM_BOOST_TYPE];

		/*
		* Scheduler boost type and boost policy might at first seem unrelated,
		@@ -53,108 +54,177 @@ static void set_boost_policy(int type)
		boost_policy = SCHED_BOOST_ON_ALL;
		}

		enum sched_boost_policy sched_boost_policy(void)
		static bool verify_boost_params(int type)
		{
		return boost_policy;
		return type >= RESTRAINED_BOOST_DISABLE && type <= RESTRAINED_BOOST;
		}

		static bool verify_boost_params(int type)
		static void sched_no_boost_nop(void)
		{
		return type >= RESTRAINED_BOOST_DISABLE && type <= RESTRAINED_BOOST;
		}

		static void _sched_set_boost(int type)
		static void sched_full_throttle_boost_enter(void)
		{
		core_ctl_set_boost(true);
		walt_enable_frequency_aggregation(true);
		}

		static void sched_full_throttle_boost_exit(void)
		{
		switch (type) {
		case NO_BOOST: /* All boost clear */
		if (boost_refcount[FULL_THROTTLE_BOOST] > 0) {
		core_ctl_set_boost(false);
		walt_enable_frequency_aggregation(false);
		boost_refcount[FULL_THROTTLE_BOOST] = 0;
		}
		if (boost_refcount[CONSERVATIVE_BOOST] > 0) {
		restore_cgroup_boost_settings();
		boost_refcount[CONSERVATIVE_BOOST] = 0;
		}
		if (boost_refcount[RESTRAINED_BOOST] > 0) {
		walt_enable_frequency_aggregation(false);
		boost_refcount[RESTRAINED_BOOST] = 0;

		static void sched_conservative_boost_enter(void)
		{
		update_cgroup_boost_settings();
		}
		break;

		case FULL_THROTTLE_BOOST:
		boost_refcount[FULL_THROTTLE_BOOST]++;
		if (boost_refcount[FULL_THROTTLE_BOOST] == 1) {
		core_ctl_set_boost(true);
		static void sched_conservative_boost_exit(void)
		{
		restore_cgroup_boost_settings();
		if (!boost_refcount[RESTRAINED_BOOST])
		walt_enable_frequency_aggregation(true);
		}
		break;

		case CONSERVATIVE_BOOST:
		boost_refcount[CONSERVATIVE_BOOST]++;
		if ((boost_refcount[CONSERVATIVE_BOOST] == 1) &&
		!boost_refcount[FULL_THROTTLE_BOOST])
		update_cgroup_boost_settings();
		break;

		case RESTRAINED_BOOST:
		boost_refcount[RESTRAINED_BOOST]++;
		if (boost_refcount[RESTRAINED_BOOST] == 1 &&
		!boost_refcount[FULL_THROTTLE_BOOST])
		static void sched_restrained_boost_enter(void)
		{
		walt_enable_frequency_aggregation(true);
		break;
		}

		case FULL_THROTTLE_BOOST_DISABLE:
		if (boost_refcount[FULL_THROTTLE_BOOST] >= 1) {
		boost_refcount[FULL_THROTTLE_BOOST]--;
		if (!boost_refcount[FULL_THROTTLE_BOOST]) {
		core_ctl_set_boost(false);
		if (boost_refcount[CONSERVATIVE_BOOST] >= 1)
		update_cgroup_boost_settings();
		if (!boost_refcount[RESTRAINED_BOOST])
		walt_enable_frequency_aggregation(
		false);
		static void sched_restrained_boost_exit(void)
		{
		walt_enable_frequency_aggregation(false);
		}

		struct sched_boost_data {
		int refcount;
		void (*enter)(void);
		void (*exit)(void);
		};

		static struct sched_boost_data sched_boosts[] = {
		[NO_BOOST] = {
		.refcount = 0,
		.enter = sched_no_boost_nop,
		.exit = sched_no_boost_nop,
		},
		[FULL_THROTTLE_BOOST] = {
		.refcount = 0,
		.enter = sched_full_throttle_boost_enter,
		.exit = sched_full_throttle_boost_exit,
		},
		[CONSERVATIVE_BOOST] = {
		.refcount = 0,
		.enter = sched_conservative_boost_enter,
		.exit = sched_conservative_boost_exit,
		},
		[RESTRAINED_BOOST] = {
		.refcount = 0,
		.enter = sched_restrained_boost_enter,
		.exit = sched_restrained_boost_exit,
		},
		};

		#define SCHED_BOOST_START FULL_THROTTLE_BOOST
		#define SCHED_BOOST_END (RESTRAINED_BOOST + 1)

		static int sched_effective_boost(void)
		{
		int i;

		/*
		* The boosts are sorted in descending order by
		* priority.
		*/
		for (i = SCHED_BOOST_START; i < SCHED_BOOST_END; i++) {
		if (sched_boosts[i].refcount >= 1)
		return i;
		}
		break;

		case CONSERVATIVE_BOOST_DISABLE:
		if (boost_refcount[CONSERVATIVE_BOOST] >= 1) {
		boost_refcount[CONSERVATIVE_BOOST]--;
		if (!boost_refcount[CONSERVATIVE_BOOST])
		restore_cgroup_boost_settings();
		return NO_BOOST;
		}
		break;

		case RESTRAINED_BOOST_DISABLE:
		if (boost_refcount[RESTRAINED_BOOST] >= 1) {
		boost_refcount[RESTRAINED_BOOST]--;
		if (!boost_refcount[RESTRAINED_BOOST] &&
		!boost_refcount[FULL_THROTTLE_BOOST])
		walt_enable_frequency_aggregation(false);
		static void sched_boost_disable(int type)
		{
		struct sched_boost_data *sb = &sched_boosts[type];
		int next_boost;

		if (sb->refcount <= 0)
		return;

		sb->refcount--;

		if (sb->refcount)
		return;

		/*
		* This boost's refcount becomes zero, so it must
		* be disabled. Disable it first and then apply
		* the next boost.
		*/
		sb->exit();

		next_boost = sched_effective_boost();
		sched_boosts[next_boost].enter();
		}
		break;

		default:
		WARN_ON(1);
		static void sched_boost_enable(int type)
		{
		struct sched_boost_data *sb = &sched_boosts[type];
		int next_boost, prev_boost = sched_boost_type;

		sb->refcount++;

		if (sb->refcount != 1)
		return;

		/*
		* This boost enable request did not come before.
		* Take this new request and find the next boost
		* by aggregating all the enabled boosts. If there
		* is a change, disable the previous boost and enable
		* the next boost.
		*/

		next_boost = sched_effective_boost();
		if (next_boost == prev_boost)
		return;

		sched_boosts[prev_boost].exit();
		sched_boosts[next_boost].enter();
		}

		static void sched_boost_disable_all(void)
		{
		int i;

		for (i = SCHED_BOOST_START; i < SCHED_BOOST_END; i++) {
		if (sched_boosts[i].refcount > 0) {
		sched_boosts[i].exit();
		sched_boosts[i].refcount = 0;
		}
		}
		}

		/* Aggregate final boost type */
		if (boost_refcount[FULL_THROTTLE_BOOST] >= 1)
		type = FULL_THROTTLE_BOOST;
		else if (boost_refcount[CONSERVATIVE_BOOST] >= 1)
		type = CONSERVATIVE_BOOST;
		else if (boost_refcount[RESTRAINED_BOOST] >= 1)
		type = RESTRAINED_BOOST;
		static void _sched_set_boost(int type)
		{
		if (type == 0)
		sched_boost_disable_all();
		else if (type > 0)
		sched_boost_enable(type);
		else
		type = NO_BOOST;
		sched_boost_disable(-type);

		/*
		* sysctl_sched_boost holds the boost request from
		* user space which could be different from the
		* effectively enabled boost. Update the effective
		* boost here.
		*/

		set_boost_policy(type);
		sysctl_sched_boost = type;
		trace_sched_set_boost(type);
		sched_boost_type = sched_effective_boost();
		sysctl_sched_boost = sched_boost_type;
		set_boost_policy(sysctl_sched_boost);
		trace_sched_set_boost(sysctl_sched_boost);
		}

		void sched_boost_parse_dt(void)
		@@ -210,8 +280,3 @@ int sched_boost_handler(struct ctl_table *table, int write,
		mutex_unlock(&boost_mutex);
		return ret;
		}

		int sched_boost(void)
		{
		return sysctl_sched_boost;
		}

kernel/sched/fair.c

+1 −1

Original line number	Diff line number	Diff line
		@@ -7277,7 +7277,7 @@ static inline int wake_to_idle(struct task_struct *p)
		#ifdef CONFIG_SCHED_WALT
		static inline bool is_task_util_above_min_thresh(struct task_struct *p)
		{
		unsigned int threshold = (sysctl_sched_boost == CONSERVATIVE_BOOST) ?
		unsigned int threshold = (sched_boost() == CONSERVATIVE_BOOST) ?
		sysctl_sched_min_task_util_for_boost :
		sysctl_sched_min_task_util_for_colocation;

kernel/sched/sched.h

+13 −7

Original line number	Diff line number	Diff line
		@@ -2682,11 +2682,6 @@ enum sched_boost_policy {
		SCHED_BOOST_ON_ALL,
		};

		#define NO_BOOST 0
		#define FULL_THROTTLE_BOOST 1
		#define CONSERVATIVE_BOOST 2
		#define RESTRAINED_BOOST 3

		/*
		* Returns the rq capacity of any rq in a group. This does not play
		* well with groups where rq capacity can change independently.
		@@ -2989,7 +2984,18 @@ static inline int same_freq_domain(int src_cpu, int dst_cpu)

		#define CPU_RESERVED 1

		extern int sched_boost(void);
		extern enum sched_boost_policy boost_policy;
		static inline enum sched_boost_policy sched_boost_policy(void)
		{
		return boost_policy;
		}

		extern unsigned int sched_boost_type;
		static inline int sched_boost(void)
		{
		return sched_boost_type;
		}

		extern int preferred_cluster(struct sched_cluster *cluster,
		struct task_struct *p);
		extern struct sched_cluster rq_cluster(struct rq rq);
		@@ -3096,7 +3102,7 @@ static inline enum sched_boost_policy task_boost_policy(struct task_struct *p)
		* Filter out tasks less than min task util threshold
		* under conservative boost.
		*/
		if (sysctl_sched_boost == CONSERVATIVE_BOOST &&
		if (sched_boost() == CONSERVATIVE_BOOST &&
		task_util(p) <=
		sysctl_sched_min_task_util_for_boost)
		policy = SCHED_BOOST_NONE;

kernel/sched/walt.c

+1 −1

Original line number	Diff line number	Diff line
		@@ -3120,7 +3120,7 @@ static void walt_update_coloc_boost_load(void)
		struct sched_cluster *cluster;

		if (!sysctl_sched_little_cluster_coloc_fmin_khz \|\|
		sysctl_sched_boost == CONSERVATIVE_BOOST)
		sched_boost() == CONSERVATIVE_BOOST)
		return;

		grp = lookup_related_thread_group(DEFAULT_CGROUP_COLOC_ID);