Merge "sched: Improve the scheduler" (817ef2e4) · Commits · e / devices / android_kernel_oneplus_sm8150

kernel/sched/boost.c

+162 −141

Original line number	Original line	Diff line number	Diff line
	@@ -23,52 +23,12 @@
	* boost is responsible for disabling it as well.		* boost is responsible for disabling it as well.
	*/		*/

	unsigned int sysctl_sched_boost;		unsigned int sysctl_sched_boost; /* To/from userspace */
	static enum sched_boost_policy boost_policy;		unsigned int sched_boost_type; /* currently activated sched boost */
			enum sched_boost_policy boost_policy;

	static enum sched_boost_policy boost_policy_dt = SCHED_BOOST_NONE;		static enum sched_boost_policy boost_policy_dt = SCHED_BOOST_NONE;
	static DEFINE_MUTEX(boost_mutex);		static DEFINE_MUTEX(boost_mutex);
	static int boost_refcount[MAX_NUM_BOOST_TYPE];

	static inline void boost_kick(int cpu)
	{
	struct rq *rq = cpu_rq(cpu);

	if (!test_and_set_bit(BOOST_KICK, &rq->walt_flags))
	smp_send_reschedule(cpu);
	}

	static void boost_kick_cpus(void)
	{
	int i;
	struct cpumask kick_mask;

	if (boost_policy != SCHED_BOOST_ON_BIG)
	return;

	cpus_read_lock();
	cpumask_andnot(&kick_mask, cpu_online_mask, cpu_isolated_mask);

	for_each_cpu(i, &kick_mask) {
	if (cpu_capacity(i) != max_capacity)
	boost_kick(i);
	}
	cpus_read_unlock();
	}

	int got_boost_kick(void)
	{
	int cpu = smp_processor_id();
	struct rq *rq = cpu_rq(cpu);

	return test_bit(BOOST_KICK, &rq->walt_flags);
	}

	void clear_boost_kick(int cpu)
	{
	struct rq *rq = cpu_rq(cpu);

	clear_bit(BOOST_KICK, &rq->walt_flags);
	}

	/*		/*
	* Scheduler boost type and boost policy might at first seem unrelated,		* Scheduler boost type and boost policy might at first seem unrelated,
	@@ -101,111 +61,177 @@ static void set_boost_policy(int type)
	boost_policy = SCHED_BOOST_ON_ALL;		boost_policy = SCHED_BOOST_ON_ALL;
	}		}

	enum sched_boost_policy sched_boost_policy(void)		static bool verify_boost_params(int type)
	{		{
	return boost_policy;		return type >= RESTRAINED_BOOST_DISABLE && type <= RESTRAINED_BOOST;
	}		}

	static bool verify_boost_params(int type)		static void sched_no_boost_nop(void)
	{		{
	return type >= RESTRAINED_BOOST_DISABLE && type <= RESTRAINED_BOOST;
	}		}

	static void _sched_set_boost(int type)		static void sched_full_throttle_boost_enter(void)
			{
			core_ctl_set_boost(true);
			walt_enable_frequency_aggregation(true);
			}

			static void sched_full_throttle_boost_exit(void)
	{		{
	switch (type) {
	case NO_BOOST: /* All boost clear */
	if (boost_refcount[FULL_THROTTLE_BOOST] > 0) {
	core_ctl_set_boost(false);		core_ctl_set_boost(false);
	walt_enable_frequency_aggregation(false);		walt_enable_frequency_aggregation(false);
	boost_refcount[FULL_THROTTLE_BOOST] = 0;
	}
	if (boost_refcount[CONSERVATIVE_BOOST] > 0) {
	restore_cgroup_boost_settings();
	boost_refcount[CONSERVATIVE_BOOST] = 0;
	}		}
	if (boost_refcount[RESTRAINED_BOOST] > 0) {
	walt_enable_frequency_aggregation(false);		static void sched_conservative_boost_enter(void)
	boost_refcount[RESTRAINED_BOOST] = 0;		{
			update_cgroup_boost_settings();
	}		}
	break;

	case FULL_THROTTLE_BOOST:		static void sched_conservative_boost_exit(void)
	boost_refcount[FULL_THROTTLE_BOOST]++;		{
	if (boost_refcount[FULL_THROTTLE_BOOST] == 1) {
	core_ctl_set_boost(true);
	restore_cgroup_boost_settings();		restore_cgroup_boost_settings();
	boost_kick_cpus();		}
	if (!boost_refcount[RESTRAINED_BOOST])
			static void sched_restrained_boost_enter(void)
			{
	walt_enable_frequency_aggregation(true);		walt_enable_frequency_aggregation(true);
	}		}
	break;

	case CONSERVATIVE_BOOST:		static void sched_restrained_boost_exit(void)
	boost_refcount[CONSERVATIVE_BOOST]++;		{
	if ((boost_refcount[CONSERVATIVE_BOOST] == 1) &&		walt_enable_frequency_aggregation(false);
	!boost_refcount[FULL_THROTTLE_BOOST]) {
	update_cgroup_boost_settings();
	boost_kick_cpus();
	}		}
	break;

	case RESTRAINED_BOOST:		struct sched_boost_data {
	boost_refcount[RESTRAINED_BOOST]++;		int refcount;
	if (boost_refcount[RESTRAINED_BOOST] == 1 &&		void (*enter)(void);
	!boost_refcount[FULL_THROTTLE_BOOST])		void (*exit)(void);
	walt_enable_frequency_aggregation(true);		};
	break;
			static struct sched_boost_data sched_boosts[] = {
			[NO_BOOST] = {
			.refcount = 0,
			.enter = sched_no_boost_nop,
			.exit = sched_no_boost_nop,
			},
			[FULL_THROTTLE_BOOST] = {
			.refcount = 0,
			.enter = sched_full_throttle_boost_enter,
			.exit = sched_full_throttle_boost_exit,
			},
			[CONSERVATIVE_BOOST] = {
			.refcount = 0,
			.enter = sched_conservative_boost_enter,
			.exit = sched_conservative_boost_exit,
			},
			[RESTRAINED_BOOST] = {
			.refcount = 0,
			.enter = sched_restrained_boost_enter,
			.exit = sched_restrained_boost_exit,
			},
			};

			#define SCHED_BOOST_START FULL_THROTTLE_BOOST
			#define SCHED_BOOST_END (RESTRAINED_BOOST + 1)

			static int sched_effective_boost(void)
			{
			int i;

	case FULL_THROTTLE_BOOST_DISABLE:		/*
	if (boost_refcount[FULL_THROTTLE_BOOST] >= 1) {		* The boosts are sorted in descending order by
	boost_refcount[FULL_THROTTLE_BOOST]--;		* priority.
	if (!boost_refcount[FULL_THROTTLE_BOOST]) {		*/
	core_ctl_set_boost(false);		for (i = SCHED_BOOST_START; i < SCHED_BOOST_END; i++) {
	if (boost_refcount[CONSERVATIVE_BOOST] >= 1)		if (sched_boosts[i].refcount >= 1)
	update_cgroup_boost_settings();		return i;
	if (!boost_refcount[RESTRAINED_BOOST])
	walt_enable_frequency_aggregation(
	false);
	}
	}		}
	break;

	case CONSERVATIVE_BOOST_DISABLE:		return NO_BOOST;
	if (boost_refcount[CONSERVATIVE_BOOST] >= 1) {
	boost_refcount[CONSERVATIVE_BOOST]--;
	if (!boost_refcount[CONSERVATIVE_BOOST])
	restore_cgroup_boost_settings();
	}		}
	break;

	case RESTRAINED_BOOST_DISABLE:		static void sched_boost_disable(int type)
	if (boost_refcount[RESTRAINED_BOOST] >= 1) {		{
	boost_refcount[RESTRAINED_BOOST]--;		struct sched_boost_data *sb = &sched_boosts[type];
	if (!boost_refcount[RESTRAINED_BOOST] &&		int next_boost;
	!boost_refcount[FULL_THROTTLE_BOOST])
	walt_enable_frequency_aggregation(false);		if (sb->refcount <= 0)
			return;

			sb->refcount--;

			if (sb->refcount)
			return;

			/*
			* This boost's refcount becomes zero, so it must
			* be disabled. Disable it first and then apply
			* the next boost.
			*/
			sb->exit();

			next_boost = sched_effective_boost();
			sched_boosts[next_boost].enter();
	}		}
	break;

	default:		static void sched_boost_enable(int type)
	WARN_ON(1);		{
			struct sched_boost_data *sb = &sched_boosts[type];
			int next_boost, prev_boost = sched_boost_type;

			sb->refcount++;

			if (sb->refcount != 1)
			return;

			/*
			* This boost enable request did not come before.
			* Take this new request and find the next boost
			* by aggregating all the enabled boosts. If there
			* is a change, disable the previous boost and enable
			* the next boost.
			*/

			next_boost = sched_effective_boost();
			if (next_boost == prev_boost)
	return;		return;

			sched_boosts[prev_boost].exit();
			sched_boosts[next_boost].enter();
			}

			static void sched_boost_disable_all(void)
			{
			int i;

			for (i = SCHED_BOOST_START; i < SCHED_BOOST_END; i++) {
			if (sched_boosts[i].refcount > 0) {
			sched_boosts[i].exit();
			sched_boosts[i].refcount = 0;
			}
			}
	}		}

	/* Aggregate final boost type */		static void _sched_set_boost(int type)
	if (boost_refcount[FULL_THROTTLE_BOOST] >= 1)		{
	type = FULL_THROTTLE_BOOST;		if (type == 0)
	else if (boost_refcount[CONSERVATIVE_BOOST] >= 1)		sched_boost_disable_all();
	type = CONSERVATIVE_BOOST;		else if (type > 0)
	else if (boost_refcount[RESTRAINED_BOOST] >= 1)		sched_boost_enable(type);
	type = RESTRAINED_BOOST;
	else		else
	type = NO_BOOST;		sched_boost_disable(-type);

	set_boost_policy(type);		/*
	sysctl_sched_boost = type;		* sysctl_sched_boost holds the boost request from
	trace_sched_set_boost(type);		* user space which could be different from the
			* effectively enabled boost. Update the effective
			* boost here.
			*/

			sched_boost_type = sched_effective_boost();
			sysctl_sched_boost = sched_boost_type;
			set_boost_policy(sysctl_sched_boost);
			trace_sched_set_boost(sysctl_sched_boost);
	}		}

	void sched_boost_parse_dt(void)		void sched_boost_parse_dt(void)
	@@ -261,8 +287,3 @@ int sched_boost_handler(struct ctl_table *table, int write,
	mutex_unlock(&boost_mutex);		mutex_unlock(&boost_mutex);
	return ret;		return ret;
	}		}

	int sched_boost(void)
	{
	return sysctl_sched_boost;
	}

kernel/sched/core.c

+1 −10

Original line number	Original line	Diff line number	Diff line
	@@ -1837,18 +1837,9 @@ void scheduler_ipi(void)
	*/		*/
	preempt_fold_need_resched();		preempt_fold_need_resched();

	if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick()		if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick())
	&& !got_boost_kick())
	return;		return;

	if (got_boost_kick()) {
	struct rq *rq = cpu_rq(cpu);

	if (rq->curr->sched_class == &fair_sched_class)
	check_for_migration(rq, rq->curr);
	clear_boost_kick(cpu);
	}

	/*		/*
	* Not all reschedule IPI handlers call irq_enter/irq_exit, since		* Not all reschedule IPI handlers call irq_enter/irq_exit, since
	* traditionally all their work was done from the interrupt return		* traditionally all their work was done from the interrupt return

kernel/sched/fair.c

+1 −4

Original line number	Original line	Diff line number	Diff line
	@@ -7981,7 +7981,7 @@ static inline int wake_to_idle(struct task_struct *p)
	#ifdef CONFIG_SCHED_WALT		#ifdef CONFIG_SCHED_WALT
	static inline bool is_task_util_above_min_thresh(struct task_struct *p)		static inline bool is_task_util_above_min_thresh(struct task_struct *p)
	{		{
	unsigned int threshold = (sysctl_sched_boost == CONSERVATIVE_BOOST) ?		unsigned int threshold = (sched_boost() == CONSERVATIVE_BOOST) ?
	sysctl_sched_min_task_util_for_boost :		sysctl_sched_min_task_util_for_boost :
	sysctl_sched_min_task_util_for_colocation;		sysctl_sched_min_task_util_for_colocation;

	@@ -12847,9 +12847,6 @@ static void walt_check_for_rotation(struct rq *src_rq)
	if (!walt_rotation_enabled)		if (!walt_rotation_enabled)
	return;		return;

	if (got_boost_kick())
	return;

	if (!is_min_capacity_cpu(src_cpu))		if (!is_min_capacity_cpu(src_cpu))
	return;		return;

kernel/sched/sched.h

+13 −17

Original line number	Original line	Diff line number	Diff line
	@@ -2522,11 +2522,6 @@ enum sched_boost_policy {
	SCHED_BOOST_ON_ALL,		SCHED_BOOST_ON_ALL,
	};		};

	#define NO_BOOST 0
	#define FULL_THROTTLE_BOOST 1
	#define CONSERVATIVE_BOOST 2
	#define RESTRAINED_BOOST 3

	/*		/*
	* Returns the rq capacity of any rq in a group. This does not play		* Returns the rq capacity of any rq in a group. This does not play
	* well with groups where rq capacity can change independently.		* well with groups where rq capacity can change independently.
	@@ -2827,10 +2822,20 @@ static inline int same_freq_domain(int src_cpu, int dst_cpu)
	return cpumask_test_cpu(dst_cpu, &rq->freq_domain_cpumask);		return cpumask_test_cpu(dst_cpu, &rq->freq_domain_cpumask);
	}		}

	#define BOOST_KICK 0
	#define CPU_RESERVED 1		#define CPU_RESERVED 1

	extern int sched_boost(void);		extern enum sched_boost_policy boost_policy;
			static inline enum sched_boost_policy sched_boost_policy(void)
			{
			return boost_policy;
			}

			extern unsigned int sched_boost_type;
			static inline int sched_boost(void)
			{
			return sched_boost_type;
			}

	extern int preferred_cluster(struct sched_cluster *cluster,		extern int preferred_cluster(struct sched_cluster *cluster,
	struct task_struct *p);		struct task_struct *p);
	extern struct sched_cluster rq_cluster(struct rq rq);		extern struct sched_cluster rq_cluster(struct rq rq);
	@@ -2907,8 +2912,6 @@ extern unsigned long thermal_cap(int cpu);

	extern void clear_walt_request(int cpu);		extern void clear_walt_request(int cpu);

	extern int got_boost_kick(void);
	extern void clear_boost_kick(int cpu);
	extern enum sched_boost_policy sched_boost_policy(void);		extern enum sched_boost_policy sched_boost_policy(void);
	extern void sched_boost_parse_dt(void);		extern void sched_boost_parse_dt(void);
	extern void clear_ed_task(struct task_struct p, struct rq rq);		extern void clear_ed_task(struct task_struct p, struct rq rq);
	@@ -2940,7 +2943,7 @@ static inline enum sched_boost_policy task_boost_policy(struct task_struct *p)
	* Filter out tasks less than min task util threshold		* Filter out tasks less than min task util threshold
	* under conservative boost.		* under conservative boost.
	*/		*/
	if (sysctl_sched_boost == CONSERVATIVE_BOOST &&		if (sched_boost() == CONSERVATIVE_BOOST &&
	task_util(p) <=		task_util(p) <=
	sysctl_sched_min_task_util_for_boost)		sysctl_sched_min_task_util_for_boost)
	policy = SCHED_BOOST_NONE;		policy = SCHED_BOOST_NONE;
	@@ -3074,13 +3077,6 @@ static inline int is_reserved(int cpu)
	return 0;		return 0;
	}		}

	static inline int got_boost_kick(void)
	{
	return 0;
	}

	static inline void clear_boost_kick(int cpu) { }

	static inline enum sched_boost_policy sched_boost_policy(void)		static inline enum sched_boost_policy sched_boost_policy(void)
	{		{
	return SCHED_BOOST_NONE;		return SCHED_BOOST_NONE;

kernel/sched/tune.c

+1 −34

Original line number	Original line	Diff line number	Diff line
	@@ -43,12 +43,6 @@ struct schedtune {
	*/		*/
	bool sched_boost_enabled;		bool sched_boost_enabled;

	/*
	* This tracks the default value of sched_boost_enabled and is used
	* restore the value following any temporary changes to that flag.
	*/
	bool sched_boost_enabled_backup;

	/*		/*
	* Controls whether tasks of this cgroup should be colocated with each		* Controls whether tasks of this cgroup should be colocated with each
	* other and tasks of other cgroups that have the same flag turned on.		* other and tasks of other cgroups that have the same flag turned on.
	@@ -94,7 +88,6 @@ root_schedtune = {
	#ifdef CONFIG_SCHED_WALT		#ifdef CONFIG_SCHED_WALT
	.sched_boost_no_override = false,		.sched_boost_no_override = false,
	.sched_boost_enabled = true,		.sched_boost_enabled = true,
	.sched_boost_enabled_backup = true,
	.colocate = false,		.colocate = false,
	.colocate_update_disabled = false,		.colocate_update_disabled = false,
	#endif		#endif
	@@ -153,7 +146,6 @@ static inline void init_sched_boost(struct schedtune *st)
	{		{
	st->sched_boost_no_override = false;		st->sched_boost_no_override = false;
	st->sched_boost_enabled = true;		st->sched_boost_enabled = true;
	st->sched_boost_enabled_backup = st->sched_boost_enabled;
	st->colocate = false;		st->colocate = false;
	st->colocate_update_disabled = false;		st->colocate_update_disabled = false;
	}		}
	@@ -186,8 +178,7 @@ void restore_cgroup_boost_settings(void)
	if (!allocated_group[i])		if (!allocated_group[i])
	break;		break;

	allocated_group[i]->sched_boost_enabled =		allocated_group[i]->sched_boost_enabled = true;
	allocated_group[i]->sched_boost_enabled_backup;
	}		}
	}		}

	@@ -461,25 +452,6 @@ int schedtune_can_attach(struct cgroup_taskset *tset)
	}		}

	#ifdef CONFIG_SCHED_WALT		#ifdef CONFIG_SCHED_WALT
	static u64 sched_boost_enabled_read(struct cgroup_subsys_state *css,
	struct cftype *cft)
	{
	struct schedtune *st = css_st(css);

	return st->sched_boost_enabled;
	}

	static int sched_boost_enabled_write(struct cgroup_subsys_state *css,
	struct cftype *cft, u64 enable)
	{
	struct schedtune *st = css_st(css);

	st->sched_boost_enabled = !!enable;
	st->sched_boost_enabled_backup = st->sched_boost_enabled;

	return 0;
	}

	static u64 sched_colocate_read(struct cgroup_subsys_state *css,		static u64 sched_colocate_read(struct cgroup_subsys_state *css,
	struct cftype *cft)		struct cftype *cft)
	{		{
	@@ -667,11 +639,6 @@ static struct cftype files[] = {
	.read_u64 = sched_boost_override_read,		.read_u64 = sched_boost_override_read,
	.write_u64 = sched_boost_override_write,		.write_u64 = sched_boost_override_write,
	},		},
	{
	.name = "sched_boost_enabled",
	.read_u64 = sched_boost_enabled_read,
	.write_u64 = sched_boost_enabled_write,
	},
	{		{
	.name = "colocate",		.name = "colocate",
	.read_u64 = sched_colocate_read,		.read_u64 = sched_colocate_read,