Merge "sched/hmp: Enhance co-location and scheduler boost features" (54e5bae2) · Commits · e / devices / android_kernel_sony_msm8998

include/linux/sched/sysctl.h

+2 −0

Original line number	Diff line number	Diff line
		@@ -53,6 +53,8 @@ extern unsigned int sysctl_sched_spill_nr_run;
		extern unsigned int sysctl_sched_spill_load_pct;
		extern unsigned int sysctl_sched_upmigrate_pct;
		extern unsigned int sysctl_sched_downmigrate_pct;
		extern unsigned int sysctl_sched_group_upmigrate_pct;
		extern unsigned int sysctl_sched_group_downmigrate_pct;
		extern unsigned int sysctl_early_detection_duration;
		extern unsigned int sysctl_sched_boost;
		extern unsigned int sysctl_sched_small_wakee_task_load_pct;

include/trace/events/sched.h

+15 −10

Original line number	Diff line number	Diff line
		@@ -133,6 +133,7 @@ TRACE_EVENT(sched_task_load,
		__field( u32, flags )
		__field( int, best_cpu )
		__field( u64, latency )
		__field( int, grp_id )
		),

		TP_fast_assign(
		@@ -148,12 +149,13 @@ TRACE_EVENT(sched_task_load,
		__entry->latency = p->state == TASK_WAKING ?
		sched_ktime_clock() -
		p->ravg.mark_start : 0;
		__entry->grp_id = p->grp ? p->grp->id : 0;
		),

		TP_printk("%d (%s): demand=%u boost=%d reason=%d sync=%d need_idle=%d flags=%x best_cpu=%d latency=%llu",
		TP_printk("%d (%s): demand=%u boost=%d reason=%d sync=%d need_idle=%d flags=%x grp=%d best_cpu=%d latency=%llu",
		__entry->pid, __entry->comm, __entry->demand,
		__entry->boost, __entry->reason, __entry->sync,
		__entry->need_idle, __entry->flags,
		__entry->need_idle, __entry->flags, __entry->grp_id,
		__entry->best_cpu, __entry->latency)
		);

		@@ -167,6 +169,9 @@ TRACE_EVENT(sched_set_preferred_cluster,
		__field( int, id )
		__field( u64, demand )
		__field( int, cluster_first_cpu )
		__array( char, comm, TASK_COMM_LEN )
		__field( pid_t, pid )
		__field(unsigned int, task_demand )
		),

		TP_fast_assign(
		@@ -245,19 +250,19 @@ DEFINE_EVENT(sched_cpu_load, sched_cpu_load_cgroup,

		TRACE_EVENT(sched_set_boost,

		TP_PROTO(int ref_count),
		TP_PROTO(int type),

		TP_ARGS(ref_count),
		TP_ARGS(type),

		TP_STRUCT__entry(
		__field(unsigned int, ref_count )
		__field(int, type )
		),

		TP_fast_assign(
		__entry->ref_count = ref_count;
		__entry->type = type;
		),

		TP_printk("ref_count=%d", __entry->ref_count)
		TP_printk("type %d", __entry->type)
		);

		#if defined(CREATE_TRACE_POINTS) && defined(CONFIG_SCHED_HMP)

kernel/sched/Makefile

+1 −1

Original line number	Diff line number	Diff line
		@@ -15,7 +15,7 @@ obj-y += core.o loadavg.o clock.o cputime.o
		obj-y += idle_task.o fair.o rt.o deadline.o stop_task.o
		obj-y += wait.o completion.o idle.o sched_avg.o
		obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o
		obj-$(CONFIG_SCHED_HMP) += hmp.o
		obj-$(CONFIG_SCHED_HMP) += hmp.o boost.o
		obj-$(CONFIG_SCHED_AUTOGROUP) += auto_group.o
		obj-$(CONFIG_SCHEDSTATS) += stats.o
		obj-$(CONFIG_SCHED_DEBUG) += debug.o

kernel/sched/boost.c

0 → 100644

+226 −0

Original line number	Diff line number	Diff line
		/* Copyright (c) 2012-2016, The Linux Foundation. All rights reserved.
		*
		* This program is free software; you can redistribute it and/or modify
		* it under the terms of the GNU General Public License version 2 and
		* only version 2 as published by the Free Software Foundation.
		*
		* This program is distributed in the hope that it will be useful,
		* but WITHOUT ANY WARRANTY; without even the implied warranty of
		* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		* GNU General Public License for more details.
		*/

		#include "sched.h"
		#include <linux/of.h>
		#include <linux/sched/core_ctl.h>
		#include <trace/events/sched.h>

		/*
		* Scheduler boost is a mechanism to temporarily place tasks on CPUs
		* with higher capacity than those where a task would have normally
		* ended up with their load characteristics. Any entity enabling
		* boost is responsible for disabling it as well.
		*/

		unsigned int sysctl_sched_boost;
		static enum sched_boost_policy boost_policy;
		static enum sched_boost_policy boost_policy_dt = SCHED_BOOST_NONE;
		static DEFINE_MUTEX(boost_mutex);
		static unsigned int freq_aggr_threshold_backup;

		static inline void boost_kick(int cpu)
		{
		struct rq *rq = cpu_rq(cpu);

		if (!test_and_set_bit(BOOST_KICK, &rq->hmp_flags))
		smp_send_reschedule(cpu);
		}

		static void boost_kick_cpus(void)
		{
		int i;
		struct cpumask kick_mask;

		if (boost_policy != SCHED_BOOST_ON_BIG)
		return;

		cpumask_andnot(&kick_mask, cpu_online_mask, cpu_isolated_mask);

		for_each_cpu(i, &kick_mask) {
		if (cpu_capacity(i) != max_capacity)
		boost_kick(i);
		}
		}

		int got_boost_kick(void)
		{
		int cpu = smp_processor_id();
		struct rq *rq = cpu_rq(cpu);

		return test_bit(BOOST_KICK, &rq->hmp_flags);
		}

		void clear_boost_kick(int cpu)
		{
		struct rq *rq = cpu_rq(cpu);

		clear_bit(BOOST_KICK, &rq->hmp_flags);
		}

		/*
		* Scheduler boost type and boost policy might at first seem unrelated,
		* however, there exists a connection between them that will allow us
		* to use them interchangeably during placement decisions. We'll explain
		* the connection here in one possible way so that the implications are
		* clear when looking at placement policies.
		*
		* When policy = SCHED_BOOST_NONE, type is either none or RESTRAINED
		* When policy = SCHED_BOOST_ON_ALL or SCHED_BOOST_ON_BIG, type can
		* neither be none nor RESTRAINED.
		*/
		static void set_boost_policy(int type)
		{
		if (type == SCHED_BOOST_NONE \|\| type == RESTRAINED_BOOST) {
		boost_policy = SCHED_BOOST_NONE;
		return;
		}

		if (boost_policy_dt) {
		boost_policy = boost_policy_dt;
		return;
		}

		if (min_possible_efficiency != max_possible_efficiency) {
		boost_policy = SCHED_BOOST_ON_BIG;
		return;
		}

		boost_policy = SCHED_BOOST_ON_ALL;
		}

		enum sched_boost_policy sched_boost_policy(void)
		{
		return boost_policy;
		}

		static bool verify_boost_params(int old_val, int new_val)
		{
		/*
		* Boost can only be turned on or off. There is no possiblity of
		* switching from one boost type to another or to set the same
		* kind of boost several times.
		*/
		return !(!!old_val == !!new_val);
		}

		static void _sched_set_boost(int old_val, int type)
		{
		switch (type) {
		case NO_BOOST:
		if (old_val == FULL_THROTTLE_BOOST)
		core_ctl_set_boost(false);
		else if (old_val == CONSERVATIVE_BOOST)
		restore_cgroup_boost_settings();
		else
		update_freq_aggregate_threshold(
		freq_aggr_threshold_backup);
		break;

		case FULL_THROTTLE_BOOST:
		core_ctl_set_boost(true);
		boost_kick_cpus();
		break;

		case CONSERVATIVE_BOOST:
		update_cgroup_boost_settings();
		boost_kick_cpus();
		break;

		case RESTRAINED_BOOST:
		freq_aggr_threshold_backup =
		update_freq_aggregate_threshold(1);
		break;

		default:
		WARN_ON(1);
		return;
		}

		set_boost_policy(type);
		sysctl_sched_boost = type;
		trace_sched_set_boost(type);
		}

		void sched_boost_parse_dt(void)
		{
		struct device_node *sn;
		const char *boost_policy;

		if (!sched_enable_hmp)
		return;

		sn = of_find_node_by_path("/sched-hmp");
		if (!sn)
		return;

		if (!of_property_read_string(sn, "boost-policy", &boost_policy)) {
		if (!strcmp(boost_policy, "boost-on-big"))
		boost_policy_dt = SCHED_BOOST_ON_BIG;
		else if (!strcmp(boost_policy, "boost-on-all"))
		boost_policy_dt = SCHED_BOOST_ON_ALL;
		}
		}

		int sched_set_boost(int type)
		{
		int ret = 0;

		if (!sched_enable_hmp)
		return -EINVAL;

		mutex_lock(&boost_mutex);

		if (verify_boost_params(sysctl_sched_boost, type))
		_sched_set_boost(sysctl_sched_boost, type);
		else
		ret = -EINVAL;

		mutex_unlock(&boost_mutex);
		return ret;
		}

		int sched_boost_handler(struct ctl_table *table, int write,
		void __user buffer, size_t lenp,
		loff_t *ppos)
		{
		int ret;
		unsigned int data = (unsigned int )table->data;
		unsigned int old_val;

		if (!sched_enable_hmp)
		return -EINVAL;

		mutex_lock(&boost_mutex);

		old_val = *data;
		ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);

		if (ret \|\| !write)
		goto done;

		if (verify_boost_params(old_val, *data)) {
		_sched_set_boost(old_val, *data);
		} else {
		*data = old_val;
		ret = -EINVAL;
		}

		done:
		mutex_unlock(&boost_mutex);
		return ret;
		}

		int sched_boost(void)
		{
		return sysctl_sched_boost;
		}

kernel/sched/core.c

+1 −2

Original line number	Diff line number	Diff line
		@@ -7846,7 +7846,6 @@ void __init sched_init_smp(void)
		hotcpu_notifier(cpuset_cpu_inactive, CPU_PRI_CPUSET_INACTIVE);

		update_cluster_topology();
		init_sched_hmp_boost_policy();

		init_hrtick();

		@@ -7895,7 +7894,7 @@ void __init sched_init(void)

		BUG_ON(num_possible_cpus() > BITS_PER_LONG);

		sched_hmp_parse_dt();
		sched_boost_parse_dt();
		init_clusters();

		#ifdef CONFIG_FAIR_GROUP_SCHED