Merge "core_ctl: un-isolate BIG CPUs more aggressively" (4e7effc0) · Commits · e / devices / android_kernel_fairphone_FP3

include/linux/sched.h

+3 −1

Original line number	Diff line number	Diff line
		@@ -178,7 +178,9 @@ extern u64 nr_running_integral(unsigned int cpu);
		#endif

		extern void sched_update_nr_prod(int cpu, long delta, bool inc);
		extern void sched_get_nr_running_avg(int avg, int iowait_avg, int *big_avg);
		extern void sched_get_nr_running_avg(int avg, int iowait_avg, int *big_avg,
		unsigned int *max_nr,
		unsigned int *big_max_nr);
		extern unsigned int sched_get_cpu_util(int cpu);

		extern void calc_global_load(unsigned long ticks);

include/trace/events/sched.h

+10 −4

Original line number	Diff line number	Diff line
		@@ -1834,24 +1834,30 @@ TRACE_EVENT(sched_overutilized,

		TRACE_EVENT(sched_get_nr_running_avg,

		TP_PROTO(int avg, int big_avg, int iowait_avg),
		TP_PROTO(int avg, int big_avg, int iowait_avg,
		unsigned int max_nr, unsigned int big_max_nr),

		TP_ARGS(avg, big_avg, iowait_avg),
		TP_ARGS(avg, big_avg, iowait_avg, max_nr, big_max_nr),

		TP_STRUCT__entry(
		__field( int, avg )
		__field( int, big_avg )
		__field( int, iowait_avg )
		__field( unsigned int, max_nr )
		__field( unsigned int, big_max_nr )
		),

		TP_fast_assign(
		__entry->avg = avg;
		__entry->big_avg = big_avg;
		__entry->iowait_avg = iowait_avg;
		__entry->max_nr = max_nr;
		__entry->big_max_nr = big_max_nr;
		),

		TP_printk("avg=%d big_avg=%d iowait_avg=%d",
		__entry->avg, __entry->big_avg, __entry->iowait_avg)
		TP_printk("avg=%d big_avg=%d iowait_avg=%d max_nr=%u big_max_nr=%u",
		__entry->avg, __entry->big_avg, __entry->iowait_avg,
		__entry->max_nr, __entry->big_max_nr)
		);

		TRACE_EVENT(core_ctl_eval_need,

kernel/sched/core_ctl.c

+15 −28

Original line number	Diff line number	Diff line
		@@ -39,6 +39,7 @@ struct cluster_data {
		cpumask_t cpu_mask;
		unsigned int need_cpus;
		unsigned int task_thres;
		unsigned int max_nr;
		s64 need_ts;
		struct list_head lru;
		bool pending;
		@@ -458,47 +459,25 @@ static struct kobj_type ktype_core_ctl = {

		/* ==================== runqueue based core count =================== */

		#define NR_RUNNING_TOLERANCE 5

		static void update_running_avg(void)
		{
		int avg, iowait_avg, big_avg;
		int max_nr, big_max_nr;
		struct cluster_data *cluster;
		unsigned int index = 0;

		sched_get_nr_running_avg(&avg, &iowait_avg, &big_avg);

		/*
		* Round up to the next integer if the average nr running tasks
		* is within NR_RUNNING_TOLERANCE/100 of the next integer.
		* If normal rounding up is used, it will allow a transient task
		* to trigger online event. By the time core is onlined, the task
		* has finished.
		* Rounding to closest suffers same problem because scheduler
		* might only provide running stats per jiffy, and a transient
		* task could skew the number for one jiffy. If core control
		* samples every 2 jiffies, it will observe 0.5 additional running
		* average which rounds up to 1 task.
		*/
		avg = (avg + NR_RUNNING_TOLERANCE) / 100;
		big_avg = (big_avg + NR_RUNNING_TOLERANCE) / 100;
		sched_get_nr_running_avg(&avg, &iowait_avg, &big_avg,
		&max_nr, &big_max_nr);

		for_each_cluster(cluster, index) {
		if (!cluster->inited)
		continue;
		/*
		* Big cluster only need to take care of big tasks, but if
		* there are not enough big cores, big tasks need to be run
		* on little as well. Thus for little's runqueue stat, it
		* has to use overall runqueue average, or derive what big
		* tasks would have to be run on little. The latter approach
		* is not easy to get given core control reacts much slower
		* than scheduler, and can't predict scheduler's behavior.
		*/
		cluster->nrrun = cluster->is_big_cluster ? big_avg : avg;
		cluster->max_nr = cluster->is_big_cluster ? big_max_nr : max_nr;
		}
		}

		#define MAX_NR_THRESHOLD 4
		/* adjust needed CPUs based on current runqueue information */
		static unsigned int apply_task_need(const struct cluster_data *cluster,
		unsigned int new_need)
		@@ -509,7 +488,15 @@ static unsigned int apply_task_need(const struct cluster_data *cluster,

		/* only unisolate more cores if there are tasks to run */
		if (cluster->nrrun > new_need)
		return new_need + 1;
		new_need = new_need + 1;

		/*
		* We don't want tasks to be overcrowded in a cluster.
		* If any CPU has more than MAX_NR_THRESHOLD in the last
		* window, bring another CPU to help out.
		*/
		if (cluster->max_nr > MAX_NR_THRESHOLD)
		new_need = new_need + 1;

		return new_need;
		}

kernel/sched/sched.h

+7 −0

Original line number	Diff line number	Diff line
		@@ -2451,6 +2451,11 @@ static inline bool hmp_capable(void)
		return max_possible_capacity != min_max_possible_capacity;
		}

		static inline bool is_max_capacity_cpu(int cpu)
		{
		return cpu_max_possible_capacity(cpu) == max_possible_capacity;
		}

		/*
		* 'load' is in reference to "best cpu" at its best frequency.
		* Scale that in reference to a given cpu, accounting for how bad it is
		@@ -2719,6 +2724,8 @@ static inline int is_task_migration_throttled(struct task_struct *p)
		return 0;
		}

		static inline bool is_max_capacity_cpu(int cpu) { return true; }

		static inline void
		inc_rq_hmp_stats(struct rq rq, struct task_struct p, int change_cra) { }

kernel/sched/sched_avg.c

+32 −6

Original line number	Diff line number	Diff line
		@@ -27,11 +27,13 @@ static DEFINE_PER_CPU(u64, nr_prod_sum);
		static DEFINE_PER_CPU(u64, last_time);
		static DEFINE_PER_CPU(u64, nr_big_prod_sum);
		static DEFINE_PER_CPU(u64, nr);
		static DEFINE_PER_CPU(u64, nr_max);

		static DEFINE_PER_CPU(unsigned long, iowait_prod_sum);
		static DEFINE_PER_CPU(spinlock_t, nr_lock) = __SPIN_LOCK_UNLOCKED(nr_lock);
		static s64 last_get_time;

		#define DIV64_U64_ROUNDUP(X, Y) div64_u64((X) + (Y - 1), Y)
		/**
		* sched_get_nr_running_avg
		* @return: Average nr_running, iowait and nr_big_tasks value since last poll.
		@@ -41,7 +43,8 @@ static s64 last_get_time;
		* Obtains the average nr_running value since the last poll.
		* This function may not be called concurrently with itself
		*/
		void sched_get_nr_running_avg(int avg, int iowait_avg, int *big_avg)
		void sched_get_nr_running_avg(int avg, int iowait_avg, int *big_avg,
		unsigned int max_nr, unsigned int big_max_nr)
		{
		int cpu;
		u64 curr_time = sched_clock();
		@@ -51,6 +54,8 @@ void sched_get_nr_running_avg(int avg, int iowait_avg, int *big_avg)
		*avg = 0;
		*iowait_avg = 0;
		*big_avg = 0;
		*max_nr = 0;
		*big_max_nr = 0;

		if (!diff)
		return;
		@@ -79,17 +84,35 @@ void sched_get_nr_running_avg(int avg, int iowait_avg, int *big_avg)
		per_cpu(nr_big_prod_sum, cpu) = 0;
		per_cpu(iowait_prod_sum, cpu) = 0;

		if (*max_nr < per_cpu(nr_max, cpu))
		*max_nr = per_cpu(nr_max, cpu);

		if (is_max_capacity_cpu(cpu)) {
		if (*big_max_nr < per_cpu(nr_max, cpu))
		*big_max_nr = per_cpu(nr_max, cpu);
		}

		per_cpu(nr_max, cpu) = per_cpu(nr, cpu);
		spin_unlock_irqrestore(&per_cpu(nr_lock, cpu), flags);
		}

		diff = curr_time - last_get_time;
		last_get_time = curr_time;

		avg = (int)div64_u64(tmp_avg 100, diff);
		big_avg = (int)div64_u64(tmp_big_avg 100, diff);
		iowait_avg = (int)div64_u64(tmp_iowait 100, diff);
		/*
		* Any task running on BIG cluster and BIG tasks running on little
		* cluster contributes to big_avg. Small or medium tasks can also
		* run on BIG cluster when co-location and scheduler boost features
		* are activated. We don't want these tasks to downmigrate to little
		* cluster when BIG CPUs are available but isolated. Round up the
		* average values so that core_ctl aggressively unisolate BIG CPUs.
		*/
		*avg = (int)DIV64_U64_ROUNDUP(tmp_avg, diff);
		*big_avg = (int)DIV64_U64_ROUNDUP(tmp_big_avg, diff);
		*iowait_avg = (int)DIV64_U64_ROUNDUP(tmp_iowait, diff);

		trace_sched_get_nr_running_avg(avg, big_avg, *iowait_avg);
		trace_sched_get_nr_running_avg(avg, big_avg, *iowait_avg,
		max_nr, big_max_nr);

		BUG_ON(avg < 0 \|\| big_avg < 0 \|\| *iowait_avg < 0);
		pr_debug("%s - avg:%d big_avg:%d iowait_avg:%d\n",
		@@ -122,6 +145,9 @@ void sched_update_nr_prod(int cpu, long delta, bool inc)

		BUG_ON((s64)per_cpu(nr, cpu) < 0);

		if (per_cpu(nr, cpu) > per_cpu(nr_max, cpu))
		per_cpu(nr_max, cpu) = per_cpu(nr, cpu);

		per_cpu(nr_prod_sum, cpu) += nr_running * diff;
		per_cpu(nr_big_prod_sum, cpu) += nr_eligible_big_tasks(cpu) * diff;
		per_cpu(iowait_prod_sum, cpu) += nr_iowait_cpu(cpu) * diff;