Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 00530928 authored by Pavankumar Kondeti's avatar Pavankumar Kondeti
Browse files

core_ctl: un-isolate BIG CPUs more aggressively 



The current algorithm to bring additional BIG CPUs is very
conservative. It works when BIG tasks alone run on BIG
cluster. When co-location and scheduler boost features
are activated, small/medium tasks also run on BIG cluster.
We don't want these tasks to downmigrate, when BIG CPUs are
available but isolated. The following changes are done to
un-isolate CPUs more aggressively.

(1) Round up the big_avg. When the big_avg indicates that
there are 1.5 tasks on an average in the last window, it
indicates that we need 2 BIG CPUs not 1 BIG CPU.

(2) Track the maximum number of running tasks in the last
window on all CPUs. If any of the CPU in a cluster has more
than 4 runnable tasks in the last window, bring an additional
CPU to help out.

Change-Id: Id05d9983af290760cec6d93d1bdc45bc5e924cce
Signed-off-by: default avatarPavankumar Kondeti <pkondeti@codeaurora.org>
parent b00a905f

include/linux/sched.h

+3 −1
Original line number Diff line number Diff line
@@ -178,7 +178,9 @@ extern u64 nr_running_integral(unsigned int cpu); 
#endif



extern void sched_update_nr_prod(int cpu, long delta, bool inc);

extern void sched_get_nr_running_avg(int *avg, int *iowait_avg, int *big_avg);

extern void sched_get_nr_running_avg(int *avg, int *iowait_avg, int *big_avg,

				     unsigned int *max_nr,

				     unsigned int *big_max_nr);

extern unsigned int sched_get_cpu_util(int cpu);



extern void calc_global_load(unsigned long ticks);

include/trace/events/sched.h

+10 −4
Original line number Diff line number Diff line
@@ -1834,24 +1834,30 @@ TRACE_EVENT(sched_overutilized, 


TRACE_EVENT(sched_get_nr_running_avg,



	TP_PROTO(int avg, int big_avg, int iowait_avg),

	TP_PROTO(int avg, int big_avg, int iowait_avg,

		 unsigned int max_nr, unsigned int big_max_nr),



	TP_ARGS(avg, big_avg, iowait_avg),

	TP_ARGS(avg, big_avg, iowait_avg, max_nr, big_max_nr),



	TP_STRUCT__entry(

		__field( int,	avg			)

		__field( int,	big_avg			)

		__field( int,	iowait_avg		)

		__field( unsigned int,	max_nr		)

		__field( unsigned int,	big_max_nr	)

	),



	TP_fast_assign(

		__entry->avg		= avg;

		__entry->big_avg	= big_avg;

		__entry->iowait_avg	= iowait_avg;

		__entry->max_nr		= max_nr;

		__entry->big_max_nr	= big_max_nr;

	),



	TP_printk("avg=%d big_avg=%d iowait_avg=%d",

		__entry->avg, __entry->big_avg, __entry->iowait_avg)

	TP_printk("avg=%d big_avg=%d iowait_avg=%d max_nr=%u big_max_nr=%u",

		__entry->avg, __entry->big_avg, __entry->iowait_avg,

		__entry->max_nr, __entry->big_max_nr)

);



TRACE_EVENT(core_ctl_eval_need,

kernel/sched/core_ctl.c

+15 −28
Original line number Diff line number Diff line
@@ -39,6 +39,7 @@ struct cluster_data { 
	cpumask_t cpu_mask;

	unsigned int need_cpus;

	unsigned int task_thres;

	unsigned int max_nr;

	s64 need_ts;

	struct list_head lru;

	bool pending;
@@ -458,47 +459,25 @@ static struct kobj_type ktype_core_ctl = { 


/* ==================== runqueue based core count =================== */



#define NR_RUNNING_TOLERANCE 5



static void update_running_avg(void)

{

	int avg, iowait_avg, big_avg;

	int max_nr, big_max_nr;

	struct cluster_data *cluster;

	unsigned int index = 0;



	sched_get_nr_running_avg(&avg, &iowait_avg, &big_avg);



	/*

	 * Round up to the next integer if the average nr running tasks

	 * is within NR_RUNNING_TOLERANCE/100 of the next integer.

	 * If normal rounding up is used, it will allow a transient task

	 * to trigger online event. By the time core is onlined, the task

	 * has finished.

	 * Rounding to closest suffers same problem because scheduler

	 * might only provide running stats per jiffy, and a transient

	 * task could skew the number for one jiffy. If core control

	 * samples every 2 jiffies, it will observe 0.5 additional running

	 * average which rounds up to 1 task.

	 */

	avg = (avg + NR_RUNNING_TOLERANCE) / 100;

	big_avg = (big_avg + NR_RUNNING_TOLERANCE) / 100;

	sched_get_nr_running_avg(&avg, &iowait_avg, &big_avg,

				 &max_nr, &big_max_nr);



	for_each_cluster(cluster, index) {

		if (!cluster->inited)

			continue;

		/*

		 * Big cluster only need to take care of big tasks, but if

		 * there are not enough big cores, big tasks need to be run

		 * on little as well. Thus for little's runqueue stat, it

		 * has to use overall runqueue average, or derive what big

		 * tasks would have to be run on little. The latter approach

		 * is not easy to get given core control reacts much slower

		 * than scheduler, and can't predict scheduler's behavior.

		 */

		cluster->nrrun = cluster->is_big_cluster ? big_avg : avg;

		cluster->max_nr = cluster->is_big_cluster ? big_max_nr : max_nr;

	}

}



#define MAX_NR_THRESHOLD	4

/* adjust needed CPUs based on current runqueue information */

static unsigned int apply_task_need(const struct cluster_data *cluster,

				    unsigned int new_need)
@@ -509,7 +488,15 @@ static unsigned int apply_task_need(const struct cluster_data *cluster, 


	/* only unisolate more cores if there are tasks to run */

	if (cluster->nrrun > new_need)

		return new_need + 1;

		new_need = new_need + 1;



	/*

	 * We don't want tasks to be overcrowded in a cluster.

	 * If any CPU has more than MAX_NR_THRESHOLD in the last

	 * window, bring another CPU to help out.

	 */

	if (cluster->max_nr > MAX_NR_THRESHOLD)

		new_need = new_need + 1;



	return new_need;

}

kernel/sched/sched.h

+7 −0
Original line number Diff line number Diff line
@@ -2451,6 +2451,11 @@ static inline bool hmp_capable(void) 
	return max_possible_capacity != min_max_possible_capacity;

}



static inline bool is_max_capacity_cpu(int cpu)

{

	return cpu_max_possible_capacity(cpu) == max_possible_capacity;

}



/*

 * 'load' is in reference to "best cpu" at its best frequency.

 * Scale that in reference to a given cpu, accounting for how bad it is
@@ -2719,6 +2724,8 @@ static inline int is_task_migration_throttled(struct task_struct *p) 
	return 0;

}



static inline bool is_max_capacity_cpu(int cpu) { return true; }



static inline void

inc_rq_hmp_stats(struct rq *rq, struct task_struct *p, int change_cra) { }

kernel/sched/sched_avg.c

+32 −6
Original line number Diff line number Diff line
@@ -27,11 +27,13 @@ static DEFINE_PER_CPU(u64, nr_prod_sum); 
static DEFINE_PER_CPU(u64, last_time);

static DEFINE_PER_CPU(u64, nr_big_prod_sum);

static DEFINE_PER_CPU(u64, nr);

static DEFINE_PER_CPU(u64, nr_max);



static DEFINE_PER_CPU(unsigned long, iowait_prod_sum);

static DEFINE_PER_CPU(spinlock_t, nr_lock) = __SPIN_LOCK_UNLOCKED(nr_lock);

static s64 last_get_time;



#define DIV64_U64_ROUNDUP(X, Y) div64_u64((X) + (Y - 1), Y)

/**

 * sched_get_nr_running_avg

 * @return: Average nr_running, iowait and nr_big_tasks value since last poll.
@@ -41,7 +43,8 @@ static s64 last_get_time; 
 * Obtains the average nr_running value since the last poll.

 * This function may not be called concurrently with itself

 */

void sched_get_nr_running_avg(int *avg, int *iowait_avg, int *big_avg)

void sched_get_nr_running_avg(int *avg, int *iowait_avg, int *big_avg,

			      unsigned int *max_nr, unsigned int *big_max_nr)

{

	int cpu;

	u64 curr_time = sched_clock();
@@ -51,6 +54,8 @@ void sched_get_nr_running_avg(int *avg, int *iowait_avg, int *big_avg) 
	*avg = 0;

	*iowait_avg = 0;

	*big_avg = 0;

	*max_nr = 0;

	*big_max_nr = 0;



	if (!diff)

		return;
@@ -79,17 +84,35 @@ void sched_get_nr_running_avg(int *avg, int *iowait_avg, int *big_avg) 
		per_cpu(nr_big_prod_sum, cpu) = 0;

		per_cpu(iowait_prod_sum, cpu) = 0;



		if (*max_nr < per_cpu(nr_max, cpu))

			*max_nr = per_cpu(nr_max, cpu);



		if (is_max_capacity_cpu(cpu)) {

			if (*big_max_nr < per_cpu(nr_max, cpu))

				*big_max_nr = per_cpu(nr_max, cpu);

		}



		per_cpu(nr_max, cpu) = per_cpu(nr, cpu);

		spin_unlock_irqrestore(&per_cpu(nr_lock, cpu), flags);

	}



	diff = curr_time - last_get_time;

	last_get_time = curr_time;



	*avg = (int)div64_u64(tmp_avg * 100, diff);

	*big_avg = (int)div64_u64(tmp_big_avg * 100, diff);

	*iowait_avg = (int)div64_u64(tmp_iowait * 100, diff);

	/*

	 * Any task running on BIG cluster and BIG tasks running on little

	 * cluster contributes to big_avg. Small or medium tasks can also

	 * run on BIG cluster when co-location and scheduler boost features

	 * are activated. We don't want these tasks to downmigrate to little

	 * cluster when BIG CPUs are available but isolated. Round up the

	 * average values so that core_ctl aggressively unisolate BIG CPUs.

	 */

	*avg = (int)DIV64_U64_ROUNDUP(tmp_avg, diff);

	*big_avg = (int)DIV64_U64_ROUNDUP(tmp_big_avg, diff);

	*iowait_avg = (int)DIV64_U64_ROUNDUP(tmp_iowait, diff);



	trace_sched_get_nr_running_avg(*avg, *big_avg, *iowait_avg);

	trace_sched_get_nr_running_avg(*avg, *big_avg, *iowait_avg,

				       *max_nr, *big_max_nr);



	BUG_ON(*avg < 0 || *big_avg < 0 || *iowait_avg < 0);

	pr_debug("%s - avg:%d big_avg:%d iowait_avg:%d\n",
@@ -122,6 +145,9 @@ void sched_update_nr_prod(int cpu, long delta, bool inc) 


	BUG_ON((s64)per_cpu(nr, cpu) < 0);



	if (per_cpu(nr, cpu) > per_cpu(nr_max, cpu))

		per_cpu(nr_max, cpu) = per_cpu(nr, cpu);



	per_cpu(nr_prod_sum, cpu) += nr_running * diff;

	per_cpu(nr_big_prod_sum, cpu) += nr_eligible_big_tasks(cpu) * diff;

	per_cpu(iowait_prod_sum, cpu) += nr_iowait_cpu(cpu) * diff;