Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 0bdf0847 authored by Srivatsa Vaddagiri's avatar Srivatsa Vaddagiri Committed by Zvikomborero Vincent Zvikaramba
Browse files

sched: Consolidate hmp stats into their own struct 



Key hmp stats (nr_big_tasks, nr_small_tasks and
cumulative_runnable_average) are currently maintained per-cpu in
'struct rq'. Merge those stats in their own structure (struct
hmp_sched_stats) and modify impacted functions to deal with the newly
introduced structure. This cleanup is required for a subsequent patch
which fixes various issues with use of CFS_BANDWIDTH feature in HMP
scheduler.

Change-Id: Ieffc10a3b82a102f561331bc385d042c15a33998
Signed-off-by: default avatarSrivatsa Vaddagiri <vatsa@codeaurora.org>
parent 962d21f8

include/trace/events/sched.h

+5 −5
Original line number Diff line number Diff line
@@ -182,11 +182,11 @@ TRACE_EVENT(sched_cpu_load, 
		__entry->idle			= idle;

		__entry->mostly_idle		= mostly_idle;

		__entry->nr_running		= rq->nr_running;

		__entry->nr_big_tasks		= rq->nr_big_tasks;

		__entry->nr_small_tasks		= rq->nr_small_tasks;

		__entry->nr_big_tasks		= rq->hmp_stats.nr_big_tasks;

		__entry->nr_small_tasks		= rq->hmp_stats.nr_small_tasks;

		__entry->load_scale_factor	= rq->load_scale_factor;

		__entry->capacity		= rq->capacity;

		__entry->cumulative_runnable_avg = rq->cumulative_runnable_avg;

		__entry->cumulative_runnable_avg = rq->hmp_stats.cumulative_runnable_avg;

		__entry->cur_freq		= rq->cur_freq;

		__entry->max_freq		= rq->max_freq;

		__entry->power_cost		= power_cost;
@@ -317,8 +317,8 @@ TRACE_EVENT(sched_update_history, 
		__entry->demand         = p->ravg.demand;

		memcpy(__entry->hist, p->ravg.sum_history,

					RAVG_HIST_SIZE_MAX * sizeof(u32));

		__entry->nr_big_tasks   = rq->nr_big_tasks;

		__entry->nr_small_tasks = rq->nr_small_tasks;

		__entry->nr_big_tasks   = rq->hmp_stats.nr_big_tasks;

		__entry->nr_small_tasks = rq->hmp_stats.nr_small_tasks;

		__entry->cpu            = rq->cpu;

	),

kernel/sched/core.c

+11 −26
Original line number Diff line number Diff line
@@ -872,7 +872,6 @@ static void enqueue_task(struct rq *rq, struct task_struct *p, int flags) 
	sched_info_queued(p);

	p->sched_class->enqueue_task(rq, p, flags);

	trace_sched_enq_deq_task(p, 1, cpumask_bits(&p->cpus_allowed)[0]);

	inc_cumulative_runnable_avg(rq, p);

}



static void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
@@ -881,7 +880,6 @@ static void dequeue_task(struct rq *rq, struct task_struct *p, int flags) 
	sched_info_dequeued(p);

	p->sched_class->dequeue_task(rq, p, flags);

	trace_sched_enq_deq_task(p, 0, cpumask_bits(&p->cpus_allowed)[0]);

	dec_cumulative_runnable_avg(rq, p);

}



void activate_task(struct rq *rq, struct task_struct *p, int flags)
@@ -1709,12 +1707,8 @@ static void update_history(struct rq *rq, struct task_struct *p, 
	}



	p->ravg.sum = 0;

	if (p->on_rq) {

		rq->cumulative_runnable_avg -= p->ravg.demand;

		BUG_ON((s64)rq->cumulative_runnable_avg < 0);

		if (p->sched_class == &fair_sched_class)

			dec_nr_big_small_task(rq, p);

	}

	if (p->on_rq)

		p->sched_class->dec_hmp_sched_stats(rq, p);



	avg = div64_u64(sum, sched_ravg_hist_size);
@@ -1729,11 +1723,8 @@ static void update_history(struct rq *rq, struct task_struct *p, 


	p->ravg.demand = demand;



	if (p->on_rq) {

		rq->cumulative_runnable_avg += p->ravg.demand;

		if (p->sched_class == &fair_sched_class)

			inc_nr_big_small_task(rq, p);

	}

	if (p->on_rq)

		p->sched_class->inc_hmp_sched_stats(rq, p);



done:

	trace_sched_update_history(rq, p, runtime, samples, event);
@@ -2097,7 +2088,7 @@ void reset_all_window_stats(u64 window_start, unsigned int window_size) 
#ifdef CONFIG_SCHED_FREQ_INPUT

		rq->curr_runnable_sum = rq->prev_runnable_sum = 0;

#endif

		rq->cumulative_runnable_avg = 0;

		rq->hmp_stats.cumulative_runnable_avg = 0;

		fixup_nr_big_small_task(cpu);

	}
@@ -2264,11 +2255,8 @@ static void fixup_busy_time(struct task_struct *p, int new_cpu) 
	 * reflect new demand. Restore load temporarily for such

	 * task on its runqueue

	 */

	if (p->on_rq) {

		inc_cumulative_runnable_avg(src_rq, p);

		if (p->sched_class == &fair_sched_class)

			inc_nr_big_small_task(src_rq, p);

	}

	if (p->on_rq)

		p->sched_class->inc_hmp_sched_stats(src_rq, p);



	update_task_ravg(p, task_rq(p), TASK_MIGRATE,

			 wallclock, 0);
@@ -2277,11 +2265,8 @@ static void fixup_busy_time(struct task_struct *p, int new_cpu) 
	 * Remove task's load from rq as its now migrating to

	 * another cpu.

	 */

	if (p->on_rq) {

		dec_cumulative_runnable_avg(src_rq, p);

		if (p->sched_class == &fair_sched_class)

			dec_nr_big_small_task(src_rq, p);

	}

	if (p->on_rq)

		p->sched_class->dec_hmp_sched_stats(src_rq, p);



	if (p->ravg.curr_window) {

		src_rq->curr_runnable_sum -= p->ravg.curr_window;
@@ -9063,12 +9048,12 @@ void __init sched_init(void) 
		rq->min_freq = 1;

		rq->max_possible_freq = 1;

		rq->max_possible_capacity = 0;

		rq->cumulative_runnable_avg = 0;

		rq->hmp_stats.cumulative_runnable_avg = 0;

		rq->efficiency = 1024;

		rq->capacity = 1024;

		rq->load_scale_factor = 1024;

		rq->window_start = 0;

		rq->nr_small_tasks = rq->nr_big_tasks = 0;

		rq->hmp_stats.nr_small_tasks = rq->hmp_stats.nr_big_tasks = 0;

		rq->hmp_flags = 0;

#ifdef CONFIG_SCHED_FREQ_INPUT

		rq->old_busy_time = 0;

kernel/sched/debug.c

+2 −2
Original line number Diff line number Diff line
@@ -314,8 +314,8 @@ do { \ 
	P(max_freq);

#endif

#ifdef CONFIG_SCHED_HMP

	P(nr_big_tasks);

	P(nr_small_tasks);

	P(hmp_stats.nr_big_tasks);

	P(hmp_stats.nr_small_tasks);

#endif

#undef P

#undef PN

kernel/sched/fair.c

+82 −34
Original line number Diff line number Diff line
@@ -1251,7 +1251,7 @@ unsigned int __read_mostly sched_enable_hmp = 0; 
/* A cpu can no longer accomodate more tasks if:

 *

 *	rq->nr_running > sysctl_sched_spill_nr_run ||

 *	rq->cumulative_runnable_avg > sched_spill_load

 *	rq->hmp_stats.cumulative_runnable_avg > sched_spill_load

 */

unsigned int __read_mostly sysctl_sched_spill_nr_run = 10;
@@ -1424,7 +1424,7 @@ static inline u64 cpu_load(int cpu) 
{

	struct rq *rq = cpu_rq(cpu);



	return scale_load_to_cpu(rq->cumulative_runnable_avg, cpu);

	return scale_load_to_cpu(rq->hmp_stats.cumulative_runnable_avg, cpu);

}



static inline u64 cpu_load_sync(int cpu, int sync)
@@ -1432,7 +1432,7 @@ static inline u64 cpu_load_sync(int cpu, int sync) 
	struct rq *rq = cpu_rq(cpu);

	u64 load;



	load = rq->cumulative_runnable_avg;

	load = rq->hmp_stats.cumulative_runnable_avg;



	/*

	 * If load is being checked in a sync wakeup environment,
@@ -2076,28 +2076,42 @@ done: 
	return best_cpu;

}



void inc_nr_big_small_task(struct rq *rq, struct task_struct *p)

static void

inc_nr_big_small_task(struct hmp_sched_stats *stats, struct task_struct *p)

{

	if (!sched_enable_hmp || sched_disable_window_stats)

		return;



	if (is_big_task(p))

		rq->nr_big_tasks++;

		stats->nr_big_tasks++;

	else if (is_small_task(p))

		rq->nr_small_tasks++;

		stats->nr_small_tasks++;

}



void dec_nr_big_small_task(struct rq *rq, struct task_struct *p)

static void

dec_nr_big_small_task(struct hmp_sched_stats *stats, struct task_struct *p)

{

	if (!sched_enable_hmp || sched_disable_window_stats)

		return;



	if (is_big_task(p))

		rq->nr_big_tasks--;

		stats->nr_big_tasks--;

	else if (is_small_task(p))

		rq->nr_small_tasks--;

		stats->nr_small_tasks--;



	BUG_ON(stats->nr_big_tasks < 0 || stats->nr_small_tasks < 0);

}



static void inc_rq_hmp_stats(struct rq *rq, struct task_struct *p)

{

	inc_cumulative_runnable_avg(&rq->hmp_stats, p);

	inc_nr_big_small_task(&rq->hmp_stats, p);

}



	BUG_ON(rq->nr_big_tasks < 0 || rq->nr_small_tasks < 0);

static void dec_rq_hmp_stats(struct rq *rq, struct task_struct *p)

{

	dec_cumulative_runnable_avg(&rq->hmp_stats, p);

	dec_nr_big_small_task(&rq->hmp_stats, p);

}



/*
@@ -2109,10 +2123,10 @@ void fixup_nr_big_small_task(int cpu) 
	struct rq *rq = cpu_rq(cpu);

	struct task_struct *p;



	rq->nr_big_tasks = 0;

	rq->nr_small_tasks = 0;

	rq->hmp_stats.nr_big_tasks = 0;

	rq->hmp_stats.nr_small_tasks = 0;

	list_for_each_entry(p, &rq->cfs_tasks, se.group_node)

		inc_nr_big_small_task(rq, p);

		inc_nr_big_small_task(&rq->hmp_stats, p);

}



/* Disable interrupts and grab runqueue lock of all cpus listed in @cpus */
@@ -2476,7 +2490,21 @@ static inline int capacity(struct rq *rq) 


static inline int nr_big_tasks(struct rq *rq)

{

	return rq->nr_big_tasks;

	return rq->hmp_stats.nr_big_tasks;

}



static void

inc_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p)

{

	inc_cumulative_runnable_avg(&rq->hmp_stats, p);

	inc_nr_big_small_task(&rq->hmp_stats, p);

}



static void

dec_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p)

{

	dec_cumulative_runnable_avg(&rq->hmp_stats, p);

	dec_nr_big_small_task(&rq->hmp_stats, p);

}



#else	/* CONFIG_SCHED_HMP */
@@ -2534,6 +2562,19 @@ static inline int capacity(struct rq *rq) 
	return SCHED_LOAD_SCALE;

}



static inline void inc_rq_hmp_stats(struct rq *rq, struct task_struct *p) { }

static inline void dec_rq_hmp_stats(struct rq *rq, struct task_struct *p) { }



static inline void

inc_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p)

{

}



static inline void

dec_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p)

{

}



#endif	/* CONFIG_SCHED_HMP */



#ifdef CONFIG_SCHED_HMP
@@ -2835,18 +2876,14 @@ static inline void update_entity_load_avg(struct sched_entity *se, 
	 */

	if (entity_is_task(se)) {

		now = cfs_rq_clock_task(cfs_rq);

		if (se->on_rq) {

			dec_cumulative_runnable_avg(rq_of(cfs_rq), task_of(se));

			dec_nr_big_small_task(rq_of(cfs_rq), task_of(se));

		}

		if (se->on_rq)

			dec_hmp_sched_stats_fair(rq_of(cfs_rq), task_of(se));

	} else

		now = cfs_rq_clock_task(group_cfs_rq(se));



	decayed = __update_entity_runnable_avg(cpu, now, &se->avg, se->on_rq);

	if (entity_is_task(se) && se->on_rq) {

		inc_cumulative_runnable_avg(rq_of(cfs_rq), task_of(se));

		inc_nr_big_small_task(rq_of(cfs_rq), task_of(se));

	}

	if (entity_is_task(se) && se->on_rq)

		inc_hmp_sched_stats_fair(rq_of(cfs_rq), task_of(se));



	if (!decayed)

		return;
@@ -2992,6 +3029,10 @@ static inline void dequeue_entity_load_avg(struct cfs_rq *cfs_rq, 
					   int sleep) {}

static inline void update_cfs_rq_blocked_load(struct cfs_rq *cfs_rq,

					      int force_update) {}



static inline void inc_rq_hmp_stats(struct rq *rq, struct task_struct *p) { }

static inline void dec_rq_hmp_stats(struct rq *rq, struct task_struct *p) { }



#endif



#ifdef CONFIG_SCHED_HMP
@@ -4327,7 +4368,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) 
	if (!se) {

		update_rq_runnable_avg(rq, rq->nr_running);

		inc_nr_running(rq);

		inc_nr_big_small_task(rq, p);

		inc_rq_hmp_stats(rq, p);

	}

	hrtick_update(rq);

}
@@ -4389,7 +4430,7 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) 
	if (!se) {

		dec_nr_running(rq);

		update_rq_runnable_avg(rq, 1);

		dec_nr_big_small_task(rq, p);

		dec_rq_hmp_stats(rq, p);

	}

	hrtick_update(rq);

}
@@ -6043,8 +6084,8 @@ static inline void update_sg_lb_stats(struct lb_env *env, 
		sgs->group_load += load;

		sgs->sum_nr_running += nr_running;

#ifdef CONFIG_SCHED_HMP

		sgs->sum_nr_big_tasks += rq->nr_big_tasks;

		sgs->sum_nr_small_tasks += rq->nr_small_tasks;

		sgs->sum_nr_big_tasks += rq->hmp_stats.nr_big_tasks;

		sgs->sum_nr_small_tasks += rq->hmp_stats.nr_small_tasks;

		sgs->group_cpu_load += cpu_load(i);

#endif

		sgs->sum_weighted_load += weighted_cpuload(i);
@@ -6533,18 +6574,20 @@ ret: 
static struct rq *find_busiest_queue_hmp(struct lb_env *env,

				     struct sched_group *group)

{

	struct rq *busiest = NULL, *rq;

	struct rq *busiest = NULL;

	u64 max_runnable_avg = 0;

	int i;



	for_each_cpu(i, sched_group_cpus(group)) {

		struct rq *rq = cpu_rq(i);

		u64 cumulative_runnable_avg =

				rq->hmp_stats.cumulative_runnable_avg;



		if (!cpumask_test_cpu(i, env->cpus))

			continue;



		rq = cpu_rq(i);



		if (rq->cumulative_runnable_avg > max_runnable_avg) {

			max_runnable_avg = rq->cumulative_runnable_avg;

		if (cumulative_runnable_avg > max_runnable_avg) {

			max_runnable_avg = cumulative_runnable_avg;

			busiest = rq;

		}

	}
@@ -7403,9 +7446,10 @@ static inline int _nohz_kick_needed_hmp(struct rq *rq, int cpu, int *type) 
	struct sched_domain *sd;

	int i;



	if (rq->nr_running >= 2 && (rq->nr_running - rq->nr_small_tasks >= 2 ||

	     rq->nr_running > sysctl_sched_mostly_idle_nr_run ||

		cpu_load(cpu) > sched_mostly_idle_load)) {

	if (rq->nr_running >= 2 &&

		(rq->nr_running - rq->hmp_stats.nr_small_tasks >= 2 ||

		rq->nr_running > rq->mostly_idle_nr_run ||

		cpu_load(cpu) > rq->mostly_idle_load)) {



		if (rq->capacity == max_capacity)

			return 1;
@@ -8009,6 +8053,10 @@ const struct sched_class fair_sched_class = { 
#ifdef CONFIG_FAIR_GROUP_SCHED

	.task_move_group	= task_move_group_fair,

#endif

#ifdef CONFIG_SCHED_HMP

	.inc_hmp_sched_stats	= inc_hmp_sched_stats_fair,

	.dec_hmp_sched_stats	= dec_hmp_sched_stats_fair,

#endif

};



#ifdef CONFIG_SCHED_DEBUG

kernel/sched/idle_task.c

+18 −0
Original line number Diff line number Diff line
@@ -84,6 +84,20 @@ static unsigned int get_rr_interval_idle(struct rq *rq, struct task_struct *task 
	return 0;

}



#ifdef CONFIG_SCHED_HMP



static void

inc_hmp_sched_stats_idle(struct rq *rq, struct task_struct *p)

{

}



static void

dec_hmp_sched_stats_idle(struct rq *rq, struct task_struct *p)

{

}



#endif



/*

 * Simple, special scheduling class for the per-CPU idle tasks:

 */
@@ -112,4 +126,8 @@ const struct sched_class idle_sched_class = { 


	.prio_changed		= prio_changed_idle,

	.switched_to		= switched_to_idle,

#ifdef CONFIG_SCHED_HMP

	.inc_hmp_sched_stats	= inc_hmp_sched_stats_idle,

	.dec_hmp_sched_stats	= dec_hmp_sched_stats_idle,

#endif

};