Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e97b8e77 authored by Srivatsa Vaddagiri's avatar Srivatsa Vaddagiri Committed by Matt Wagantall
Browse files

sched: Consolidate hmp stats into their own struct 



Key hmp stats (nr_big_tasks, nr_small_tasks and
cumulative_runnable_average) are currently maintained per-cpu in
'struct rq'. Merge those stats in their own structure (struct
hmp_sched_stats) and modify impacted functions to deal with the newly
introduced structure. This cleanup is required for a subsequent patch
which fixes various issues with use of CFS_BANDWIDTH feature in HMP
scheduler.

Change-Id: Ieffc10a3b82a102f561331bc385d042c15a33998
Signed-off-by: default avatarSrivatsa Vaddagiri <vatsa@codeaurora.org>
[rameezmustafa@codeaurora.org: Port to msm-3.18]
Signed-off-by: default avatarSyed Rameez Mustafa <rameezmustafa@codeaurora.org>
parent 19430adb

include/trace/events/sched.h

+5 −5
Original line number Diff line number Diff line
@@ -185,11 +185,11 @@ TRACE_EVENT(sched_cpu_load, 
		__entry->idle			= idle;

		__entry->mostly_idle		= mostly_idle;

		__entry->nr_running		= rq->nr_running;

		__entry->nr_big_tasks		= rq->nr_big_tasks;

		__entry->nr_small_tasks		= rq->nr_small_tasks;

		__entry->nr_big_tasks		= rq->hmp_stats.nr_big_tasks;

		__entry->nr_small_tasks		= rq->hmp_stats.nr_small_tasks;

		__entry->load_scale_factor	= rq->load_scale_factor;

		__entry->capacity		= rq->capacity;

		__entry->cumulative_runnable_avg = rq->cumulative_runnable_avg;

		__entry->cumulative_runnable_avg = rq->hmp_stats.cumulative_runnable_avg;

		__entry->irqload		= irqload;

		__entry->cur_freq		= rq->cur_freq;

		__entry->max_freq		= rq->max_freq;
@@ -323,8 +323,8 @@ TRACE_EVENT(sched_update_history, 
		__entry->demand         = p->ravg.demand;

		memcpy(__entry->hist, p->ravg.sum_history,

					RAVG_HIST_SIZE_MAX * sizeof(u32));

		__entry->nr_big_tasks   = rq->nr_big_tasks;

		__entry->nr_small_tasks = rq->nr_small_tasks;

		__entry->nr_big_tasks   = rq->hmp_stats.nr_big_tasks;

		__entry->nr_small_tasks = rq->hmp_stats.nr_small_tasks;

		__entry->cpu            = rq->cpu;

	),

kernel/sched/core.c

+11 −26
Original line number Diff line number Diff line
@@ -866,7 +866,6 @@ static void enqueue_task(struct rq *rq, struct task_struct *p, int flags) 
	sched_info_queued(rq, p);

	p->sched_class->enqueue_task(rq, p, flags);

	trace_sched_enq_deq_task(p, 1, cpumask_bits(&p->cpus_allowed)[0]);

	inc_cumulative_runnable_avg(rq, p);

}



static void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
@@ -875,7 +874,6 @@ static void dequeue_task(struct rq *rq, struct task_struct *p, int flags) 
	sched_info_dequeued(rq, p);

	p->sched_class->dequeue_task(rq, p, flags);

	trace_sched_enq_deq_task(p, 0, cpumask_bits(&p->cpus_allowed)[0]);

	dec_cumulative_runnable_avg(rq, p);

}



void activate_task(struct rq *rq, struct task_struct *p, int flags)
@@ -1682,12 +1680,8 @@ static void update_history(struct rq *rq, struct task_struct *p, 
	}



	p->ravg.sum = 0;

	if (p->on_rq) {

		rq->cumulative_runnable_avg -= p->ravg.demand;

		BUG_ON((s64)rq->cumulative_runnable_avg < 0);

		if (p->sched_class == &fair_sched_class)

			dec_nr_big_small_task(rq, p);

	}

	if (p->on_rq)

		p->sched_class->dec_hmp_sched_stats(rq, p);



	avg = div64_u64(sum, sched_ravg_hist_size);
@@ -1702,11 +1696,8 @@ static void update_history(struct rq *rq, struct task_struct *p, 


	p->ravg.demand = demand;



	if (p->on_rq) {

		rq->cumulative_runnable_avg += p->ravg.demand;

		if (p->sched_class == &fair_sched_class)

			inc_nr_big_small_task(rq, p);

	}

	if (p->on_rq)

		p->sched_class->inc_hmp_sched_stats(rq, p);



done:

	trace_sched_update_history(rq, p, runtime, samples, event);
@@ -2092,7 +2083,7 @@ void reset_all_window_stats(u64 window_start, unsigned int window_size) 
#ifdef CONFIG_SCHED_FREQ_INPUT

		rq->curr_runnable_sum = rq->prev_runnable_sum = 0;

#endif

		rq->cumulative_runnable_avg = 0;

		rq->hmp_stats.cumulative_runnable_avg = 0;

		fixup_nr_big_small_task(cpu);

	}
@@ -2246,11 +2237,8 @@ static void fixup_busy_time(struct task_struct *p, int new_cpu) 
	 * reflect new demand. Restore load temporarily for such

	 * task on its runqueue

	 */

	if (p->on_rq) {

		inc_cumulative_runnable_avg(src_rq, p);

		if (p->sched_class == &fair_sched_class)

			inc_nr_big_small_task(src_rq, p);

	}

	if (p->on_rq)

		p->sched_class->inc_hmp_sched_stats(src_rq, p);



	update_task_ravg(p, task_rq(p), TASK_MIGRATE,

			 wallclock, 0);
@@ -2259,11 +2247,8 @@ static void fixup_busy_time(struct task_struct *p, int new_cpu) 
	 * Remove task's load from rq as its now migrating to

	 * another cpu.

	 */

	if (p->on_rq) {

		dec_cumulative_runnable_avg(src_rq, p);

		if (p->sched_class == &fair_sched_class)

			dec_nr_big_small_task(src_rq, p);

	}

	if (p->on_rq)

		p->sched_class->dec_hmp_sched_stats(src_rq, p);



	if (p->ravg.curr_window) {

		src_rq->curr_runnable_sum -= p->ravg.curr_window;
@@ -8804,12 +8789,12 @@ void __init sched_init(void) 
		rq->min_freq = 1;

		rq->max_possible_freq = 1;

		rq->max_possible_capacity = 0;

		rq->cumulative_runnable_avg = 0;

		rq->hmp_stats.cumulative_runnable_avg = 0;

		rq->efficiency = 1024;

		rq->capacity = 1024;

		rq->load_scale_factor = 1024;

		rq->window_start = 0;

		rq->nr_small_tasks = rq->nr_big_tasks = 0;

		rq->hmp_stats.nr_small_tasks = rq->hmp_stats.nr_big_tasks = 0;

		rq->hmp_flags = 0;

		rq->mostly_idle_load = pct_to_real(20);

		rq->mostly_idle_nr_run = 3;

kernel/sched/debug.c

+2 −2
Original line number Diff line number Diff line
@@ -318,8 +318,8 @@ do { \ 
	P(max_freq);

#endif

#ifdef CONFIG_SCHED_HMP

	P(nr_big_tasks);

	P(nr_small_tasks);

	P(hmp_stats.nr_big_tasks);

	P(hmp_stats.nr_small_tasks);

#endif

#undef P

#undef PN

kernel/sched/fair.c

+78 −33
Original line number Diff line number Diff line
@@ -2356,7 +2356,7 @@ unsigned int __read_mostly sched_enable_hmp = 0; 
/* A cpu can no longer accomodate more tasks if:

 *

 *	rq->nr_running > sysctl_sched_spill_nr_run ||

 *	rq->cumulative_runnable_avg > sched_spill_load

 *	rq->hmp_stats.cumulative_runnable_avg > sched_spill_load

 */

unsigned int __read_mostly sysctl_sched_spill_nr_run = 10;
@@ -2602,7 +2602,7 @@ static inline u64 cpu_load(int cpu) 
{

	struct rq *rq = cpu_rq(cpu);



	return scale_load_to_cpu(rq->cumulative_runnable_avg, cpu);

	return scale_load_to_cpu(rq->hmp_stats.cumulative_runnable_avg, cpu);

}



static inline u64 cpu_load_sync(int cpu, int sync)
@@ -2610,7 +2610,7 @@ static inline u64 cpu_load_sync(int cpu, int sync) 
	struct rq *rq = cpu_rq(cpu);

	u64 load;



	load = rq->cumulative_runnable_avg;

	load = rq->hmp_stats.cumulative_runnable_avg;



	/*

	 * If load is being checked in a sync wakeup environment,
@@ -3235,28 +3235,42 @@ done: 
	return best_cpu;

}



void inc_nr_big_small_task(struct rq *rq, struct task_struct *p)

static void

inc_nr_big_small_task(struct hmp_sched_stats *stats, struct task_struct *p)

{

	if (!sched_enable_hmp || sched_disable_window_stats)

		return;



	if (is_big_task(p))

		rq->nr_big_tasks++;

		stats->nr_big_tasks++;

	else if (is_small_task(p))

		rq->nr_small_tasks++;

		stats->nr_small_tasks++;

}



void dec_nr_big_small_task(struct rq *rq, struct task_struct *p)

static void

dec_nr_big_small_task(struct hmp_sched_stats *stats, struct task_struct *p)

{

	if (!sched_enable_hmp || sched_disable_window_stats)

		return;



	if (is_big_task(p))

		rq->nr_big_tasks--;

		stats->nr_big_tasks--;

	else if (is_small_task(p))

		rq->nr_small_tasks--;

		stats->nr_small_tasks--;



	BUG_ON(rq->nr_big_tasks < 0 || rq->nr_small_tasks < 0);

	BUG_ON(stats->nr_big_tasks < 0 || stats->nr_small_tasks < 0);

}



static void inc_rq_hmp_stats(struct rq *rq, struct task_struct *p)

{

	inc_cumulative_runnable_avg(&rq->hmp_stats, p);

	inc_nr_big_small_task(&rq->hmp_stats, p);

}



static void dec_rq_hmp_stats(struct rq *rq, struct task_struct *p)

{

	dec_cumulative_runnable_avg(&rq->hmp_stats, p);

	dec_nr_big_small_task(&rq->hmp_stats, p);

}



/*
@@ -3268,10 +3282,10 @@ void fixup_nr_big_small_task(int cpu) 
	struct rq *rq = cpu_rq(cpu);

	struct task_struct *p;



	rq->nr_big_tasks = 0;

	rq->nr_small_tasks = 0;

	rq->hmp_stats.nr_big_tasks = 0;

	rq->hmp_stats.nr_small_tasks = 0;

	list_for_each_entry(p, &rq->cfs_tasks, se.group_node)

		inc_nr_big_small_task(rq, p);

		inc_nr_big_small_task(&rq->hmp_stats, p);

}



/* Disable interrupts and grab runqueue lock of all cpus listed in @cpus */
@@ -3606,7 +3620,7 @@ void check_for_migration(struct rq *rq, struct task_struct *p) 


static inline int nr_big_tasks(struct rq *rq)

{

	return rq->nr_big_tasks;

	return rq->hmp_stats.nr_big_tasks;

}



static inline int is_cpu_throttling_imminent(int cpu)
@@ -3639,6 +3653,19 @@ unsigned int cpu_temp(int cpu) 
		return 0;

}



static void

inc_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p)

{

	inc_cumulative_runnable_avg(&rq->hmp_stats, p);

	inc_nr_big_small_task(&rq->hmp_stats, p);

}



static void

dec_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p)

{

	dec_cumulative_runnable_avg(&rq->hmp_stats, p);

	dec_nr_big_small_task(&rq->hmp_stats, p);

}



#else	/* CONFIG_SCHED_HMP */
@@ -3711,6 +3738,18 @@ unsigned int cpu_temp(int cpu) 
	return 0;

}



static inline void inc_rq_hmp_stats(struct rq *rq, struct task_struct *p) { }

static inline void dec_rq_hmp_stats(struct rq *rq, struct task_struct *p) { }



static inline void

inc_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p)

{

}



static inline void

dec_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p)

{

}



#endif	/* CONFIG_SCHED_HMP */
@@ -4036,18 +4075,14 @@ static inline void update_entity_load_avg(struct sched_entity *se, 
	 */

	if (entity_is_task(se)) {

		now = cfs_rq_clock_task(cfs_rq);

		if (se->on_rq) {

			dec_cumulative_runnable_avg(rq_of(cfs_rq), task_of(se));

			dec_nr_big_small_task(rq_of(cfs_rq), task_of(se));

		}

		if (se->on_rq)

			dec_hmp_sched_stats_fair(rq_of(cfs_rq), task_of(se));

	} else

		now = cfs_rq_clock_task(group_cfs_rq(se));



	decayed = __update_entity_runnable_avg(cpu, now, &se->avg, se->on_rq);

	if (entity_is_task(se) && se->on_rq) {

		inc_cumulative_runnable_avg(rq_of(cfs_rq), task_of(se));

		inc_nr_big_small_task(rq_of(cfs_rq), task_of(se));

	}

	if (entity_is_task(se) && se->on_rq)

		inc_hmp_sched_stats_fair(rq_of(cfs_rq), task_of(se));



	if (!decayed)

		return;
@@ -4200,6 +4235,9 @@ static inline int idle_balance(struct rq *rq) 
	return 0;

}



static inline void inc_rq_hmp_stats(struct rq *rq, struct task_struct *p) { }

static inline void dec_rq_hmp_stats(struct rq *rq, struct task_struct *p) { }



#endif /* CONFIG_SMP */



#ifdef CONFIG_SCHED_HMP
@@ -5563,7 +5601,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) 
	if (!se) {

		update_rq_runnable_avg(rq, rq->nr_running);

		add_nr_running(rq, 1);

		inc_nr_big_small_task(rq, p);

		inc_rq_hmp_stats(rq, p);

	}

	hrtick_update(rq);

}
@@ -5625,7 +5663,7 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) 
	if (!se) {

		sub_nr_running(rq, 1);

		update_rq_runnable_avg(rq, 1);

		dec_nr_big_small_task(rq, p);

		dec_rq_hmp_stats(rq, p);

	}

	hrtick_update(rq);

}
@@ -7697,8 +7735,8 @@ static inline void update_sg_lb_stats(struct lb_env *env, 
			*overload = true;



#ifdef CONFIG_SCHED_HMP

		sgs->sum_nr_big_tasks += rq->nr_big_tasks;

		sgs->sum_nr_small_tasks += rq->nr_small_tasks;

		sgs->sum_nr_big_tasks += rq->hmp_stats.nr_big_tasks;

		sgs->sum_nr_small_tasks += rq->hmp_stats.nr_small_tasks;

		sgs->group_cpu_load += cpu_load(i);

#endif
@@ -8201,18 +8239,20 @@ out_balanced: 
static struct rq *find_busiest_queue_hmp(struct lb_env *env,

				     struct sched_group *group)

{

	struct rq *busiest = NULL, *rq;

	struct rq *busiest = NULL;

	u64 max_runnable_avg = 0;

	int i;



	for_each_cpu(i, sched_group_cpus(group)) {

		struct rq *rq = cpu_rq(i);

		u64 cumulative_runnable_avg =

				rq->hmp_stats.cumulative_runnable_avg;



		if (!cpumask_test_cpu(i, env->cpus))

			continue;



		rq = cpu_rq(i);



		if (rq->cumulative_runnable_avg > max_runnable_avg) {

			max_runnable_avg = rq->cumulative_runnable_avg;

		if (cumulative_runnable_avg > max_runnable_avg) {

			max_runnable_avg = cumulative_runnable_avg;

			busiest = rq;

		}

	}
@@ -9300,7 +9340,8 @@ static inline int _nohz_kick_needed_hmp(struct rq *rq, int cpu, int *type) 
		 && rq->max_freq > rq->mostly_idle_freq)

			return 0;



	if (rq->nr_running >= 2 && (rq->nr_running - rq->nr_small_tasks >= 2 ||

	if (rq->nr_running >= 2 &&

		(rq->nr_running - rq->hmp_stats.nr_small_tasks >= 2 ||

		rq->nr_running > rq->mostly_idle_nr_run ||

		cpu_load(cpu) > rq->mostly_idle_load)) {
@@ -9928,6 +9969,10 @@ const struct sched_class fair_sched_class = { 
#ifdef CONFIG_FAIR_GROUP_SCHED

	.task_move_group	= task_move_group_fair,

#endif

#ifdef CONFIG_SCHED_HMP

	.inc_hmp_sched_stats	= inc_hmp_sched_stats_fair,

	.dec_hmp_sched_stats	= dec_hmp_sched_stats_fair,

#endif

};



#ifdef CONFIG_SCHED_DEBUG

kernel/sched/idle_task.c

+18 −0
Original line number Diff line number Diff line
@@ -79,6 +79,20 @@ static void update_curr_idle(struct rq *rq) 
{

}



#ifdef CONFIG_SCHED_HMP



static void

inc_hmp_sched_stats_idle(struct rq *rq, struct task_struct *p)

{

}



static void

dec_hmp_sched_stats_idle(struct rq *rq, struct task_struct *p)

{

}



#endif



/*

 * Simple, special scheduling class for the per-CPU idle tasks:

 */
@@ -106,4 +120,8 @@ const struct sched_class idle_sched_class = { 
	.prio_changed		= prio_changed_idle,

	.switched_to		= switched_to_idle,

	.update_curr		= update_curr_idle,

#ifdef CONFIG_SCHED_HMP

	.inc_hmp_sched_stats	= inc_hmp_sched_stats_idle,

	.dec_hmp_sched_stats	= dec_hmp_sched_stats_idle,

#endif

};