Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 84a8088e authored by Joonwoo Park's avatar Joonwoo Park
Browse files

sched: WALT: account cumulative window demand 



Adequate task packing brings a significant amount of power saving
at cost of minimum wake up latency.  But we need to be extra careful as
excessive task packing can introduce negative impact not only to wake
up latency but also to power aspect as excessive task packing can lead
higher CPU frequency.

The most challenging part is frequency estimation at the time of
task placement to avoid placing on a busy CPU increases CPU's frequency
whereas placing the task on a idle CPU could sustain lower frequency.

Today, EAS utilizes CPU's demand which is accumulated instantaneous
tasks' demands.  However it's not appropriate and can easily lead to
over packing from CPU frequency guidance aspect due to the discrepancy
between instantaneous load and total execution time at the window
boundary.

As a result, introduce a new accounting unit 'cumulative window demand'.
The cumulative window demand tracks all the tasks' demands have seen in
current window which is neither instantaneous nor actual execution time.
It aims to estimate CPU's frequency at the end of current WALT window.

Change-Id: I9908c77ead9973a26dea2b36c001c2baf944d4f5
Signed-off-by: default avatarJoonwoo Park <joonwoop@codeaurora.org>
parent 7b211a4e

include/linux/sched.h

+1 −0
Original line number Diff line number Diff line
@@ -1661,6 +1661,7 @@ struct task_struct { 
	const struct sched_class *sched_class;

	struct sched_entity se;

	struct sched_rt_entity rt;

	u64 last_sleep_ts;

#ifdef CONFIG_SCHED_WALT

	struct ravg ravg;

	/*

kernel/sched/core.c

+13 −0
Original line number Diff line number Diff line
@@ -2194,6 +2194,9 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) 
		notif_required = true;

	}



	if (!__task_in_cum_window_demand(cpu_rq(cpu), p))

		inc_cum_window_demand(cpu_rq(cpu), p, task_load(p));



	note_task_waking(p, wallclock);

#endif /* CONFIG_SMP */
@@ -2266,6 +2269,8 @@ static void try_to_wake_up_local(struct task_struct *p, struct pin_cookie cookie 


		update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0);

		update_task_ravg(p, rq, TASK_WAKE, wallclock, 0);

		if (!__task_in_cum_window_demand(rq, p))

			inc_cum_window_demand(rq, p, task_load(p));

		cpufreq_update_util(rq, 0);

		ttwu_activate(rq, p, ENQUEUE_WAKEUP);

		note_task_waking(p, wallclock);
@@ -2353,6 +2358,8 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p) 
	p->se.prev_sum_exec_runtime	= 0;

	p->se.nr_migrations		= 0;

	p->se.vruntime			= 0;

	p->last_sleep_ts		= 0;



	INIT_LIST_HEAD(&p->se.group_node);



#ifdef CONFIG_FAIR_GROUP_SCHED
@@ -3680,6 +3687,9 @@ static void __sched notrace __schedule(bool preempt) 


	wallclock = sched_ktime_clock();

	if (likely(prev != next)) {

		if (!prev->on_rq)

			prev->last_sleep_ts = wallclock;



		update_task_ravg(prev, rq, PUT_PREV_TASK, wallclock, 0);

		update_task_ravg(next, rq, PICK_NEXT_TASK, wallclock, 0);

		cpufreq_update_util(rq, 0);
@@ -8354,6 +8364,7 @@ void __init sched_init(void) 
		cpumask_set_cpu(i, &rq->freq_domain_cpumask);

		rq->hmp_stats.cumulative_runnable_avg = 0;

		rq->window_start = 0;

		rq->cum_window_start = 0;

		rq->hmp_stats.nr_big_tasks = 0;

		rq->hmp_flags = 0;

		rq->cur_irqload = 0;
@@ -8395,6 +8406,7 @@ void __init sched_init(void) 


			clear_top_tasks_bitmap(rq->top_tasks_bitmap[j]);

		}

		rq->cum_window_demand = 0;

#endif

		INIT_LIST_HEAD(&rq->cfs_tasks);
@@ -9647,6 +9659,7 @@ void sched_exit(struct task_struct *p) 
	update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0);

	dequeue_task(rq, p, 0);

	reset_task_stats(p);

	dec_cum_window_demand(rq, p);

	p->ravg.mark_start = wallclock;

	p->ravg.sum_history[0] = EXITING_TASK_MARKER;

	free_task_load_ptrs(p);

kernel/sched/sched.h

+39 −0
Original line number Diff line number Diff line
@@ -773,6 +773,7 @@ struct rq { 


	int cstate, wakeup_latency, wakeup_energy;

	u64 window_start;

	s64 cum_window_start;

	u64 load_reported_window;

	unsigned long hmp_flags;
@@ -788,6 +789,7 @@ struct rq { 
	u64 prev_runnable_sum;

	u64 nt_curr_runnable_sum;

	u64 nt_prev_runnable_sum;

	u64 cum_window_demand;

	struct group_cpu_time grp_time;

	struct load_subtractions load_subs[NUM_TRACKED_WINDOWS];

	DECLARE_BITMAP_ARRAY(top_tasks_bitmap,
@@ -2598,6 +2600,31 @@ static inline void clear_reserved(int cpu) 
	clear_bit(CPU_RESERVED, &rq->hmp_flags);

}



static inline bool

__task_in_cum_window_demand(struct rq *rq, struct task_struct *p)

{

	return (p->on_rq || p->last_sleep_ts >= rq->window_start);

}



static inline bool

task_in_cum_window_demand(struct rq *rq, struct task_struct *p)

{

	return cpu_of(rq) == task_cpu(p) && __task_in_cum_window_demand(rq, p);

}



static inline void

dec_cum_window_demand(struct rq *rq, struct task_struct *p)

{

	rq->cum_window_demand -= p->ravg.demand;

	WARN_ON_ONCE(rq->cum_window_demand < 0);

}



static inline void

inc_cum_window_demand(struct rq *rq, struct task_struct *p, s64 delta)

{

	rq->cum_window_demand += delta;

}



#else	/* CONFIG_SCHED_WALT */



struct hmp_sched_stats;
@@ -2708,6 +2735,18 @@ static inline int alloc_related_thread_groups(void) { return 0; } 
#define trace_sched_cpu_load_cgroup(...)

#define trace_sched_cpu_load_wakeup(...)



static inline bool

__task_in_cum_window_demand(struct rq *rq, struct task_struct *p)

{

	return 0;

}



static inline void

dec_cum_window_demand(struct rq *rq, struct task_struct *p) { }



static inline void

inc_cum_window_demand(struct rq *rq, struct task_struct *p, s64 delta) { }



#endif	/* CONFIG_SCHED_WALT */



#ifdef CONFIG_SCHED_HMP

kernel/sched/walt.c

+17 −2
Original line number Diff line number Diff line
@@ -239,7 +239,7 @@ void reset_hmp_stats(struct hmp_sched_stats *stats, int reset_cra) 
__read_mostly unsigned int sched_freq_aggregate = 1;



static void

update_window_start(struct rq *rq, u64 wallclock)

update_window_start(struct rq *rq, u64 wallclock, int event)

{

	s64 delta;

	int nr_windows;
@@ -256,6 +256,10 @@ update_window_start(struct rq *rq, u64 wallclock) 


	nr_windows = div64_u64(delta, sched_ravg_window);

	rq->window_start += (u64)nr_windows * (u64)sched_ravg_window;



	rq->cum_window_demand = rq->hmp_stats.cumulative_runnable_avg;

	if (event == PUT_PREV_TASK)

		rq->cum_window_demand += rq->curr->ravg.demand;

}



int register_cpu_cycle_counter_cb(struct cpu_cycle_counter_cb *cb)
@@ -659,6 +663,11 @@ void fixup_busy_time(struct task_struct *p, int new_cpu) 


	update_task_cpu_cycles(p, new_cpu);



	if (__task_in_cum_window_demand(src_rq, p)) {

		dec_cum_window_demand(src_rq, p);

		inc_cum_window_demand(dest_rq, p, p->ravg.demand);

	}



	new_task = is_new_task(p);

	/* Protected by rq_lock */

	grp = p->grp;
@@ -1485,11 +1494,14 @@ static void update_history(struct rq *rq, struct task_struct *p, 
	int ridx, widx;

	u32 max = 0, avg, demand, pred_demand;

	u64 sum = 0;

	u64 prev_demand;



	/* Ignore windows where task had no activity */

	if (!runtime || is_idle_task(p) || exiting_task(p) || !samples)

		goto done;



	prev_demand = p->ravg.demand;



	/* Push new 'runtime' value onto stack */

	widx = sched_ravg_hist_size - 1;

	ridx = widx - samples;
@@ -1535,6 +1547,9 @@ static void update_history(struct rq *rq, struct task_struct *p, 
	p->ravg.demand = demand;

	p->ravg.pred_demand = pred_demand;



	if (__task_in_cum_window_demand(rq, p))

		inc_cum_window_demand(rq, p, p->ravg.demand - prev_demand);



done:

	trace_sched_update_history(rq, p, runtime, samples, event);

}
@@ -1728,7 +1743,7 @@ void update_task_ravg(struct task_struct *p, struct rq *rq, int event, 


	lockdep_assert_held(&rq->lock);



	update_window_start(rq, wallclock);

	update_window_start(rq, wallclock, event);



	if (!p->ravg.mark_start) {

		update_task_cpu_cycles(p, cpu_of(rq));