Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 6668d20d authored by Joonwoo Park's avatar Joonwoo Park
Browse files

sched: select task's prev_cpu as the best CPU when it was chosen recently 



Select given task's prev_cpu when the task slept for short period to
reduce latency of task placement and migrations.  A new tunable
/proc/sys/kernel/sched_select_prev_cpu_us introduced to determine whether
tasks are eligible to go through fast path.

CRs-fixed: 947467
Change-Id: Ia507665b91f4e9f0e6ee1448d8df8994ead9739a
Signed-off-by: default avatarJoonwoo Park <joonwoop@codeaurora.org>
parent 8556dbe9

Documentation/scheduler/sched-zone.txt

+8 −0
Original line number Diff line number Diff line
@@ -719,6 +719,14 @@ c. PF_WAKE_UP_IDLE 
	term "PF_WAKE_UP_IDLE wakeup" to signify wakeups involving a task with

	PF_WAKE_UP_IDLE flag set.



d. /proc/sys/kernel/sched_select_prev_cpu_us

	This threshold controls whether task placement goes through fast path or

	not. If task's wakeup time since last sleep is short there are high

	chances that it's better to place the task on its previous CPU. This

	reduces task placement latency, cache miss and number of migrations.

	Default value of sched_select_prev_cpu_us is 2000 (2ms).  This can be

	turned off by setting it to 0.



**** 5.2.4 Wakeup Logic for Task "p"



Wakeup task placement logic is as follows:

include/linux/sched.h

+1 −0
Original line number Diff line number Diff line
@@ -1315,6 +1315,7 @@ struct task_struct { 
	 */

	u32 init_load_pct;

	u64 last_wake_ts;

	u64 last_switch_out_ts;

#ifdef CONFIG_SCHED_QHMP

	u64 run_start;

#endif

include/linux/sched/sysctl.h

+1 −0
Original line number Diff line number Diff line
@@ -75,6 +75,7 @@ extern unsigned int sysctl_sched_small_task_pct; 
#else

extern unsigned int sysctl_sched_lowspill_freq;

extern unsigned int sysctl_sched_pack_freq;

extern unsigned int sysctl_sched_select_prev_cpu_us;

#if defined(CONFIG_SCHED_FREQ_INPUT)

extern unsigned int sysctl_sched_new_task_windows;

#endif

include/trace/events/sched.h

+7 −4
Original line number Diff line number Diff line
@@ -115,9 +115,9 @@ TRACE_EVENT(sched_enq_deq_task, 
TRACE_EVENT(sched_task_load,



	TP_PROTO(struct task_struct *p, bool boost, int reason,

		 bool sync, bool need_idle, int best_cpu),

		 bool sync, bool need_idle, bool fast_path, int best_cpu),



	TP_ARGS(p, boost, reason, sync, need_idle, best_cpu),

	TP_ARGS(p, boost, reason, sync, need_idle, fast_path, best_cpu),



	TP_STRUCT__entry(

		__array(	char,	comm,	TASK_COMM_LEN	)
@@ -127,6 +127,7 @@ TRACE_EVENT(sched_task_load, 
		__field(	int,	reason			)

		__field(	bool,	sync			)

		__field(	bool,	need_idle		)

		__field(	bool,	fast_path		)

		__field(	int,	best_cpu		)

		__field(	u64,	latency			)

	),
@@ -139,16 +140,18 @@ TRACE_EVENT(sched_task_load, 
		__entry->reason		= reason;

		__entry->sync		= sync;

		__entry->need_idle	= need_idle;

		__entry->fast_path	= fast_path;

		__entry->best_cpu	= best_cpu;

		__entry->latency	= p->state == TASK_WAKING ?

						      sched_ktime_clock() -

						      p->ravg.mark_start : 0;

	),



	TP_printk("%d (%s): demand=%u boost=%d reason=%d sync=%d need_idle=%d best_cpu=%d latency=%llu",

	TP_printk("%d (%s): demand=%u boost=%d reason=%d sync=%d need_idle=%d fast_path=%d best_cpu=%d latency=%llu",

		__entry->pid, __entry->comm, __entry->demand,

		__entry->boost, __entry->reason, __entry->sync,

		__entry->need_idle, __entry->best_cpu, __entry->latency)

		__entry->need_idle, __entry->fast_path,

		__entry->best_cpu, __entry->latency)

);



DECLARE_EVENT_CLASS(sched_cpu_load,

kernel/sched/core.c

+11 −0
Original line number Diff line number Diff line
@@ -872,6 +872,12 @@ static inline void set_task_last_wake(struct task_struct *p, u64 wallclock) 
{

	p->last_wake_ts = wallclock;

}



static inline void set_task_last_switch_out(struct task_struct *p,

					    u64 wallclock)

{

	p->last_switch_out_ts = wallclock;

}

#else

u64 sched_ktime_clock(void)

{
@@ -880,6 +886,8 @@ u64 sched_ktime_clock(void) 


static inline void clear_ed_task(struct task_struct *p, struct rq *rq) {}

static inline void set_task_last_wake(struct task_struct *p, u64 wallclock) {}

static inline void set_task_last_switch_out(struct task_struct *p,

					    u64 wallclock) {}

#endif



#if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \
@@ -2199,6 +2207,7 @@ static inline void mark_task_starting(struct task_struct *p) 


	wallclock = sched_ktime_clock();

	p->ravg.mark_start = p->last_wake_ts = wallclock;

	p->last_switch_out_ts = 0;

}



static inline void set_window_start(struct rq *rq)
@@ -4937,6 +4946,8 @@ need_resched: 
		rq->curr = next;

		++*switch_count;



		set_task_last_switch_out(prev, wallclock);



		context_switch(rq, prev, next); /* unlocks the rq */

		/*

		 * The context switch have flipped the stack from under us