Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit d1122dab authored by Syed Rameez Mustafa's avatar Syed Rameez Mustafa
Browse files

sched: Notify cpufreq governor early about potential big tasks 



Tasks that are on the runqueue continuously for a certain amount of time
have the potential to be big tasks at the end of the window in which they
are runnable. In such scenarios ramping the CPU frequency early can
boost performance rather than waiting till the end of a window for the
governor to query load. Notify the governor early at every tick when a
task has been observed to execute beyond some percentage of the tick
period.

The threshold beyond which a task is eligible for early detection can be
changed via the tunable sched_early_detection_duration. The feature itself
is enabled only when scheduler boost is in effect.

Change-Id: I528b72bbc79a55b4593d1b8ab45450411c6d70f3
Signed-off-by: default avatarSyed Rameez Mustafa <rameezmustafa@codeaurora.org>
parent 50e0173f

include/linux/sched.h

+1 −0
Original line number Diff line number Diff line
@@ -1314,6 +1314,7 @@ struct task_struct { 
	 * of this task

	 */

	u32 init_load_pct;

	u64 last_wake_ts;

#ifdef CONFIG_SCHED_QHMP

	u64 run_start;

#endif

include/linux/sched/sysctl.h

+1 −0
Original line number Diff line number Diff line
@@ -68,6 +68,7 @@ extern int sysctl_sched_upmigrate_min_nice; 
extern unsigned int sysctl_sched_powerband_limit_pct;

extern unsigned int sysctl_sched_boost;

extern unsigned int sysctl_power_aware_timer_migration;

extern unsigned int sysctl_early_detection_duration;



#ifdef CONFIG_SCHED_QHMP

extern unsigned int sysctl_sched_min_runtime;

include/trace/events/sched.h

+6 −4
Original line number Diff line number Diff line
@@ -411,24 +411,26 @@ TRACE_EVENT(sched_migration_update_sum, 


TRACE_EVENT(sched_get_busy,



	TP_PROTO(int cpu, u64 load, u64 nload),

	TP_PROTO(int cpu, u64 load, u64 nload, int early),



	TP_ARGS(cpu, load, nload),

	TP_ARGS(cpu, load, nload, early),



	TP_STRUCT__entry(

		__field(	int,	cpu			)

		__field(	u64,	load			)

		__field(	u64,	nload			)

		__field(	int,	early			)

	),



	TP_fast_assign(

		__entry->cpu		= cpu;

		__entry->load		= load;

		__entry->nload		= nload;

		__entry->early		= early;

	),



	TP_printk("cpu %d load %lld new_task_load %lld",

		__entry->cpu, __entry->load, __entry->nload)

	TP_printk("cpu %d load %lld new_task_load %lld early %d",

		__entry->cpu, __entry->load, __entry->nload, __entry->early)

);



TRACE_EVENT(sched_freq_alert,

kernel/sched/core.c

+95 −5
Original line number Diff line number Diff line
@@ -892,6 +892,22 @@ void sched_set_cluster_dstate(const cpumask_t *cluster_cpus, int dstate, 


#endif /* CONFIG_SMP */



#ifdef CONFIG_SCHED_HMP

static inline void clear_ed_task(struct task_struct *p, struct rq *rq)

{

	if (p == rq->ed_task)

		rq->ed_task = NULL;

}



static inline void set_task_last_wake(struct task_struct *p, u64 wallclock)

{

	p->last_wake_ts = wallclock;

}

#else

static inline void clear_ed_task(struct task_struct *p, struct rq *rq) {}

static inline void set_task_last_wake(struct task_struct *p, u64 wallclock) {}

#endif



#if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \

			(defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH)))

/*
@@ -984,6 +1000,9 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags) 
	if (task_contributes_to_load(p))

		rq->nr_uninterruptible++;



	if (flags & DEQUEUE_SLEEP)

		clear_ed_task(p, rq);



	dequeue_task(rq, p, flags);

}
@@ -1288,6 +1307,13 @@ static inline void clear_hmp_request(int cpu) { } 
 * IMPORTANT: Initialize both copies to same value!!

 */



/*

 * Tasks that are runnable continuously for a period greather than

 * sysctl_early_detection_duration can be flagged early as potential

 * high load tasks.

 */

__read_mostly unsigned int sysctl_early_detection_duration = 9500000;



static __read_mostly unsigned int sched_ravg_hist_size = 5;

__read_mostly unsigned int sysctl_sched_ravg_hist_size = 5;
@@ -2193,7 +2219,7 @@ static inline void mark_task_starting(struct task_struct *p) 
		return;

	}



	p->ravg.mark_start = wallclock;

	p->ravg.mark_start = p->last_wake_ts = wallclock;

}



static inline void set_window_start(struct rq *rq)
@@ -2266,6 +2292,7 @@ void sched_exit(struct task_struct *p) 
	p->ravg.mark_start = wallclock;

	p->ravg.sum_history[0] = EXITING_TASK_MARKER;

	enqueue_task(rq, p, 0);

	clear_ed_task(p, rq);

	raw_spin_unlock_irqrestore(&rq->lock, flags);



	put_cpu();
@@ -2411,6 +2438,7 @@ void sched_get_cpus_busy(struct sched_load *busy, 
	u64 load[cpus], nload[cpus];

	unsigned int cur_freq[cpus], max_freq[cpus];

	int notifier_sent[cpus];

	int early_detection[cpus];

	int cpu, i = 0;

	unsigned int window_size;
@@ -2444,6 +2472,7 @@ void sched_get_cpus_busy(struct sched_load *busy, 
		nload[i] = scale_load_to_cpu(nload[i], cpu);



		notifier_sent[i] = rq->notifier_sent;

		early_detection[i] = (rq->ed_task != NULL);

		rq->notifier_sent = 0;

		cur_freq[i] = rq->cur_freq;

		max_freq[i] = rq->max_freq;
@@ -2458,6 +2487,13 @@ void sched_get_cpus_busy(struct sched_load *busy, 
	for_each_cpu(cpu, query_cpus) {

		rq = cpu_rq(cpu);



		if (early_detection[i]) {

			busy[i].prev_load = div64_u64(sched_ravg_window,

							NSEC_PER_USEC);

			busy[i].new_task_load = 0;

			goto exit_early;

		}



		if (!notifier_sent[i]) {

			load[i] = scale_load_to_freq(load[i], max_freq[i],

						     cur_freq[i]);
@@ -2482,8 +2518,9 @@ void sched_get_cpus_busy(struct sched_load *busy, 
		busy[i].prev_load = div64_u64(load[i], NSEC_PER_USEC);

		busy[i].new_task_load = div64_u64(nload[i], NSEC_PER_USEC);



exit_early:

		trace_sched_get_busy(cpu, busy[i].prev_load,

				     busy[i].new_task_load);

				     busy[i].new_task_load, early_detection[i]);

		i++;

	}

}
@@ -2551,9 +2588,14 @@ static void fixup_busy_time(struct task_struct *p, int new_cpu) 
	bool new_task;



	if (!sched_enable_hmp || !sched_migration_fixup ||

		 exiting_task(p) || (!p->on_rq && p->state != TASK_WAKING))

		 (!p->on_rq && p->state != TASK_WAKING))

			return;



	if (exiting_task(p)) {

		clear_ed_task(p, src_rq);

		return;

	}



	if (p->state == TASK_WAKING)

		double_rq_lock(src_rq, dest_rq);
@@ -2591,6 +2633,12 @@ static void fixup_busy_time(struct task_struct *p, int new_cpu) 
		}

	}



	if (p == src_rq->ed_task) {

		src_rq->ed_task = NULL;

		if (!dest_rq->ed_task)

			dest_rq->ed_task = p;

	}



	BUG_ON((s64)src_rq->prev_runnable_sum < 0);

	BUG_ON((s64)src_rq->curr_runnable_sum < 0);

	BUG_ON((s64)src_rq->nt_prev_runnable_sum < 0);
@@ -3698,8 +3746,9 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) 
		wake_flags |= WF_MIGRATED;

		set_task_cpu(p, cpu);

	}

#endif /* CONFIG_SMP */



	set_task_last_wake(p, wallclock);

#endif /* CONFIG_SMP */

	ttwu_queue(p, cpu);

stat:

	ttwu_stat(p, cpu, wake_flags);
@@ -3774,6 +3823,7 @@ static void try_to_wake_up_local(struct task_struct *p) 
		update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0);

		update_task_ravg(p, rq, TASK_WAKE, wallclock, 0);

		ttwu_activate(rq, p, ENQUEUE_WAKEUP);

		set_task_last_wake(p, wallclock);

	}



	ttwu_do_wakeup(rq, p, 0);
@@ -4538,6 +4588,38 @@ unsigned long long task_sched_runtime(struct task_struct *p) 
	return ns;

}



#ifdef CONFIG_SCHED_HMP

static bool early_detection_notify(struct rq *rq, u64 wallclock)

{

	struct task_struct *p;

	int loop_max = 10;



	if (!sched_boost() || !rq->cfs.h_nr_running)

		return 0;



	rq->ed_task = NULL;

	list_for_each_entry(p, &rq->cfs_tasks, se.group_node) {

		if (!loop_max)

			break;



		if (wallclock - p->last_wake_ts >=

				sysctl_early_detection_duration) {

			rq->ed_task = p;

			return 1;

		}



		loop_max--;

	}



	return 0;

}

#else /* CONFIG_SCHED_HMP */

static bool early_detection_notify(struct rq *rq, u64 wallclock)

{

	return 0;

}

#endif /* CONFIG_SCHED_HMP */



/*

 * This function gets called by the timer code, with HZ frequency.

 * We call it with interrupts disabled.
@@ -4547,6 +4629,8 @@ void scheduler_tick(void) 
	int cpu = smp_processor_id();

	struct rq *rq = cpu_rq(cpu);

	struct task_struct *curr = rq->curr;

	u64 wallclock;

	bool early_notif;



	sched_clock_tick();
@@ -4555,9 +4639,15 @@ void scheduler_tick(void) 
	update_rq_clock(rq);

	curr->sched_class->task_tick(rq, curr, 0);

	update_cpu_load_active(rq);

	update_task_ravg(rq->curr, rq, TASK_UPDATE, sched_clock(), 0);

	wallclock = sched_clock();

	update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0);

	early_notif = early_detection_notify(rq, wallclock);

	raw_spin_unlock(&rq->lock);



	if (early_notif)

		atomic_notifier_call_chain(&load_alert_notifier_head,

					0, (void *)(long)cpu);



	perf_event_task_tick();



#ifdef CONFIG_SMP

kernel/sched/sched.h

+1 −0
Original line number Diff line number Diff line
@@ -660,6 +660,7 @@ struct rq { 
	u64 irqload_ts;

	unsigned int static_cpu_pwr_cost;

	unsigned int static_cluster_pwr_cost;

	struct task_struct *ed_task;



#ifdef CONFIG_SCHED_FREQ_INPUT

	unsigned int old_busy_time;