Merge "sched: Consolidate hmp stats into their own struct" (54dd18b6) · Commits · e / devices / android_kernel_sony_msm8994

include/trace/events/sched.h

+5 −5

Original line number	Diff line number	Diff line
		@@ -186,11 +186,11 @@ TRACE_EVENT(sched_cpu_load,
		__entry->idle = idle;
		__entry->mostly_idle = mostly_idle;
		__entry->nr_running = rq->nr_running;
		__entry->nr_big_tasks = rq->nr_big_tasks;
		__entry->nr_small_tasks = rq->nr_small_tasks;
		__entry->nr_big_tasks = rq->hmp_stats.nr_big_tasks;
		__entry->nr_small_tasks = rq->hmp_stats.nr_small_tasks;
		__entry->load_scale_factor = rq->load_scale_factor;
		__entry->capacity = rq->capacity;
		__entry->cumulative_runnable_avg = rq->cumulative_runnable_avg;
		__entry->cumulative_runnable_avg = rq->hmp_stats.cumulative_runnable_avg;
		__entry->irqload = irqload;
		__entry->cur_freq = rq->cur_freq;
		__entry->max_freq = rq->max_freq;
		@@ -324,8 +324,8 @@ TRACE_EVENT(sched_update_history,
		__entry->demand = p->ravg.demand;
		memcpy(__entry->hist, p->ravg.sum_history,
		RAVG_HIST_SIZE_MAX * sizeof(u32));
		__entry->nr_big_tasks = rq->nr_big_tasks;
		__entry->nr_small_tasks = rq->nr_small_tasks;
		__entry->nr_big_tasks = rq->hmp_stats.nr_big_tasks;
		__entry->nr_small_tasks = rq->hmp_stats.nr_small_tasks;
		__entry->cpu = rq->cpu;
		),

kernel/sched/core.c

+11 −26

Original line number	Diff line number	Diff line
		@@ -840,7 +840,6 @@ static void enqueue_task(struct rq rq, struct task_struct p, int flags)
		sched_info_queued(p);
		p->sched_class->enqueue_task(rq, p, flags);
		trace_sched_enq_deq_task(p, 1, cpumask_bits(&p->cpus_allowed)[0]);
		inc_cumulative_runnable_avg(rq, p);
		}

		static void dequeue_task(struct rq rq, struct task_struct p, int flags)
		@@ -849,7 +848,6 @@ static void dequeue_task(struct rq rq, struct task_struct p, int flags)
		sched_info_dequeued(p);
		p->sched_class->dequeue_task(rq, p, flags);
		trace_sched_enq_deq_task(p, 0, cpumask_bits(&p->cpus_allowed)[0]);
		dec_cumulative_runnable_avg(rq, p);
		}

		void activate_task(struct rq rq, struct task_struct p, int flags)
		@@ -1665,12 +1663,8 @@ static void update_history(struct rq rq, struct task_struct p,
		}

		p->ravg.sum = 0;
		if (p->on_rq) {
		rq->cumulative_runnable_avg -= p->ravg.demand;
		BUG_ON((s64)rq->cumulative_runnable_avg < 0);
		if (p->sched_class == &fair_sched_class)
		dec_nr_big_small_task(rq, p);
		}
		if (p->on_rq)
		p->sched_class->dec_hmp_sched_stats(rq, p);

		avg = div64_u64(sum, sched_ravg_hist_size);

		@@ -1685,11 +1679,8 @@ static void update_history(struct rq rq, struct task_struct p,

		p->ravg.demand = demand;

		if (p->on_rq) {
		rq->cumulative_runnable_avg += p->ravg.demand;
		if (p->sched_class == &fair_sched_class)
		inc_nr_big_small_task(rq, p);
		}
		if (p->on_rq)
		p->sched_class->inc_hmp_sched_stats(rq, p);

		done:
		trace_sched_update_history(rq, p, runtime, samples, event);
		@@ -2075,7 +2066,7 @@ void reset_all_window_stats(u64 window_start, unsigned int window_size)
		#ifdef CONFIG_SCHED_FREQ_INPUT
		rq->curr_runnable_sum = rq->prev_runnable_sum = 0;
		#endif
		rq->cumulative_runnable_avg = 0;
		rq->hmp_stats.cumulative_runnable_avg = 0;
		fixup_nr_big_small_task(cpu);
		}

		@@ -2229,11 +2220,8 @@ static void fixup_busy_time(struct task_struct *p, int new_cpu)
		* reflect new demand. Restore load temporarily for such
		* task on its runqueue
		*/
		if (p->on_rq) {
		inc_cumulative_runnable_avg(src_rq, p);
		if (p->sched_class == &fair_sched_class)
		inc_nr_big_small_task(src_rq, p);
		}
		if (p->on_rq)
		p->sched_class->inc_hmp_sched_stats(src_rq, p);

		update_task_ravg(p, task_rq(p), TASK_MIGRATE,
		wallclock, 0);
		@@ -2242,11 +2230,8 @@ static void fixup_busy_time(struct task_struct *p, int new_cpu)
		* Remove task's load from rq as its now migrating to
		* another cpu.
		*/
		if (p->on_rq) {
		dec_cumulative_runnable_avg(src_rq, p);
		if (p->sched_class == &fair_sched_class)
		dec_nr_big_small_task(src_rq, p);
		}
		if (p->on_rq)
		p->sched_class->dec_hmp_sched_stats(src_rq, p);

		if (p->ravg.curr_window) {
		src_rq->curr_runnable_sum -= p->ravg.curr_window;
		@@ -9032,12 +9017,12 @@ void __init sched_init(void)
		rq->min_freq = 1;
		rq->max_possible_freq = 1;
		rq->max_possible_capacity = 0;
		rq->cumulative_runnable_avg = 0;
		rq->hmp_stats.cumulative_runnable_avg = 0;
		rq->efficiency = 1024;
		rq->capacity = 1024;
		rq->load_scale_factor = 1024;
		rq->window_start = 0;
		rq->nr_small_tasks = rq->nr_big_tasks = 0;
		rq->hmp_stats.nr_small_tasks = rq->hmp_stats.nr_big_tasks = 0;
		rq->hmp_flags = 0;
		rq->mostly_idle_load = pct_to_real(20);
		rq->mostly_idle_nr_run = 3;

kernel/sched/debug.c

+2 −2

Original line number	Diff line number	Diff line
		@@ -316,8 +316,8 @@ do { \
		P(max_freq);
		#endif
		#ifdef CONFIG_SCHED_HMP
		P(nr_big_tasks);
		P(nr_small_tasks);
		P(hmp_stats.nr_big_tasks);
		P(hmp_stats.nr_small_tasks);
		#endif
		#undef P
		#undef PN

kernel/sched/fair.c

+79 −33

Original line number	Diff line number	Diff line
		@@ -1281,7 +1281,7 @@ unsigned int __read_mostly sched_enable_hmp = 0;
		/* A cpu can no longer accomodate more tasks if:
		*
		* rq->nr_running > sysctl_sched_spill_nr_run \|\|
		* rq->cumulative_runnable_avg > sched_spill_load
		* rq->hmp_stats.cumulative_runnable_avg > sched_spill_load
		*/
		unsigned int __read_mostly sysctl_sched_spill_nr_run = 10;

		@@ -1527,7 +1527,7 @@ static inline u64 cpu_load(int cpu)
		{
		struct rq *rq = cpu_rq(cpu);

		return scale_load_to_cpu(rq->cumulative_runnable_avg, cpu);
		return scale_load_to_cpu(rq->hmp_stats.cumulative_runnable_avg, cpu);
		}

		static inline u64 cpu_load_sync(int cpu, int sync)
		@@ -1535,7 +1535,7 @@ static inline u64 cpu_load_sync(int cpu, int sync)
		struct rq *rq = cpu_rq(cpu);
		u64 load;

		load = rq->cumulative_runnable_avg;
		load = rq->hmp_stats.cumulative_runnable_avg;

		/*
		* If load is being checked in a sync wakeup environment,
		@@ -2160,28 +2160,42 @@ done:
		return best_cpu;
		}

		void inc_nr_big_small_task(struct rq rq, struct task_struct p)
		static void
		inc_nr_big_small_task(struct hmp_sched_stats stats, struct task_struct p)
		{
		if (!sched_enable_hmp \|\| sched_disable_window_stats)
		return;

		if (is_big_task(p))
		rq->nr_big_tasks++;
		stats->nr_big_tasks++;
		else if (is_small_task(p))
		rq->nr_small_tasks++;
		stats->nr_small_tasks++;
		}

		void dec_nr_big_small_task(struct rq rq, struct task_struct p)
		static void
		dec_nr_big_small_task(struct hmp_sched_stats stats, struct task_struct p)
		{
		if (!sched_enable_hmp \|\| sched_disable_window_stats)
		return;

		if (is_big_task(p))
		rq->nr_big_tasks--;
		stats->nr_big_tasks--;
		else if (is_small_task(p))
		rq->nr_small_tasks--;
		stats->nr_small_tasks--;

		BUG_ON(stats->nr_big_tasks < 0 \|\| stats->nr_small_tasks < 0);
		}

		static void inc_rq_hmp_stats(struct rq rq, struct task_struct p)
		{
		inc_cumulative_runnable_avg(&rq->hmp_stats, p);
		inc_nr_big_small_task(&rq->hmp_stats, p);
		}

		BUG_ON(rq->nr_big_tasks < 0 \|\| rq->nr_small_tasks < 0);
		static void dec_rq_hmp_stats(struct rq rq, struct task_struct p)
		{
		dec_cumulative_runnable_avg(&rq->hmp_stats, p);
		dec_nr_big_small_task(&rq->hmp_stats, p);
		}

		/*
		@@ -2193,10 +2207,10 @@ void fixup_nr_big_small_task(int cpu)
		struct rq *rq = cpu_rq(cpu);
		struct task_struct *p;

		rq->nr_big_tasks = 0;
		rq->nr_small_tasks = 0;
		rq->hmp_stats.nr_big_tasks = 0;
		rq->hmp_stats.nr_small_tasks = 0;
		list_for_each_entry(p, &rq->cfs_tasks, se.group_node)
		inc_nr_big_small_task(rq, p);
		inc_nr_big_small_task(&rq->hmp_stats, p);
		}

		/* Disable interrupts and grab runqueue lock of all cpus listed in @cpus */
		@@ -2530,7 +2544,7 @@ void check_for_migration(struct rq rq, struct task_struct p)

		static inline int nr_big_tasks(struct rq *rq)
		{
		return rq->nr_big_tasks;
		return rq->hmp_stats.nr_big_tasks;
		}

		static inline int is_cpu_throttling_imminent(int cpu)
		@@ -2563,6 +2577,19 @@ unsigned int cpu_temp(int cpu)
		return 0;
		}

		static void
		inc_hmp_sched_stats_fair(struct rq rq, struct task_struct p)
		{
		inc_cumulative_runnable_avg(&rq->hmp_stats, p);
		inc_nr_big_small_task(&rq->hmp_stats, p);
		}

		static void
		dec_hmp_sched_stats_fair(struct rq rq, struct task_struct p)
		{
		dec_cumulative_runnable_avg(&rq->hmp_stats, p);
		dec_nr_big_small_task(&rq->hmp_stats, p);
		}

		#else /* CONFIG_SCHED_HMP */

		@@ -2635,6 +2662,18 @@ unsigned int cpu_temp(int cpu)
		return 0;
		}

		static inline void inc_rq_hmp_stats(struct rq rq, struct task_struct p) { }
		static inline void dec_rq_hmp_stats(struct rq rq, struct task_struct p) { }

		static inline void
		inc_hmp_sched_stats_fair(struct rq rq, struct task_struct p)
		{
		}

		static inline void
		dec_hmp_sched_stats_fair(struct rq rq, struct task_struct p)
		{
		}

		#endif /* CONFIG_SCHED_HMP */

		@@ -2949,18 +2988,14 @@ static inline void update_entity_load_avg(struct sched_entity *se,
		*/
		if (entity_is_task(se)) {
		now = cfs_rq_clock_task(cfs_rq);
		if (se->on_rq) {
		dec_cumulative_runnable_avg(rq_of(cfs_rq), task_of(se));
		dec_nr_big_small_task(rq_of(cfs_rq), task_of(se));
		}
		if (se->on_rq)
		dec_hmp_sched_stats_fair(rq_of(cfs_rq), task_of(se));
		} else
		now = cfs_rq_clock_task(group_cfs_rq(se));

		decayed = __update_entity_runnable_avg(cpu, now, &se->avg, se->on_rq);
		if (entity_is_task(se) && se->on_rq) {
		inc_cumulative_runnable_avg(rq_of(cfs_rq), task_of(se));
		inc_nr_big_small_task(rq_of(cfs_rq), task_of(se));
		}
		if (entity_is_task(se) && se->on_rq)
		inc_hmp_sched_stats_fair(rq_of(cfs_rq), task_of(se));

		if (!decayed)
		return;
		@@ -3106,6 +3141,10 @@ static inline void dequeue_entity_load_avg(struct cfs_rq *cfs_rq,
		int sleep) {}
		static inline void update_cfs_rq_blocked_load(struct cfs_rq *cfs_rq,
		int force_update) {}

		static inline void inc_rq_hmp_stats(struct rq rq, struct task_struct p) { }
		static inline void dec_rq_hmp_stats(struct rq rq, struct task_struct p) { }

		#endif

		#ifdef CONFIG_SCHED_HMP
		@@ -4441,7 +4480,7 @@ enqueue_task_fair(struct rq rq, struct task_struct p, int flags)
		if (!se) {
		update_rq_runnable_avg(rq, rq->nr_running);
		inc_nr_running(rq);
		inc_nr_big_small_task(rq, p);
		inc_rq_hmp_stats(rq, p);
		}
		hrtick_update(rq);
		}
		@@ -4503,7 +4542,7 @@ static void dequeue_task_fair(struct rq rq, struct task_struct p, int flags)
		if (!se) {
		dec_nr_running(rq);
		update_rq_runnable_avg(rq, 1);
		dec_nr_big_small_task(rq, p);
		dec_rq_hmp_stats(rq, p);
		}
		hrtick_update(rq);
		}
		@@ -6165,8 +6204,8 @@ static inline void update_sg_lb_stats(struct lb_env *env,
		sgs->group_load += load;
		sgs->sum_nr_running += nr_running;
		#ifdef CONFIG_SCHED_HMP
		sgs->sum_nr_big_tasks += rq->nr_big_tasks;
		sgs->sum_nr_small_tasks += rq->nr_small_tasks;
		sgs->sum_nr_big_tasks += rq->hmp_stats.nr_big_tasks;
		sgs->sum_nr_small_tasks += rq->hmp_stats.nr_small_tasks;
		sgs->group_cpu_load += cpu_load(i);
		#endif
		sgs->sum_weighted_load += weighted_cpuload(i);
		@@ -6678,18 +6717,20 @@ ret:
		static struct rq find_busiest_queue_hmp(struct lb_env env,
		struct sched_group *group)
		{
		struct rq busiest = NULL, rq;
		struct rq *busiest = NULL;
		u64 max_runnable_avg = 0;
		int i;

		for_each_cpu(i, sched_group_cpus(group)) {
		struct rq *rq = cpu_rq(i);
		u64 cumulative_runnable_avg =
		rq->hmp_stats.cumulative_runnable_avg;

		if (!cpumask_test_cpu(i, env->cpus))
		continue;

		rq = cpu_rq(i);

		if (rq->cumulative_runnable_avg > max_runnable_avg) {
		max_runnable_avg = rq->cumulative_runnable_avg;
		if (cumulative_runnable_avg > max_runnable_avg) {
		max_runnable_avg = cumulative_runnable_avg;
		busiest = rq;
		}
		}
		@@ -7556,7 +7597,8 @@ static inline int _nohz_kick_needed_hmp(struct rq rq, int cpu, int type)
		&& rq->max_freq > rq->mostly_idle_freq)
		return 0;

		if (rq->nr_running >= 2 && (rq->nr_running - rq->nr_small_tasks >= 2 \|\|
		if (rq->nr_running >= 2 &&
		(rq->nr_running - rq->hmp_stats.nr_small_tasks >= 2 \|\|
		rq->nr_running > rq->mostly_idle_nr_run \|\|
		cpu_load(cpu) > rq->mostly_idle_load)) {

		@@ -8162,6 +8204,10 @@ const struct sched_class fair_sched_class = {
		#ifdef CONFIG_FAIR_GROUP_SCHED
		.task_move_group = task_move_group_fair,
		#endif
		#ifdef CONFIG_SCHED_HMP
		.inc_hmp_sched_stats = inc_hmp_sched_stats_fair,
		.dec_hmp_sched_stats = dec_hmp_sched_stats_fair,
		#endif
		};

		#ifdef CONFIG_SCHED_DEBUG

kernel/sched/idle_task.c

+18 −0

Original line number	Diff line number	Diff line
		@@ -84,6 +84,20 @@ static unsigned int get_rr_interval_idle(struct rq rq, struct task_struct task
		return 0;
		}

		#ifdef CONFIG_SCHED_HMP

		static void
		inc_hmp_sched_stats_idle(struct rq rq, struct task_struct p)
		{
		}

		static void
		dec_hmp_sched_stats_idle(struct rq rq, struct task_struct p)
		{
		}

		#endif

		/*
		* Simple, special scheduling class for the per-CPU idle tasks:
		*/
		@@ -112,4 +126,8 @@ const struct sched_class idle_sched_class = {

		.prio_changed = prio_changed_idle,
		.switched_to = switched_to_idle,
		#ifdef CONFIG_SCHED_HMP
		.inc_hmp_sched_stats = inc_hmp_sched_stats_idle,
		.dec_hmp_sched_stats = dec_hmp_sched_stats_idle,
		#endif
		};