Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 824efcd5 authored by Syed Rameez Mustafa's avatar Syed Rameez Mustafa
Browse files

sched: remove the notion of small tasks and small task packing 



Task packing will now be determined solely on the basis of the
power cost of task placement. All tasks are eligible for packing.
Remove the notion of "small" tasks from the scheduler.

Change-Id: I72d52d04b2677c6a8d0bc6aa7d50ff0f1a4f5ebb
Signed-off-by: default avatarSyed Rameez Mustafa <rameezmustafa@codeaurora.org>
parent 9a38a91b

include/linux/sched/sysctl.h

+0 −1
Original line number Diff line number Diff line
@@ -62,7 +62,6 @@ extern int sysctl_sched_freq_dec_notify; 
#ifdef CONFIG_SCHED_HMP

extern unsigned int sysctl_sched_spill_nr_run;

extern unsigned int sysctl_sched_spill_load_pct;

extern unsigned int sysctl_sched_small_task_pct;

extern unsigned int sysctl_sched_upmigrate_pct;

extern unsigned int sysctl_sched_downmigrate_pct;

extern int sysctl_sched_upmigrate_min_nice;

include/trace/events/sched.h

+11 −20
Original line number Diff line number Diff line
@@ -115,10 +115,10 @@ TRACE_EVENT(sched_enq_deq_task, 


TRACE_EVENT(sched_task_load,



	TP_PROTO(struct task_struct *p, int small_task, int boost, int reason,

	TP_PROTO(struct task_struct *p, int boost, int reason,

		 int sync, int prefer_idle),



	TP_ARGS(p, small_task, boost, reason, sync, prefer_idle),

	TP_ARGS(p, boost, reason, sync, prefer_idle),



	TP_STRUCT__entry(

		__array(	char,	comm,	TASK_COMM_LEN	)
@@ -127,7 +127,6 @@ TRACE_EVENT(sched_task_load, 
		__field(unsigned int,	sum_scaled		)

		__field(unsigned int,	period			)

		__field(unsigned int,	demand			)

		__field(	int,	small_task		)

		__field(	int,	boost			)

		__field(	int,	reason			)

		__field(	int,	sync			)
@@ -141,18 +140,17 @@ TRACE_EVENT(sched_task_load, 
		__entry->sum_scaled	= p->se.avg.runnable_avg_sum_scaled;

		__entry->period		= p->se.avg.runnable_avg_period;

		__entry->demand		= p->ravg.demand;

		__entry->small_task	= small_task;

		__entry->boost		= boost;

		__entry->reason		= reason;

		__entry->sync		= sync;

		__entry->prefer_idle	= prefer_idle;

	),



	TP_printk("%d (%s): sum=%u, sum_scaled=%u, period=%u demand=%u small=%d boost=%d reason=%d sync=%d prefer_idle=%d",

	TP_printk("%d (%s): sum=%u, sum_scaled=%u, period=%u demand=%u boost=%d reason=%d sync=%d prefer_idle=%d",

		__entry->pid, __entry->comm, __entry->sum,

		__entry->sum_scaled, __entry->period, __entry->demand,

		__entry->small_task, __entry->boost, __entry->reason,

		__entry->sync, __entry->prefer_idle)

		__entry->boost, __entry->reason, __entry->sync,

		__entry->prefer_idle)

);



TRACE_EVENT(sched_cpu_load,
@@ -168,7 +166,6 @@ TRACE_EVENT(sched_cpu_load, 
		__field(unsigned int, mostly_idle		)

		__field(unsigned int, nr_running		)

		__field(unsigned int, nr_big_tasks		)

		__field(unsigned int, nr_small_tasks		)

		__field(unsigned int, load_scale_factor		)

		__field(unsigned int, capacity			)

		__field(	 u64, cumulative_runnable_avg	)
@@ -186,7 +183,6 @@ TRACE_EVENT(sched_cpu_load, 
		__entry->mostly_idle		= mostly_idle;

		__entry->nr_running		= rq->nr_running;

		__entry->nr_big_tasks		= rq->hmp_stats.nr_big_tasks;

		__entry->nr_small_tasks		= rq->hmp_stats.nr_small_tasks;

		__entry->load_scale_factor	= rq->load_scale_factor;

		__entry->capacity		= rq->capacity;

		__entry->cumulative_runnable_avg = rq->hmp_stats.cumulative_runnable_avg;
@@ -198,13 +194,11 @@ TRACE_EVENT(sched_cpu_load, 
		__entry->temp			= temp;

	),



	TP_printk("cpu %u idle %d mostly_idle %d nr_run %u nr_big %u nr_small %u lsf %u capacity %u cr_avg %llu irqload %llu fcur %u fmax %u power_cost %u cstate %d temp %d",

	TP_printk("cpu %u idle %d mostly_idle %d nr_run %u nr_big %u lsf %u capacity %u cr_avg %llu irqload %llu fcur %u fmax %u power_cost %u cstate %d temp %d",

	__entry->cpu, __entry->idle, __entry->mostly_idle, __entry->nr_running,

	__entry->nr_big_tasks, __entry->nr_small_tasks,

	__entry->load_scale_factor, __entry->capacity,

	__entry->cumulative_runnable_avg, __entry->irqload,

	__entry->cur_freq, __entry->max_freq,

	__entry->power_cost, __entry->cstate, __entry->temp)

	__entry->nr_big_tasks, __entry->load_scale_factor, __entry->capacity,

	__entry->cumulative_runnable_avg, __entry->irqload, __entry->cur_freq,

	__entry->max_freq, __entry->power_cost, __entry->cstate, __entry->temp)

);



TRACE_EVENT(sched_set_boost,
@@ -310,7 +304,6 @@ TRACE_EVENT(sched_update_history, 
		__field(unsigned int,	demand			)

		__array(	 u32,	hist, RAVG_HIST_SIZE_MAX)

		__field(unsigned int,	nr_big_tasks		)

		__field(unsigned int,	nr_small_tasks		)

		__field(	 int,	cpu			)

	),
@@ -324,18 +317,16 @@ TRACE_EVENT(sched_update_history, 
		memcpy(__entry->hist, p->ravg.sum_history,

					RAVG_HIST_SIZE_MAX * sizeof(u32));

		__entry->nr_big_tasks   = rq->hmp_stats.nr_big_tasks;

		__entry->nr_small_tasks = rq->hmp_stats.nr_small_tasks;

		__entry->cpu            = rq->cpu;

	),



	TP_printk("%d (%s): runtime %u samples %d event %s demand %u (hist: %u %u %u %u %u) cpu %d nr_big %u nr_small %u",

	TP_printk("%d (%s): runtime %u samples %d event %s demand %u (hist: %u %u %u %u %u) cpu %d nr_big %u",

		__entry->pid, __entry->comm,

		__entry->runtime, __entry->samples,

		task_event_names[__entry->evt],

		__entry->demand, __entry->hist[0],

		__entry->hist[1], __entry->hist[2], __entry->hist[3],

		__entry->hist[4], __entry->cpu, __entry->nr_big_tasks,

		__entry->nr_small_tasks)

		__entry->hist[4], __entry->cpu, __entry->nr_big_tasks)

);



TRACE_EVENT(sched_reset_all_window_stats,

kernel/sched/core.c

+11 −11
Original line number Diff line number Diff line
@@ -2103,7 +2103,7 @@ void reset_all_window_stats(u64 window_start, unsigned int window_size) 
#endif

		reset_cpu_hmp_stats(cpu, 1);



		fixup_nr_big_small_task(cpu, 0);

		fixup_nr_big_task(cpu, 0);

	}



	if (sched_window_stats_policy != sysctl_sched_window_stats_policy) {
@@ -2525,23 +2525,23 @@ static int cpufreq_notifier_policy(struct notifier_block *nb, 
	 * A changed min_max_freq or max_possible_freq (possible during bootup)

	 * needs to trigger re-computation of load_scale_factor and capacity for

	 * all possible cpus (even those offline). It also needs to trigger

	 * re-computation of nr_big/small_task count on all online cpus.

	 * re-computation of nr_big_task count on all online cpus.

	 *

	 * A changed rq->max_freq otoh needs to trigger re-computation of

	 * load_scale_factor and capacity for just the cluster of cpus involved.

	 * Since small task definition depends on max_load_scale_factor, a

	 * changed load_scale_factor of one cluster could influence small_task

	 * changed load_scale_factor of one cluster could influence

	 * classification of tasks in another cluster. Hence a changed

	 * rq->max_freq will need to trigger re-computation of nr_big/small_task

	 * rq->max_freq will need to trigger re-computation of nr_big_task

	 * count on all online cpus.

	 *

	 * While it should be sufficient for nr_big/small_tasks to be

	 * While it should be sufficient for nr_big_tasks to be

	 * re-computed for only online cpus, we have inadequate context

	 * information here (in policy notifier) with regard to hotplug-safety

	 * context in which notification is issued. As a result, we can't use

	 * get_online_cpus() here, as it can lead to deadlock. Until cpufreq is

	 * fixed up to issue notification always in hotplug-safe context,

	 * re-compute nr_big/small_task for all possible cpus.

	 * re-compute nr_big_task for all possible cpus.

	 */



	if (orig_min_max_freq != min_max_freq ||
@@ -2555,7 +2555,7 @@ static int cpufreq_notifier_policy(struct notifier_block *nb, 
	 * big or small. Make this change "atomic" so that tasks are accounted

	 * properly due to changed load_scale_factor

	 */

	pre_big_small_task_count_change(cpu_possible_mask);

	pre_big_task_count_change(cpu_possible_mask);

	for_each_cpu(i, cpus) {

		struct rq *rq = cpu_rq(i);
@@ -2592,7 +2592,7 @@ static int cpufreq_notifier_policy(struct notifier_block *nb, 


	__update_min_max_capacity();

	check_for_up_down_migrate_update(policy->related_cpus);

	post_big_small_task_count_change(cpu_possible_mask);

	post_big_task_count_change(cpu_possible_mask);



	return 0;

}
@@ -8933,7 +8933,7 @@ void __init sched_init(void) 
		rq->capacity = 1024;

		rq->load_scale_factor = 1024;

		rq->window_start = 0;

		rq->hmp_stats.nr_small_tasks = rq->hmp_stats.nr_big_tasks = 0;

		rq->hmp_stats.nr_big_tasks = 0;

		rq->hmp_flags = 0;

		rq->mostly_idle_load = pct_to_real(20);

		rq->mostly_idle_nr_run = 3;
@@ -9808,11 +9808,11 @@ static int cpu_upmigrate_discourage_write_u64(struct cgroup_subsys_state *css, 
	 * classification.

	 */

	get_online_cpus();

	pre_big_small_task_count_change(cpu_online_mask);

	pre_big_task_count_change(cpu_online_mask);



	tg->upmigrate_discouraged = discourage;



	post_big_small_task_count_change(cpu_online_mask);

	post_big_task_count_change(cpu_online_mask);

	put_online_cpus();



	return 0;

kernel/sched/debug.c

+0 −4
Original line number Diff line number Diff line
@@ -237,8 +237,6 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) 
#ifdef CONFIG_SCHED_HMP

	SEQ_printf(m, "  .%-30s: %d\n", "nr_big_tasks",

			cfs_rq->hmp_stats.nr_big_tasks);

	SEQ_printf(m, "  .%-30s: %d\n", "nr_small_tasks",

			cfs_rq->hmp_stats.nr_small_tasks);

	SEQ_printf(m, "  .%-30s: %llu\n", "cumulative_runnable_avg",

			cfs_rq->hmp_stats.cumulative_runnable_avg);

#endif
@@ -329,7 +327,6 @@ do { \ 
#endif

#ifdef CONFIG_SCHED_HMP

	P(hmp_stats.nr_big_tasks);

	P(hmp_stats.nr_small_tasks);

	SEQ_printf(m, "  .%-30s: %llu\n", "hmp_stats.cumulative_runnable_avg",

			rq->hmp_stats.cumulative_runnable_avg);

#endif
@@ -413,7 +410,6 @@ static void sched_debug_header(struct seq_file *m) 
	P(sysctl_sched_child_runs_first);

	P(sysctl_sched_features);

#ifdef CONFIG_SCHED_HMP

	P(sched_small_task);

	P(sched_upmigrate);

	P(sched_downmigrate);

	P(sched_init_task_load_windows);

kernel/sched/fair.c

+43 −206
Original line number Diff line number Diff line
@@ -2493,13 +2493,6 @@ unsigned int __read_mostly sysctl_sched_powerband_limit_pct; 
unsigned int __read_mostly sched_spill_load;

unsigned int __read_mostly sysctl_sched_spill_load_pct = 100;



/*

 * Tasks whose bandwidth consumption on a cpu is less than

 * sched_small_task are considered as small tasks.

 */

unsigned int __read_mostly sched_small_task;

unsigned int __read_mostly sysctl_sched_small_task_pct = 10;



/*

 * Tasks with demand >= sched_heavy_task will have their

 * window-based demand added to the previous window's CPU
@@ -2593,9 +2586,6 @@ void set_hmp_defaults(void) 
	sched_spill_load =

		pct_to_real(sysctl_sched_spill_load_pct);



	sched_small_task =

		pct_to_real(sysctl_sched_small_task_pct);



	update_up_down_migrate();



#ifdef CONFIG_SCHED_FREQ_INPUT
@@ -2732,15 +2722,6 @@ static inline int is_big_task(struct task_struct *p) 
	return load > sched_upmigrate;

}



/* Is a task "small" on the minimum capacity CPU */

static inline int is_small_task(struct task_struct *p)

{

	u64 load = task_load(p);

	load *= (u64)max_load_scale_factor;

	load /= 1024;

	return load < sched_small_task;

}



static inline u64 cpu_load(int cpu)

{

	struct rq *rq = cpu_rq(cpu);
@@ -3013,105 +2994,6 @@ static unsigned int power_cost(u64 total_load, int cpu) 
	return power_cost_at_freq(cpu, task_freq);

}



static int best_small_task_cpu(struct task_struct *p, int sync)

{

	int best_busy_cpu = -1, fallback_cpu = -1;

	int min_cstate_cpu = -1;

	int min_cstate = INT_MAX;

	int cpu_cost, min_cost = INT_MAX;

	int i = task_cpu(p), prev_cpu;

	int hmp_capable;

	u64 tload, cpu_load, min_load = ULLONG_MAX;

	cpumask_t temp;

	cpumask_t search_cpu;

	cpumask_t fb_search_cpu = CPU_MASK_NONE;

	struct rq *rq;



	cpumask_and(&temp, &mpc_mask, cpu_possible_mask);

	hmp_capable = !cpumask_full(&temp);



	cpumask_and(&search_cpu, tsk_cpus_allowed(p), cpu_online_mask);

	if (unlikely(!cpumask_test_cpu(i, &search_cpu)))

		i = cpumask_first(&search_cpu);



	do {

		rq = cpu_rq(i);



		cpumask_clear_cpu(i, &search_cpu);



		trace_sched_cpu_load(rq, idle_cpu(i),

		 mostly_idle_cpu_sync(i, cpu_load_sync(i, sync), sync),

		 sched_irqload(i), power_cost(scale_load_to_cpu(task_load(p),

				i) + cpu_load_sync(i, sync), i), cpu_temp(i));



		if (rq->max_possible_capacity == max_possible_capacity &&

		    hmp_capable) {

			cpumask_and(&fb_search_cpu, &search_cpu,

				    &rq->freq_domain_cpumask);

			cpumask_andnot(&search_cpu, &search_cpu,

				       &rq->freq_domain_cpumask);

			continue;

		}



		if (sched_cpu_high_irqload(i))

			continue;



		if (idle_cpu(i) && rq->cstate) {

			if (rq->cstate < min_cstate) {

				min_cstate_cpu = i;

				min_cstate = rq->cstate;

			}

			continue;

		}



		cpu_load = cpu_load_sync(i, sync);

		if (mostly_idle_cpu_sync(i, cpu_load, sync))

			return i;

	} while ((i = cpumask_first(&search_cpu)) < nr_cpu_ids);



	if (min_cstate_cpu != -1)

		return min_cstate_cpu;



	cpumask_and(&search_cpu, tsk_cpus_allowed(p), cpu_online_mask);

	cpumask_andnot(&search_cpu, &search_cpu, &fb_search_cpu);

	for_each_cpu(i, &search_cpu) {

		rq = cpu_rq(i);

		prev_cpu = (i == task_cpu(p));



		if (sched_cpu_high_irqload(i))

			continue;



		tload = scale_load_to_cpu(task_load(p), i);

		cpu_load = cpu_load_sync(i, sync);

		if (!spill_threshold_crossed(tload, cpu_load, rq)) {

			if (cpu_load < min_load ||

			    (prev_cpu && cpu_load == min_load)) {

				min_load = cpu_load;

				best_busy_cpu = i;

			}

		}

	}



	if (best_busy_cpu != -1)

		return best_busy_cpu;



	for_each_cpu(i, &fb_search_cpu) {

		rq = cpu_rq(i);

		prev_cpu = (i == task_cpu(p));



		tload = scale_load_to_cpu(task_load(p), i);

		cpu_load = cpu_load_sync(i, sync);

		cpu_cost = power_cost(tload + cpu_load, i);

		if (cpu_cost < min_cost ||

		   (prev_cpu && cpu_cost == min_cost)) {

			fallback_cpu = i;

			min_cost = cpu_cost;

		}

	}



	return fallback_cpu;

}



#define UP_MIGRATION		1

#define DOWN_MIGRATION		2

#define IRQLOAD_MIGRATION	4
@@ -3225,7 +3107,6 @@ static int select_best_cpu(struct task_struct *p, int target, int reason, 
	int min_idle_cost = INT_MAX, min_busy_cost = INT_MAX;

	u64 tload, cpu_load;

	u64 min_load = ULLONG_MAX, min_fallback_load = ULLONG_MAX;

	int small_task = is_small_task(p);

	int boost = sched_boost();

	int cstate, min_cstate = INT_MAX;

	int prefer_idle = -1;
@@ -3241,7 +3122,6 @@ static int select_best_cpu(struct task_struct *p, int target, int reason, 
	if (wake_to_idle(p)) {

		prefer_idle = 1;

		prefer_idle_override = 1;

		small_task = 0;

		/*

		 * If wake to idle and sync are both set prefer wake to idle

		 * since sync is a weak hint that might not always be correct.
@@ -3249,12 +3129,6 @@ static int select_best_cpu(struct task_struct *p, int target, int reason, 
		sync = 0;

	}



	if (small_task && !boost) {

		best_cpu = best_small_task_cpu(p, sync);

		prefer_idle = 0;	/* For sched_task_load tracepoint */

		goto done;

	}



	trq = task_rq(p);

	cpumask_and(&search_cpus, tsk_cpus_allowed(p), cpu_online_mask);

	for_each_cpu(i, &search_cpus) {
@@ -3399,7 +3273,7 @@ static int select_best_cpu(struct task_struct *p, int target, int reason, 
	if (min_cstate_cpu >= 0 && (prefer_idle > 0 || best_cpu < 0 ||

			!mostly_idle_cpu_sync(best_cpu, min_load, sync)))

		best_cpu = min_cstate_cpu;

done:



	if (best_cpu < 0) {

		if (unlikely(fallback_idle_cpu < 0))

			/*
@@ -3420,41 +3294,37 @@ done: 
	 * tracepoint towards end to capture prefer_idle flag used for this

	 * instance of wakeup.

	 */

	trace_sched_task_load(p, small_task, boost, reason, sync, prefer_idle);

	trace_sched_task_load(p, boost, reason, sync, prefer_idle);



	return best_cpu;

}



static void

inc_nr_big_small_task(struct hmp_sched_stats *stats, struct task_struct *p)

inc_nr_big_task(struct hmp_sched_stats *stats, struct task_struct *p)

{

	if (!sched_enable_hmp || sched_disable_window_stats)

		return;



	if (is_big_task(p))

		stats->nr_big_tasks++;

	else if (is_small_task(p))

		stats->nr_small_tasks++;

}



static void

dec_nr_big_small_task(struct hmp_sched_stats *stats, struct task_struct *p)

dec_nr_big_task(struct hmp_sched_stats *stats, struct task_struct *p)

{

	if (!sched_enable_hmp || sched_disable_window_stats)

		return;



	if (is_big_task(p))

		stats->nr_big_tasks--;

	else if (is_small_task(p))

		stats->nr_small_tasks--;



	BUG_ON(stats->nr_big_tasks < 0 || stats->nr_small_tasks < 0);

	BUG_ON(stats->nr_big_tasks < 0);

}



static void

inc_rq_hmp_stats(struct rq *rq, struct task_struct *p, int change_cra)

{

	inc_nr_big_small_task(&rq->hmp_stats, p);

	inc_nr_big_task(&rq->hmp_stats, p);

	if (change_cra)

		inc_cumulative_runnable_avg(&rq->hmp_stats, p);

}
@@ -3462,14 +3332,14 @@ inc_rq_hmp_stats(struct rq *rq, struct task_struct *p, int change_cra) 
static void

dec_rq_hmp_stats(struct rq *rq, struct task_struct *p, int change_cra)

{

	dec_nr_big_small_task(&rq->hmp_stats, p);

	dec_nr_big_task(&rq->hmp_stats, p);

	if (change_cra)

		dec_cumulative_runnable_avg(&rq->hmp_stats, p);

}



static void reset_hmp_stats(struct hmp_sched_stats *stats, int reset_cra)

{

	stats->nr_big_tasks = stats->nr_small_tasks = 0;

	stats->nr_big_tasks = 0;

	if (reset_cra)

		stats->cumulative_runnable_avg = 0;

}
@@ -3512,29 +3382,23 @@ static inline void reset_cfs_rq_hmp_stats(int cpu, int reset_cra) { } 
 * Return total number of tasks "eligible" to run on highest capacity cpu

 *

 * This is simply nr_big_tasks for cpus which are not of max_capacity and

 * (nr_running - nr_small_tasks) for cpus of max_capacity

 * nr_running for cpus of max_capacity

 */

unsigned int nr_eligible_big_tasks(int cpu)

{

	struct rq *rq = cpu_rq(cpu);

	int nr_big = rq->hmp_stats.nr_big_tasks;

	int nr = rq->nr_running;

	int nr_small = rq->hmp_stats.nr_small_tasks;



	if (rq->max_possible_capacity != max_possible_capacity)

		return nr_big;



	/* Consider all (except small) tasks on max_capacity cpu as big tasks */

	nr_big = nr - nr_small;

	if (nr_big < 0)

		nr_big = 0;



	return nr_big;

	return nr;

}



/*

 * reset_cpu_hmp_stats - reset HMP stats for a cpu

 *	nr_big_tasks, nr_small_tasks

 *	nr_big_tasks

 *	cumulative_runnable_avg (iff reset_cra is true)

 */

void reset_cpu_hmp_stats(int cpu, int reset_cra)
@@ -3561,7 +3425,7 @@ _inc_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p, int change_cra) 


	/*

	 * Although below check is not strictly required  (as

	 * inc/dec_nr_big_small_task and inc/dec_cumulative_runnable_avg called

	 * inc/dec_nr_big_task and inc/dec_cumulative_runnable_avg called

	 * from inc_cfs_rq_hmp_stats() have similar checks), we gain a bit on

	 * efficiency by short-circuiting for_each_sched_entity() loop when

	 * !sched_enable_hmp || sched_disable_window_stats
@@ -3624,10 +3488,10 @@ static void fixup_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p, 
	for_each_sched_entity(se) {

		cfs_rq = cfs_rq_of(se);



		dec_nr_big_small_task(&cfs_rq->hmp_stats, p);

		dec_nr_big_task(&cfs_rq->hmp_stats, p);

		fixup_cumulative_runnable_avg(&cfs_rq->hmp_stats, p,

					      new_task_load);

		inc_nr_big_small_task(&cfs_rq->hmp_stats, p);

		inc_nr_big_task(&cfs_rq->hmp_stats, p);

		if (cfs_rq_throttled(cfs_rq))

			break;

		/*
@@ -3639,9 +3503,9 @@ static void fixup_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p, 


	/* Fix up rq->hmp_stats only if we didn't find any throttled cfs_rq */

	if (!se) {

		dec_nr_big_small_task(&rq->hmp_stats, p);

		dec_nr_big_task(&rq->hmp_stats, p);

		fixup_cumulative_runnable_avg(&rq->hmp_stats, p, new_task_load);

		inc_nr_big_small_task(&rq->hmp_stats, p);

		inc_nr_big_task(&rq->hmp_stats, p);

	}

}
@@ -3652,14 +3516,14 @@ static int task_will_be_throttled(struct task_struct *p); 
static void

inc_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p)

{

	inc_nr_big_small_task(&rq->hmp_stats, p);

	inc_nr_big_task(&rq->hmp_stats, p);

	inc_cumulative_runnable_avg(&rq->hmp_stats, p);

}



static void

dec_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p)

{

	dec_nr_big_small_task(&rq->hmp_stats, p);

	dec_nr_big_task(&rq->hmp_stats, p);

	dec_cumulative_runnable_avg(&rq->hmp_stats, p);

}
@@ -3667,9 +3531,9 @@ static void 
fixup_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p,

			   u32 new_task_load)

{

	dec_nr_big_small_task(&rq->hmp_stats, p);

	dec_nr_big_task(&rq->hmp_stats, p);

	fixup_cumulative_runnable_avg(&rq->hmp_stats, p, new_task_load);

	inc_nr_big_small_task(&rq->hmp_stats, p);

	inc_nr_big_task(&rq->hmp_stats, p);

}



static inline int task_will_be_throttled(struct task_struct *p)
@@ -3680,21 +3544,20 @@ static inline int task_will_be_throttled(struct task_struct *p) 
static void

_inc_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p, int change_cra)

{

	inc_nr_big_small_task(&rq->hmp_stats, p);

	inc_nr_big_task(&rq->hmp_stats, p);

}



#endif	/* CONFIG_CFS_BANDWIDTH */



/*

 * Walk runqueue of cpu and re-initialize 'nr_big_tasks' and 'nr_small_tasks'

 * counters.

 * Walk runqueue of cpu and re-initialize 'nr_big_tasks' counters.

 */

void fixup_nr_big_small_task(int cpu, int reset_stats)

void fixup_nr_big_task(int cpu, int reset_stats)

{

	struct rq *rq = cpu_rq(cpu);

	struct task_struct *p;



	/* fixup_nr_big_small_task() is called from two functions. In one of

	/* fixup_nr_big_task() is called from two functions. In one of

	 * them stats are already reset, don't waste time resetting them again

	 */

	if (reset_stats) {
@@ -3707,7 +3570,7 @@ void fixup_nr_big_small_task(int cpu, int reset_stats) 
}



/* Disable interrupts and grab runqueue lock of all cpus listed in @cpus */

void pre_big_small_task_count_change(const struct cpumask *cpus)

void pre_big_task_count_change(const struct cpumask *cpus)

{

	int i;
@@ -3718,16 +3581,15 @@ void pre_big_small_task_count_change(const struct cpumask *cpus) 
}



/*

 * Reinitialize 'nr_big_tasks' and 'nr_small_tasks' counters on all affected

 * cpus

 * Reinitialize 'nr_big_tasks' counters on all affected cpus

 */

void post_big_small_task_count_change(const struct cpumask *cpus)

void post_big_task_count_change(const struct cpumask *cpus)

{

	int i;



	/* Assumes local_irq_disable() keeps online cpumap stable */

	for_each_cpu(i, cpus)

		fixup_nr_big_small_task(i, 1);

		fixup_nr_big_task(i, 1);



	for_each_cpu(i, cpus)

		raw_spin_unlock(&cpu_rq(i)->lock);
@@ -3852,24 +3714,22 @@ int sched_hmp_proc_update_handler(struct ctl_table *table, int write, 
	}



	/*

	 * Big/Small task tunable change will need to re-classify tasks on

	 * runqueue as big and small and set their counters appropriately.

	 * Big task tunable change will need to re-classify tasks on

	 * runqueue as big and set their counters appropriately.

	 * sysctl interface affects secondary variables (*_pct), which is then

	 * "atomically" carried over to the primary variables. Atomic change

	 * includes taking runqueue lock of all online cpus and re-initiatizing

	 * their big/small counter values based on changed criteria.

	 * their big counter values based on changed criteria.

	 */

	if ((data == &sysctl_sched_upmigrate_pct ||

	     data == &sysctl_sched_small_task_pct || update_min_nice)) {

	if ((data == &sysctl_sched_upmigrate_pct || update_min_nice)) {

		get_online_cpus();

		pre_big_small_task_count_change(cpu_online_mask);

		pre_big_task_count_change(cpu_online_mask);

	}



	set_hmp_defaults();



	if ((data == &sysctl_sched_upmigrate_pct ||

	     data == &sysctl_sched_small_task_pct || update_min_nice)) {

		post_big_small_task_count_change(cpu_online_mask);

	if ((data == &sysctl_sched_upmigrate_pct || update_min_nice)) {

		post_big_task_count_change(cpu_online_mask);

		put_online_cpus();

	}
@@ -3957,9 +3817,6 @@ static inline int migration_needed(struct rq *rq, struct task_struct *p) 
		return 0;

	}



	if (is_small_task(p))

		return 0;



	if (sched_cpu_high_irqload(cpu_of(rq)))

		return IRQLOAD_MIGRATION;
@@ -4080,11 +3937,6 @@ static inline int sched_boost(void) 
	return 0;

}



static inline int is_small_task(struct task_struct *p)

{

	return 0;

}



static inline int is_big_task(struct task_struct *p)

{

	return 0;
@@ -4661,14 +4513,13 @@ static inline void decay_scaled_stat(struct sched_avg *sa, u64 periods) 
static void init_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq)

{

	cfs_rq->hmp_stats.nr_big_tasks = 0;

	cfs_rq->hmp_stats.nr_small_tasks = 0;

	cfs_rq->hmp_stats.cumulative_runnable_avg = 0;

}



static void inc_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq,

		 struct task_struct *p, int change_cra)

{

	inc_nr_big_small_task(&cfs_rq->hmp_stats, p);

	inc_nr_big_task(&cfs_rq->hmp_stats, p);

	if (change_cra)

		inc_cumulative_runnable_avg(&cfs_rq->hmp_stats, p);

}
@@ -4676,7 +4527,7 @@ static void inc_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq, 
static void dec_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq,

		 struct task_struct *p, int change_cra)

{

	dec_nr_big_small_task(&cfs_rq->hmp_stats, p);

	dec_nr_big_task(&cfs_rq->hmp_stats, p);

	if (change_cra)

		dec_cumulative_runnable_avg(&cfs_rq->hmp_stats, p);

}
@@ -4685,7 +4536,6 @@ static void inc_throttled_cfs_rq_hmp_stats(struct hmp_sched_stats *stats, 
			 struct cfs_rq *cfs_rq)

{

	stats->nr_big_tasks += cfs_rq->hmp_stats.nr_big_tasks;

	stats->nr_small_tasks += cfs_rq->hmp_stats.nr_small_tasks;

	stats->cumulative_runnable_avg +=

				cfs_rq->hmp_stats.cumulative_runnable_avg;

}
@@ -4694,11 +4544,10 @@ static void dec_throttled_cfs_rq_hmp_stats(struct hmp_sched_stats *stats, 
				 struct cfs_rq *cfs_rq)

{

	stats->nr_big_tasks -= cfs_rq->hmp_stats.nr_big_tasks;

	stats->nr_small_tasks -= cfs_rq->hmp_stats.nr_small_tasks;

	stats->cumulative_runnable_avg -=

				cfs_rq->hmp_stats.cumulative_runnable_avg;



	BUG_ON(stats->nr_big_tasks < 0 || stats->nr_small_tasks < 0 ||

	BUG_ON(stats->nr_big_tasks < 0 ||

		(s64)stats->cumulative_runnable_avg < 0);

}
@@ -7224,7 +7073,6 @@ enum fbq_type { regular, remote, all }; 
#define LBF_NEED_BREAK	0x02

#define LBF_DST_PINNED  0x04

#define LBF_SOME_PINNED	0x08

#define LBF_IGNORE_SMALL_TASKS 0x10

#define LBF_SCHED_BOOST_ACTIVE_BALANCE 0x40

#define LBF_BIG_TASK_ACTIVE_BALANCE 0x80

#define LBF_HMP_ACTIVE_BALANCE (LBF_SCHED_BOOST_ACTIVE_BALANCE | \
@@ -7442,9 +7290,6 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) 
			!is_big_task(p))

		return 0;



	if (env->flags & LBF_IGNORE_SMALL_TASKS && is_small_task(p))

		return 0;



	twf = task_will_fit(p, env->dst_cpu);



	/*
@@ -7562,9 +7407,7 @@ static int detach_tasks(struct lb_env *env) 
	if (env->imbalance <= 0)

		return 0;



	if (capacity(env->dst_rq) > capacity(env->src_rq))

		env->flags |= LBF_IGNORE_SMALL_TASKS;

	else if (capacity(env->dst_rq) < capacity(env->src_rq) &&

	if (capacity(env->dst_rq) < capacity(env->src_rq) &&

							!sched_boost())

		env->flags |= LBF_IGNORE_BIG_TASKS;
@@ -7624,10 +7467,9 @@ next: 
		list_move_tail(&p->se.group_node, tasks);

	}



	if (env->flags & (LBF_IGNORE_SMALL_TASKS | LBF_IGNORE_BIG_TASKS)

							     && !detached) {

	if (env->flags & LBF_IGNORE_BIG_TASKS && !detached) {

		tasks = &env->src_rq->cfs_tasks;

		env->flags &= ~(LBF_IGNORE_SMALL_TASKS | LBF_IGNORE_BIG_TASKS);

		env->flags &= ~LBF_IGNORE_BIG_TASKS;

		env->loop = orig_loop;

		goto redo;

	}
@@ -7823,7 +7665,7 @@ struct sg_lb_stats { 
	unsigned long group_capacity;

	unsigned int sum_nr_running; /* Nr tasks running in the group */

#ifdef CONFIG_SCHED_HMP

	unsigned long sum_nr_big_tasks, sum_nr_small_tasks;

	unsigned long sum_nr_big_tasks;

	u64 group_cpu_load; /* Scaled load of all CPUs of the group */

#endif

	unsigned int group_capacity_factor;
@@ -8231,7 +8073,6 @@ static inline void update_sg_lb_stats(struct lb_env *env, 


#ifdef CONFIG_SCHED_HMP

		sgs->sum_nr_big_tasks += rq->hmp_stats.nr_big_tasks;

		sgs->sum_nr_small_tasks += rq->hmp_stats.nr_small_tasks;

		sgs->group_cpu_load += cpu_load(i);

#endif
@@ -9829,11 +9670,7 @@ static inline int _nohz_kick_needed_hmp(struct rq *rq, int cpu, int *type) 
		 && rq->max_freq > rq->mostly_idle_freq)

			return 0;



	if (rq->nr_running >= 2 &&

		(rq->nr_running - rq->hmp_stats.nr_small_tasks >= 2 ||

		rq->nr_running > rq->mostly_idle_nr_run ||

		cpu_load(cpu) > rq->mostly_idle_load)) {



	if (rq->nr_running >= 2) {

		if (rq->capacity == max_capacity)

			return 1;