Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 2cb0176a authored by Morten Rasmussen's avatar Morten Rasmussen Committed by Satya Durga Srinivasu Prabhala
Browse files

FROMLIST: sched/fair: Add group_misfit_task load-balance type 



To maximize throughput in systems with asymmetric cpu capacities (e.g.
ARM big.LITTLE) load-balancing has to consider task and cpu utilization
as well as per-cpu compute capacity when load-balancing in addition to
the current average load based load-balancing policy. Tasks with high
utilization that are scheduled on a lower capacity cpu need to be
identified and migrated to a higher capacity cpu if possible to maximize
throughput.

To implement this additional policy an additional group_type
(load-balance scenario) is added: group_misfit_task. This represents
scenarios where a sched_group has one or more tasks that are not
suitable for its per-cpu capacity. group_misfit_task is only considered
if the system is not overloaded or imbalanced (group_imbalanced or
group_overloaded).

Identifying misfit tasks requires the rq lock to be held. To avoid
taking remote rq locks to examine source sched_groups for misfit tasks,
each cpu is responsible for tracking misfit tasks themselves and update
the rq->misfit_task flag. This means checking task utilization when
tasks are scheduled and on sched_tick.

cc: Ingo Molnar <mingo@redhat.com>
cc: Peter Zijlstra <peterz@infradead.org>

Change-Id: I71bd3a77c7088a102ba183df6ece7943aa7eb0c2
Signed-off-by: default avatarMorten Rasmussen <morten.rasmussen@arm.com>
[From https://lore.kernel.org/lkml/1530699470-29808-3-git-send-email-morten.rasmussen@arm.com/

]
[backported because some parts are already present in android]
Signed-off-by: default avatarIoan Budea <ioan.budea@arm.com>
Signed-off-by: default avatarValentin Schneider <valentin.schneider@arm.com>
Signed-off-by: default avatarChris Redpath <chris.redpath@arm.com>
Git-commit: c662b0e5
Git-repo: https://android.googlesource.com/kernel/common/


[pujag@codeaurora.org:
1. Trivial merge conflict resolution in kernel/sched/fair.c
2. Replace task_fits_capacity() with task_fits_max()]
Signed-off-by: default avatarPuja Gupta <pujag@codeaurora.org>
parent 2929d0b0

kernel/sched/fair.c

+41 −38
Original line number Diff line number Diff line
@@ -780,6 +780,7 @@ static u64 sched_vslice(struct cfs_rq *cfs_rq, struct sched_entity *se) 


static int select_idle_sibling(struct task_struct *p, int prev_cpu, int cpu);

static unsigned long task_h_load(struct task_struct *p);

static unsigned long capacity_of(int cpu);



/* Give new sched_entity start runnable values to heavy its load in infant time */

void init_entity_runnable_average(struct sched_entity *se)
@@ -3643,6 +3644,27 @@ static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq) 


static int idle_balance(struct rq *this_rq, struct rq_flags *rf);



static inline bool task_fits_capacity(struct task_struct *p, long capacity,

								int cpu);



static inline void update_misfit_status(struct task_struct *p, struct rq *rq)

{

	if (!static_branch_unlikely(&sched_asym_cpucapacity))

		return;



	if (!p) {

		rq->misfit_task_load = 0;

		return;

	}



	if (task_fits_max(p, cpu_of(rq))) {

		rq->misfit_task_load = 0;

		return;

	}



	rq->misfit_task_load = task_h_load(p);

}



#else /* CONFIG_SMP */



static inline int
@@ -3680,6 +3702,8 @@ static inline int idle_balance(struct rq *rq, struct rq_flags *rf) 
	return 0;

}



static inline void update_misfit_status(struct task_struct *p, struct rq *rq) {}



#endif /* CONFIG_SMP */



static void check_spread(struct cfs_rq *cfs_rq, struct sched_entity *se)
@@ -8253,29 +8277,6 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_ 
		set_last_buddy(se);

}



static inline void update_misfit_task(struct rq *rq, struct task_struct *p)

{

#ifdef CONFIG_SMP

	rq->misfit_task = !task_fits_max(p, rq->cpu);

#endif

}



static inline void clear_rq_misfit(struct rq *rq)

{

#ifdef CONFIG_SMP

	rq->misfit_task = 0;

#endif

}



static inline unsigned int rq_has_misfit(struct rq *rq)

{

#ifdef CONFIG_SMP

	return rq->misfit_task;

#else

	return 0;

#endif

}



static struct task_struct *

pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)

{
@@ -8366,7 +8367,7 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf 
	if (hrtick_enabled(rq))

		hrtick_start_fair(rq, p);



	update_misfit_task(rq, p);

	update_misfit_status(p, rq);



	return p;

simple:
@@ -8385,12 +8386,12 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf 
	if (hrtick_enabled(rq))

		hrtick_start_fair(rq, p);



	update_misfit_task(rq, p);

	update_misfit_status(p, rq);



	return p;



idle:

	clear_rq_misfit(rq);

	update_misfit_status(NULL, rq);

	new_tasks = idle_balance(rq, rf);



	/*
@@ -9191,7 +9192,8 @@ struct sg_lb_stats { 
	unsigned int group_weight;

	enum group_type group_type;

	int group_no_capacity;

	int group_misfit_task; /* A cpu has a task too big for its capacity */

	/* A cpu has a task too big for its capacity */

	unsigned long group_misfit_task_load;

#ifdef CONFIG_NUMA_BALANCING

	unsigned int nr_numa_running;

	unsigned int nr_preferred_running;
@@ -9520,7 +9522,7 @@ group_type group_classify(struct sched_group *group, 
	if (sg_imbalanced(group))

		return group_imbalanced;



	if (sgs->group_misfit_task)

	if (sgs->group_misfit_task_load)

		return group_misfit_task;



	return group_other;
@@ -9578,13 +9580,13 @@ static inline void update_sg_lb_stats(struct lb_env *env, 
			sgs->idle_cpus++;



		if (env->sd->flags & SD_ASYM_CPUCAPACITY &&

		    !sgs->group_misfit_task && rq_has_misfit(rq))

			sgs->group_misfit_task = capacity_of(i);

		    sgs->group_misfit_task_load < rq->misfit_task_load)

			sgs->group_misfit_task_load = rq->misfit_task_load;



		if (cpu_overutilized(i)) {

			*overutilized = true;



			if (rq_has_misfit(rq))

			if (rq->misfit_task_load)

				*misfit_task = true;

		}

	}
@@ -10094,7 +10096,7 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s 
	/* Boost imbalance to allow misfit task to be balanced. */

	if (busiest->group_type == group_misfit_task) {

		env->imbalance = max_t(long, env->imbalance,

				       busiest->group_misfit_task);

				       busiest->group_misfit_task_load);

	}



	/*
@@ -10286,7 +10288,8 @@ static struct rq *find_busiest_queue(struct lb_env *env, 
		 * For ASYM_CPUCAPACITY domains with misfit tasks we ignore

		 * load.

		 */

		if (env->src_grp_type == group_misfit_task && rq_has_misfit(rq))

		if (env->src_grp_type == group_misfit_task &&

		    rq->misfit_task_load)

			return rq;



		capacity = capacity_of(i);
@@ -10736,7 +10739,7 @@ get_sd_balance_interval(struct sched_domain *sd, int cpu_busy) 
	if (energy_aware() && sd_overutilized(sd)) {

		/* we know the root is overutilized, let's check for a misfit task */

		for_each_cpu(cpu, sched_domain_span(sd)) {

			if (rq_has_misfit(cpu_rq(cpu)))

			if (cpu_rq(cpu)->misfit_task_load)

				return 1;

		}

	}
@@ -11585,10 +11588,10 @@ static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued) 
	if (static_branch_unlikely(&sched_numa_balancing))

		task_tick_numa(rq, curr);



	update_misfit_task(rq, curr);

	update_misfit_status(curr, rq);



#ifdef CONFIG_SCHED_WALT

	misfit = rq->misfit_task;

	misfit = rq->misfit_task_load;



	if (old_misfit != misfit) {

		walt_fixup_nr_big_tasks(rq, curr, 1, misfit);
@@ -12395,7 +12398,7 @@ static void walt_check_for_rotation(struct rq *src_rq) 
		if (is_reserved(i))

			continue;



		if (!rq->misfit_task || rq->curr->sched_class !=

		if (!rq->misfit_task_load || rq->curr->sched_class !=

						&fair_sched_class)

			continue;
@@ -12475,7 +12478,7 @@ void check_for_migration(struct rq *rq, struct task_struct *p) 
	int prev_cpu = task_cpu(p);

	struct sched_domain *sd = NULL;



	if (rq->misfit_task) {

	if (rq->misfit_task_load) {

		if (rq->curr->state != TASK_RUNNING ||

		    rq->curr->nr_cpus_allowed == 1)

			return;

kernel/sched/sched.h

+2 −1
Original line number Diff line number Diff line
@@ -825,7 +825,8 @@ struct rq { 


	unsigned char idle_balance;



	unsigned int misfit_task;

	unsigned long misfit_task_load;



	/* For active balancing */

	int active_balance;

	int push_cpu;