Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 43a7d245 authored by qctecmdr Service's avatar qctecmdr Service Committed by Gerrit - the friendly Code Review server
Browse files

Merge "FROMLIST: sched/fair: Add group_misfit_task load-balance type"

parents ebfa6a4f 2cb0176a

kernel/sched/fair.c

+44 −38
Original line number Diff line number Diff line
@@ -780,6 +780,7 @@ static u64 sched_vslice(struct cfs_rq *cfs_rq, struct sched_entity *se) 


static int select_idle_sibling(struct task_struct *p, int prev_cpu, int cpu);

static unsigned long task_h_load(struct task_struct *p);

static unsigned long capacity_of(int cpu);



/* Give new sched_entity start runnable values to heavy its load in infant time */

void init_entity_runnable_average(struct sched_entity *se)
@@ -3643,6 +3644,27 @@ static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq) 


static int idle_balance(struct rq *this_rq, struct rq_flags *rf);



static inline bool task_fits_capacity(struct task_struct *p, long capacity,

								int cpu);



static inline void update_misfit_status(struct task_struct *p, struct rq *rq)

{

	if (!static_branch_unlikely(&sched_asym_cpucapacity))

		return;



	if (!p) {

		rq->misfit_task_load = 0;

		return;

	}



	if (task_fits_max(p, cpu_of(rq))) {

		rq->misfit_task_load = 0;

		return;

	}



	rq->misfit_task_load = task_h_load(p);

}



#else /* CONFIG_SMP */



static inline int
@@ -3680,6 +3702,8 @@ static inline int idle_balance(struct rq *rq, struct rq_flags *rf) 
	return 0;

}



static inline void update_misfit_status(struct task_struct *p, struct rq *rq) {}



#endif /* CONFIG_SMP */



static void check_spread(struct cfs_rq *cfs_rq, struct sched_entity *se)
@@ -7523,6 +7547,9 @@ static int wake_cap(struct task_struct *p, int cpu, int prev_cpu) 
{

	long min_cap, max_cap;



	if (!static_branch_unlikely(&sched_asym_cpucapacity))

		return 0;



	min_cap = min(capacity_orig_of(prev_cpu), capacity_orig_of(cpu));

	max_cap = cpu_rq(cpu)->rd->max_cpu_capacity;
@@ -8250,29 +8277,6 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_ 
		set_last_buddy(se);

}



static inline void update_misfit_task(struct rq *rq, struct task_struct *p)

{

#ifdef CONFIG_SMP

	rq->misfit_task = !task_fits_max(p, rq->cpu);

#endif

}



static inline void clear_rq_misfit(struct rq *rq)

{

#ifdef CONFIG_SMP

	rq->misfit_task = 0;

#endif

}



static inline unsigned int rq_has_misfit(struct rq *rq)

{

#ifdef CONFIG_SMP

	return rq->misfit_task;

#else

	return 0;

#endif

}



static struct task_struct *

pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)

{
@@ -8363,7 +8367,7 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf 
	if (hrtick_enabled(rq))

		hrtick_start_fair(rq, p);



	update_misfit_task(rq, p);

	update_misfit_status(p, rq);



	return p;

simple:
@@ -8382,12 +8386,12 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf 
	if (hrtick_enabled(rq))

		hrtick_start_fair(rq, p);



	update_misfit_task(rq, p);

	update_misfit_status(p, rq);



	return p;



idle:

	clear_rq_misfit(rq);

	update_misfit_status(NULL, rq);

	new_tasks = idle_balance(rq, rf);



	/*
@@ -9188,7 +9192,8 @@ struct sg_lb_stats { 
	unsigned int group_weight;

	enum group_type group_type;

	int group_no_capacity;

	int group_misfit_task; /* A cpu has a task too big for its capacity */

	/* A cpu has a task too big for its capacity */

	unsigned long group_misfit_task_load;

#ifdef CONFIG_NUMA_BALANCING

	unsigned int nr_numa_running;

	unsigned int nr_preferred_running;
@@ -9517,7 +9522,7 @@ group_type group_classify(struct sched_group *group, 
	if (sg_imbalanced(group))

		return group_imbalanced;



	if (sgs->group_misfit_task)

	if (sgs->group_misfit_task_load)

		return group_misfit_task;



	return group_other;
@@ -9575,13 +9580,13 @@ static inline void update_sg_lb_stats(struct lb_env *env, 
			sgs->idle_cpus++;



		if (env->sd->flags & SD_ASYM_CPUCAPACITY &&

		    !sgs->group_misfit_task && rq_has_misfit(rq))

			sgs->group_misfit_task = capacity_of(i);

		    sgs->group_misfit_task_load < rq->misfit_task_load)

			sgs->group_misfit_task_load = rq->misfit_task_load;



		if (cpu_overutilized(i)) {

			*overutilized = true;



			if (rq_has_misfit(rq))

			if (rq->misfit_task_load)

				*misfit_task = true;

		}

	}
@@ -10091,7 +10096,7 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s 
	/* Boost imbalance to allow misfit task to be balanced. */

	if (busiest->group_type == group_misfit_task) {

		env->imbalance = max_t(long, env->imbalance,

				       busiest->group_misfit_task);

				       busiest->group_misfit_task_load);

	}



	/*
@@ -10283,7 +10288,8 @@ static struct rq *find_busiest_queue(struct lb_env *env, 
		 * For ASYM_CPUCAPACITY domains with misfit tasks we ignore

		 * load.

		 */

		if (env->src_grp_type == group_misfit_task && rq_has_misfit(rq))

		if (env->src_grp_type == group_misfit_task &&

		    rq->misfit_task_load)

			return rq;



		capacity = capacity_of(i);
@@ -10733,7 +10739,7 @@ get_sd_balance_interval(struct sched_domain *sd, int cpu_busy) 
	if (energy_aware() && sd_overutilized(sd)) {

		/* we know the root is overutilized, let's check for a misfit task */

		for_each_cpu(cpu, sched_domain_span(sd)) {

			if (rq_has_misfit(cpu_rq(cpu)))

			if (cpu_rq(cpu)->misfit_task_load)

				return 1;

		}

	}
@@ -11582,10 +11588,10 @@ static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued) 
	if (static_branch_unlikely(&sched_numa_balancing))

		task_tick_numa(rq, curr);



	update_misfit_task(rq, curr);

	update_misfit_status(curr, rq);



#ifdef CONFIG_SCHED_WALT

	misfit = rq->misfit_task;

	misfit = rq->misfit_task_load;



	if (old_misfit != misfit) {

		walt_fixup_nr_big_tasks(rq, curr, 1, misfit);
@@ -12392,7 +12398,7 @@ static void walt_check_for_rotation(struct rq *src_rq) 
		if (is_reserved(i))

			continue;



		if (!rq->misfit_task || rq->curr->sched_class !=

		if (!rq->misfit_task_load || rq->curr->sched_class !=

						&fair_sched_class)

			continue;
@@ -12472,7 +12478,7 @@ void check_for_migration(struct rq *rq, struct task_struct *p) 
	int prev_cpu = task_cpu(p);

	struct sched_domain *sd = NULL;



	if (rq->misfit_task) {

	if (rq->misfit_task_load) {

		if (rq->curr->state != TASK_RUNNING ||

		    rq->curr->nr_cpus_allowed == 1)

			return;

kernel/sched/sched.h

+3 −1
Original line number Diff line number Diff line
@@ -825,7 +825,8 @@ struct rq { 


	unsigned char idle_balance;



	unsigned int misfit_task;

	unsigned long misfit_task_load;



	/* For active balancing */

	int active_balance;

	int push_cpu;
@@ -1190,6 +1191,7 @@ DECLARE_PER_CPU(struct sched_domain *, sd_numa); 
DECLARE_PER_CPU(struct sched_domain *, sd_asym);

DECLARE_PER_CPU(struct sched_domain *, sd_ea);

DECLARE_PER_CPU(struct sched_domain *, sd_scs);

extern struct static_key_false sched_asym_cpucapacity;



struct sched_group_capacity {

	atomic_t ref;

kernel/sched/topology.c

+19 −0
Original line number Diff line number Diff line
@@ -418,6 +418,7 @@ DEFINE_PER_CPU(struct sched_domain *, sd_numa); 
DEFINE_PER_CPU(struct sched_domain *, sd_asym);

DEFINE_PER_CPU(struct sched_domain *, sd_ea);

DEFINE_PER_CPU(struct sched_domain *, sd_scs);

DEFINE_STATIC_KEY_FALSE(sched_asym_cpucapacity);



static void update_top_cache_domain(int cpu)

{
@@ -457,6 +458,21 @@ static void update_top_cache_domain(int cpu) 
	rcu_assign_pointer(per_cpu(sd_scs, cpu), sd);

}



static void update_asym_cpucapacity(int cpu)

{

	int enable = false;



	rcu_read_lock();

	if (lowest_flag_domain(cpu, SD_ASYM_CPUCAPACITY))

		enable = true;

	rcu_read_unlock();



	if (enable) {

		/* This expects to be hotplug-safe */

		static_branch_enable_cpuslocked(&sched_asym_cpucapacity);

	}

}



/*

 * Attach the domain 'sd' to 'cpu' as its base domain. Callers must

 * hold the hotplug lock.
@@ -1868,6 +1884,9 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att 
	}

	rcu_read_unlock();



	if (!cpumask_empty(cpu_map))

		update_asym_cpucapacity(cpumask_first(cpu_map));



	if (rq && sched_debug_enabled) {

		pr_info("span: %*pbl (max cpu_capacity = %lu)\n",

			cpumask_pr_args(cpu_map), rq->rd->max_cpu_capacity);