Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 0587a6af authored by Pavankumar Kondeti's avatar Pavankumar Kondeti Committed by Joonwoo Park
Browse files

sched: EAS: add core isolation support 



Isolated CPUs are only allowed to run pinned kernel threads, timers
and hrtimers. Don't consider isolated CPUs during energy aware task
placement for fair and RT class tasks.

Change-Id: I6c53f2ce2bcb2355f9d0d99e1ba502a91074adac
Signed-off-by: default avatarPavankumar Kondeti <pkondeti@codeaurora.org>
parent cf80de5a

kernel/sched/fair.c

+45 −8
Original line number Diff line number Diff line
@@ -6331,7 +6331,7 @@ static int select_idle_core(struct task_struct *p, struct sched_domain *sd, int 
				idle = false;

		}



		if (idle)

		if (!cpu_isolated(cpu) && idle)

			return core;

	}
@@ -6356,6 +6356,8 @@ static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int t 
	for_each_cpu(cpu, cpu_smt_mask(target)) {

		if (!cpumask_test_cpu(cpu, tsk_cpus_allowed(p)))

			continue;

		if (cpu_isolated(cpu))

			continue;

		if (idle_cpu(cpu))

			return cpu;

	}
@@ -6408,6 +6410,8 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t 
	for_each_cpu_wrap(cpu, sched_domain_span(sd), target, wrap) {

		if (!cpumask_test_cpu(cpu, tsk_cpus_allowed(p)))

			continue;

		if (cpu_isolated(cpu))

			continue;

		if (idle_cpu(cpu))

			break;

	}
@@ -6433,13 +6437,14 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) 
	int best_idle_capacity = INT_MAX;



	if (!sysctl_sched_cstate_aware) {

		if (idle_cpu(target))

		if (idle_cpu(target) && !cpu_isolated(target))

			return target;



		/*

		 * If the prevous cpu is cache affine and idle, don't be stupid.

		 */

		if (i != target && cpus_share_cache(i, target) && idle_cpu(i))

		if (i != target && cpus_share_cache(i, target) &&

				idle_cpu(i) && !cpu_isolated(i))

			return i;



		sd = rcu_dereference(per_cpu(sd_llc, target));
@@ -6477,6 +6482,10 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) 
					int idle_idx = idle_get_state_idx(rq);

					unsigned long new_usage = boosted_task_util(p);

					unsigned long capacity_orig = capacity_orig_of(i);



					if (cpu_isolated(i))

						continue;



					if (new_usage > capacity_orig || !idle_cpu(i))

						goto next;
@@ -6491,6 +6500,9 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) 
				}

			} else {

				for_each_cpu(i, sched_group_cpus(sg)) {

					if (cpu_isolated(i))

						continue;



					if (i == target || !idle_cpu(i))

						goto next;

				}
@@ -6707,6 +6719,7 @@ static int energy_aware_wake_cpu(struct task_struct *p, int target, int sync) 
	long target_cpu_new_util_cum = LONG_MAX;

	struct cpumask *rtg_target = NULL;

	bool wake_on_sibling = false;

	int isolated_candidate = -1;

	bool need_idle;

	bool skip_ediff = false;
@@ -6752,8 +6765,23 @@ static int energy_aware_wake_cpu(struct task_struct *p, int target, int sync) 
		 * point.

		 */

		do {

			int max_cap_cpu;

			cpumask_t avail_cpus;



			/* Are all CPUs isolated in this group? */

			if (unlikely(!sg->group_weight))

				continue;



			/* Can this task run on any CPUs of this group? */

			cpumask_and(&avail_cpus, sched_group_cpus(sg),

							tsk_cpus_allowed(p));

			cpumask_andnot(&avail_cpus, &avail_cpus,

							cpu_isolated_mask);

			if (cpumask_empty(&avail_cpus))

				continue;



			/* Assuming all cpus are the same in group */

			int max_cap_cpu = group_first_cpu(sg);

			max_cap_cpu = group_first_cpu(sg);



			/*

			 * Assume smaller max capacity means more energy-efficient.
@@ -6801,9 +6829,18 @@ static int energy_aware_wake_cpu(struct task_struct *p, int target, int sync) 


		/* Find cpu with sufficient capacity */

		for_each_cpu_and(i, tsk_cpus_allowed(p), sched_group_cpus(sg_target)) {

			if (cpu_isolated(i))

				continue;



			if (isolated_candidate == -1)

				isolated_candidate = i;



			if (is_reserved(i))

				continue;



			if (sched_cpu_high_irqload(cpu))

				continue;



			/*

			 * p's blocked utilization is still accounted for on prev_cpu

			 * so prev_cpu will receive a negative bias due to the double
@@ -6824,9 +6861,6 @@ static int energy_aware_wake_cpu(struct task_struct *p, int target, int sync) 
			trace_sched_cpu_util(p, i, task_util_boosted, curr_util,

					     new_util_cum, sync);



			if (sched_cpu_high_irqload(cpu))

				continue;



			/*

			 * Ensure minimum capacity to grant the required boost.

			 * The target CPU can be already at a capacity level higher
@@ -6920,6 +6954,9 @@ static int energy_aware_wake_cpu(struct task_struct *p, int target, int sync) 
					  targeted_cpus))) {

			if (likely(min_util_cpu != -1))

				target_cpu = min_util_cpu;

			else if (cpu_isolated(task_cpu(p)) &&

					isolated_candidate != -1)

				target_cpu = isolated_candidate;

			else

				target_cpu = task_cpu(p);

		}
@@ -6951,7 +6988,7 @@ static int energy_aware_wake_cpu(struct task_struct *p, int target, int sync) 
		return target_cpu;

	}



	if (target_cpu != task_cpu(p)) {

	if (target_cpu != task_cpu(p) && !cpu_isolated(task_cpu(p))) {

		struct energy_env eenv = {

			.util_delta	= task_util(p),

			.src_cpu	= task_cpu(p),

kernel/sched/hmp.c

+0 −24
Original line number Diff line number Diff line
@@ -186,30 +186,6 @@ unsigned long __weak arch_get_cpu_efficiency(int cpu) 
	return SCHED_CAPACITY_SCALE;

}



/* Clear any HMP scheduler related requests pending from or on cpu */

void clear_hmp_request(int cpu)

{

	struct rq *rq = cpu_rq(cpu);

	unsigned long flags;



	clear_boost_kick(cpu);

	clear_reserved(cpu);

	if (rq->push_task) {

		struct task_struct *push_task = NULL;



		raw_spin_lock_irqsave(&rq->lock, flags);

		if (rq->push_task) {

			clear_reserved(rq->push_cpu);

			push_task = rq->push_task;

			rq->push_task = NULL;

		}

		rq->active_balance = 0;

		raw_spin_unlock_irqrestore(&rq->lock, flags);

		if (push_task)

			put_task_struct(push_task);

	}

}



int sched_set_static_cpu_pwr_cost(int cpu, unsigned int cost)

{

	struct rq *rq = cpu_rq(cpu);

kernel/sched/rt.c

+3 −0
Original line number Diff line number Diff line
@@ -1902,6 +1902,9 @@ static int find_lowest_rq(struct task_struct *task) 
			 */

			util = cpu_util(cpu);

			if (!cpu_overutilized(cpu)) {

				if (cpu_isolated(cpu))

					continue;



				if (sched_cpu_high_irqload(cpu))

					continue;

kernel/sched/sched.h

+7 −6
Original line number Diff line number Diff line
@@ -2303,7 +2303,6 @@ extern unsigned int __read_mostly sched_load_granule; 


extern u64 sched_ktime_clock(void);

extern int register_cpu_cycle_counter_cb(struct cpu_cycle_counter_cb *cb);

extern void clear_boost_kick(int cpu);

extern void reset_cpu_hmp_stats(int cpu, int reset_cra);

extern int update_preferred_cluster(struct related_thread_group *grp,

			struct task_struct *p, u32 old_load);
@@ -2649,6 +2648,8 @@ extern void update_cpu_cluster_capacity(const cpumask_t *cpus); 


extern unsigned long thermal_cap(int cpu);



extern void clear_hmp_request(int cpu);



#else	/* CONFIG_SCHED_WALT */



struct hmp_sched_stats;
@@ -2660,8 +2661,6 @@ static inline bool task_sched_boost(struct task_struct *p) 
	return true;

}



static inline void clear_boost_kick(int cpu) { }



static inline void check_for_migration(struct rq *rq, struct task_struct *p) { }



static inline int task_will_fit(struct task_struct *p, int cpu)
@@ -2780,6 +2779,8 @@ static inline unsigned long thermal_cap(int cpu) 
}

#endif



static inline void clear_hmp_request(int cpu) { }



#endif	/* CONFIG_SCHED_WALT */



#ifdef CONFIG_SCHED_HMP
@@ -2789,7 +2790,6 @@ extern int is_big_task(struct task_struct *p); 
extern unsigned int pct_task_load(struct task_struct *p);

extern void notify_migration(int src_cpu, int dest_cpu,

			bool src_cpu_dead, struct task_struct *p);

extern void clear_hmp_request(int cpu);

extern void note_task_waking(struct task_struct *p, u64 wallclock);

extern void

check_for_freq_change(struct rq *rq, bool check_pred, bool check_groups);
@@ -2837,6 +2837,8 @@ static inline bool is_short_burst_task(struct task_struct *p) 
	return p->ravg.avg_burst < sysctl_sched_short_burst &&

	       p->ravg.avg_sleep_time > sysctl_sched_short_sleep;

}



extern void clear_boost_kick(int cpu);

#else

static inline bool energy_aware(void)

{
@@ -2848,8 +2850,6 @@ static inline int pct_task_load(struct task_struct *p) { return 0; } 
static inline void notify_migration(int src_cpu, int dest_cpu,

			bool src_cpu_dead, struct task_struct *p) { }



static inline void clear_hmp_request(int cpu) { }



static inline void note_task_waking(struct task_struct *p, u64 wallclock) { }



static inline void
@@ -2909,4 +2909,5 @@ inc_nr_big_task(struct hmp_sched_stats *stats, struct task_struct *p) { } 
static inline void

dec_nr_big_task(struct hmp_sched_stats *stats, struct task_struct *p) { }



static inline void clear_boost_kick(int cpu) { }

#endif /* CONFIG_SCHED_HMP */

kernel/sched/walt.c

+23 −0
Original line number Diff line number Diff line
@@ -298,6 +298,29 @@ void sched_account_irqstart(int cpu, struct task_struct *curr, u64 wallclock) 
	}

}



void clear_hmp_request(int cpu)

{

	struct rq *rq = cpu_rq(cpu);

	unsigned long flags;



	clear_boost_kick(cpu);

	clear_reserved(cpu);

	if (rq->push_task) {

		struct task_struct *push_task = NULL;



		raw_spin_lock_irqsave(&rq->lock, flags);

		if (rq->push_task) {

			clear_reserved(rq->push_cpu);

			push_task = rq->push_task;

			rq->push_task = NULL;

		}

		rq->active_balance = 0;

		raw_spin_unlock_irqrestore(&rq->lock, flags);

		if (push_task)

			put_task_struct(push_task);

	}

}



void sched_account_irqtime(int cpu, struct task_struct *curr,

				 u64 delta, u64 wallclock)

{