Merge "sched: walt: Add CFS_BANDWIDTH support" (9c5aa539) · Commits · e / devices / android_kernel_fairphone_FP3

kernel/sched/fair.c

+205 −7

Original line number	Diff line number	Diff line
		@@ -38,9 +38,38 @@
		#include <trace/events/sched.h>

		#ifdef CONFIG_SCHED_WALT

		static inline bool task_fits_max(struct task_struct *p, int cpu);
		static void walt_fixup_sched_stats_fair(struct rq rq, struct task_struct p,
		u32 new_task_load, u32 new_pred_demand);
		static void walt_fixup_nr_big_tasks(struct rq rq, struct task_struct p,
		int delta, bool inc);
		#endif /* CONFIG_SCHED_WALT */

		#if defined(CONFIG_SCHED_WALT) && defined(CONFIG_CFS_BANDWIDTH)

		static void walt_init_cfs_rq_stats(struct cfs_rq *cfs_rq);
		static void walt_inc_cfs_rq_stats(struct cfs_rq *cfs_rq,
		struct task_struct *p);
		static void walt_dec_cfs_rq_stats(struct cfs_rq *cfs_rq,
		struct task_struct *p);
		static void walt_inc_throttled_cfs_rq_stats(struct walt_sched_stats *stats,
		struct cfs_rq *cfs_rq);
		static void walt_dec_throttled_cfs_rq_stats(struct walt_sched_stats *stats,
		struct cfs_rq *cfs_rq);
		#else
		static inline void walt_init_cfs_rq_stats(struct cfs_rq *cfs_rq) {}
		static inline void
		walt_inc_cfs_rq_stats(struct cfs_rq cfs_rq, struct task_struct p) {}
		static inline void
		walt_dec_cfs_rq_stats(struct cfs_rq cfs_rq, struct task_struct p) {}

		#define walt_inc_throttled_cfs_rq_stats(...)
		#define walt_dec_throttled_cfs_rq_stats(...)

		#endif


		/*
		* Targeted preemption latency for CPU-bound tasks:
		* (default: 6ms * (1 + ilog(ncpus)), units: nanoseconds)
		@@ -4007,13 +4036,16 @@ static void throttle_cfs_rq(struct cfs_rq *cfs_rq)
		if (dequeue)
		dequeue_entity(qcfs_rq, se, DEQUEUE_SLEEP);
		qcfs_rq->h_nr_running -= task_delta;
		walt_dec_throttled_cfs_rq_stats(&qcfs_rq->walt_stats, cfs_rq);

		if (qcfs_rq->load.weight)
		dequeue = 0;
		}

		if (!se)
		if (!se) {
		sub_nr_running(rq, task_delta);
		walt_dec_throttled_cfs_rq_stats(&rq->walt_stats, cfs_rq);
		}

		cfs_rq->throttled = 1;
		cfs_rq->throttled_clock = rq_clock(rq);
		@@ -4071,13 +4103,16 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
		if (enqueue)
		enqueue_entity(cfs_rq, se, ENQUEUE_WAKEUP);
		cfs_rq->h_nr_running += task_delta;
		walt_inc_throttled_cfs_rq_stats(&cfs_rq->walt_stats, tcfs_rq);

		if (cfs_rq_throttled(cfs_rq))
		break;
		}

		if (!se)
		if (!se) {
		add_nr_running(rq, task_delta);
		walt_inc_throttled_cfs_rq_stats(&rq->walt_stats, tcfs_rq);
		}

		/* determine whether we need to wake up potentially idle cpu */
		if (rq->curr == rq->idle && rq->cfs.nr_running)
		@@ -4419,6 +4454,7 @@ static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq)
		{
		cfs_rq->runtime_enabled = 0;
		INIT_LIST_HEAD(&cfs_rq->throttled_list);
		walt_init_cfs_rq_stats(cfs_rq);
		}

		void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
		@@ -4613,6 +4649,9 @@ enqueue_task_fair(struct rq rq, struct task_struct p, int flags)
		int task_wakeup = flags & ENQUEUE_WAKEUP;
		#endif

		#ifdef CONFIG_SCHED_WALT
		p->misfit = !task_fits_max(p, rq->cpu);
		#endif
		/*
		* If in_iowait is set, the code below may not trigger any cpufreq
		* utilization updates, so do it here explicitly with the IOWAIT flag
		@@ -4636,6 +4675,7 @@ enqueue_task_fair(struct rq rq, struct task_struct p, int flags)
		if (cfs_rq_throttled(cfs_rq))
		break;
		cfs_rq->h_nr_running++;
		walt_inc_cfs_rq_stats(cfs_rq, p);

		flags = ENQUEUE_WAKEUP;
		}
		@@ -4643,6 +4683,7 @@ enqueue_task_fair(struct rq rq, struct task_struct p, int flags)
		for_each_sched_entity(se) {
		cfs_rq = cfs_rq_of(se);
		cfs_rq->h_nr_running++;
		walt_inc_cfs_rq_stats(cfs_rq, p);

		if (cfs_rq_throttled(cfs_rq))
		break;
		@@ -4653,9 +4694,6 @@ enqueue_task_fair(struct rq rq, struct task_struct p, int flags)

		if (!se) {
		add_nr_running(rq, 1);
		#ifdef CONFIG_SCHED_WALT
		p->misfit = !task_fits_max(p, rq->cpu);
		#endif
		inc_rq_walt_stats(rq, p);
		}

		@@ -4712,6 +4750,7 @@ static void dequeue_task_fair(struct rq rq, struct task_struct p, int flags)
		if (cfs_rq_throttled(cfs_rq))
		break;
		cfs_rq->h_nr_running--;
		walt_dec_cfs_rq_stats(cfs_rq, p);

		/* Don't dequeue parent if it has other entities besides us */
		if (cfs_rq->load.weight) {
		@@ -4731,6 +4770,7 @@ static void dequeue_task_fair(struct rq rq, struct task_struct p, int flags)
		for_each_sched_entity(se) {
		cfs_rq = cfs_rq_of(se);
		cfs_rq->h_nr_running--;
		walt_dec_cfs_rq_stats(cfs_rq, p);

		if (cfs_rq_throttled(cfs_rq))
		break;
		@@ -10348,7 +10388,7 @@ static void task_tick_fair(struct rq rq, struct task_struct curr, int queued)
		rq->misfit_task = misfit;

		if (old_misfit != misfit) {
		walt_adjust_nr_big_tasks(rq, 1, misfit);
		walt_fixup_nr_big_tasks(rq, curr, 1, misfit);
		curr->misfit = misfit;
		}
		#endif
		@@ -10813,7 +10853,7 @@ const struct sched_class fair_sched_class = {
		.task_change_group = task_change_group_fair,
		#endif
		#ifdef CONFIG_SCHED_WALT
		.fixup_walt_sched_stats = fixup_walt_sched_stats_common,
		.fixup_walt_sched_stats = walt_fixup_sched_stats_fair,
		#endif
		};

		@@ -10865,6 +10905,161 @@ __init void init_sched_fair_class(void)
		/* WALT sched implementation begins here */
		#ifdef CONFIG_SCHED_WALT

		#ifdef CONFIG_CFS_BANDWIDTH

		static void walt_init_cfs_rq_stats(struct cfs_rq *cfs_rq)
		{
		cfs_rq->walt_stats.nr_big_tasks = 0;
		cfs_rq->walt_stats.cumulative_runnable_avg = 0;
		cfs_rq->walt_stats.pred_demands_sum = 0;
		}

		static void walt_inc_cfs_rq_stats(struct cfs_rq cfs_rq, struct task_struct p)
		{
		inc_nr_big_task(&cfs_rq->walt_stats, p);
		fixup_cumulative_runnable_avg(&cfs_rq->walt_stats, p->ravg.demand,
		p->ravg.pred_demand);
		}

		static void walt_dec_cfs_rq_stats(struct cfs_rq cfs_rq, struct task_struct p)
		{
		dec_nr_big_task(&cfs_rq->walt_stats, p);
		fixup_cumulative_runnable_avg(&cfs_rq->walt_stats, -(s64)p->ravg.demand,
		-(s64)p->ravg.pred_demand);
		}

		static void walt_inc_throttled_cfs_rq_stats(struct walt_sched_stats *stats,
		struct cfs_rq *tcfs_rq)
		{
		struct rq *rq = rq_of(tcfs_rq);

		stats->nr_big_tasks += tcfs_rq->walt_stats.nr_big_tasks;
		fixup_cumulative_runnable_avg(stats,
		tcfs_rq->walt_stats.cumulative_runnable_avg,
		tcfs_rq->walt_stats.pred_demands_sum);

		if (stats == &rq->walt_stats)
		walt_fixup_cum_window_demand(rq,
		tcfs_rq->walt_stats.cumulative_runnable_avg);

		}

		static void walt_dec_throttled_cfs_rq_stats(struct walt_sched_stats *stats,
		struct cfs_rq *tcfs_rq)
		{
		struct rq *rq = rq_of(tcfs_rq);

		stats->nr_big_tasks -= tcfs_rq->walt_stats.nr_big_tasks;
		fixup_cumulative_runnable_avg(stats,
		-tcfs_rq->walt_stats.cumulative_runnable_avg,
		-tcfs_rq->walt_stats.pred_demands_sum);

		/*
		* We remove the throttled cfs_rq's tasks's contribution from the
		* cumulative window demand so that the same can be added
		* unconditionally when the cfs_rq is unthrottled.
		*/
		if (stats == &rq->walt_stats)
		walt_fixup_cum_window_demand(rq,
		-tcfs_rq->walt_stats.cumulative_runnable_avg);
		}

		static void walt_fixup_sched_stats_fair(struct rq rq, struct task_struct p,
		u32 new_task_load, u32 new_pred_demand)
		{
		struct cfs_rq *cfs_rq;
		struct sched_entity *se = &p->se;
		s64 task_load_delta = (s64)new_task_load - task_load(p);
		s64 pred_demand_delta = PRED_DEMAND_DELTA;

		for_each_sched_entity(se) {
		cfs_rq = cfs_rq_of(se);

		fixup_cumulative_runnable_avg(&cfs_rq->walt_stats,
		task_load_delta,
		pred_demand_delta);
		if (cfs_rq_throttled(cfs_rq))
		break;
		}

		/* Fix up rq->walt_stats only if we didn't find any throttled cfs_rq */
		if (!se) {
		fixup_cumulative_runnable_avg(&rq->walt_stats,
		task_load_delta,
		pred_demand_delta);
		walt_fixup_cum_window_demand(rq, task_load_delta);
		}
		}

		static void walt_fixup_nr_big_tasks(struct rq rq, struct task_struct p,
		int delta, bool inc)
		{
		struct cfs_rq *cfs_rq;
		struct sched_entity *se = &p->se;

		for_each_sched_entity(se) {
		cfs_rq = cfs_rq_of(se);

		cfs_rq->walt_stats.nr_big_tasks += inc ? delta : -delta;
		BUG_ON(cfs_rq->walt_stats.nr_big_tasks < 0);

		if (cfs_rq_throttled(cfs_rq))
		break;
		}

		/* Fix up rq->walt_stats only if we didn't find any throttled cfs_rq */
		if (!se)
		walt_adjust_nr_big_tasks(rq, delta, inc);
		}

		/*
		* Check if task is part of a hierarchy where some cfs_rq does not have any
		* runtime left.
		*
		* We can't rely on throttled_hierarchy() to do this test, as
		* cfs_rq->throttle_count will not be updated yet when this function is called
		* from scheduler_tick()
		*/
		static int task_will_be_throttled(struct task_struct *p)
		{
		struct sched_entity *se = &p->se;
		struct cfs_rq *cfs_rq;

		if (!cfs_bandwidth_used())
		return 0;

		for_each_sched_entity(se) {
		cfs_rq = cfs_rq_of(se);
		if (!cfs_rq->runtime_enabled)
		continue;
		if (cfs_rq->runtime_remaining <= 0)
		return 1;
		}

		return 0;
		}

		#else /* CONFIG_CFS_BANDWIDTH */

		static void walt_fixup_sched_stats_fair(struct rq rq, struct task_struct p,
		u32 new_task_load, u32 new_pred_demand)
		{
		fixup_walt_sched_stats_common(rq, p, new_task_load, new_pred_demand);
		}

		static void walt_fixup_nr_big_tasks(struct rq rq, struct task_struct p,
		int delta, bool inc)
		{
		walt_adjust_nr_big_tasks(rq, delta, inc);
		}

		static int task_will_be_throttled(struct task_struct *p)
		{
		return false;
		}

		#endif /* CONFIG_CFS_BANDWIDTH */

		static inline int
		kick_active_balance(struct rq rq, struct task_struct p, int new_cpu)
		{
		@@ -10897,6 +11092,9 @@ void check_for_migration(struct rq rq, struct task_struct p)
		rq->curr->nr_cpus_allowed == 1)
		return;

		if (task_will_be_throttled(p))
		return;

		raw_spin_lock(&migration_lock);
		rcu_read_lock();
		new_cpu = energy_aware_wake_cpu(p, cpu, 0);

kernel/sched/sched.h

+5 −0

Original line number	Diff line number	Diff line
		@@ -510,6 +510,11 @@ struct cfs_rq {
		struct task_group tg; / group that "owns" this runqueue */

		#ifdef CONFIG_CFS_BANDWIDTH

		#ifdef CONFIG_SCHED_WALT
		struct walt_sched_stats walt_stats;
		#endif

		int runtime_enabled;
		u64 runtime_expires;
		s64 runtime_remaining;