Merge "sched: walt: Optimize cpu_util() and cpu_util_cum()" (4402e6fd) · Commits · e / devices / android_kernel_oneplus_sm8150

include/linux/sched.h

+4 −1

Original line number	Diff line number	Diff line
		@@ -591,6 +591,8 @@ struct ravg {
		*
		* 'busy_buckets' groups historical busy time into different buckets
		* used for prediction
		*
		* 'demand_scaled' represents task's demand scaled to 1024
		*/
		u64 mark_start;
		u32 sum, demand;
		@@ -601,6 +603,8 @@ struct ravg {
		u16 active_windows;
		u32 pred_demand;
		u8 busy_buckets[NUM_BUSY_BUCKETS];
		u16 demand_scaled;
		u16 pred_demand_scaled;
		};
		#else
		static inline void sched_exit(struct task_struct *p) { }
		@@ -782,7 +786,6 @@ struct task_struct {
		*/
		u32 init_load_pct;
		u64 last_wake_ts;
		u64 last_switch_out_ts;
		u64 last_enqueued_ts;
		struct related_thread_group *grp;
		struct list_head grp_list;

include/trace/events/walt.h

+2 −2

Original line number	Diff line number	Diff line
		@@ -470,7 +470,7 @@ DECLARE_EVENT_CLASS(sched_cpu_load,
		__entry->nr_big_tasks = rq->walt_stats.nr_big_tasks;
		__entry->load_scale_factor = cpu_load_scale_factor(rq->cpu);
		__entry->capacity = cpu_capacity(rq->cpu);
		__entry->cumulative_runnable_avg = rq->walt_stats.cumulative_runnable_avg;
		__entry->cumulative_runnable_avg = rq->walt_stats.cumulative_runnable_avg_scaled;
		__entry->irqload = irqload;
		__entry->max_freq = cpu_max_freq(rq->cpu);
		__entry->power_cost = power_cost;
		@@ -532,7 +532,7 @@ TRACE_EVENT(sched_load_to_gov,
		__entry->grp_rq_ps = rq->grp_time.prev_runnable_sum;
		__entry->nt_ps = rq->nt_prev_runnable_sum;
		__entry->grp_nt_ps = rq->grp_time.nt_prev_runnable_sum;
		__entry->pl = rq->walt_stats.pred_demands_sum;
		__entry->pl = rq->walt_stats.pred_demands_sum_scaled;
		__entry->load = load;
		__entry->big_task_rotation = big_task_rotation;
		__entry->sysctl_sched_little_cluster_coloc_fmin_khz =

kernel/sched/core.c

+1 −1

Original line number	Diff line number	Diff line
		@@ -6412,7 +6412,7 @@ void __init sched_init(void)
		rq->avg_idle = 2*sysctl_sched_migration_cost;
		rq->max_idle_balance_cost = sysctl_sched_migration_cost;
		rq->push_task = NULL;
		walt_sched_init(rq);
		walt_sched_init_rq(rq);

		INIT_LIST_HEAD(&rq->cfs_tasks);

kernel/sched/debug.c

+1 −1

Original line number	Diff line number	Diff line
		@@ -759,7 +759,7 @@ do { \
		P(cluster->exec_scale_factor);
		P(walt_stats.nr_big_tasks);
		SEQ_printf(m, " .%-30s: %llu\n", "walt_stats.cumulative_runnable_avg",
		rq->walt_stats.cumulative_runnable_avg);
		rq->walt_stats.cumulative_runnable_avg_scaled);
		#endif
		#undef P
		#undef PN

kernel/sched/fair.c

+26 −19

Original line number	Diff line number	Diff line
		@@ -47,7 +47,8 @@ static inline bool task_fits_max(struct task_struct *p, int cpu);
		#ifdef CONFIG_SCHED_WALT

		static void walt_fixup_sched_stats_fair(struct rq rq, struct task_struct p,
		u32 new_task_load, u32 new_pred_demand);
		u16 updated_demand_scaled,
		u16 updated_pred_demand_scaled);
		static void walt_fixup_nr_big_tasks(struct rq rq, struct task_struct p,
		int delta, bool inc);
		#endif /* CONFIG_SCHED_WALT */
		@@ -3726,8 +3727,7 @@ static inline unsigned long task_util_est(struct task_struct *p)
		{
		#ifdef CONFIG_SCHED_WALT
		if (likely(!walt_disabled && sysctl_sched_use_walt_task_util))
		return (p->ravg.demand /
		(sched_ravg_window >> SCHED_CAPACITY_SHIFT));
		return p->ravg.demand_scaled;
		#endif
		return max(task_util(p), _task_util_est(p));
		}
		@@ -12519,22 +12519,24 @@ __init void init_sched_fair_class(void)
		static void walt_init_cfs_rq_stats(struct cfs_rq *cfs_rq)
		{
		cfs_rq->walt_stats.nr_big_tasks = 0;
		cfs_rq->walt_stats.cumulative_runnable_avg = 0;
		cfs_rq->walt_stats.cumulative_runnable_avg_scaled = 0;
		cfs_rq->walt_stats.pred_demands_sum = 0;
		}

		static void walt_inc_cfs_rq_stats(struct cfs_rq cfs_rq, struct task_struct p)
		{
		inc_nr_big_task(&cfs_rq->walt_stats, p);
		fixup_cumulative_runnable_avg(&cfs_rq->walt_stats, p->ravg.demand,
		p->ravg.pred_demand);
		fixup_cumulative_runnable_avg(&cfs_rq->walt_stats,
		p->ravg.demand_scaled,
		p->ravg.pred_demand_scaled);
		}

		static void walt_dec_cfs_rq_stats(struct cfs_rq cfs_rq, struct task_struct p)
		{
		dec_nr_big_task(&cfs_rq->walt_stats, p);
		fixup_cumulative_runnable_avg(&cfs_rq->walt_stats, -(s64)p->ravg.demand,
		-(s64)p->ravg.pred_demand);
		fixup_cumulative_runnable_avg(&cfs_rq->walt_stats,
		-(s64)p->ravg.demand_scaled,
		-(s64)p->ravg.pred_demand_scaled);
		}

		static void walt_inc_throttled_cfs_rq_stats(struct walt_sched_stats *stats,
		@@ -12544,12 +12546,12 @@ static void walt_inc_throttled_cfs_rq_stats(struct walt_sched_stats *stats,

		stats->nr_big_tasks += tcfs_rq->walt_stats.nr_big_tasks;
		fixup_cumulative_runnable_avg(stats,
		tcfs_rq->walt_stats.cumulative_runnable_avg,
		tcfs_rq->walt_stats.pred_demands_sum);
		tcfs_rq->walt_stats.cumulative_runnable_avg_scaled,
		tcfs_rq->walt_stats.pred_demands_sum_scaled);

		if (stats == &rq->walt_stats)
		walt_fixup_cum_window_demand(rq,
		tcfs_rq->walt_stats.cumulative_runnable_avg);
		tcfs_rq->walt_stats.cumulative_runnable_avg_scaled);

		}

		@@ -12560,8 +12562,8 @@ static void walt_dec_throttled_cfs_rq_stats(struct walt_sched_stats *stats,

		stats->nr_big_tasks -= tcfs_rq->walt_stats.nr_big_tasks;
		fixup_cumulative_runnable_avg(stats,
		-tcfs_rq->walt_stats.cumulative_runnable_avg,
		-tcfs_rq->walt_stats.pred_demands_sum);
		-tcfs_rq->walt_stats.cumulative_runnable_avg_scaled,
		-tcfs_rq->walt_stats.pred_demands_sum_scaled);

		/*
		* We remove the throttled cfs_rq's tasks's contribution from the
		@@ -12570,16 +12572,19 @@ static void walt_dec_throttled_cfs_rq_stats(struct walt_sched_stats *stats,
		*/
		if (stats == &rq->walt_stats)
		walt_fixup_cum_window_demand(rq,
		-tcfs_rq->walt_stats.cumulative_runnable_avg);
		-tcfs_rq->walt_stats.cumulative_runnable_avg_scaled);
		}

		static void walt_fixup_sched_stats_fair(struct rq rq, struct task_struct p,
		u32 new_task_load, u32 new_pred_demand)
		u16 updated_demand_scaled,
		u16 updated_pred_demand_scaled)
		{
		struct cfs_rq *cfs_rq;
		struct sched_entity *se = &p->se;
		s64 task_load_delta = (s64)new_task_load - task_load(p);
		s64 pred_demand_delta = PRED_DEMAND_DELTA;
		s64 task_load_delta = (s64)updated_demand_scaled -
		p->ravg.demand_scaled;
		s64 pred_demand_delta = (s64)updated_pred_demand_scaled -
		p->ravg.pred_demand_scaled;

		for_each_sched_entity(se) {
		cfs_rq = cfs_rq_of(se);
		@@ -12651,9 +12656,11 @@ static int task_will_be_throttled(struct task_struct *p)
		#else /* CONFIG_CFS_BANDWIDTH */

		static void walt_fixup_sched_stats_fair(struct rq rq, struct task_struct p,
		u32 new_task_load, u32 new_pred_demand)
		u16 updated_demand_scaled,
		u16 updated_pred_demand_scaled)
		{
		fixup_walt_sched_stats_common(rq, p, new_task_load, new_pred_demand);
		fixup_walt_sched_stats_common(rq, p, updated_demand_scaled,
		updated_pred_demand_scaled);
		}

		static void walt_fixup_nr_big_tasks(struct rq rq, struct task_struct p,