Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 4dd4632f authored by Joonwoo Park's avatar Joonwoo Park
Browse files

sched: preserve CPU cycle counter in rq 



Preserve cycle counter in rq in preparation for wait time accounting
while CPU idle fix.

Change-Id: I469263c90e12f39bb36bde5ed26298b7c1c77597
Signed-off-by: default avatarJoonwoo Park <joonwoop@codeaurora.org>
parent 556f086b

kernel/sched/core.c

+49 −70
Original line number Diff line number Diff line
@@ -1663,11 +1663,6 @@ int register_cpu_cycle_counter_cb(struct cpu_cycle_counter_cb *cb) 


#define SCHED_MIN_FREQ 1



struct cpu_cycle {

	u64 cycles;

	u64 time;

};



#if defined(CONFIG_SCHED_HMP)



/*
@@ -1866,14 +1861,13 @@ update_window_start(struct rq *rq, u64 wallclock) 


#define DIV64_U64_ROUNDUP(X, Y) div64_u64((X) + (Y - 1), Y)



static inline u64 scale_exec_time(u64 delta, struct rq *rq,

				  const struct cpu_cycle *cc)

static inline u64 scale_exec_time(u64 delta, struct rq *rq)

{

	int cpu = cpu_of(rq);

	int sf;



	delta = DIV64_U64_ROUNDUP(delta * cc->cycles,

				  max_possible_freq * cc->time);

	delta = DIV64_U64_ROUNDUP(delta * rq->cc.cycles,

				  max_possible_freq * rq->cc.time);

	sf = DIV_ROUND_UP(cpu_efficiency(cpu) * 1024, max_possible_efficiency);



	delta *= sf;
@@ -2317,8 +2311,7 @@ void update_task_pred_demand(struct rq *rq, struct task_struct *p, int event) 
 * Account cpu activity in its busy time counters (rq->curr/prev_runnable_sum)

 */

static void update_cpu_busy_time(struct task_struct *p, struct rq *rq,

				 int event, u64 wallclock, u64 irqtime,

				 const struct cpu_cycle *cc)

				 int event, u64 wallclock, u64 irqtime)

{

	int new_window, nr_full_windows = 0;

	int p_is_curr_task = (p == rq->curr);
@@ -2451,7 +2444,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, 
			delta = wallclock - mark_start;

		else

			delta = irqtime;

		delta = scale_exec_time(delta, rq, cc);

		delta = scale_exec_time(delta, rq);

		*curr_runnable_sum += delta;

		if (new_task)

			*nt_curr_runnable_sum += delta;
@@ -2477,15 +2470,14 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, 
		if (!nr_full_windows) {

			/* A full window hasn't elapsed, account partial

			 * contribution to previous completed window. */

			delta = scale_exec_time(window_start - mark_start, rq,

						cc);

			delta = scale_exec_time(window_start - mark_start, rq);

			if (!exiting_task(p))

				p->ravg.prev_window += delta;

		} else {

			/* Since at least one full window has elapsed,

			 * the contribution to the previous window is the

			 * full window (window_size). */

			delta = scale_exec_time(window_size, rq, cc);

			delta = scale_exec_time(window_size, rq);

			if (!exiting_task(p))

				p->ravg.prev_window = delta;

		}
@@ -2495,7 +2487,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, 
			*nt_prev_runnable_sum += delta;



		/* Account piece of busy time in the current window. */

		delta = scale_exec_time(wallclock - window_start, rq, cc);

		delta = scale_exec_time(wallclock - window_start, rq);

		*curr_runnable_sum += delta;

		if (new_task)

			*nt_curr_runnable_sum += delta;
@@ -2522,15 +2514,14 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, 
		if (!nr_full_windows) {

			/* A full window hasn't elapsed, account partial

			 * contribution to previous completed window. */

			delta = scale_exec_time(window_start - mark_start, rq,

						cc);

			delta = scale_exec_time(window_start - mark_start, rq);

			if (!is_idle_task(p) && !exiting_task(p))

				p->ravg.prev_window += delta;

		} else {

			/* Since at least one full window has elapsed,

			 * the contribution to the previous window is the

			 * full window (window_size). */

			delta = scale_exec_time(window_size, rq, cc);

			delta = scale_exec_time(window_size, rq);

			if (!is_idle_task(p) && !exiting_task(p))

				p->ravg.prev_window = delta;

		}
@@ -2542,7 +2533,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, 
			*nt_prev_runnable_sum += delta;



		/* Account piece of busy time in the current window. */

		delta = scale_exec_time(wallclock - window_start, rq, cc);

		delta = scale_exec_time(wallclock - window_start, rq);

		*curr_runnable_sum += delta;

		if (new_task)

			*nt_curr_runnable_sum += delta;
@@ -2570,7 +2561,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, 
		/* Roll window over. If IRQ busy time was just in the current

		 * window then that is all that need be accounted. */

		if (mark_start > window_start) {

			*curr_runnable_sum = scale_exec_time(irqtime, rq, cc);

			*curr_runnable_sum = scale_exec_time(irqtime, rq);

			return;

		}
@@ -2579,12 +2570,12 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, 
		delta = window_start - mark_start;

		if (delta > window_size)

			delta = window_size;

		delta = scale_exec_time(delta, rq, cc);

		delta = scale_exec_time(delta, rq);

		*prev_runnable_sum += delta;



		/* Process the remaining IRQ busy time in the current window. */

		delta = wallclock - window_start;

		rq->curr_runnable_sum = scale_exec_time(delta, rq, cc);

		rq->curr_runnable_sum = scale_exec_time(delta, rq);



		return;

	}
@@ -2704,7 +2695,7 @@ update_task_pred_demand(struct rq *rq, struct task_struct *p, int event) 
}



static inline void update_cpu_busy_time(struct task_struct *p, struct rq *rq,

	     int event, u64 wallclock, u64 irqtime, const struct cpu_cycle *cc)

	     int event, u64 wallclock, u64 irqtime)

{

}
@@ -2723,34 +2714,33 @@ static void update_task_cpu_cycles(struct task_struct *p, int cpu) 
		p->cpu_cycles = cpu_cycle_counter_cb.get_cpu_cycle_counter(cpu);

}



static struct cpu_cycle

get_task_cpu_cycles(struct task_struct *p, struct rq *rq, int event,

static void

update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event,

			  u64 wallclock)

{

	u64 cur_cycles;

	struct cpu_cycle cc;

	int cpu = cpu_of(rq);



	lockdep_assert_held(&rq->lock);



	if (!use_cycle_counter) {

		cc.cycles = cpu_cur_freq(cpu);

		cc.time = 1;

		return cc;

		rq->cc.cycles = cpu_cur_freq(cpu);

		rq->cc.time = 1;

		return;

	}



	cur_cycles = cpu_cycle_counter_cb.get_cpu_cycle_counter(cpu);

	if (unlikely(cur_cycles < p->cpu_cycles))

		cc.cycles = cur_cycles + (U64_MAX - p->cpu_cycles);

		rq->cc.cycles = cur_cycles + (U64_MAX - p->cpu_cycles);

	else

		cc.cycles = cur_cycles - p->cpu_cycles;

	cc.cycles = cc.cycles * NSEC_PER_MSEC;

	cc.time = wallclock - p->ravg.mark_start;

	BUG_ON((s64)cc.time < 0);

		rq->cc.cycles = cur_cycles - p->cpu_cycles;

	rq->cc.cycles = rq->cc.cycles * NSEC_PER_MSEC;

	rq->cc.time = wallclock - p->ravg.mark_start;

	BUG_ON((s64)rq->cc.time < 0);



	p->cpu_cycles = cur_cycles;



	trace_sched_get_task_cpu_cycles(cpu, event, cc.cycles, cc.time);



	return cc;

	trace_sched_get_task_cpu_cycles(cpu, event, rq->cc.cycles, rq->cc.time);

}



static int account_busy_for_task_demand(struct task_struct *p, int event)
@@ -2838,10 +2828,9 @@ done: 
	trace_sched_update_history(rq, p, runtime, samples, event);

}



static void add_to_task_demand(struct rq *rq, struct task_struct *p,

				u64 delta, const struct cpu_cycle *cc)

static void add_to_task_demand(struct rq *rq, struct task_struct *p, u64 delta)

{

	delta = scale_exec_time(delta, rq, cc);

	delta = scale_exec_time(delta, rq);

	p->ravg.sum += delta;

	if (unlikely(p->ravg.sum > sched_ravg_window))

		p->ravg.sum = sched_ravg_window;
@@ -2898,8 +2887,7 @@ static void add_to_task_demand(struct rq *rq, struct task_struct *p, 
 * depends on it!

 */

static void update_task_demand(struct task_struct *p, struct rq *rq,

			       int event, u64 wallclock,

			       const struct cpu_cycle *cc)

			       int event, u64 wallclock)

{

	u64 mark_start = p->ravg.mark_start;

	u64 delta, window_start = rq->window_start;
@@ -2922,7 +2910,7 @@ static void update_task_demand(struct task_struct *p, struct rq *rq, 
	if (!new_window) {

		/* The simple case - busy time contained within the existing

		 * window. */

		add_to_task_demand(rq, p, wallclock - mark_start, cc);

		add_to_task_demand(rq, p, wallclock - mark_start);

		return;

	}
@@ -2933,12 +2921,12 @@ static void update_task_demand(struct task_struct *p, struct rq *rq, 
	window_start -= (u64)nr_full_windows * (u64)window_size;



	/* Process (window_start - mark_start) first */

	add_to_task_demand(rq, p, window_start - mark_start, cc);

	add_to_task_demand(rq, p, window_start - mark_start);



	/* Push new sample(s) into task's demand history */

	update_history(rq, p, p->ravg.sum, 1, event);

	if (nr_full_windows)

		update_history(rq, p, scale_exec_time(window_size, rq, cc),

		update_history(rq, p, scale_exec_time(window_size, rq),

			       nr_full_windows, event);



	/* Roll window_start back to current to process any remainder
@@ -2947,18 +2935,16 @@ static void update_task_demand(struct task_struct *p, struct rq *rq, 


	/* Process (wallclock - window_start) next */

	mark_start = window_start;

	add_to_task_demand(rq, p, wallclock - mark_start, cc);

	add_to_task_demand(rq, p, wallclock - mark_start);

}



/* Reflect task activity on its demand and cpu's busy time statistics */

static struct cpu_cycle

static void

update_task_ravg(struct task_struct *p, struct rq *rq, int event,

		 u64 wallclock, u64 irqtime)

{

	struct cpu_cycle cc = { .cycles = SCHED_MIN_FREQ, .time = 1 };



	if (sched_use_pelt || !rq->window_start || sched_disable_window_stats)

		return cc;

		return;



	lockdep_assert_held(&rq->lock);
@@ -2969,18 +2955,16 @@ update_task_ravg(struct task_struct *p, struct rq *rq, int event, 
		goto done;

	}



	cc = get_task_cpu_cycles(p, rq, event, wallclock);

	update_task_demand(p, rq, event, wallclock, &cc);

	update_cpu_busy_time(p, rq, event, wallclock, irqtime, &cc);

	update_task_rq_cpu_cycles(p, rq, event, wallclock);

	update_task_demand(p, rq, event, wallclock);

	update_cpu_busy_time(p, rq, event, wallclock, irqtime);

	update_task_pred_demand(rq, p, event);

done:

	trace_sched_update_task_ravg(p, rq, event, wallclock, irqtime,

				     cc.cycles, cc.time,

				     rq->cc.cycles, rq->cc.time,

				     _group_cpu_time(p->grp, cpu_of(rq)));



	p->ravg.mark_start = wallclock;



	return cc;

}



void sched_account_irqtime(int cpu, struct task_struct *curr,
@@ -3274,7 +3258,6 @@ void sched_get_cpus_busy(struct sched_load *busy, 
	int early_detection[cpus];

	int cpu, i = 0;

	unsigned int window_size;

	struct cpu_cycle cc;

	u64 max_prev_sum = 0;

	int max_busy_cpu = cpumask_first(query_cpus);

	struct related_thread_group *grp;
@@ -3298,9 +3281,9 @@ void sched_get_cpus_busy(struct sched_load *busy, 
	for_each_cpu(cpu, query_cpus) {

		rq = cpu_rq(cpu);



		cc = update_task_ravg(rq->curr, rq, TASK_UPDATE,

				      sched_ktime_clock(), 0);

		cur_freq[i] = cpu_cycles_to_freq(i, cc.cycles, cc.time);

		update_task_ravg(rq->curr, rq, TASK_UPDATE, sched_ktime_clock(),

				 0);

		cur_freq[i] = cpu_cycles_to_freq(i, rq->cc.cycles, rq->cc.time);



		load[i] = rq->old_busy_time = rq->prev_runnable_sum;

		nload[i] = rq->nt_prev_runnable_sum;
@@ -3659,7 +3642,7 @@ static void set_preferred_cluster(struct related_thread_group *grp) 


#ifdef CONFIG_SCHED_FREQ_INPUT



static struct cpu_cycle

static void

update_task_ravg(struct task_struct *p, struct rq *rq,

		 int event, u64 wallclock, u64 irqtime);
@@ -4273,16 +4256,10 @@ static inline int update_preferred_cluster(struct related_thread_group *grp, 


static inline void fixup_busy_time(struct task_struct *p, int new_cpu) { }



static struct cpu_cycle

static void

update_task_ravg(struct task_struct *p, struct rq *rq,

			 int event, u64 wallclock, u64 irqtime)

{

	static const struct cpu_cycle cc = {

		.cycles = SCHED_MIN_FREQ,

		.time = 1

	};



	return cc;

}



static inline void mark_task_starting(struct task_struct *p) {}
@@ -10639,6 +10616,8 @@ void __init sched_init(void) 
		rq->avg_irqload = 0;

		rq->irqload_ts = 0;

		rq->static_cpu_pwr_cost = 0;

		rq->cc.cycles = SCHED_MIN_FREQ;

		rq->cc.time = 1;



		/*

		 * All cpus part of same cluster by default. This avoids the

kernel/sched/sched.h

+6 −0
Original line number Diff line number Diff line
@@ -398,6 +398,11 @@ extern struct sched_cluster *sched_cluster[NR_CPUS]; 
extern int group_will_fit(struct sched_cluster *cluster,

		 struct related_thread_group *grp, u64 demand);



struct cpu_cycle {

	u64 cycles;

	u64 time;

};



#define for_each_sched_cluster(cluster) \

	list_for_each_entry_rcu(cluster, &cluster_head, list)
@@ -711,6 +716,7 @@ struct rq { 
	u64 irqload_ts;

	unsigned int static_cpu_pwr_cost;

	struct task_struct *ed_task;

	struct cpu_cycle cc;



#ifdef CONFIG_SCHED_FREQ_INPUT

	u64 old_busy_time, old_busy_time_group;