Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 723dab78 authored by Vincent Guittot's avatar Vincent Guittot Committed by Andres Oportus
Browse files

BACKPORT: sched/fair: Factorize PELT update 



Every time we modify load/utilization of sched_entity, we start to
sync it with its cfs_rq. This update is done in different ways:

 - when attaching/detaching a sched_entity, we update cfs_rq and then
   we sync the entity with the cfs_rq.

 - when enqueueing/dequeuing the sched_entity, we update both
   sched_entity and cfs_rq metrics to now.

Use update_load_avg() everytime we have to update and sync cfs_rq and
sched_entity before changing the state of a sched_enity.

Change-Id: Ibde9a7e07ac80e9d5753bb4a0c30dfb3643cc666
Signed-off-by: default avatarVincent Guittot <vincent.guittot@linaro.org>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: default avatarDietmar Eggemann <dietmar.eggemann@arm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Morten.Rasmussen@arm.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: bsegall@google.com
Cc: kernellwp@gmail.com
Cc: pjt@google.com
Cc: yuyang.du@intel.com
Link: http://lkml.kernel.org/r/1478598827-32372-4-git-send-email-vincent.guittot@linaro.org


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
[backported FROMLIST]
Signed-off-by: default avatarAndres Oportus <andresoportus@google.com>
(cherry picked from commit d31b1a66cbe0931733583ad9d9e8c6cfd710907d)
Signed-off-by: default avatarChris Redpath <chris.redpath@arm.com>
parent 18d09a45

kernel/sched/fair.c

+28 −50
Original line number Diff line number Diff line
@@ -2916,8 +2916,14 @@ update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq, bool update_freq) 
	return decayed || removed;

}



/*

 * Optional action to be done while updating the load average

 */

#define UPDATE_TG	0x1

#define SKIP_AGE_LOAD	0x2



/* Update task and its cfs_rq load average */

static inline void update_load_avg(struct sched_entity *se, int update_tg)

static inline void update_load_avg(struct sched_entity *se, int flags)

{

	struct cfs_rq *cfs_rq = cfs_rq_of(se);

	u64 now = cfs_rq_clock_task(cfs_rq);
@@ -2927,11 +2933,13 @@ static inline void update_load_avg(struct sched_entity *se, int update_tg) 
	 * Track task load average for carrying it to new CPU after migrated, and

	 * track group sched_entity load average for task_h_load calc in migration

	 */

	if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD)) {

		__update_load_avg(now, cpu, &se->avg,

			  se->on_rq * scale_load_down(se->load.weight),

			  cfs_rq->curr == se, NULL);

	}



	if (update_cfs_rq_load_avg(now, cfs_rq, true) && update_tg)

	if (update_cfs_rq_load_avg(now, cfs_rq, true) && (flags & UPDATE_TG))

		update_tg_load_avg(cfs_rq, 0);



	if (entity_is_task(se))
@@ -2948,24 +2956,6 @@ static inline void update_load_avg(struct sched_entity *se, int update_tg) 
 */

static void attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)

{

	if (!sched_feat(ATTACH_AGE_LOAD))

		goto skip_aging;



	/*

	 * If we got migrated (either between CPUs or between cgroups) we'll

	 * have aged the average right before clearing @last_update_time.

	 */

	if (se->avg.last_update_time) {

		__update_load_avg(cfs_rq->avg.last_update_time, cpu_of(rq_of(cfs_rq)),

				  &se->avg, 0, 0, NULL);



		/*

		 * XXX: we could have just aged the entire load away if we've been

		 * absent from the fair class for too long.

		 */

	}



skip_aging:

	se->avg.last_update_time = cfs_rq->avg.last_update_time;

	cfs_rq->avg.load_avg += se->avg.load_avg;

	cfs_rq->avg.load_sum += se->avg.load_sum;
@@ -2985,9 +2975,6 @@ skip_aging: 
 */

static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)

{

	__update_load_avg(cfs_rq->avg.last_update_time, cpu_of(rq_of(cfs_rq)),

			  &se->avg, se->on_rq * scale_load_down(se->load.weight),

			  cfs_rq->curr == se, NULL);



	sub_positive(&cfs_rq->avg.load_avg, se->avg.load_avg);

	sub_positive(&cfs_rq->avg.load_sum, se->avg.load_sum);
@@ -3002,34 +2989,20 @@ static inline void 
enqueue_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)

{

	struct sched_avg *sa = &se->avg;

	u64 now = cfs_rq_clock_task(cfs_rq);

	int migrated, decayed;



	migrated = !sa->last_update_time;

	if (!migrated) {

		__update_load_avg(now, cpu_of(rq_of(cfs_rq)), sa,

			se->on_rq * scale_load_down(se->load.weight),

			cfs_rq->curr == se, NULL);

	}



	decayed = update_cfs_rq_load_avg(now, cfs_rq, !migrated);



	cfs_rq->runnable_load_avg += sa->load_avg;

	cfs_rq->runnable_load_sum += sa->load_sum;



	if (migrated)

	if (!sa->last_update_time) {

		attach_entity_load_avg(cfs_rq, se);



	if (decayed || migrated)

		update_tg_load_avg(cfs_rq, 0);

	}

}



/* Remove the runnable load generated by se from cfs_rq's runnable load average */

static inline void

dequeue_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)

{

	update_load_avg(se, 1);



	cfs_rq->runnable_load_avg =

		max_t(long, cfs_rq->runnable_load_avg - se->avg.load_avg, 0);

	cfs_rq->runnable_load_sum =
@@ -3122,11 +3095,16 @@ static int idle_balance(struct rq *this_rq); 


#else /* CONFIG_SMP */



static inline void update_load_avg(struct sched_entity *se, int update_tg)

static inline int

update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq, bool update_freq)

{

	cpufreq_update_util(rq_of(cfs_rq_of(se)), 0);

	return 0;

}



#define UPDATE_TG	0x0

#define SKIP_AGE_LOAD	0x0



static inline void update_load_avg(struct sched_entity *se, int not_used1){}

static inline void

enqueue_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {}

static inline void
@@ -3269,6 +3247,7 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 
	 * Update run-time statistics of the 'current'.

	 */

	update_curr(cfs_rq);

	update_load_avg(se, UPDATE_TG);

	enqueue_entity_load_avg(cfs_rq, se);

	account_entity_enqueue(cfs_rq, se);

	update_cfs_shares(cfs_rq);
@@ -3344,6 +3323,7 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 
	 * Update run-time statistics of the 'current'.

	 */

	update_curr(cfs_rq);

	update_load_avg(se, UPDATE_TG);

	dequeue_entity_load_avg(cfs_rq, se);



	update_stats_dequeue(cfs_rq, se);
@@ -3434,7 +3414,7 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 
		 */

		update_stats_wait_end(cfs_rq, se);

		__dequeue_entity(cfs_rq, se);

		update_load_avg(se, 1);

		update_load_avg(se, UPDATE_TG);

	}



	update_stats_curr_start(cfs_rq, se);
@@ -3550,7 +3530,7 @@ entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued) 
	/*

	 * Ensure that runnable average is periodically updated.

	 */

	update_load_avg(curr, 1);

	update_load_avg(curr, UPDATE_TG);

	update_cfs_shares(cfs_rq);



#ifdef CONFIG_SCHED_HRTICK
@@ -4479,7 +4459,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) 
		if (cfs_rq_throttled(cfs_rq))

			break;



		update_load_avg(se, 1);

		update_load_avg(se, UPDATE_TG);

		update_cfs_shares(cfs_rq);

	}
@@ -4581,7 +4561,7 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) 
		if (cfs_rq_throttled(cfs_rq))

			break;



		update_load_avg(se, 1);

		update_load_avg(se, UPDATE_TG);

		update_cfs_shares(cfs_rq);

	}
@@ -9391,10 +9371,9 @@ static inline bool vruntime_normalized(struct task_struct *p) 
static void detach_entity_cfs_rq(struct sched_entity *se)

{

	struct cfs_rq *cfs_rq = cfs_rq_of(se);

	u64 now = cfs_rq_clock_task(cfs_rq);



	/* Catch up with the cfs_rq and remove our load when we leave */

	update_cfs_rq_load_avg(now, cfs_rq, false);

	update_load_avg(se, 0);

	detach_entity_load_avg(cfs_rq, se);

	update_tg_load_avg(cfs_rq, false);

}
@@ -9402,7 +9381,6 @@ static void detach_entity_cfs_rq(struct sched_entity *se) 
static void attach_entity_cfs_rq(struct sched_entity *se)

{

	struct cfs_rq *cfs_rq = cfs_rq_of(se);

	u64 now = cfs_rq_clock_task(cfs_rq);



#ifdef CONFIG_FAIR_GROUP_SCHED

	/*
@@ -9413,7 +9391,7 @@ static void attach_entity_cfs_rq(struct sched_entity *se) 
#endif



	/* Synchronize entity with its cfs_rq */

	update_cfs_rq_load_avg(now, cfs_rq, false);

	update_load_avg(se, sched_feat(ATTACH_AGE_LOAD) ? 0 : SKIP_AGE_LOAD);

	attach_entity_load_avg(cfs_rq, se);

	update_tg_load_avg(cfs_rq, false);

}