Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 1029d12a authored by Srivatsa Vaddagiri's avatar Srivatsa Vaddagiri Committed by Matt Wagantall
Browse files

sched: Fix bug in average nr_running and nr_iowait calculation 



sched_get_nr_running_avg() returns average nr_running and nr_iowait
task count since it was last invoked. Fix several bugs in their
calculation.

* sched_update_nr_prod() needs to consider that nr_running count can
  change by more than 1 when CFS_BANDWIDTH feature is used

* sched_get_nr_running_avg() needs to sum up nr_iowait count across
  all cpus, rather than just one

* sched_get_nr_running_avg() could race with sched_update_nr_prod(),
  as a result of which it could use curr_time which is behind a cpu's
  'last_time' value. That would lead to erroneous calculation of
  average nr_running or nr_iowait.

While at it, fix also a bug in BUG_ON() check in
sched_update_nr_prod() function and remove unnecessary nr_running
argument to sched_update_nr_prod() function.

Change-Id: I46737614737292fae0d7204c4648fb9b862f65b2
Signed-off-by: default avatarSrivatsa Vaddagiri <vatsa@codeaurora.org>
[rameezmustafa@codeaurora.org: Port to msm-3.18]
Signed-off-by: default avatarSyed Rameez Mustafa <rameezmustafa@codeaurora.org>
parent 65a443c5

include/linux/sched.h

+1 −1
Original line number Diff line number Diff line
@@ -173,7 +173,7 @@ extern unsigned long nr_iowait(void); 
extern unsigned long nr_iowait_cpu(int cpu);

extern void get_iowait_load(unsigned long *nr_waiters, unsigned long *load);



extern void sched_update_nr_prod(int cpu, unsigned long nr, bool inc);

extern void sched_update_nr_prod(int cpu, long delta, bool inc);

extern void sched_get_nr_running_avg(int *avg, int *iowait_avg);



extern void calc_global_load(unsigned long ticks);

kernel/sched/sched.h

+2 −2
Original line number Diff line number Diff line
@@ -1539,7 +1539,7 @@ static inline void add_nr_running(struct rq *rq, unsigned count) 
{

	unsigned prev_nr = rq->nr_running;



	sched_update_nr_prod(cpu_of(rq), rq->nr_running, true);

	sched_update_nr_prod(cpu_of(rq), count, true);

	rq->nr_running = prev_nr + count;



	if (prev_nr < 2 && rq->nr_running >= 2) {
@@ -1566,7 +1566,7 @@ static inline void add_nr_running(struct rq *rq, unsigned count) 


static inline void sub_nr_running(struct rq *rq, unsigned count)

{

	sched_update_nr_prod(cpu_of(rq), rq->nr_running, false);

	sched_update_nr_prod(cpu_of(rq), count, false);

	rq->nr_running -= count;

}

kernel/sched/sched_avg.c

+13 −9
Original line number Diff line number Diff line 
/* Copyright (c) 2012, The Linux Foundation. All rights reserved.

/* Copyright (c) 2012, 2015, The Linux Foundation. All rights reserved.

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License version 2 and
@@ -48,16 +48,16 @@ void sched_get_nr_running_avg(int *avg, int *iowait_avg) 
	if (!diff)

		return;



	last_get_time = curr_time;

	/* read and reset nr_running counts */

	for_each_possible_cpu(cpu) {

		unsigned long flags;



		spin_lock_irqsave(&per_cpu(nr_lock, cpu), flags);

		curr_time = sched_clock();

		tmp_avg += per_cpu(nr_prod_sum, cpu);

		tmp_avg += per_cpu(nr, cpu) *

			(curr_time - per_cpu(last_time, cpu));

		tmp_iowait = per_cpu(iowait_prod_sum, cpu);

		tmp_iowait += per_cpu(iowait_prod_sum, cpu);

		tmp_iowait +=  nr_iowait_cpu(cpu) *

			(curr_time - per_cpu(last_time, cpu));

		per_cpu(last_time, cpu) = curr_time;
@@ -66,38 +66,42 @@ void sched_get_nr_running_avg(int *avg, int *iowait_avg) 
		spin_unlock_irqrestore(&per_cpu(nr_lock, cpu), flags);

	}



	diff = curr_time - last_get_time;

	last_get_time = curr_time;



	*avg = (int)div64_u64(tmp_avg * 100, diff);

	*iowait_avg = (int)div64_u64(tmp_iowait * 100, diff);



	BUG_ON(*avg < 0);

	pr_debug("%s - avg:%d\n", __func__, *avg);

	BUG_ON(*iowait_avg < 0);

	pr_debug("%s - avg:%d\n", __func__, *iowait_avg);

	pr_debug("%s - iowait_avg:%d\n", __func__, *iowait_avg);

}

EXPORT_SYMBOL(sched_get_nr_running_avg);



/**

 * sched_update_nr_prod

 * @cpu: The core id of the nr running driver.

 * @nr: Updated nr running value for cpu.

 * @delta: Adjust nr by 'delta' amount

 * @inc: Whether we are increasing or decreasing the count

 * @return: N/A

 *

 * Update average with latest nr_running value for CPU

 */

void sched_update_nr_prod(int cpu, unsigned long nr_running, bool inc)

void sched_update_nr_prod(int cpu, long delta, bool inc)

{

	int diff;

	s64 curr_time;

	unsigned long flags;

	unsigned long flags, nr_running;



	spin_lock_irqsave(&per_cpu(nr_lock, cpu), flags);

	nr_running = per_cpu(nr, cpu);

	curr_time = sched_clock();

	diff = curr_time - per_cpu(last_time, cpu);

	per_cpu(last_time, cpu) = curr_time;

	per_cpu(nr, cpu) = nr_running + (inc ? 1 : -1);

	per_cpu(nr, cpu) = nr_running + (inc ? delta : -delta);



	BUG_ON(per_cpu(nr, cpu) < 0);

	BUG_ON((s64)per_cpu(nr, cpu) < 0);



	per_cpu(nr_prod_sum, cpu) += nr_running * diff;

	per_cpu(iowait_prod_sum, cpu) += nr_iowait_cpu(cpu) * diff;