Merge "sched: Fix bug in average nr_running and nr_iowait calculation" (37b4f543) · Commits · e / devices / android_kernel_sony_msm8994

include/linux/sched.h

+1 −1

Original line number	Diff line number	Diff line
		@@ -163,7 +163,7 @@ extern unsigned long nr_iowait(void);
		extern unsigned long nr_iowait_cpu(int cpu);
		extern unsigned long this_cpu_load(void);

		extern void sched_update_nr_prod(int cpu, unsigned long nr, bool inc);
		extern void sched_update_nr_prod(int cpu, long delta, bool inc);
		extern void sched_get_nr_running_avg(int avg, int iowait_avg);

		extern void calc_global_load(unsigned long ticks);

kernel/sched/fair.c

+2 −0

Original line number	Diff line number	Diff line
		@@ -4157,6 +4157,7 @@ static void throttle_cfs_rq(struct cfs_rq *cfs_rq)
		}

		if (!se) {
		sched_update_nr_prod(cpu_of(rq), task_delta, false);
		rq->nr_running -= task_delta;
		dec_throttled_cfs_rq_hmp_stats(&rq->hmp_stats, cfs_rq);
		}
		@@ -4217,6 +4218,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
		}

		if (!se) {
		sched_update_nr_prod(cpu_of(rq), task_delta, true);
		rq->nr_running += task_delta;
		inc_throttled_cfs_rq_hmp_stats(&rq->hmp_stats, tcfs_rq);
		}

kernel/sched/sched.h

+2 −2

Original line number	Diff line number	Diff line
		@@ -1381,7 +1381,7 @@ static inline u64 steal_ticks(u64 steal)

		static inline void inc_nr_running(struct rq *rq)
		{
		sched_update_nr_prod(cpu_of(rq), rq->nr_running, true);
		sched_update_nr_prod(cpu_of(rq), 1, true);
		rq->nr_running++;

		#ifdef CONFIG_NO_HZ_FULL
		@@ -1397,7 +1397,7 @@ static inline void inc_nr_running(struct rq *rq)

		static inline void dec_nr_running(struct rq *rq)
		{
		sched_update_nr_prod(cpu_of(rq), rq->nr_running, false);
		sched_update_nr_prod(cpu_of(rq), 1, false);
		rq->nr_running--;
		}

kernel/sched/sched_avg.c

+13 −9

Original line number	Diff line number	Diff line
		/* Copyright (c) 2012, The Linux Foundation. All rights reserved.
		/* Copyright (c) 2012, 2015, The Linux Foundation. All rights reserved.
		*
		* This program is free software; you can redistribute it and/or modify
		* it under the terms of the GNU General Public License version 2 and
		@@ -48,16 +48,16 @@ void sched_get_nr_running_avg(int avg, int iowait_avg)
		if (!diff)
		return;

		last_get_time = curr_time;
		/* read and reset nr_running counts */
		for_each_possible_cpu(cpu) {
		unsigned long flags;

		spin_lock_irqsave(&per_cpu(nr_lock, cpu), flags);
		curr_time = sched_clock();
		tmp_avg += per_cpu(nr_prod_sum, cpu);
		tmp_avg += per_cpu(nr, cpu) *
		(curr_time - per_cpu(last_time, cpu));
		tmp_iowait = per_cpu(iowait_prod_sum, cpu);
		tmp_iowait += per_cpu(iowait_prod_sum, cpu);
		tmp_iowait += nr_iowait_cpu(cpu) *
		(curr_time - per_cpu(last_time, cpu));
		per_cpu(last_time, cpu) = curr_time;
		@@ -66,38 +66,42 @@ void sched_get_nr_running_avg(int avg, int iowait_avg)
		spin_unlock_irqrestore(&per_cpu(nr_lock, cpu), flags);
		}

		diff = curr_time - last_get_time;
		last_get_time = curr_time;

		avg = (int)div64_u64(tmp_avg 100, diff);
		iowait_avg = (int)div64_u64(tmp_iowait 100, diff);

		BUG_ON(*avg < 0);
		pr_debug("%s - avg:%d\n", __func__, *avg);
		BUG_ON(*iowait_avg < 0);
		pr_debug("%s - avg:%d\n", __func__, *iowait_avg);
		pr_debug("%s - iowait_avg:%d\n", __func__, *iowait_avg);
		}
		EXPORT_SYMBOL(sched_get_nr_running_avg);

		/**
		* sched_update_nr_prod
		* @cpu: The core id of the nr running driver.
		* @nr: Updated nr running value for cpu.
		* @delta: Adjust nr by 'delta' amount
		* @inc: Whether we are increasing or decreasing the count
		* @return: N/A
		*
		* Update average with latest nr_running value for CPU
		*/
		void sched_update_nr_prod(int cpu, unsigned long nr_running, bool inc)
		void sched_update_nr_prod(int cpu, long delta, bool inc)
		{
		int diff;
		s64 curr_time;
		unsigned long flags;
		unsigned long flags, nr_running;

		spin_lock_irqsave(&per_cpu(nr_lock, cpu), flags);
		nr_running = per_cpu(nr, cpu);
		curr_time = sched_clock();
		diff = curr_time - per_cpu(last_time, cpu);
		per_cpu(last_time, cpu) = curr_time;
		per_cpu(nr, cpu) = nr_running + (inc ? 1 : -1);
		per_cpu(nr, cpu) = nr_running + (inc ? delta : -delta);

		BUG_ON(per_cpu(nr, cpu) < 0);
		BUG_ON((s64)per_cpu(nr, cpu) < 0);

		per_cpu(nr_prod_sum, cpu) += nr_running * diff;
		per_cpu(iowait_prod_sum, cpu) += nr_iowait_cpu(cpu) * diff;