Merge "sched: Make RAVG_HIST_SIZE tunable" (740bb30f) · Commits · e / devices / android_kernel_sony_msm8994

include/linux/sched.h

+5 −4

Original line number	Diff line number	Diff line
		@@ -1045,7 +1045,7 @@ struct sched_statistics {
		};
		#endif

		#define RAVG_HIST_SIZE 5
		#define RAVG_HIST_SIZE_MAX 5

		/* ravg represents frequency scaled cpu-demand of tasks */
		struct ravg {
		@@ -1061,8 +1061,9 @@ struct ravg {
		* RAVG_HIST_SIZE windows. Windows where task was entirely sleeping are
		* ignored.
		*
		* 'demand' represents maximum sum seen over previous RAVG_HIST_SIZE
		* windows. 'demand' could drive frequency demand for tasks.
		* 'demand' represents maximum sum seen over previous
		* sysctl_sched_ravg_hist_size windows. 'demand' could drive frequency
		* demand for tasks.
		*
		* 'prev_window' is the history in the most recent window. This value
		* may be zero if there was no task activity in that window - that is
		@@ -1074,7 +1075,7 @@ struct ravg {
		*/
		u64 mark_start;
		u32 sum, demand, prev_window, partial_demand, flags;
		u32 sum_history[RAVG_HIST_SIZE];
		u32 sum_history[RAVG_HIST_SIZE_MAX];
		};

		struct sched_entity {

include/linux/sched/sysctl.h

+4 −0

Original line number	Diff line number	Diff line
		@@ -39,6 +39,7 @@ extern unsigned int sysctl_sched_wake_to_idle;
		extern unsigned int sysctl_sched_wakeup_load_threshold;
		extern unsigned int sysctl_sched_window_stats_policy;
		extern unsigned int sysctl_sched_account_wait_time;
		extern unsigned int sysctl_sched_ravg_hist_size;

		#if defined(CONFIG_SCHED_FREQ_INPUT) \|\| defined(CONFIG_SCHED_HMP)
		extern unsigned int sysctl_sched_init_task_load_pct;
		@@ -109,6 +110,9 @@ extern int sched_acct_wait_time_update_handler(struct ctl_table *table,
		extern int sched_window_stats_policy_update_handler(struct ctl_table *table,
		int write, void __user buffer, size_t lenp, loff_t *ppos);

		extern int sched_ravg_hist_size_update_handler(struct ctl_table *table,
		int write, void __user buffer, size_t lenp, loff_t *ppos);

		#ifdef CONFIG_SCHED_DEBUG
		static inline unsigned int get_sysctl_timer_migration(void)
		{

include/trace/events/sched.h

+2 −2

Original line number	Diff line number	Diff line
		@@ -287,7 +287,7 @@ TRACE_EVENT(sched_update_history,
		__field(enum task_event, evt )
		__field(unsigned int, partial_demand )
		__field(unsigned int, demand )
		__array( u32, hist, RAVG_HIST_SIZE )
		__array( u32, hist, RAVG_HIST_SIZE_MAX)
		__field(unsigned int, nr_big_tasks )
		__field(unsigned int, nr_small_tasks )
		__field( int, cpu )
		@@ -304,7 +304,7 @@ TRACE_EVENT(sched_update_history,
		__entry->partial_demand = p->ravg.partial_demand;
		__entry->demand = p->ravg.demand;
		memcpy(__entry->hist, p->ravg.sum_history,
		RAVG_HIST_SIZE*sizeof(u32));
		RAVG_HIST_SIZE_MAX * sizeof(u32));
		#ifdef CONFIG_SCHED_HMP
		__entry->nr_big_tasks = rq->nr_big_tasks;
		__entry->nr_small_tasks = rq->nr_small_tasks;

kernel/sched/core.c

+32 −13

Original line number	Diff line number	Diff line
		@@ -1103,6 +1103,17 @@ static inline void clear_hmp_request(int cpu) { }

		#if defined(CONFIG_SCHED_FREQ_INPUT) \|\| defined(CONFIG_SCHED_HMP)

		__read_mostly unsigned int sysctl_sched_ravg_hist_size = 5;

		/*
		* copy of sysctl_sched_ravg_hist_size. Required for atomically
		* changing the ravg history size (see sched_ravg_hist_size_update_handler()
		* for details).
		*
		* Initialize both to same value!!
		*/
		static __read_mostly unsigned int sched_ravg_hist_size = 5;

		/* Window size (in ns) */
		__read_mostly unsigned int sched_ravg_window = 10000000;

		@@ -1225,7 +1236,7 @@ update_history(struct rq rq, struct task_struct p, u32 runtime, int samples,
		return;

		if (!new_window) {
		for (ridx = 0; ridx < RAVG_HIST_SIZE - 1; ++ridx) {
		for (ridx = 0; ridx < sched_ravg_hist_size - 1; ++ridx) {
		sum += hist[ridx];
		if (hist[ridx] > max)
		max = hist[ridx];
		@@ -1237,21 +1248,25 @@ update_history(struct rq rq, struct task_struct p, u32 runtime, int samples,
		}

		/* Push new 'runtime' value onto stack */
		widx = RAVG_HIST_SIZE - 1;
		widx = RAVG_HIST_SIZE_MAX - 1;
		ridx = widx - samples;
		for (; ridx >= 0; --widx, --ridx) {
		hist[widx] = hist[ridx];
		if (widx < sched_ravg_hist_size) {
		sum += hist[widx];
		if (hist[widx] > max)
		max = hist[widx];
		}
		}

		for (widx = 0; widx < samples && widx < RAVG_HIST_SIZE; widx++) {
		for (widx = 0; widx < samples && widx < RAVG_HIST_SIZE_MAX; widx++) {
		hist[widx] = runtime;
		if (widx < sched_ravg_hist_size) {
		sum += hist[widx];
		if (hist[widx] > max)
		max = hist[widx];
		}
		}

		p->ravg.sum = 0;
		if (p->on_rq) {
		@@ -1262,7 +1277,7 @@ update_history(struct rq rq, struct task_struct p, u32 runtime, int samples,
		}

		compute_demand:
		avg = div64_u64(sum, RAVG_HIST_SIZE);
		avg = div64_u64(sum, sched_ravg_hist_size);

		if (sched_window_stats_policy == WINDOW_STATS_USE_RECENT)
		demand = runtime;
		@@ -1630,7 +1645,8 @@ unsigned long sched_get_busy(int cpu)

		/* Called with IRQs disabled */
		void reset_all_window_stats(u64 window_start, unsigned int window_size,
		int policy, int acct_wait_time)
		int policy, int acct_wait_time,
		unsigned int ravg_hist_size)
		{
		int cpu;
		u64 wallclock;
		@@ -1657,7 +1673,7 @@ void reset_all_window_stats(u64 window_start, unsigned int window_size,
		p->ravg.partial_demand = 0;
		p->ravg.prev_window = 0;
		p->ravg.flags = 0;
		for (i = 0; i < RAVG_HIST_SIZE; ++i)
		for (i = 0; i < RAVG_HIST_SIZE_MAX; ++i)
		p->ravg.sum_history[i] = 0;
		p->ravg.mark_start = wallclock;
		} while_each_thread(g, p);
		@@ -1679,6 +1695,9 @@ void reset_all_window_stats(u64 window_start, unsigned int window_size,
		if (acct_wait_time >= 0)
		sched_account_wait_time = acct_wait_time;

		if (ravg_hist_size > 0)
		sched_ravg_hist_size = ravg_hist_size;

		for_each_online_cpu(cpu) {
		struct rq *rq = cpu_rq(cpu);
		raw_spin_unlock(&rq->lock);
		@@ -1716,7 +1735,7 @@ int sched_set_window(u64 window_start, unsigned int window_size)

		BUG_ON(sched_clock() < ws);

		reset_all_window_stats(ws, window_size, -1, -1);
		reset_all_window_stats(ws, window_size, -1, -1, 0);

		local_irq_restore(flags);

kernel/sched/fair.c

+41 −3

Original line number	Diff line number	Diff line
		@@ -1910,7 +1910,45 @@ int sched_acct_wait_time_update_handler(struct ctl_table *table, int write,

		local_irq_save(flags);

		reset_all_window_stats(0, 0, -1, sysctl_sched_account_wait_time);
		reset_all_window_stats(0, 0, -1, sysctl_sched_account_wait_time, 0);

		local_irq_restore(flags);

		done:
		mutex_unlock(&policy_mutex);

		return ret;
		}

		int sched_ravg_hist_size_update_handler(struct ctl_table *table, int write,
		void __user buffer, size_t lenp,
		loff_t *ppos)
		{
		int ret;
		unsigned int data = (unsigned int )table->data;
		unsigned int old_val;
		unsigned long flags;

		if (!sched_enable_hmp)
		return -EINVAL;

		mutex_lock(&policy_mutex);

		old_val = *data;

		ret = proc_dointvec(table, write, buffer, lenp, ppos);
		if (ret \|\| !write \|\| (write && (old_val == *data)))
		goto done;

		if (data > RAVG_HIST_SIZE_MAX \|\| data < 1) {
		*data = old_val;
		ret = -EINVAL;
		goto done;
		}

		local_irq_save(flags);

		reset_all_window_stats(0, 0, -1, -1, sysctl_sched_ravg_hist_size);

		local_irq_restore(flags);

		@@ -1942,7 +1980,7 @@ int sched_window_stats_policy_update_handler(struct ctl_table *table, int write,

		local_irq_save(flags);

		reset_all_window_stats(0, 0, sysctl_sched_window_stats_policy, -1);
		reset_all_window_stats(0, 0, sysctl_sched_window_stats_policy, -1, 0);

		local_irq_restore(flags);

		@@ -2240,7 +2278,7 @@ void init_new_task_load(struct task_struct *p)
		p->ravg.sum = 0;
		p->ravg.flags = 0;

		for (i = 0; i < RAVG_HIST_SIZE; ++i)
		for (i = 0; i < RAVG_HIST_SIZE_MAX; ++i)
		p->ravg.sum_history[i] = sched_init_task_load_windows;
		p->se.avg.runnable_avg_period =
		sysctl_sched_init_task_load_pct ? LOAD_AVG_MAX : 0;