Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit d556a8f2 authored by Linux Build Service Account's avatar Linux Build Service Account Committed by Gerrit - the friendly Code Review server
Browse files

Merge changes Ifbdd3ecc,Ieca413c1,Id2ba91cc,Ib83207b9 into msm-3.10 

* changes:
  cpufreq: cpu-boost: Handle wakeup hints received for foreground tasks
  sched: Call the notify_on_migrate notifier chain for wakeups as well
  cpufreq: cpu-boost: Introduce scheduler assisted load based syncs
  sched: window-based load stats for tasks
parents 0a7bf3e4 c09decae

drivers/cpufreq/cpu-boost.c

+35 −11
Original line number Diff line number Diff line
@@ -37,6 +37,7 @@ struct cpu_sync { 
	int src_cpu;

	unsigned int boost_min;

	unsigned int input_boost_min;

	unsigned int task_load;

};



static DEFINE_PER_CPU(struct cpu_sync, sync_info);
@@ -56,6 +57,12 @@ module_param(input_boost_freq, uint, 0644); 
static unsigned int input_boost_ms = 40;

module_param(input_boost_ms, uint, 0644);



static unsigned int migration_load_threshold = 15;

module_param(migration_load_threshold, uint, 0644);



static bool load_based_syncs;

module_param(load_based_syncs, bool, 0644);



static u64 last_input_time;

#define MIN_INPUT_INTERVAL (150 * USEC_PER_MSEC)
@@ -139,6 +146,7 @@ static int boost_mig_sync_thread(void *data) 
	struct cpufreq_policy dest_policy;

	struct cpufreq_policy src_policy;

	unsigned long flags;

	unsigned int req_freq;



	while (1) {

		wait_event(s->sync_wq, s->pending || kthread_should_stop());
@@ -159,17 +167,20 @@ static int boost_mig_sync_thread(void *data) 
		if (ret)

			continue;



		if (src_policy.cur == src_policy.cpuinfo.min_freq) {

			pr_debug("No sync. Source CPU%d@%dKHz at min freq\n",

				 src_cpu, src_policy.cur);

		req_freq = max((dest_policy.max * s->task_load) / 100,

							src_policy.cur);



		if (req_freq <= dest_policy.cpuinfo.min_freq) {

			pr_debug("No sync. Sync Freq:%u\n", req_freq);

			continue;

		}



		cancel_delayed_work_sync(&s->boost_rem);

		if (sync_threshold)

			s->boost_min = min(sync_threshold, src_policy.cur);

		else

			s->boost_min = src_policy.cur;

			req_freq = min(sync_threshold, req_freq);



		cancel_delayed_work_sync(&s->boost_rem);



		s->boost_min = req_freq;



		/* Force policy re-evaluation to trigger adjust notifier. */

		get_online_cpus();
@@ -198,10 +209,22 @@ static int boost_mig_sync_thread(void *data) 
}



static int boost_migration_notify(struct notifier_block *nb,

				unsigned long dest_cpu, void *arg)

				unsigned long unused, void *arg)

{

	struct migration_notify_data *mnd = arg;

	unsigned long flags;

	struct cpu_sync *s = &per_cpu(sync_info, dest_cpu);

	struct cpu_sync *s = &per_cpu(sync_info, mnd->dest_cpu);



	if (load_based_syncs && (mnd->load <= migration_load_threshold))

		return NOTIFY_OK;



	if (load_based_syncs && ((mnd->load < 0) || (mnd->load > 100))) {

		pr_err("cpu-boost:Invalid load: %d\n", mnd->load);

		return NOTIFY_OK;

	}



	if (!load_based_syncs && (mnd->src_cpu == mnd->dest_cpu))

		return NOTIFY_OK;



	if (!boost_ms)

		return NOTIFY_OK;
@@ -210,10 +233,11 @@ static int boost_migration_notify(struct notifier_block *nb, 
	if (s->thread == current)

		return NOTIFY_OK;



	pr_debug("Migration: CPU%d --> CPU%d\n", (int) arg, (int) dest_cpu);

	pr_debug("Migration: CPU%d --> CPU%d\n", mnd->src_cpu, mnd->dest_cpu);

	spin_lock_irqsave(&s->lock, flags);

	s->pending = true;

	s->src_cpu = (int) arg;

	s->src_cpu = mnd->src_cpu;

	s->task_load = load_based_syncs ? mnd->load : 0;

	spin_unlock_irqrestore(&s->lock, flags);

	wake_up(&s->sync_wq);

include/linux/sched.h

+36 −0
Original line number Diff line number Diff line
@@ -969,8 +969,39 @@ struct sched_statistics { 
};

#endif



#define RAVG_HIST_SIZE  5



/* ravg represents frequency scaled cpu-demand of tasks */

struct ravg {

	/*

	 * 'window_start' marks the beginning of new window

	 *

	 * 'mark_start' marks the beginning of an event (task waking up, task

	 * starting to execute, task being preempted) within a window

	 *

	 * 'sum' represents how runnable a task has been within current

	 * window. It incorporates both running time and wait time and is

	 * frequency scaled.

	 *

	 * 'sum_history' keeps track of history of 'sum' seen over previous

	 * RAVG_HIST_SIZE windows. Windows where task was entirely sleeping are

	 * ignored.

	 *

	 * 'demand' represents maximum sum seen over previous RAVG_HIST_SIZE

	 * windows. 'demand' could drive frequency demand for tasks.

	 */

	u64 window_start, mark_start;

	u32 sum, demand;

	u32 sum_history[RAVG_HIST_SIZE];

};



struct sched_entity {

	struct load_weight	load;		/* for load-balancing */

	/*

	 * Todo : Move ravg to 'struct task_struct', as this is common for both

	 * real-time and non-realtime tasks

	 */

	struct ravg		ravg;

	struct rb_node		run_node;

	struct list_head	group_node;

	unsigned int		on_rq;
@@ -2568,6 +2599,11 @@ static inline void set_task_cpu(struct task_struct *p, unsigned int cpu) 
#endif /* CONFIG_SMP */



extern struct atomic_notifier_head migration_notifier_head;

struct migration_notify_data {

	int src_cpu;

	int dest_cpu;

	int load;

};



extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask);

extern long sched_getaffinity(pid_t pid, struct cpumask *mask);

include/linux/sched/sysctl.h

+2 −0
Original line number Diff line number Diff line
@@ -36,6 +36,8 @@ extern unsigned int sysctl_sched_min_granularity; 
extern unsigned int sysctl_sched_wakeup_granularity;

extern unsigned int sysctl_sched_child_runs_first;

extern unsigned int sysctl_sched_wake_to_idle;

extern unsigned int sysctl_sched_ravg_window;

extern unsigned int sysctl_sched_wakeup_load_threshold;



enum sched_tunable_scaling {

	SCHED_TUNABLESCALING_NONE,

kernel/sched/core.c

+161 −8
Original line number Diff line number Diff line
@@ -784,6 +784,7 @@ static void enqueue_task(struct rq *rq, struct task_struct *p, int flags) 
	sched_info_queued(p);

	p->sched_class->enqueue_task(rq, p, flags);

	trace_sched_enq_deq_task(p, 1);

	rq->cumulative_runnable_avg += p->se.ravg.demand;

}



static void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
@@ -792,6 +793,8 @@ static void dequeue_task(struct rq *rq, struct task_struct *p, int flags) 
	sched_info_dequeued(p);

	p->sched_class->dequeue_task(rq, p, flags);

	trace_sched_enq_deq_task(p, 0);

	rq->cumulative_runnable_avg -= p->se.ravg.demand;

	BUG_ON((s64)rq->cumulative_runnable_avg < 0);

}



void activate_task(struct rq *rq, struct task_struct *p, int flags)
@@ -1343,6 +1346,110 @@ static void ttwu_activate(struct rq *rq, struct task_struct *p, int en_flags) 
		wq_worker_waking_up(p, cpu_of(rq));

}



/*

 * Called when new window is starting for a task, to record cpu usage over

 * recently concluded window(s). Normally 'samples' should be 1. It can be > 1

 * when, say, a real-time task runs without preemption for several windows at a

 * stretch.

 */

static inline void

update_history(struct rq *rq, struct task_struct *p, u32 runtime, int samples)

{

	u32 *hist = &p->se.ravg.sum_history[0];

	int ridx, widx;

	u32 max = 0;



	/* Ignore windows where task had no activity */

	if (!runtime)

		return;



	/* Push new 'runtime' value onto stack */

	widx = RAVG_HIST_SIZE - 1;

	ridx = widx - samples;

	for (; ridx >= 0; --widx, --ridx) {

		hist[widx] = hist[ridx];

		if  (hist[widx] > max)

			max = hist[widx];

	}



	for (widx = 0; widx < samples && widx < RAVG_HIST_SIZE; widx++) {

		hist[widx] = runtime;

		if  (hist[widx] > max)

			max = hist[widx];

	}



	p->se.ravg.sum = 0;

	if (p->on_rq) {

		rq->cumulative_runnable_avg -= p->se.ravg.demand;

		BUG_ON((s64)rq->cumulative_runnable_avg < 0);

	}

	/*

	 * Maximum demand seen over previous RAVG_HIST_SIZE windows drives

	 * frequency demand for a task. Record maximum in 'demand' attribute.

	 */

	p->se.ravg.demand = max;

	if (p->on_rq)

		rq->cumulative_runnable_avg += p->se.ravg.demand;

}



/* Window size (in ns) */

__read_mostly unsigned int sysctl_sched_ravg_window = 50000000;



void update_task_ravg(struct task_struct *p, struct rq *rq, int update_sum)

{

	u32 window_size = sysctl_sched_ravg_window;

	int new_window;

	u64 wallclock = sched_clock();



	do {

		s64 delta = 0;

		int n;

		u64 now = wallclock;



		new_window = 0;

		delta = now - p->se.ravg.window_start;

		BUG_ON(delta < 0);

		if (delta > window_size) {

			p->se.ravg.window_start += window_size;

			now = p->se.ravg.window_start;

			new_window = 1;

		}



		if (update_sum) {

			delta = now - p->se.ravg.mark_start;

			BUG_ON(delta < 0);



			if (likely(rq->cur_freq &&

					rq->cur_freq <= max_possible_freq))

				delta = div64_u64(delta  * rq->cur_freq,

							max_possible_freq);

			p->se.ravg.sum += delta;

			WARN_ON(p->se.ravg.sum > window_size);

		}



		if (!new_window)

			break;



		update_history(rq, p, p->se.ravg.sum, 1);



		delta = wallclock - p->se.ravg.window_start;

		BUG_ON(delta < 0);

		n = div64_u64(delta, window_size);

		if (n) {

			if (!update_sum)

				p->se.ravg.window_start = wallclock;

			else

				p->se.ravg.window_start += n * window_size;

			BUG_ON(p->se.ravg.window_start > wallclock);

			if (update_sum)

				update_history(rq, p, window_size, n);

		}

		p->se.ravg.mark_start =  p->se.ravg.window_start;

	} while (new_window);



	p->se.ravg.mark_start = wallclock;

}



/*

 * Mark the task runnable and perform wakeup-preemption.

 */
@@ -1352,6 +1459,7 @@ ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags) 
	check_preempt_curr(rq, p, wake_flags);

	trace_sched_wakeup(p, true);



	update_task_ravg(p, rq, 0);

	p->state = TASK_RUNNING;

#ifdef CONFIG_SMP

	if (p->sched_class->task_woken)
@@ -1484,6 +1592,7 @@ static void ttwu_queue(struct task_struct *p, int cpu) 
	raw_spin_unlock(&rq->lock);

}



__read_mostly unsigned int sysctl_sched_wakeup_load_threshold = 60;

/**

 * try_to_wake_up - wake up a thread

 * @p: the thread to be awakened
@@ -1550,9 +1659,28 @@ stat: 
out:

	raw_spin_unlock_irqrestore(&p->pi_lock, flags);



	if (src_cpu != cpu && task_notify_on_migrate(p))

	if (task_notify_on_migrate(p)) {

		struct migration_notify_data mnd;



		mnd.src_cpu = src_cpu;

		mnd.dest_cpu = cpu;

		if (sysctl_sched_ravg_window)

			mnd.load = div64_u64((u64)p->se.ravg.demand * 100,

				(u64)(sysctl_sched_ravg_window));

		else

			mnd.load = 0;

		/*

		 * Call the migration notifier with mnd for foreground task

		 * migrations as well as for wakeups if their load is above

		 * sysctl_sched_wakeup_load_threshold. This would prompt the

		 * cpu-boost to boost the CPU frequency on wake up of a heavy

		 * weight foreground task

		 */

		if ((src_cpu != cpu) || (mnd.load >

					sysctl_sched_wakeup_load_threshold))

			atomic_notifier_call_chain(&migration_notifier_head,

					   cpu, (void *)src_cpu);

					   0, (void *)&mnd);

	}

	return success;

}
@@ -1626,6 +1754,8 @@ int wake_up_state(struct task_struct *p, unsigned int state) 
 */

static void __sched_fork(struct task_struct *p)

{

	int i;



	p->on_rq			= 0;



	p->se.on_rq			= 0;
@@ -1634,6 +1764,13 @@ static void __sched_fork(struct task_struct *p) 
	p->se.prev_sum_exec_runtime	= 0;

	p->se.nr_migrations		= 0;

	p->se.vruntime			= 0;

	p->se.ravg.sum			= 0;

	p->se.ravg.demand		= 0;

	p->se.ravg.window_start		= 0;

	p->se.ravg.mark_start		= 0;

	for (i = 0; i < RAVG_HIST_SIZE; ++i)

		p->se.ravg.sum_history[i] = 0;



	INIT_LIST_HEAD(&p->se.group_node);



/*
@@ -1777,6 +1914,7 @@ void wake_up_new_task(struct task_struct *p) 
{

	unsigned long flags;

	struct rq *rq;

	u64 wallclock = sched_clock();



	raw_spin_lock_irqsave(&p->pi_lock, flags);

#ifdef CONFIG_SMP
@@ -1790,6 +1928,8 @@ void wake_up_new_task(struct task_struct *p) 


	rq = __task_rq_lock(p);

	activate_task(rq, p, 0);

	p->se.ravg.window_start	= wallclock;

	p->se.ravg.mark_start	= wallclock;

	p->on_rq = 1;

	trace_sched_wakeup_new(p, true);

	check_preempt_curr(rq, p, WF_FORK);
@@ -2904,6 +3044,7 @@ static inline void schedule_debug(struct task_struct *prev) 


static void put_prev_task(struct rq *rq, struct task_struct *prev)

{

	update_task_ravg(prev, rq, 1);

	if (prev->on_rq || rq->skip_clock_update < 0)

		update_rq_clock(rq);

	prev->sched_class->put_prev_task(rq, prev);
@@ -2924,15 +3065,19 @@ pick_next_task(struct rq *rq) 
	 */

	if (likely(rq->nr_running == rq->cfs.h_nr_running)) {

		p = fair_sched_class.pick_next_task(rq);

		if (likely(p))

		if (likely(p)) {

			update_task_ravg(p, rq, 1);

			return p;

		}

	}



	for_each_class(class) {

		p = class->pick_next_task(rq);

		if (p)

		if (p) {

			update_task_ravg(p, rq, 1);

			return p;

		}

	}



	BUG(); /* the idle class will always have a runnable task */

}
@@ -4913,10 +5058,17 @@ fail: 
	double_rq_unlock(rq_src, rq_dest);

	raw_spin_unlock(&p->pi_lock);

	if (moved && task_notify_on_migrate(p)) {

		unsigned long _src_cpu;

		_src_cpu = src_cpu;

		struct migration_notify_data mnd;



		mnd.src_cpu = src_cpu;

		mnd.dest_cpu = dest_cpu;

		if (sysctl_sched_ravg_window)

			mnd.load = div64_u64((u64)p->se.ravg.demand * 100,

				(u64)(sysctl_sched_ravg_window));

		else

			mnd.load = 0;

		atomic_notifier_call_chain(&migration_notifier_head,

					   dest_cpu, (void *)_src_cpu);

					   0, (void *)&mnd);

	}

	return ret;

}
@@ -7135,6 +7287,7 @@ void __init sched_init(void) 
		rq->cur_freq = 0;

		rq->max_freq = 0;

		rq->min_freq = 0;

		rq->cumulative_runnable_avg = 0;



		INIT_LIST_HEAD(&rq->cfs_tasks);

kernel/sched/fair.c

+27 −12
Original line number Diff line number Diff line
@@ -3984,7 +3984,7 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) 
 *

 * Called with both runqueues locked.

 */

static int move_one_task(struct lb_env *env)

static int move_one_task(struct lb_env *env, int *total_run_moved)

{

	struct task_struct *p, *n;
@@ -3999,6 +3999,10 @@ static int move_one_task(struct lb_env *env) 
		 * stats here rather than inside move_task().

		 */

		schedstat_inc(env->sd, lb_gained[env->idle]);

		if (sysctl_sched_ravg_window)

			*total_run_moved += div64_u64((u64)p->se.ravg.demand *

					100, (u64)(sysctl_sched_ravg_window));



		return 1;

	}

	return 0;
@@ -4015,7 +4019,7 @@ static const unsigned int sched_nr_migrate_break = 32; 
 *

 * Called with both runqueues locked.

 */

static int move_tasks(struct lb_env *env)

static int move_tasks(struct lb_env *env, int *total_run_moved)

{

	struct list_head *tasks = &env->src_rq->cfs_tasks;

	struct task_struct *p;
@@ -4054,6 +4058,9 @@ static int move_tasks(struct lb_env *env) 
		move_task(p, env);

		pulled++;

		env->imbalance -= load;

		if (sysctl_sched_ravg_window)

			*total_run_moved += div64_u64((u64)p->se.ravg.demand *

					100, (u64)(sysctl_sched_ravg_window));



#ifdef CONFIG_PREEMPT

		/*
@@ -5026,6 +5033,7 @@ static int load_balance(int this_cpu, struct rq *this_rq, 
			int *balance)

{

	int ld_moved, cur_ld_moved, active_balance = 0;

	int total_run_moved = 0;

	struct sched_group *group;

	struct rq *busiest = NULL;

	unsigned long flags;
@@ -5095,7 +5103,7 @@ more_balance: 
		 * cur_ld_moved - load moved in current iteration

		 * ld_moved     - cumulative load moved across iterations

		 */

		cur_ld_moved = move_tasks(&env);

		cur_ld_moved = move_tasks(&env, &total_run_moved);

		ld_moved += cur_ld_moved;

		double_rq_unlock(env.dst_rq, busiest);

		local_irq_restore(flags);
@@ -5213,12 +5221,15 @@ more_balance: 
	} else {

		sd->nr_balance_failed = 0;

		if (per_cpu(dbs_boost_needed, this_cpu)) {

			unsigned long _busy_cpu;

			_busy_cpu = cpu_of(busiest);

			struct migration_notify_data mnd;



			per_cpu(dbs_boost_needed, this_cpu) = false;



			mnd.src_cpu = cpu_of(busiest);

			mnd.dest_cpu = this_cpu;

			mnd.load = total_run_moved;

			atomic_notifier_call_chain(&migration_notifier_head,

						   this_cpu,

						   (void *)_busy_cpu);

						   0, (void *)&mnd);

		}

	}

	if (likely(!active_balance)) {
@@ -5326,6 +5337,7 @@ static int active_load_balance_cpu_stop(void *data) 
	struct rq *busiest_rq = data;

	int busiest_cpu = cpu_of(busiest_rq);

	int target_cpu = busiest_rq->push_cpu;

	int total_run_moved = 0;

	struct rq *target_rq = cpu_rq(target_cpu);

	struct sched_domain *sd;
@@ -5370,7 +5382,7 @@ static int active_load_balance_cpu_stop(void *data) 


		schedstat_inc(sd, alb_count);



		if (move_one_task(&env))

		if (move_one_task(&env, &total_run_moved))

			schedstat_inc(sd, alb_pushed);

		else

			schedstat_inc(sd, alb_failed);
@@ -5381,12 +5393,15 @@ out_unlock: 
	busiest_rq->active_balance = 0;

	raw_spin_unlock_irq(&busiest_rq->lock);

	if (per_cpu(dbs_boost_needed, target_cpu)) {

		unsigned long _busy_cpu;

		struct migration_notify_data mnd;



		per_cpu(dbs_boost_needed, target_cpu) = false;

		_busy_cpu = cpu_of(busiest_rq);



		mnd.src_cpu = cpu_of(busiest_rq);

		mnd.dest_cpu = target_cpu;

		mnd.load = total_run_moved;

		atomic_notifier_call_chain(&migration_notifier_head,

					   target_cpu,

					   (void *)_busy_cpu);

					   0, (void *)&mnd);

	}

	return 0;

}