Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 594f0bb4 authored by Kyle Yan's avatar Kyle Yan Committed by Gerrit - the friendly Code Review server
Browse files

Merge "sched: Introduce HMP scheduling" into msm-4.8

parents 28ea5bf4 dddcab78
Loading
Loading
Loading
Loading
+1432 −0

File added.

Preview size limit exceeded, changes collapsed.

+15 −0
Original line number Diff line number Diff line
@@ -28,6 +28,21 @@ struct cpu {
	struct device dev;
};

struct cpu_pstate_pwr {
	unsigned int freq;
	uint32_t power;
};

struct cpu_pwr_stats {
	int cpu;
	long temp;
	struct cpu_pstate_pwr *ptable;
	bool throttling;
	int len;
};

extern struct cpu_pwr_stats *get_cpu_pwr_stats(void);

extern void boot_cpu_init(void);
extern void boot_cpu_state_init(void);

+184 −1
Original line number Diff line number Diff line
@@ -174,6 +174,9 @@ extern unsigned long nr_iowait(void);
extern unsigned long nr_iowait_cpu(int cpu);
extern void get_iowait_load(unsigned long *nr_waiters, unsigned long *load);

extern void sched_update_nr_prod(int cpu, long delta, bool inc);
extern void sched_get_nr_running_avg(int *avg, int *iowait_avg, int *big_avg);

extern void calc_global_load(unsigned long ticks);

#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
@@ -315,6 +318,25 @@ extern char ___assert_task_state[1 - 2*!!(
/* Task command name length */
#define TASK_COMM_LEN 16

extern const char *sched_window_reset_reasons[];

enum task_event {
	PUT_PREV_TASK   = 0,
	PICK_NEXT_TASK  = 1,
	TASK_WAKE       = 2,
	TASK_MIGRATE    = 3,
	TASK_UPDATE     = 4,
	IRQ_UPDATE	= 5,
};

/* Note: this need to be in sync with migrate_type_names array */
enum migrate_types {
	GROUP_TO_RQ,
	RQ_TO_GROUP,
	RQ_TO_RQ,
	GROUP_TO_GROUP,
};

#include <linux/spinlock.h>

/*
@@ -1335,6 +1357,47 @@ struct sched_statistics {
};
#endif

#define RAVG_HIST_SIZE_MAX  5
#define NUM_BUSY_BUCKETS 10

/* ravg represents frequency scaled cpu-demand of tasks */
struct ravg {
	/*
	 * 'mark_start' marks the beginning of an event (task waking up, task
	 * starting to execute, task being preempted) within a window
	 *
	 * 'sum' represents how runnable a task has been within current
	 * window. It incorporates both running time and wait time and is
	 * frequency scaled.
	 *
	 * 'sum_history' keeps track of history of 'sum' seen over previous
	 * RAVG_HIST_SIZE windows. Windows where task was entirely sleeping are
	 * ignored.
	 *
	 * 'demand' represents maximum sum seen over previous
	 * sysctl_sched_ravg_hist_size windows. 'demand' could drive frequency
	 * demand for tasks.
	 *
	 * 'curr_window' represents task's contribution to cpu busy time
	 * statistics (rq->curr_runnable_sum) in current window
	 *
	 * 'prev_window' represents task's contribution to cpu busy time
	 * statistics (rq->prev_runnable_sum) in previous window
	 *
	 * 'pred_demand' represents task's current predicted cpu busy time
	 *
	 * 'busy_buckets' groups historical busy time into different buckets
	 * used for prediction
	 */
	u64 mark_start;
	u32 sum, demand;
	u32 sum_history[RAVG_HIST_SIZE_MAX];
	u32 curr_window, prev_window;
	u16 active_windows;
	u32 pred_demand;
	u8 busy_buckets[NUM_BUSY_BUCKETS];
};

struct sched_entity {
	struct load_weight	load;		/* for load-balancing */
	struct rb_node		run_node;
@@ -1505,6 +1568,20 @@ struct task_struct {
	const struct sched_class *sched_class;
	struct sched_entity se;
	struct sched_rt_entity rt;
#ifdef CONFIG_SCHED_HMP
	struct ravg ravg;
	/*
	 * 'init_load_pct' represents the initial task load assigned to children
	 * of this task
	 */
	u32 init_load_pct;
	u64 last_wake_ts;
	u64 last_switch_out_ts;
	u64 last_cpu_selected_ts;
	struct related_thread_group *grp;
	struct list_head grp_list;
	u64 cpu_cycles;
#endif
#ifdef CONFIG_CGROUP_SCHED
	struct task_group *sched_task_group;
#endif
@@ -2254,6 +2331,7 @@ extern void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut,
/*
 * Per process flags
 */
#define PF_WAKE_UP_IDLE 0x00000002	/* try to wake up on an idle CPU */
#define PF_EXITING	0x00000004	/* getting shut down */
#define PF_EXITPIDONE	0x00000008	/* pi exit done on shut down */
#define PF_VCPU		0x00000010	/* I'm a virtual CPU */
@@ -2436,6 +2514,93 @@ static inline int set_cpus_allowed_ptr(struct task_struct *p,
}
#endif

struct sched_load {
	unsigned long prev_load;
	unsigned long new_task_load;
	unsigned long predicted_load;
};

extern int sched_set_wake_up_idle(struct task_struct *p, int wake_up_idle);
extern u32 sched_get_wake_up_idle(struct task_struct *p);

struct cpu_cycle_counter_cb {
	u64 (*get_cpu_cycle_counter)(int cpu);
};

#ifdef CONFIG_SCHED_HMP
extern int sched_set_window(u64 window_start, unsigned int window_size);
extern unsigned long sched_get_busy(int cpu);
extern void sched_get_cpus_busy(struct sched_load *busy,
				const struct cpumask *query_cpus);
extern void sched_set_io_is_busy(int val);
extern int sched_set_boost(int enable);
extern int sched_set_init_task_load(struct task_struct *p, int init_load_pct);
extern u32 sched_get_init_task_load(struct task_struct *p);
extern int sched_set_static_cpu_pwr_cost(int cpu, unsigned int cost);
extern unsigned int sched_get_static_cpu_pwr_cost(int cpu);
extern int sched_set_static_cluster_pwr_cost(int cpu, unsigned int cost);
extern unsigned int sched_get_static_cluster_pwr_cost(int cpu);
extern int sched_update_freq_max_load(const cpumask_t *cpumask);
extern void sched_update_cpu_freq_min_max(const cpumask_t *cpus,
							u32 fmin, u32 fmax);
extern void sched_set_cpu_cstate(int cpu, int cstate,
			 int wakeup_energy, int wakeup_latency);
extern void sched_set_cluster_dstate(const cpumask_t *cluster_cpus, int dstate,
				int wakeup_energy, int wakeup_latency);
extern int register_cpu_cycle_counter_cb(struct cpu_cycle_counter_cb *cb);
extern u64 sched_ktime_clock(void);
extern int sched_set_group_id(struct task_struct *p, unsigned int group_id);
extern unsigned int sched_get_group_id(struct task_struct *p);

#else /* CONFIG_SCHED_HMP */
static inline u64 sched_ktime_clock(void)
{
	return 0;
}

static inline int
register_cpu_cycle_counter_cb(struct cpu_cycle_counter_cb *cb)
{
	return 0;
}

static inline int sched_set_window(u64 window_start, unsigned int window_size)
{
	return -EINVAL;
}
static inline unsigned long sched_get_busy(int cpu)
{
	return 0;
}
static inline void sched_get_cpus_busy(struct sched_load *busy,
				       const struct cpumask *query_cpus) {};

static inline void sched_set_io_is_busy(int val) {};

static inline int sched_set_boost(int enable)
{
	return -EINVAL;
}

static inline int sched_update_freq_max_load(const cpumask_t *cpumask)
{
	return 0;
}

static inline void sched_update_cpu_freq_min_max(const cpumask_t *cpus,
					u32 fmin, u32 fmax) { }

static inline void
sched_set_cpu_cstate(int cpu, int cstate, int wakeup_energy, int wakeup_latency)
{
}

static inline void sched_set_cluster_dstate(const cpumask_t *cluster_cpus,
			int dstate, int wakeup_energy, int wakeup_latency)
{
}
#endif /* CONFIG_SCHED_HMP */

#ifdef CONFIG_NO_HZ_COMMON
void calc_load_enter_idle(void);
void calc_load_exit_idle(void);
@@ -2444,6 +2609,14 @@ static inline void calc_load_enter_idle(void) { }
static inline void calc_load_exit_idle(void) { }
#endif /* CONFIG_NO_HZ_COMMON */

static inline void set_wake_up_idle(bool enabled)
{
	if (enabled)
		current->flags |= PF_WAKE_UP_IDLE;
	else
		current->flags &= ~PF_WAKE_UP_IDLE;
}

/*
 * Do not use outside of architecture code which knows its limitations.
 *
@@ -2461,6 +2634,7 @@ extern u64 sched_clock_cpu(int cpu);


extern void sched_clock_init(void);
extern int sched_clock_initialized(void);

#ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
static inline void sched_clock_tick(void)
@@ -2537,7 +2711,7 @@ extern unsigned long long
task_sched_runtime(struct task_struct *task);

/* sched_exec is called by processes performing an exec */
#ifdef CONFIG_SMP
#if defined(CONFIG_SMP) && !defined(CONFIG_SCHED_HMP)
extern void sched_exec(void);
#else
#define sched_exec()   {}
@@ -2670,6 +2844,7 @@ extern void xtime_update(unsigned long ticks);

extern int wake_up_state(struct task_struct *tsk, unsigned int state);
extern int wake_up_process(struct task_struct *tsk);
extern int wake_up_process_no_notif(struct task_struct *tsk);
extern void wake_up_new_task(struct task_struct *tsk);
#ifdef CONFIG_SMP
 extern void kick_process(struct task_struct *tsk);
@@ -2678,6 +2853,12 @@ extern void wake_up_new_task(struct task_struct *tsk);
#endif
extern int sched_fork(unsigned long clone_flags, struct task_struct *p);
extern void sched_dead(struct task_struct *p);
#ifdef CONFIG_SCHED_HMP
extern void sched_exit(struct task_struct *p);
#else
static inline void sched_exit(struct task_struct *p) { }
#endif


extern void proc_caches_init(void);
extern void flush_signals(struct task_struct *);
@@ -3506,6 +3687,8 @@ static inline void set_task_cpu(struct task_struct *p, unsigned int cpu)

#endif /* CONFIG_SMP */

extern struct atomic_notifier_head load_alert_notifier_head;

extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask);
extern long sched_getaffinity(pid_t pid, struct cpumask *mask);

+38 −0
Original line number Diff line number Diff line
@@ -18,6 +18,32 @@ extern unsigned int sysctl_sched_latency;
extern unsigned int sysctl_sched_min_granularity;
extern unsigned int sysctl_sched_wakeup_granularity;
extern unsigned int sysctl_sched_child_runs_first;
extern unsigned int sysctl_sched_wake_to_idle;

#ifdef CONFIG_SCHED_HMP
extern int sysctl_sched_freq_inc_notify;
extern int sysctl_sched_freq_dec_notify;
extern unsigned int sysctl_sched_window_stats_policy;
extern unsigned int sysctl_sched_ravg_hist_size;
extern unsigned int sysctl_sched_cpu_high_irqload;
extern unsigned int sysctl_sched_init_task_load_pct;
extern unsigned int sysctl_sched_spill_nr_run;
extern unsigned int sysctl_sched_spill_load_pct;
extern unsigned int sysctl_sched_upmigrate_pct;
extern unsigned int sysctl_sched_downmigrate_pct;
extern unsigned int sysctl_early_detection_duration;
extern unsigned int sysctl_sched_boost;
extern unsigned int sysctl_sched_small_wakee_task_load_pct;
extern unsigned int sysctl_sched_big_waker_task_load_pct;
extern unsigned int sysctl_sched_select_prev_cpu_us;
extern unsigned int sysctl_sched_enable_colocation;
extern unsigned int sysctl_sched_restrict_cluster_spill;
extern unsigned int sysctl_sched_new_task_windows;
extern unsigned int sysctl_sched_pred_alert_freq;
extern unsigned int sysctl_sched_freq_aggregate;
extern unsigned int sysctl_sched_enable_thread_grouping;
extern unsigned int sysctl_sched_freq_aggregate_threshold_pct;
#endif /* CONFIG_SCHED_HMP */

enum sched_tunable_scaling {
	SCHED_TUNABLESCALING_NONE,
@@ -43,6 +69,18 @@ int sched_proc_update_handler(struct ctl_table *table, int write,
		loff_t *ppos);
#endif

extern int sched_migrate_notify_proc_handler(struct ctl_table *table,
		int write, void __user *buffer, size_t *lenp, loff_t *ppos);

extern int sched_hmp_proc_update_handler(struct ctl_table *table,
		int write, void __user *buffer, size_t *lenp, loff_t *ppos);

extern int sched_boost_handler(struct ctl_table *table, int write,
			void __user *buffer, size_t *lenp, loff_t *ppos);

extern int sched_window_update_handler(struct ctl_table *table,
		 int write, void __user *buffer, size_t *lenp, loff_t *ppos);

/*
 *  control realtime throttling:
 *
+2 −0
Original line number Diff line number Diff line
@@ -27,6 +27,8 @@ static inline void tick_handover_do_timer(void) { }
static inline void tick_cleanup_dead_cpu(int cpu) { }
#endif /* !CONFIG_GENERIC_CLOCKEVENTS */

extern u64 jiffy_to_ktime_ns(u64 *now, u64 *jiffy_ktime_ns);

#if defined(CONFIG_GENERIC_CLOCKEVENTS) && defined(CONFIG_SUSPEND)
extern void tick_freeze(void);
extern void tick_unfreeze(void);
Loading