Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 7ebdf76d authored by Satya Durga Srinivasu Prabhala's avatar Satya Durga Srinivasu Prabhala
Browse files

sched: Add snapshot of Window Assisted Load Tracking (WALT) 



This snapshot is taken from msm-4.14 as of
commit 871eac76e6be567 ("sched: Improve the scheduler").

Change-Id: Ib4e0b39526d3009cedebb626ece5a767d8247846
Signed-off-by: default avatarSatya Durga Srinivasu Prabhala <satyap@codeaurora.org>
parent a3d9530a

include/linux/sched.h

+156 −0
Original line number Diff line number Diff line
@@ -206,6 +206,21 @@ struct task_group; 
/* Task command name length: */

#define TASK_COMM_LEN			16



enum task_event {

	PUT_PREV_TASK   = 0,

	PICK_NEXT_TASK  = 1,

	TASK_WAKE       = 2,

	TASK_MIGRATE    = 3,

	TASK_UPDATE     = 4,

	IRQ_UPDATE      = 5,

};



/* Note: this need to be in sync with migrate_type_names array */

enum migrate_types {

	GROUP_TO_RQ,

	RQ_TO_GROUP,

};



extern void scheduler_tick(void);



#define	MAX_SCHEDULE_TIMEOUT		LONG_MAX
@@ -480,6 +495,102 @@ struct sched_entity { 
#endif

};



struct sched_load {

	unsigned long prev_load;

	unsigned long new_task_load;

	unsigned long predicted_load;

};



struct cpu_cycle_counter_cb {

	u64 (*get_cpu_cycle_counter)(int cpu);

};



#define MAX_NUM_CGROUP_COLOC_ID	20



extern DEFINE_PER_CPU_READ_MOSTLY(int, sched_load_boost);



#ifdef CONFIG_SCHED_WALT

extern void sched_exit(struct task_struct *p);

extern int register_cpu_cycle_counter_cb(struct cpu_cycle_counter_cb *cb);

extern void sched_set_io_is_busy(int val);

extern int sched_set_group_id(struct task_struct *p, unsigned int group_id);

extern unsigned int sched_get_group_id(struct task_struct *p);

extern int sched_set_init_task_load(struct task_struct *p, int init_load_pct);

extern u32 sched_get_init_task_load(struct task_struct *p);

extern void sched_update_cpu_freq_min_max(const cpumask_t *cpus, u32 fmin,

					  u32 fmax);

extern int sched_set_boost(int enable);

extern void free_task_load_ptrs(struct task_struct *p);



#define RAVG_HIST_SIZE_MAX  5

#define NUM_BUSY_BUCKETS 10



/* ravg represents frequency scaled cpu-demand of tasks */

struct ravg {

	/*

	 * 'mark_start' marks the beginning of an event (task waking up, task

	 * starting to execute, task being preempted) within a window

	 *

	 * 'sum' represents how runnable a task has been within current

	 * window. It incorporates both running time and wait time and is

	 * frequency scaled.

	 *

	 * 'sum_history' keeps track of history of 'sum' seen over previous

	 * RAVG_HIST_SIZE windows. Windows where task was entirely sleeping are

	 * ignored.

	 *

	 * 'demand' represents maximum sum seen over previous

	 * sysctl_sched_ravg_hist_size windows. 'demand' could drive frequency

	 * demand for tasks.

	 *

	 * 'curr_window_cpu' represents task's contribution to cpu busy time on

	 * various CPUs in the current window

	 *

	 * 'prev_window_cpu' represents task's contribution to cpu busy time on

	 * various CPUs in the previous window

	 *

	 * 'curr_window' represents the sum of all entries in curr_window_cpu

	 *

	 * 'prev_window' represents the sum of all entries in prev_window_cpu

	 *

	 * 'pred_demand' represents task's current predicted cpu busy time

	 *

	 * 'busy_buckets' groups historical busy time into different buckets

	 * used for prediction

	 *

	 * 'demand_scaled' represents task's demand scaled to 1024

	 */

	u64 mark_start;

	u32 sum, demand;

	u32 coloc_demand;

	u32 sum_history[RAVG_HIST_SIZE_MAX];

	u32 *curr_window_cpu, *prev_window_cpu;

	u32 curr_window, prev_window;

	u16 active_windows;

	u32 pred_demand;

	u8 busy_buckets[NUM_BUSY_BUCKETS];

	u16 demand_scaled;

	u16 pred_demand_scaled;

};

#else

static inline void sched_exit(struct task_struct *p) { }

static inline int

register_cpu_cycle_counter_cb(struct cpu_cycle_counter_cb *cb)

{

	return 0;

}

static inline void sched_set_io_is_busy(int val) {};



static inline int sched_set_boost(int enable)

{

	return -EINVAL;

}

static inline void free_task_load_ptrs(struct task_struct *p) { }



static inline void sched_update_cpu_freq_min_max(const cpumask_t *cpus,

					u32 fmin, u32 fmax) { }

#endif /* CONFIG_SCHED_WALT */



struct sched_rt_entity {

	struct list_head		run_list;

	unsigned long			timeout;
@@ -644,6 +755,22 @@ struct task_struct { 
	const struct sched_class	*sched_class;

	struct sched_entity		se;

	struct sched_rt_entity		rt;

	u64 last_sleep_ts;

#ifdef CONFIG_SCHED_WALT

	struct ravg ravg;

	/*

	 * 'init_load_pct' represents the initial task load assigned to children

	 * of this task

	 */

	u32 init_load_pct;

	u64 last_wake_ts;

	u64 last_enqueued_ts;

	struct related_thread_group *grp;

	struct list_head grp_list;

	u64 cpu_cycles;

	bool misfit;

#endif



#ifdef CONFIG_CGROUP_SCHED

	struct task_group		*sched_task_group;

#endif
@@ -1394,6 +1521,7 @@ extern struct pid *cad_pid; 
#define PF_KTHREAD		0x00200000	/* I am a kernel thread */

#define PF_RANDOMIZE		0x00400000	/* Randomize virtual address space */

#define PF_SWAPWRITE		0x00800000	/* Allowed to write to swap */

#define PF_WAKE_UP_IDLE         0x01000000	/* TTWU on an idle CPU */

#define PF_NO_SETAFFINITY	0x04000000	/* Userland is not allowed to meddle with cpus_allowed */

#define PF_MCE_EARLY		0x08000000      /* Early kill for mce process policy */

#define PF_MUTEX_TESTER		0x20000000	/* Thread belongs to the rt mutex tester */
@@ -1904,4 +2032,32 @@ static inline void rseq_syscall(struct pt_regs *regs) 


#endif



static inline u32 sched_get_wake_up_idle(struct task_struct *p)

{

	u32 enabled = p->flags & PF_WAKE_UP_IDLE;



	return !!enabled;

}



static inline int sched_set_wake_up_idle(struct task_struct *p,

						int wake_up_idle)

{

	int enable = !!wake_up_idle;



	if (enable)

		p->flags |= PF_WAKE_UP_IDLE;

	else

		p->flags &= ~PF_WAKE_UP_IDLE;



	return 0;

}



static inline void set_wake_up_idle(bool enabled)

{

	if (enabled)

		current->flags |= PF_WAKE_UP_IDLE;

	else

		current->flags &= ~PF_WAKE_UP_IDLE;

}



#endif

include/linux/sched/core_ctl.h

0 → 100644
+28 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0-only */

/*

 * Copyright (c) 2016, 2018, The Linux Foundation. All rights reserved.

 */



#ifndef __CORE_CTL_H

#define __CORE_CTL_H



struct core_ctl_notif_data {

	unsigned int nr_big;

	unsigned int coloc_load_pct;

};



#ifdef CONFIG_SCHED_CORE_CTL

void core_ctl_check(u64 wallclock);

int core_ctl_set_boost(bool boost);

void core_ctl_notifier_register(struct notifier_block *n);

void core_ctl_notifier_unregister(struct notifier_block *n);

#else

static inline void core_ctl_check(u64 wallclock) {}

static inline int core_ctl_set_boost(bool boost)

{

	return 0;

}

static inline void core_ctl_notifier_register(struct notifier_block *n) {}

static inline void core_ctl_notifier_unregister(struct notifier_block *n) {}

#endif

#endif

include/linux/sched/cpufreq.h

+6 −0
Original line number Diff line number Diff line
@@ -11,6 +11,12 @@ 


#define SCHED_CPUFREQ_IOWAIT	(1U << 0)

#define SCHED_CPUFREQ_MIGRATION	(1U << 1)

#define SCHED_CPUFREQ_INTERCLUSTER_MIG (1U << 3)

#define SCHED_CPUFREQ_WALT (1U << 4)

#define SCHED_CPUFREQ_PL        (1U << 5)

#define SCHED_CPUFREQ_EARLY_DET (1U << 6)

#define SCHED_CPUFREQ_FORCE_UPDATE (1U << 7)

#define SCHED_CPUFREQ_CONTINUE (1U << 8)



#ifdef CONFIG_CPU_FREQ

struct update_util_data {

include/linux/sched/stat.h

+14 −0
Original line number Diff line number Diff line
@@ -22,6 +22,20 @@ extern unsigned long nr_iowait(void); 
extern unsigned long nr_iowait_cpu(int cpu);

extern void get_iowait_load(unsigned long *nr_waiters, unsigned long *load);



#ifdef CONFIG_SMP

extern unsigned int sched_get_cpu_util(int cpu);

extern u64 sched_get_cpu_last_busy_time(int cpu);

#else

static inline unsigned int sched_get_cpu_util(int cpu)

{

	return 0;

}

static inline u64 sched_get_cpu_last_busy_time(int cpu)

{

	return 0;

}

#endif



static inline int sched_info_on(void)

{

#ifdef CONFIG_SCHEDSTATS

include/linux/sched/sysctl.h

+27 −0
Original line number Diff line number Diff line
@@ -26,6 +26,25 @@ extern unsigned int sysctl_sched_sync_hint_enable; 
extern unsigned int sysctl_sched_cstate_aware;

extern unsigned int sysctl_sched_wakeup_granularity;

extern unsigned int sysctl_sched_child_runs_first;

#ifdef CONFIG_SCHED_WALT

extern unsigned int sysctl_sched_use_walt_cpu_util;

extern unsigned int sysctl_sched_use_walt_task_util;

extern unsigned int sysctl_sched_walt_init_task_load_pct;

extern unsigned int sysctl_sched_cpu_high_irqload;

extern unsigned int sysctl_sched_boost;

extern unsigned int sysctl_sched_group_upmigrate_pct;

extern unsigned int sysctl_sched_group_downmigrate_pct;

extern unsigned int sysctl_sched_walt_rotate_big_tasks;

extern unsigned int sysctl_sched_min_task_util_for_boost;

extern unsigned int sysctl_sched_min_task_util_for_colocation;

extern unsigned int sysctl_sched_little_cluster_coloc_fmin_khz;



extern int

walt_proc_update_handler(struct ctl_table *table, int write,

			 void __user *buffer, size_t *lenp,

			 loff_t *ppos);



#endif



enum sched_tunable_scaling {

	SCHED_TUNABLESCALING_NONE,
@@ -49,6 +68,8 @@ int sched_proc_update_handler(struct ctl_table *table, int write, 
		loff_t *ppos);

#endif



extern int sched_boost_handler(struct ctl_table *table, int write,

			void __user *buffer, size_t *lenp, loff_t *ppos);

/*

 *  control realtime throttling:

 *
@@ -85,6 +106,12 @@ extern int sysctl_schedstats(struct ctl_table *table, int write, 
				 void __user *buffer, size_t *lenp,

				 loff_t *ppos);



#ifdef CONFIG_SCHED_WALT

extern int sched_little_cluster_coloc_fmin_khz_handler(struct ctl_table *table,

					int write, void __user *buffer,

					size_t *lenp, loff_t *ppos);

#endif



#if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL)

extern unsigned int sysctl_sched_energy_aware;

extern int sched_energy_aware_handler(struct ctl_table *table, int write,