Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit af88a165 authored by Dietmar Eggemann's avatar Dietmar Eggemann Committed by Andres Oportus
Browse files

ANDROID: sched/fair: Add eas (& cas) specific rq, sd and task stats 



The statistic counter are placed in the eas (& cas) wakeup path. Each
of them has one representation for the runqueue (rq), the sched_domain
(sd) and the task.
A task counter is always incremented. A rq counter is always
incremented for the rq the scheduler is currently running on. A sd
counter is only incremented if a relation to a sd exists.

The counters are exposed:

(1) In /proc/schedstat for rq's and sd's:

$ cat /proc/schedstat
...
cpu0 71422 0 2321254 ...
eas  44144 0 0 19446 0 24698 568435 51621 156932 133 222011 17459 120279 516814 83 0 156962 359235 176439 139981
  <- runqueue for cpu0
...
domain0 3 42430 42331 ...
eas 0 0 0 14200 0 0 0 0 0 0 0 0 0 0 0 0 0 0 66355 0  <- MC sched domain for cpu0
...

The per-cpu eas vector has the following elements:

sis_attempts  sis_idle   sis_cache_affine sis_suff_cap    sis_idle_cpu    sis_count               ||
secb_attempts secb_sync  secb_idle_bt     secb_insuff_cap secb_no_nrg_sav secb_nrg_sav secb_count ||
fbt_attempts  fbt_no_cpu fbt_no_sd        fbt_pref_idle   fbt_count                               ||
cas_attempts  cas_count

The following relations exist between these counters (from cpu0 eas
vector above):

sis_attempts = sis_idle + sis_cache_affine + sis_suff_cap + sis_idle_cpu + sis_count

44144        = 0        + 0                + 19446        + 0            + 24698

secb_attempts = secb_sync + secb_idle_bt + secb_insuff_cap + secb_no_nrg_sav + secb_nrg_sav + secb_count

568435        = 51621     + 156932       + 133             + 222011          + 17459        + 120279

fbt_attempts = fbt_no_cpu + fbt_no_sd + fbt_pref_idle + fbt_count + (return -1)

516814       = 83         + 0         + 156962        + 359235    + (534)

cas_attempts = cas_count + (return -1 or smp_processor_id())

176439       = 139981    + (36458)

(2) In /proc/$PROCESS_PID/task/$TASK_PID/sched for a task.

example: main thread of system_server

$ cat /proc/1083/task/1083/sched

...
se.statistics.nr_wakeups_sis_attempts        :                  945
se.statistics.nr_wakeups_sis_idle            :                    0
se.statistics.nr_wakeups_sis_cache_affine    :                    0
se.statistics.nr_wakeups_sis_suff_cap        :                  219
se.statistics.nr_wakeups_sis_idle_cpu        :                    0
se.statistics.nr_wakeups_sis_count           :                  726
se.statistics.nr_wakeups_secb_attempts       :                10376
se.statistics.nr_wakeups_secb_sync           :                 1462
se.statistics.nr_wakeups_secb_idle_bt        :                 6984
se.statistics.nr_wakeups_secb_insuff_cap     :                    3
se.statistics.nr_wakeups_secb_no_nrg_sav     :                  927
se.statistics.nr_wakeups_secb_nrg_sav        :                  206
se.statistics.nr_wakeups_secb_count          :                  794
se.statistics.nr_wakeups_fbt_attempts        :                 8914
se.statistics.nr_wakeups_fbt_no_cpu          :                    0
se.statistics.nr_wakeups_fbt_no_sd           :                    0
se.statistics.nr_wakeups_fbt_pref_idle       :                 6987
se.statistics.nr_wakeups_fbt_count           :                 1554
se.statistics.nr_wakeups_cas_attempts        :                 3107
se.statistics.nr_wakeups_cas_count           :                 1195
...

The same relation between the counters as in the per-cpu case apply.

Change-Id: Ie7d01267c78a3f41f60a3ef52917d5a5d463f195
Signed-off-by: default avatarDietmar Eggemann <dietmar.eggemann@arm.com>
Signed-off-by: default avatarChris Redpath <chris.redpath@arm.com>
[fixed schedstat macros calls to match modifications
made in commit ae92882e]
Signed-off-by: default avatarQuentin Perret <quentin.perret@arm.com>
parent 8990dc7f

include/linux/sched.h

+62 −0
Original line number Diff line number Diff line
@@ -1108,6 +1108,37 @@ unsigned long capacity_curr_of(int cpu); 


struct sched_group;



struct eas_stats {

	/* select_idle_sibling() stats */

	u64 sis_attempts;

	u64 sis_idle;

	u64 sis_cache_affine;

	u64 sis_suff_cap;

	u64 sis_idle_cpu;

	u64 sis_count;



	/* select_energy_cpu_brute() stats */

	u64 secb_attempts;

	u64 secb_sync;

	u64 secb_idle_bt;

	u64 secb_insuff_cap;

	u64 secb_no_nrg_sav;

	u64 secb_nrg_sav;

	u64 secb_count;



	/* find_best_target() stats */

	u64 fbt_attempts;

	u64 fbt_no_cpu;

	u64 fbt_no_sd;

	u64 fbt_pref_idle;

	u64 fbt_count;



	/* cas */

	/* select_task_rq_fair() stats */

	u64 cas_attempts;

	u64 cas_count;

};



struct sched_domain_shared {

	atomic_t	ref;

	atomic_t	nr_busy_cpus;
@@ -1176,6 +1207,8 @@ struct sched_domain { 
	unsigned int ttwu_wake_remote;

	unsigned int ttwu_move_affine;

	unsigned int ttwu_move_balance;



	struct eas_stats eas_stats;

#endif

#ifdef CONFIG_SCHED_DEBUG

	char *name;
@@ -1374,6 +1407,35 @@ struct sched_statistics { 
	u64			nr_wakeups_affine_attempts;

	u64			nr_wakeups_passive;

	u64			nr_wakeups_idle;



	/* select_idle_sibling() */

	u64			nr_wakeups_sis_attempts;

	u64			nr_wakeups_sis_idle;

	u64			nr_wakeups_sis_cache_affine;

	u64			nr_wakeups_sis_suff_cap;

	u64			nr_wakeups_sis_idle_cpu;

	u64			nr_wakeups_sis_count;



	/* energy_aware_wake_cpu() */

	u64			nr_wakeups_secb_attempts;

	u64			nr_wakeups_secb_sync;

	u64			nr_wakeups_secb_idle_bt;

	u64			nr_wakeups_secb_insuff_cap;

	u64			nr_wakeups_secb_no_nrg_sav;

	u64			nr_wakeups_secb_nrg_sav;

	u64			nr_wakeups_secb_count;



	/* find_best_target() */

	u64			nr_wakeups_fbt_attempts;

	u64			nr_wakeups_fbt_no_cpu;

	u64			nr_wakeups_fbt_no_sd;

	u64			nr_wakeups_fbt_pref_idle;

	u64			nr_wakeups_fbt_count;



	/* cas */

	/* select_task_rq_fair() */

	u64			nr_wakeups_cas_attempts;

	u64			nr_wakeups_cas_count;

};

#endif

kernel/sched/debug.c

+27 −1
Original line number Diff line number Diff line
@@ -981,6 +981,32 @@ void proc_sched_show_task(struct task_struct *p, struct seq_file *m) 
		P_SCHEDSTAT(se.statistics.nr_wakeups_affine_attempts);

		P_SCHEDSTAT(se.statistics.nr_wakeups_passive);

		P_SCHEDSTAT(se.statistics.nr_wakeups_idle);

		/* eas */

		/* select_idle_sibling() */

		P_SCHEDSTAT(se.statistics.nr_wakeups_sis_attempts);

		P_SCHEDSTAT(se.statistics.nr_wakeups_sis_idle);

		P_SCHEDSTAT(se.statistics.nr_wakeups_sis_cache_affine);

		P_SCHEDSTAT(se.statistics.nr_wakeups_sis_suff_cap);

		P_SCHEDSTAT(se.statistics.nr_wakeups_sis_idle_cpu);

		P_SCHEDSTAT(se.statistics.nr_wakeups_sis_count);

		/* select_energy_cpu_brute() */

		P_SCHEDSTAT(se.statistics.nr_wakeups_secb_attempts);

		P_SCHEDSTAT(se.statistics.nr_wakeups_secb_sync);

		P_SCHEDSTAT(se.statistics.nr_wakeups_secb_idle_bt);

		P_SCHEDSTAT(se.statistics.nr_wakeups_secb_insuff_cap);

		P_SCHEDSTAT(se.statistics.nr_wakeups_secb_no_nrg_sav);

		P_SCHEDSTAT(se.statistics.nr_wakeups_secb_nrg_sav);

		P_SCHEDSTAT(se.statistics.nr_wakeups_secb_count);

		/* find_best_target() */

		P_SCHEDSTAT(se.statistics.nr_wakeups_fbt_attempts);

		P_SCHEDSTAT(se.statistics.nr_wakeups_fbt_no_cpu);

		P_SCHEDSTAT(se.statistics.nr_wakeups_fbt_no_sd);

		P_SCHEDSTAT(se.statistics.nr_wakeups_fbt_pref_idle);

		P_SCHEDSTAT(se.statistics.nr_wakeups_fbt_count);

		/* cas */

		/* select_task_rq_fair() */

		P_SCHEDSTAT(se.statistics.nr_wakeups_cas_attempts);

		P_SCHEDSTAT(se.statistics.nr_wakeups_cas_count);

 

		avg_atom = p->se.sum_exec_runtime;

		if (nr_switches)

kernel/sched/fair.c

+114 −36
Original line number Diff line number Diff line
@@ -6382,15 +6382,24 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) 
	int best_idle_cstate = INT_MAX;

	unsigned long best_idle_capacity = ULONG_MAX;



	schedstat_inc(p->se.statistics.nr_wakeups_sis_attempts);

	schedstat_inc(this_rq()->eas_stats.sis_attempts);



	if (!sysctl_sched_cstate_aware) {

		if (idle_cpu(target))

		if (idle_cpu(target)) {

			schedstat_inc(p->se.statistics.nr_wakeups_sis_idle);

			schedstat_inc(this_rq()->eas_stats.sis_idle);

			return target;

		}



		/*

		 * If the prevous cpu is cache affine and idle, don't be stupid.

		 */

		if (i != target && cpus_share_cache(i, target) && idle_cpu(i))

		if (i != target && cpus_share_cache(i, target) && idle_cpu(i)) {

			schedstat_inc(p->se.statistics.nr_wakeups_sis_cache_affine);

			schedstat_inc(this_rq()->eas_stats.sis_cache_affine);

			return i;

		}



		sd = rcu_dereference(per_cpu(sd_llc, target));

		if (!sd)
@@ -6430,8 +6439,12 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) 
					if (new_usage > capacity_orig || !idle_cpu(i))

						goto next;



					if (i == target && new_usage <= capacity_curr_of(target))

					if (i == target && new_usage <= capacity_curr_of(target)) {

						schedstat_inc(p->se.statistics.nr_wakeups_sis_suff_cap);

						schedstat_inc(this_rq()->eas_stats.sis_suff_cap);

						schedstat_inc(sd->eas_stats.sis_suff_cap);

						return target;

					}



					if (idle_idx < best_idle_cstate &&

					    capacity_orig <= best_idle_capacity) {
@@ -6448,6 +6461,9 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) 


				target = cpumask_first_and(sched_group_cpus(sg),

					tsk_cpus_allowed(p));

				schedstat_inc(p->se.statistics.nr_wakeups_sis_idle_cpu);

				schedstat_inc(this_rq()->eas_stats.sis_idle_cpu);

				schedstat_inc(sd->eas_stats.sis_idle_cpu);

				goto done;

			}

next:
@@ -6459,6 +6475,9 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) 
		target = best_idle_cpu;



done:

	schedstat_inc(p->se.statistics.nr_wakeups_sis_count);

	schedstat_inc(this_rq()->eas_stats.sis_count);



	return target;

}
@@ -6485,13 +6504,22 @@ static inline int find_best_target(struct task_struct *p, bool boosted, bool pre 
	struct sched_group *sg;

	int cpu = start_cpu(boosted);



	if (cpu < 0)

	schedstat_inc(p->se.statistics.nr_wakeups_fbt_attempts);

	schedstat_inc(this_rq()->eas_stats.fbt_attempts);



	if (cpu < 0) {

		schedstat_inc(p->se.statistics.nr_wakeups_fbt_no_cpu);

		schedstat_inc(this_rq()->eas_stats.fbt_no_cpu);

		return target_cpu;

	}



	sd = rcu_dereference(per_cpu(sd_ea, cpu));



	if (!sd)

	if (!sd) {

		schedstat_inc(p->se.statistics.nr_wakeups_fbt_no_sd);

		schedstat_inc(this_rq()->eas_stats.fbt_no_sd);

		return target_cpu;

	}



	sg = sd->groups;
@@ -6529,8 +6557,11 @@ static inline int find_best_target(struct task_struct *p, bool boosted, bool pre 
			 * Unconditionally favoring tasks that prefer idle cpus to

			 * improve latency.

			 */

			if (idle_cpu(i) && prefer_idle)

			if (idle_cpu(i) && prefer_idle) {

				schedstat_inc(p->se.statistics.nr_wakeups_fbt_pref_idle);

				schedstat_inc(this_rq()->eas_stats.fbt_pref_idle);

				return i;

			}



			cur_capacity = capacity_curr_of(i);
@@ -6567,6 +6598,11 @@ static inline int find_best_target(struct task_struct *p, bool boosted, bool pre 
	if (target_cpu < 0)

		target_cpu = best_idle_cpu >= 0 ? best_idle_cpu : backup_cpu;



	if (target_cpu >= 0) {

		schedstat_inc(p->se.statistics.nr_wakeups_fbt_count);

		schedstat_inc(this_rq()->eas_stats.fbt_count);

	}



	return target_cpu;

}
@@ -6616,12 +6652,18 @@ static int select_energy_cpu_brute(struct task_struct *p, int prev_cpu, int sync 
	int target_cpu = prev_cpu, tmp_target;

	bool boosted, prefer_idle;



	schedstat_inc(p->se.statistics.nr_wakeups_secb_attempts);

	schedstat_inc(this_rq()->eas_stats.secb_attempts);



	if (sysctl_sched_sync_hint_enable && sync) {

		int cpu = smp_processor_id();



		if (cpumask_test_cpu(cpu, tsk_cpus_allowed(p)))

		if (cpumask_test_cpu(cpu, tsk_cpus_allowed(p))) {

			schedstat_inc(p->se.statistics.nr_wakeups_secb_sync);

			schedstat_inc(this_rq()->eas_stats.secb_sync);

			return cpu;

		}

	}



	rcu_read_lock();

#ifdef CONFIG_CGROUP_SCHEDTUNE
@@ -6640,9 +6682,12 @@ static int select_energy_cpu_brute(struct task_struct *p, int prev_cpu, int sync 
		goto unlock;

	if (tmp_target >= 0) {

		target_cpu = tmp_target;

		if ((boosted || prefer_idle) && idle_cpu(target_cpu))

		if ((boosted || prefer_idle) && idle_cpu(target_cpu)) {

			schedstat_inc(p->se.statistics.nr_wakeups_secb_idle_bt);

			schedstat_inc(this_rq()->eas_stats.secb_idle_bt);

			goto unlock;

		}

	}



	if (target_cpu != prev_cpu) {

		struct energy_env eenv = {
@@ -6653,13 +6698,27 @@ static int select_energy_cpu_brute(struct task_struct *p, int prev_cpu, int sync 
		};



		/* Not enough spare capacity on previous cpu */

		if (cpu_overutilized(prev_cpu))

		if (cpu_overutilized(prev_cpu)) {

			schedstat_inc(p->se.statistics.nr_wakeups_secb_insuff_cap);

			schedstat_inc(this_rq()->eas_stats.secb_insuff_cap);

			goto unlock;

		}



		if (energy_diff(&eenv) >= 0)

		if (energy_diff(&eenv) >= 0) {

			schedstat_inc(p->se.statistics.nr_wakeups_secb_no_nrg_sav);

			schedstat_inc(this_rq()->eas_stats.secb_no_nrg_sav);

			target_cpu = prev_cpu;

			goto unlock;

		}



		schedstat_inc(p->se.statistics.nr_wakeups_secb_nrg_sav);

		schedstat_inc(this_rq()->eas_stats.secb_nrg_sav);

		goto unlock;

	}



	schedstat_inc(p->se.statistics.nr_wakeups_secb_count);

	schedstat_inc(this_rq()->eas_stats.secb_count);



unlock:

	rcu_read_unlock();

	return target_cpu;
@@ -6726,10 +6785,23 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_f 
		if (sd_flag & SD_BALANCE_WAKE) /* XXX always ? */

			new_cpu = select_idle_sibling(p, prev_cpu, new_cpu);



	} else while (sd) {

	} else {

		int wu = sd_flag & SD_BALANCE_WAKE;

		int cas_cpu = -1;



		if (wu) {

			schedstat_inc(p->se.statistics.nr_wakeups_cas_attempts);

			schedstat_inc(this_rq()->eas_stats.cas_attempts);

		}





		while (sd) {

			struct sched_group *group;

			int weight;



			if (wu)

				schedstat_inc(sd->eas_stats.cas_attempts);



			if (!(sd->flags & sd_flag)) {

				sd = sd->child;

				continue;
@@ -6749,7 +6821,7 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_f 
			}



			/* Now try balancing at a lower domain level of new_cpu */

		cpu = new_cpu;

			cpu = cas_cpu = new_cpu;

			weight = sd->span_weight;

			sd = NULL;

			for_each_domain(cpu, tmp) {
@@ -6760,6 +6832,12 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_f 
			}

			/* while loop will break here if sd == NULL */

		}



		if (wu && (cas_cpu >= 0)) {

			schedstat_inc(p->se.statistics.nr_wakeups_cas_count);

			schedstat_inc(this_rq()->eas_stats.cas_count);

		}

	}

	rcu_read_unlock();



	return new_cpu;

kernel/sched/sched.h

+2 −0
Original line number Diff line number Diff line
@@ -768,6 +768,8 @@ struct rq { 
	/* try_to_wake_up() stats */

	unsigned int ttwu_count;

	unsigned int ttwu_local;



	struct eas_stats eas_stats;

#endif



#ifdef CONFIG_SMP

kernel/sched/stats.c

+24 −0
Original line number Diff line number Diff line
@@ -12,6 +12,26 @@ 
 */

#define SCHEDSTAT_VERSION 15



static inline void show_easstat(struct seq_file *seq, struct eas_stats *stats)

{

	/* eas-specific runqueue stats */

	seq_printf(seq, "eas %llu %llu %llu %llu %llu %llu ",

	    stats->sis_attempts, stats->sis_idle, stats->sis_cache_affine,

	    stats->sis_suff_cap, stats->sis_idle_cpu, stats->sis_count);



	seq_printf(seq, "%llu %llu %llu %llu %llu %llu %llu ",

	    stats->secb_attempts, stats->secb_sync, stats->secb_idle_bt,

	    stats->secb_insuff_cap, stats->secb_no_nrg_sav,

	    stats->secb_nrg_sav, stats->secb_count);



	seq_printf(seq, "%llu %llu %llu %llu %llu ",

	    stats->fbt_attempts, stats->fbt_no_cpu, stats->fbt_no_sd,

	    stats->fbt_pref_idle, stats->fbt_count);



	seq_printf(seq, "%llu %llu\n",

	    stats->cas_attempts, stats->cas_count);

}



static int show_schedstat(struct seq_file *seq, void *v)

{

	int cpu;
@@ -39,6 +59,8 @@ static int show_schedstat(struct seq_file *seq, void *v) 


		seq_printf(seq, "\n");



		show_easstat(seq, &rq->eas_stats);



#ifdef CONFIG_SMP

		/* domain-specific stats */

		rcu_read_lock();
@@ -66,6 +88,8 @@ static int show_schedstat(struct seq_file *seq, void *v) 
			    sd->sbf_count, sd->sbf_balanced, sd->sbf_pushed,

			    sd->ttwu_wake_remote, sd->ttwu_move_affine,

			    sd->ttwu_move_balance);



			show_easstat(seq, &sd->eas_stats);

		}

		rcu_read_unlock();

#endif