Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit fca89815 authored by Syed Rameez Mustafa's avatar Syed Rameez Mustafa Committed by Joonwoo Park
Browse files

sched: Introduce a different version of freq aggregation 



This version sets up a timer every tick when RESTRAINED boost is turned
on. In the timer handler, we take all rq locks and aggregate the group
load for all clusters. This aggregate load is then reported while
restrained boost remains on. This is not perfect since it ignores the
changes in group load due to inter cluster migrations for one tick.
But this is the best we can do.

Change-Id: Iaa7aa64c8fd651a9f0b084f091d25aa4241b37dd
Signed-off-by: default avatarSyed Rameez Mustafa <rameezmustafa@codeaurora.org>
parent 5754feb3

kernel/sched/boost.c

+3 −1
Original line number Diff line number Diff line 
/* Copyright (c) 2012-2016, The Linux Foundation. All rights reserved.

/* Copyright (c) 2012-2017, The Linux Foundation. All rights reserved.

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License version 2 and
@@ -10,6 +10,7 @@ 
 * GNU General Public License for more details.

 */



#include <linux/jiffies.h>

#include "sched.h"

#include <linux/of.h>

#include <linux/sched/core_ctl.h>
@@ -139,6 +140,7 @@ static void _sched_set_boost(int old_val, int type) 
	case RESTRAINED_BOOST:

		freq_aggr_threshold_backup =

			update_freq_aggregate_threshold(1);

		mod_timer(&sched_grp_timer, jiffies + 1);

		break;



	default:

kernel/sched/sched.h

+5 −4
Original line number Diff line number Diff line
@@ -79,9 +79,12 @@ struct sched_cluster { 
	unsigned int static_cluster_pwr_cost;

	int notifier_sent;

	bool wake_up_idle;

	u64 aggr_grp_load;

};



extern unsigned int sched_disable_window_stats;



extern struct timer_list sched_grp_timer;

#endif /* CONFIG_SCHED_WALT */
@@ -1787,7 +1790,7 @@ static inline unsigned long cpu_util_cum(int cpu, int delta) 
}



#ifdef CONFIG_SCHED_WALT

u64 freq_policy_load(struct rq *rq, u64 load);

u64 freq_policy_load(struct rq *rq);

#endif



static inline unsigned long
@@ -1799,9 +1802,7 @@ cpu_util_freq(int cpu, struct sched_walt_cpu_load *walt_load) 


#ifdef CONFIG_SCHED_WALT

	if (!walt_disabled && sysctl_sched_use_walt_cpu_util) {

		util = rq->prev_runnable_sum +

		       rq->grp_time.prev_runnable_sum;

		util = freq_policy_load(rq, util);

		util = freq_policy_load(rq);

		util = div64_u64(util,

				 sched_ravg_window >> SCHED_CAPACITY_SHIFT);

kernel/sched/walt.c

+74 −33
Original line number Diff line number Diff line
@@ -22,6 +22,7 @@ 
#include <linux/syscore_ops.h>

#include <linux/cpufreq.h>

#include <linux/list_sort.h>

#include <linux/jiffies.h>

#include <trace/events/sched.h>

#include "sched.h"

#include "walt.h"
@@ -75,6 +76,65 @@ static int __init sched_init_ops(void) 
}

late_initcall(sched_init_ops);



static void acquire_rq_locks_irqsave(const cpumask_t *cpus,

				     unsigned long *flags)

{

	int cpu;



	local_irq_save(*flags);

	for_each_cpu(cpu, cpus)

		raw_spin_lock(&cpu_rq(cpu)->lock);

}



static void release_rq_locks_irqrestore(const cpumask_t *cpus,

					unsigned long *flags)

{

	int cpu;



	for_each_cpu(cpu, cpus)

		raw_spin_unlock(&cpu_rq(cpu)->lock);

	local_irq_restore(*flags);

}



struct timer_list sched_grp_timer;

static void sched_agg_grp_load(unsigned long data)

{

	struct sched_cluster *cluster;

	unsigned long flags;

	int cpu;



	acquire_rq_locks_irqsave(cpu_possible_mask, &flags);



	for_each_sched_cluster(cluster) {

		u64 aggr_grp_load = 0;



		for_each_cpu(cpu, &cluster->cpus) {

			struct rq *rq = cpu_rq(cpu);



			if (rq->curr)

				update_task_ravg(rq->curr, rq, TASK_UPDATE,

						sched_ktime_clock(), 0);

			aggr_grp_load +=

				rq->grp_time.prev_runnable_sum;

		}



		cluster->aggr_grp_load = aggr_grp_load;

	}



	release_rq_locks_irqrestore(cpu_possible_mask, &flags);



	if (sched_boost() == RESTRAINED_BOOST)

		mod_timer(&sched_grp_timer, jiffies + 1);

}



static int __init setup_sched_grp_timer(void)

{

	init_timer_deferrable(&sched_grp_timer);

	sched_grp_timer.function = sched_agg_grp_load;

	return 0;

}

late_initcall(setup_sched_grp_timer);



/* 1 -> use PELT based load stats, 0 -> use window-based load stats */

unsigned int __read_mostly walt_disabled = 0;
@@ -193,9 +253,6 @@ void reset_hmp_stats(struct hmp_sched_stats *stats, int reset_cra) 
/*

 * Demand aggregation for frequency purpose:

 *

 * 'sched_freq_aggregate' controls aggregation of cpu demand of related threads

 * for frequency determination purpose. This aggregation is done per-cluster.

 *

 * CPU demand of tasks from various related groups is aggregated per-cluster and

 * added to the "max_busy_cpu" in that cluster, where max_busy_cpu is determined

 * by just rq->prev_runnable_sum.
@@ -236,7 +293,7 @@ void reset_hmp_stats(struct hmp_sched_stats *stats, int reset_cra) 
 *	C1 busy time = 5 + 5 + 6 = 16ms

 *

 */

__read_mostly unsigned int sched_freq_aggregate = 1;

__read_mostly int sched_freq_aggregate_threshold;



static void

update_window_start(struct rq *rq, u64 wallclock, int event)
@@ -401,9 +458,17 @@ static u32 top_task_load(struct rq *rq) 
	}

}



u64 freq_policy_load(struct rq *rq, u64 load)

u64 freq_policy_load(struct rq *rq)

{

	unsigned int reporting_policy = sysctl_sched_freq_reporting_policy;

	struct sched_cluster *cluster = rq->cluster;

	u64 aggr_grp_load = cluster->aggr_grp_load;

	u64 load;



	if (aggr_grp_load > sched_freq_aggregate_threshold)

		load = rq->prev_runnable_sum + aggr_grp_load;

	else

		load = rq->prev_runnable_sum + rq->grp_time.prev_runnable_sum;



	switch (reporting_policy) {

	case FREQ_REPORT_MAX_CPU_LOAD_TOP_TASK:
@@ -716,7 +781,7 @@ void fixup_busy_time(struct task_struct *p, int new_cpu) 
	 * even for intra cluster migrations. This is because, the aggregated

	 * load has to reported on a single CPU regardless.

	 */

	if (grp && sched_freq_aggregate) {

	if (grp) {

		struct group_cpu_time *cpu_time;



		cpu_time = &src_rq->grp_time;
@@ -1273,7 +1338,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, 
		goto done;



	grp = p->grp;

	if (grp && sched_freq_aggregate) {

	if (grp) {

		struct group_cpu_time *cpu_time = &rq->grp_time;



		curr_runnable_sum = &cpu_time->curr_runnable_sum;
@@ -1797,8 +1862,7 @@ void update_task_ravg(struct task_struct *p, struct rq *rq, int event, 
	update_task_pred_demand(rq, p, event);

done:

	trace_sched_update_task_ravg(p, rq, event, wallclock, irqtime,

				     rq->cc.cycles, rq->cc.time,

				     p->grp ? &rq->grp_time : NULL);

				rq->cc.cycles, rq->cc.time, &rq->grp_time);

	trace_sched_update_task_ravg_mini(p, rq, event, wallclock, irqtime,

				rq->cc.cycles, rq->cc.time, &rq->grp_time);
@@ -1988,26 +2052,6 @@ static int compute_max_possible_capacity(struct sched_cluster *cluster) 
	return capacity;

}



static void acquire_rq_locks_irqsave(const cpumask_t *cpus,

				     unsigned long *flags)

{

	int cpu;



	local_irq_save(*flags);

	for_each_cpu(cpu, cpus)

		raw_spin_lock(&cpu_rq(cpu)->lock);

}



static void release_rq_locks_irqrestore(const cpumask_t *cpus,

					unsigned long *flags)

{

	int cpu;



	for_each_cpu(cpu, cpus)

		raw_spin_unlock(&cpu_rq(cpu)->lock);

	local_irq_restore(*flags);

}



static void update_min_max_capacity(void)

{

	unsigned long flags;
@@ -2154,6 +2198,7 @@ struct sched_cluster init_cluster = { 
	.exec_scale_factor	=	1024,

	.notifier_sent		=	0,

	.wake_up_idle		=	0,

	.aggr_grp_load		=	0,

};



void init_clusters(void)
@@ -2263,7 +2308,6 @@ static LIST_HEAD(active_related_thread_groups); 
DEFINE_RWLOCK(related_thread_group_lock);



unsigned int __read_mostly sysctl_sched_freq_aggregate_threshold_pct;

int __read_mostly sched_freq_aggregate_threshold;



/*

 * Task groups whose aggregate demand on a cpu is more than
@@ -2788,9 +2832,6 @@ static void transfer_busy_time(struct rq *rq, struct related_thread_group *grp, 
	bool new_task;

	int i;



	if (!sched_freq_aggregate)

		return;



	wallclock = sched_ktime_clock();



	update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0);

kernel/sched/walt.h

+0 −1
Original line number Diff line number Diff line
@@ -217,7 +217,6 @@ static inline void clear_top_tasks_table(u8 *table) 
	memset(table, 0, NUM_LOAD_INDICES * sizeof(u8));

}



extern u64 freq_policy_load(struct rq *rq, u64 load);

extern void update_cluster_load_subtractions(struct task_struct *p,

					int cpu, u64 ws, bool new_task);

extern void sched_account_irqstart(int cpu, struct task_struct *curr,