Revert "sched: Add Sony scheduling modifications"

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 */

/*

 * NOTE: This file has been modified by Sony Mobile Communications Inc.

 * Modifications are Copyright (c) 2017 Sony Mobile Communications Inc,

 * and licensed under the license of the file.

 */

#include "sched.h"

#include <linux/of.h>

{

	int i;

	struct cpumask kick_mask;

	u32 nr_running;

	if (boost_policy != SCHED_BOOST_ON_BIG)

		return;

	cpumask_andnot(&kick_mask, cpu_online_mask, cpu_isolated_mask);

	for_each_cpu(i, &kick_mask) {

		/*

		 * kick only "small" cluster

		 */

		if (cpu_capacity(i) != max_capacity) {

			nr_running = ACCESS_ONCE(cpu_rq(i)->nr_running);

			/*

			 * make sense to interrupt CPU if its run-queue

			 * has something running in order to check for

			 * migration afterwards, otherwise skip it.

			 */

			if (nr_running)

		if (cpu_capacity(i) != max_capacity)

			boost_kick(i);

	}

}

}

int got_boost_kick(void)

{

 *  2007-11-29  RT balancing improvements by Steven Rostedt, Gregory Haskins,

 *              Thomas Gleixner, Mike Kravetz

 */

/*

 * NOTE: This file has been modified by Sony Mobile Communications Inc.

 * Modifications are Copyright (c) 2017 Sony Mobile Communications Inc,

 * and licensed under the license of the file.

 */

#include <linux/kasan.h>

#include <linux/mm.h>

stat:

	ttwu_stat(p, cpu, wake_flags);

out:

	raw_spin_unlock(&p->pi_lock);

	raw_spin_unlock_irqrestore(&p->pi_lock, flags);

	if (freq_notif_allowed) {

		if (!same_freq_domain(src_cpu, cpu)) {

		}

	}

	local_irq_restore(flags);

	return success;

}

 *  Adaptive scheduling granularity, math enhancements by Peter Zijlstra

 *  Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra

 */

/*

 * NOTE: This file has been modified by Sony Mobile Communications Inc.

 * Modifications are Copyright (c) 2015 Sony Mobile Communications Inc,

 * and licensed under the license of the file.

 */

#include <linux/latencytop.h>

#include <linux/sched.h>

#define SBC_FLAG_BEST_SIBLING				0x200

#define SBC_FLAG_WAKER_CPU				0x400

#define SBC_FLAG_PACK_TASK				0x800

#define SBC_FLAG_SKIP_RT_TASK				0x1000

/* Cluster selection flag */

#define SBC_FLAG_COLOC_CLUSTER				0x10000

		cpumask_and(&search_cpus, &env->search_cpus, &next->cpus);

		for_each_cpu(i, &search_cpus) {

			if (trace_sched_cpu_load_wakeup_enabled()) {

			trace_sched_cpu_load_wakeup(cpu_rq(i), idle_cpu(i),

			sched_irqload(i), power_cost(i, task_load(env->p) +

					cpu_cravg_sync(i, env->sync)), 0);

			}

			update_spare_capacity(stats, env, i, next->capacity,

					  cpu_load_sync(i, env->sync));

		}

	return next;

}

/*

 * Returns true, if a current task has RT/DL class:

 * SCHED_FIFO + SCHED_RR + SCHED_DEADLINE

 */

static inline int

is_current_high_prio_class_task(int cpu)

{

	struct task_struct *curr = READ_ONCE(cpu_rq(cpu)->curr);

	return (task_has_rt_policy(curr) | task_has_dl_policy(curr));

}

#ifdef CONFIG_SCHED_HMP_CSTATE_AWARE

static void __update_cluster_stats(int cpu, struct cluster_cpu_stats *stats,

				   struct cpu_select_env *env, int cpu_cost)

		return;

	}

	/*

	 * We try to escape of selecting CPUs with running RT

	 * class tasks, if a power coast is the same. A reason

	 * is to reduce a latency, since RT task may not be

	 * preempted for a long time.

	 */

	if (is_current_high_prio_class_task(stats->best_cpu) &&

			!is_current_high_prio_class_task(cpu)) {

		stats->best_cpu_wakeup_latency = wakeup_latency;

		stats->best_load = env->cpu_load;

		stats->best_cpu = cpu;

		env->sbc_best_flag = SBC_FLAG_SKIP_RT_TASK;

		return;

	}

	if (!is_current_high_prio_class_task(stats->best_cpu) &&

			is_current_high_prio_class_task(cpu))

		return;

	/* CPU cost is the same. Start breaking the tie by C-state */

	if (wakeup_latency > stats->best_cpu_wakeup_latency)

	for_each_cpu(i, &search_cpus) {

		env->cpu_load = cpu_load_sync(i, env->sync);

		if (trace_sched_cpu_load_wakeup_enabled()) {

		trace_sched_cpu_load_wakeup(cpu_rq(i), idle_cpu(i),

			sched_irqload(i),

			power_cost(i, task_load(env->p) +

					cpu_cravg_sync(i, env->sync)), 0);

		}

		if (skip_cpu(i, env))

			continue;

	raw_spin_unlock(&cfs_b->lock);

	/* Log effect on hmp stats after throttling */

	if (trace_sched_cpu_load_cgroup_enabled()) {

	trace_sched_cpu_load_cgroup(rq, idle_cpu(cpu_of(rq)),

			     sched_irqload(cpu_of(rq)),

			     power_cost(cpu_of(rq), 0),

			     cpu_temp(cpu_of(rq)));

}

}

void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)

{

		resched_curr(rq);

	/* Log effect on hmp stats after un-throttling */

	if (trace_sched_cpu_load_cgroup_enabled()) {

	trace_sched_cpu_load_cgroup(rq, idle_cpu(cpu_of(rq)),

			     sched_irqload(cpu_of(rq)),

			     power_cost(cpu_of(rq), 0),

			     cpu_temp(cpu_of(rq)));

}

}

static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b,

		u64 remaining, u64 expires)

	if (!se) {

		add_nr_running(rq, 1);

		if (unlikely(p->nr_cpus_allowed == 1))

			rq->nr_pinned_tasks++;

		inc_rq_hmp_stats(rq, p, 1);

	}

	if (!se) {

		sub_nr_running(rq, 1);

		if (unlikely(p->nr_cpus_allowed == 1))

			rq->nr_pinned_tasks--;

		dec_rq_hmp_stats(rq, p, 1);

	}

	for (idx = 0; idx < sge->nr_cap_states; idx++) {

		if (sge->cap_states[idx].cap >= util) {

			/* Keep track of SG's capacity index */

			eenv->cap_idx = idx;

			break;

		}

		if (env->idle != CPU_NOT_IDLE && env->src_rq->nr_running <= 1)

			break;

		/*

		 * Another CPU can place tasks, since we do not hold dst_rq lock

		 * while doing balancing. If newly idle CPU already got something,

		 * give up to reduce a latency.

		 */

		if (env->idle == CPU_NEWLY_IDLE && env->dst_rq->nr_running > 0)

			break;

		p = list_first_entry(tasks, struct task_struct, se.group_node);

		env->loop++;

		if (sched_feat(LB_MIN) && load < 16 && !env->sd->nr_balance_failed)

			goto next;

		if (env->idle != CPU_NEWLY_IDLE)

		if ((load / 2) > env->imbalance)

			goto next;

	for_each_cpu_and(i, sched_group_cpus(group), env->cpus) {

		struct rq *rq = cpu_rq(i);

		if (trace_sched_cpu_load_lb_enabled()) {

		trace_sched_cpu_load_lb(cpu_rq(i), idle_cpu(i),

				     sched_irqload(i),

				     power_cost(i, 0),

				     cpu_temp(i));

		}

		if (cpu_isolated(i))

			continue;

	if (sgs->group_type < busiest->group_type)

		return false;

	if (sgs->avg_load <= busiest->avg_load)

		return false;

	/*

	 * Group has no more than one task per CPU

	 */

	if (sgs->sum_nr_running <= sgs->group_weight)

		return false;

	if (energy_aware()) {

		/*

		 * Candidate sg doesn't face any serious load-balance problems

		    !group_has_capacity(env, &sds->local_stat))

			return false;

		if (sgs->avg_load <= busiest->avg_load)

			return false;

		if (!(env->sd->flags & SD_ASYM_CPUCAPACITY))

			goto asym_packing;

		 * correctly treated as an imbalance.

		 */

		env.flags |= LBF_ALL_PINNED;

		env.loop_max  = min(sysctl_sched_nr_migrate, busiest->nr_running);

more_balance:

		raw_spin_lock_irqsave(&busiest->lock, flags);

			goto no_move;

		}

		/*

		 * Set loop_max when rq's lock is taken to prevent a race.

		 */

		env.loop_max = min(sysctl_sched_nr_migrate,

							busiest->cfs.h_nr_running);

		/*

		 * cur_ld_moved - load moved in current iteration

		 * ld_moved     - cumulative load moved across iterations

	if (p)

		attach_one_task(target_rq, p);

	local_irq_enable();

	if (moved && !same_freq_domain(busiest_cpu, target_cpu)) {

		int check_groups = !!(env.flags &

					 LBF_MOVED_RELATED_THREAD_GROUP_TASK);

		check_for_freq_change(target_rq, true, false);

	}

	local_irq_enable();

	return 0;

}

			sched_boost_policy() == SCHED_BOOST_ON_ALL)

		return 1;

	if (unlikely(rq->nr_pinned_tasks > 0)) {

		int delta = rq->nr_running - rq->nr_pinned_tasks;

		/*

		 * Check if it is possible to "unload" this CPU in case

		 * of having pinned/affine tasks. Do not disturb idle

		 * core if one of the below condition is true:

		 *

		 * - there is one pinned task and it is not "current"

		 * - all tasks are pinned to this CPU

		 */

		if (delta < 2)

			if (current->nr_cpus_allowed > 1 || !delta)

				return 0;

	}

	if (cpu_max_power_cost(cpu) == max_power_cost)

		return 1;

 * Syed Rameez Mustafa, Olav haugan, Joonwoo Park, Pavan Kumar Kondeti

 * and Vikram Mulukutla

 */

/*

 * NOTE: This file has been modified by Sony Mobile Communications Inc.

 * Modifications are Copyright (c) 2015 Sony Mobile Communications Inc,

 * and licensed under the license of the file.

 */

#include <linux/cpufreq.h>

#include <linux/list_sort.h>

static DEFINE_PER_CPU(struct freq_max_load *, freq_max_load);

static DEFINE_SPINLOCK(freq_max_load_lock);

static DEFINE_PER_CPU(u64, prev_group_runnable_sum);

struct cpu_pwr_stats __weak *get_cpu_pwr_stats(void)

{

	.dstate_wakeup_energy	=	0,

	.dstate_wakeup_latency	=	0,

	.exec_scale_factor	=	1024,

	.notifier_sent		=	0,

	.wake_up_idle		=	0,

};

	if (cluster->efficiency < min_possible_efficiency)

		min_possible_efficiency = cluster->efficiency;

	atomic_set(&cluster->notifier_sent, 0);

	cluster->notifier_sent = 0;

	return cluster;

}

void init_clusters(void)

{

	int cpu;

	bitmap_clear(all_cluster_ids, 0, NR_CPUS);

	init_cluster.cpus = *cpu_possible_mask;

	atomic_set(&init_cluster.notifier_sent, 0);

	raw_spin_lock_init(&init_cluster.load_lock);

	INIT_LIST_HEAD(&cluster_head);

	for_each_possible_cpu(cpu)

		per_cpu(prev_group_runnable_sum, cpu) = 0;

}

int register_cpu_cycle_counter_cb(struct cpu_cycle_counter_cb *cb)

	u32 freq;

	freq = cpu_cycles_to_freq(rq->cc.cycles, rq->cc.time);

	/*

	 * For some reason, current frequency estimation

	 * can be far bigger than max available frequency.

	 *

	 * TODO: need to be investigated. As for now, take

	 * min as a workaround.

	 */

	freq = min(freq, max_possible_freq);

	delta = DIV64_U64_ROUNDUP(delta * freq, max_possible_freq);

	delta *= rq->cluster->exec_scale_factor;

	delta >>= 10;

}

static inline u64 freq_policy_load(struct rq *rq, u64 load);

static inline void commit_prev_group_run_sum(struct rq *rq);

static inline u64 get_prev_group_run_sum(struct rq *rq);

/*

 * Should scheduler alert governor for changing frequency?

 *

static int send_notification(struct rq *rq, int check_pred, int check_groups)

{

	unsigned int cur_freq, freq_required;

	unsigned long flags;

	int rc = 0;

	u64 new_load, val = 0;

	u32 prev_run_sum, group_run_sum;

	u64 group_load = 0, new_load  = 0;

	if (check_pred) {

		u64 prev = rq->old_busy_time;

		if (freq_required < cur_freq + sysctl_sched_pred_alert_freq)

			return 0;

	} else {

		val = get_prev_group_run_sum(rq);

		group_run_sum = (u32) (val >> 32);

		prev_run_sum = (u32) val;

		if (check_groups)

		/*

			 * prev_run_sum and group_run_sum are synced

		 * Protect from concurrent update of rq->prev_runnable_sum and

		 * group cpu load

		 */

			new_load = prev_run_sum + group_run_sum;

		else

			new_load = prev_run_sum;

		raw_spin_lock_irqsave(&rq->lock, flags);

		if (check_groups)

			group_load = rq->grp_time.prev_runnable_sum;

		new_load = rq->prev_runnable_sum + group_load;

		new_load = freq_policy_load(rq, new_load);

		raw_spin_unlock_irqrestore(&rq->lock, flags);

		cur_freq = load_to_freq(rq, rq->old_busy_time);

		freq_required = load_to_freq(rq, new_load);

			return 0;

	}

	if (!atomic_cmpxchg(&rq->cluster->notifier_sent, 0, 1)) {

	raw_spin_lock_irqsave(&rq->lock, flags);

	if (!rq->cluster->notifier_sent) {

		rq->cluster->notifier_sent = 1;

		rc = 1;

		trace_sched_freq_alert(cpu_of(rq), check_pred, check_groups, rq,

				       new_load);

	}

	raw_spin_unlock_irqrestore(&rq->lock, flags);

	return rc;

}

/* Alert governor if there is a need to change frequency */

void check_for_freq_change(struct rq *rq, bool check_pred, bool check_groups)

{

	if (send_notification(rq, check_pred, check_groups)) {

	int cpu = cpu_of(rq);

	if (!send_notification(rq, check_pred, check_groups))

		return;

	atomic_notifier_call_chain(

		&load_alert_notifier_head, 0,

			(void *)(long) cpu_of(rq));

	}

		(void *)(long)cpu);

}

void notify_migration(int src_cpu, int dest_cpu, bool src_cpu_dead,

/*

 * Special case the last index and provide a fast path for index = 0.

 * Note that sched_load_granule can change underneath us if we are not

 * holding any runqueue locks while calling the two functions below.

 */

static u32  top_task_load(struct rq *rq)

{

	int index = rq->prev_top;

	u8 prev = 1 - rq->curr_table;

	u32 sched_granule_load;

	u32 ret_val = 0;

	sched_granule_load = READ_ONCE(sched_load_granule);

	if (!index) {

		int msb = NUM_LOAD_INDICES - 1;

		if (test_bit(msb, rq->top_tasks_bitmap[prev]))

			ret_val = sched_granule_load;

		if (!test_bit(msb, rq->top_tasks_bitmap[prev]))

			return 0;

		else

			return sched_load_granule;

	} else if (index == NUM_LOAD_INDICES - 1) {

		ret_val = sched_ravg_window;

		return sched_ravg_window;

	} else {

		ret_val = (index + 1) * sched_granule_load;

		return (index + 1) * sched_load_granule;

	}

	return ret_val;

}

static u32 load_to_index(u32 load)

	rq->grp_time.nt_curr_runnable_sum = 0;

}

static inline void

commit_prev_group_run_sum(struct rq *rq)

{

	u64 val;

	val = rq->grp_time.prev_runnable_sum;

	val = (val << 32) | rq->prev_runnable_sum;

	WRITE_ONCE(per_cpu(prev_group_runnable_sum, cpu_of(rq)), val);

}

static inline u64

get_prev_group_run_sum(struct rq *rq)

{

	return READ_ONCE(per_cpu(prev_group_runnable_sum, cpu_of(rq)));

}

/*

 * Account cpu activity in its busy time counters (rq->curr/prev_runnable_sum)

 */

		 */

		if (mark_start > window_start) {

			*curr_runnable_sum = scale_exec_time(irqtime, rq);

			goto done;

			return;

		}

		/*

		/* Process the remaining IRQ busy time in the current window. */

		delta = wallclock - window_start;

		rq->curr_runnable_sum = scale_exec_time(delta, rq);

		return;

	}

done:

	commit_prev_group_run_sum(rq);

	if (!is_idle_task(p) && !exiting_task(p))

		update_top_tasks(p, rq, old_curr_window,

					new_window, full_window);

	u32 *hist = &p->ravg.sum_history[0];

	int ridx, widx;

	u32 max = 0, avg, demand, pred_demand;

	u64 sum = 0, wma = 0, ewa = 0;

	u64 sum = 0;

	/* Ignore windows where task had no activity */

	if (!runtime || is_idle_task(p) || exiting_task(p) || !samples)

	for (; ridx >= 0; --widx, --ridx) {

		hist[widx] = hist[ridx];

		sum += hist[widx];

		wma += hist[widx] * (sched_ravg_hist_size - widx);

		ewa += hist[widx] << (sched_ravg_hist_size - widx - 1);

		if (hist[widx] > max)

			max = hist[widx];

	}

	for (widx = 0; widx < samples && widx < sched_ravg_hist_size; widx++) {

		hist[widx] = runtime;

		sum += hist[widx];

		wma += hist[widx] * (sched_ravg_hist_size - widx);

		ewa += hist[widx] << (sched_ravg_hist_size - widx - 1);

		if (hist[widx] > max)

			max = hist[widx];

	}

		demand = max;

	} else {

		avg = div64_u64(sum, sched_ravg_hist_size);

		wma = div64_u64(wma, (sched_ravg_hist_size * (sched_ravg_hist_size + 1)) / 2);

		ewa = div64_u64(ewa, (1 << sched_ravg_hist_size) - 1);

		if (sched_window_stats_policy == WINDOW_STATS_AVG)

			demand = avg;

		else if (sched_window_stats_policy == WINDOW_STATS_MAX_RECENT_WMA)

			/*

			 * WMA stands for weighted moving average. It helps

			 * to smooth load curve and react faster while ramping

			 * down comparing with basic averaging. We do it only

			 * when load trend goes down. See below example (4 HS):

			 *

			 * WMA = (P0 * 4 + P1 * 3 + P2 * 2 + P3 * 1) / (4 + 3 + 2 + 1)

			 *

			 * This is done for power saving. Means when load disappears

			 * or becomes low, this algorithm caches real bottom load faster

			 * (because of weights) then taking AVG values.

			 */

			demand = max((u32) wma, runtime);

		else if (sched_window_stats_policy == WINDOW_STATS_WMA)

			demand = (u32) wma;

		else if (sched_window_stats_policy == WINDOW_STATS_MAX_RECENT_EWA)

			/*

			 * EWA stands for exponential weighted average

			 */

			demand = max((u32) ewa, runtime);

		else if (sched_window_stats_policy == WINDOW_STATS_EWA)