sched, block: Unify cache detection (39be3501) · Commits · e / devices / android_kernel_teracube_emerald

block/blk-softirq.c

+8 −8

Original line number	Original line	Diff line number	Diff line
	@@ -8,6 +8,7 @@
	#include <linux/blkdev.h>		#include <linux/blkdev.h>
	#include <linux/interrupt.h>		#include <linux/interrupt.h>
	#include <linux/cpu.h>		#include <linux/cpu.h>
			#include <linux/sched.h>

	#include "blk.h"		#include "blk.h"

	@@ -103,9 +104,10 @@ static struct notifier_block __cpuinitdata blk_cpu_notifier = {

	void __blk_complete_request(struct request *req)		void __blk_complete_request(struct request *req)
	{		{
	int ccpu, cpu, group_cpu = NR_CPUS;		int ccpu, cpu;
	struct request_queue *q = req->q;		struct request_queue *q = req->q;
	unsigned long flags;		unsigned long flags;
			bool shared = false;

	BUG_ON(!q->softirq_done_fn);		BUG_ON(!q->softirq_done_fn);

	@@ -117,22 +119,20 @@ void __blk_complete_request(struct request *req)
	*/		*/
	if (req->cpu != -1) {		if (req->cpu != -1) {
	ccpu = req->cpu;		ccpu = req->cpu;
	if (!test_bit(QUEUE_FLAG_SAME_FORCE, &q->queue_flags)) {		if (!test_bit(QUEUE_FLAG_SAME_FORCE, &q->queue_flags))
	ccpu = blk_cpu_to_group(ccpu);		shared = cpus_share_cache(cpu, ccpu);
	group_cpu = blk_cpu_to_group(cpu);
	}
	} else		} else
	ccpu = cpu;		ccpu = cpu;

	/*		/*
	* If current CPU and requested CPU are in the same group, running		* If current CPU and requested CPU share a cache, run the softirq on
	* softirq in current CPU. One might concern this is just like		* the current CPU. One might concern this is just like
	* QUEUE_FLAG_SAME_FORCE, but actually not. blk_complete_request() is		* QUEUE_FLAG_SAME_FORCE, but actually not. blk_complete_request() is
	* running in interrupt handler, and currently I/O controller doesn't		* running in interrupt handler, and currently I/O controller doesn't
	* support multiple interrupts, so current CPU is unique actually. This		* support multiple interrupts, so current CPU is unique actually. This
	* avoids IPI sending from current CPU to the first CPU of a group.		* avoids IPI sending from current CPU to the first CPU of a group.
	*/		*/
	if (ccpu == cpu \|\| ccpu == group_cpu) {		if (ccpu == cpu \|\| shared) {
	struct list_head *list;		struct list_head *list;
	do_local:		do_local:
	list = &__get_cpu_var(blk_cpu_done);		list = &__get_cpu_var(blk_cpu_done);

block/blk.h

+0 −16

Original line number	Original line	Diff line number	Diff line
	@@ -164,22 +164,6 @@ static inline int queue_congestion_off_threshold(struct request_queue *q)
	return q->nr_congestion_off;		return q->nr_congestion_off;
	}		}

	static inline int blk_cpu_to_group(int cpu)
	{
	int group = NR_CPUS;
	#ifdef CONFIG_SCHED_MC
	const struct cpumask *mask = cpu_coregroup_mask(cpu);
	group = cpumask_first(mask);
	#elif defined(CONFIG_SCHED_SMT)
	group = cpumask_first(topology_thread_cpumask(cpu));
	#else
	return cpu;
	#endif
	if (likely(group < NR_CPUS))
	return group;
	return cpu;
	}

	/*		/*
	* Contribute to IO statistics IFF:		* Contribute to IO statistics IFF:
	*		*

include/linux/sched.h

+8 −0

Original line number	Original line	Diff line number	Diff line
	@@ -1052,6 +1052,8 @@ static inline int test_sd_parent(struct sched_domain *sd, int flag)
	unsigned long default_scale_freq_power(struct sched_domain *sd, int cpu);		unsigned long default_scale_freq_power(struct sched_domain *sd, int cpu);
	unsigned long default_scale_smt_power(struct sched_domain *sd, int cpu);		unsigned long default_scale_smt_power(struct sched_domain *sd, int cpu);

			bool cpus_share_cache(int this_cpu, int that_cpu);

	#else /* CONFIG_SMP */		#else /* CONFIG_SMP */

	struct sched_domain_attr;		struct sched_domain_attr;
	@@ -1061,6 +1063,12 @@ partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
	struct sched_domain_attr *dattr_new)		struct sched_domain_attr *dattr_new)
	{		{
	}		}

			static inline bool cpus_share_cache(int this_cpu, int that_cpu)
			{
			return true;
			}

	#endif /* !CONFIG_SMP */		#endif /* !CONFIG_SMP */

kernel/sched/core.c

+3 −3

Original line number	Original line	Diff line number	Diff line
	@@ -1507,7 +1507,7 @@ static int ttwu_activate_remote(struct task_struct *p, int wake_flags)
	}		}
	#endif /* __ARCH_WANT_INTERRUPTS_ON_CTXSW */		#endif /* __ARCH_WANT_INTERRUPTS_ON_CTXSW */

	static inline int ttwu_share_cache(int this_cpu, int that_cpu)		bool cpus_share_cache(int this_cpu, int that_cpu)
	{		{
	return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);		return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);
	}		}
	@@ -1518,7 +1518,7 @@ static void ttwu_queue(struct task_struct *p, int cpu)
	struct rq *rq = cpu_rq(cpu);		struct rq *rq = cpu_rq(cpu);

	#if defined(CONFIG_SMP)		#if defined(CONFIG_SMP)
	if (sched_feat(TTWU_QUEUE) && !ttwu_share_cache(smp_processor_id(), cpu)) {		if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) {
	sched_clock_cpu(cpu); /* sync clocks x-cpu */		sched_clock_cpu(cpu); /* sync clocks x-cpu */
	ttwu_queue_remote(p, cpu);		ttwu_queue_remote(p, cpu);
	return;		return;
	@@ -5754,7 +5754,7 @@ static void destroy_sched_domains(struct sched_domain *sd, int cpu)
	*		*
	* Also keep a unique ID per domain (we use the first cpu number in		* Also keep a unique ID per domain (we use the first cpu number in
	* the cpumask of the domain), this allows us to quickly tell if		* the cpumask of the domain), this allows us to quickly tell if
	* two cpus are in the same cache domain, see ttwu_share_cache().		* two cpus are in the same cache domain, see cpus_share_cache().
	*/		*/
	DEFINE_PER_CPU(struct sched_domain *, sd_llc);		DEFINE_PER_CPU(struct sched_domain *, sd_llc);
	DEFINE_PER_CPU(int, sd_llc_id);		DEFINE_PER_CPU(int, sd_llc_id);