Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit a619e59c authored by Benjamin Herrenschmidt's avatar Benjamin Herrenschmidt Committed by Michael Ellerman
Browse files

powerpc/mm: Optimize detection of thread local mm's 



Instead of comparing the whole CPU mask every time, let's
keep a counter of how many bits are set in the mask. Thus
testing for a local mm only requires testing if that counter
is 1 and the current CPU bit is set in the mask.

Signed-off-by: default avatarBenjamin Herrenschmidt <benh@kernel.crashing.org>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
parent b426e4bd

arch/powerpc/include/asm/book3s/64/mmu.h

+3 −0
Original line number Diff line number Diff line
@@ -83,6 +83,9 @@ typedef struct { 
	mm_context_id_t id;

	u16 user_psize;		/* page size index */



	/* Number of bits in the mm_cpumask */

	atomic_t active_cpus;



	/* NPU NMMU context */

	struct npu_context *npu_context;

arch/powerpc/include/asm/mmu_context.h

+9 −0
Original line number Diff line number Diff line
@@ -96,6 +96,14 @@ static inline void switch_mm_pgdir(struct task_struct *tsk, 
				   struct mm_struct *mm) { }

#endif



#ifdef CONFIG_PPC_BOOK3S_64

static inline void inc_mm_active_cpus(struct mm_struct *mm)

{

	atomic_inc(&mm->context.active_cpus);

}

#else

static inline void inc_mm_active_cpus(struct mm_struct *mm) { }

#endif



/*

 * switch_mm is the entry point called from the architecture independent
@@ -110,6 +118,7 @@ static inline void switch_mm_irqs_off(struct mm_struct *prev, 
	/* Mark this context has been used on the new CPU */

	if (!cpumask_test_cpu(smp_processor_id(), mm_cpumask(next))) {

		cpumask_set_cpu(smp_processor_id(), mm_cpumask(next));

		inc_mm_active_cpus(next);



		/*

		 * This full barrier orders the store to the cpumask above vs

arch/powerpc/include/asm/tlb.h

+10 −1
Original line number Diff line number Diff line
@@ -69,13 +69,22 @@ static inline int mm_is_core_local(struct mm_struct *mm) 
			      topology_sibling_cpumask(smp_processor_id()));

}



#ifdef CONFIG_PPC_BOOK3S_64

static inline int mm_is_thread_local(struct mm_struct *mm)

{

	if (atomic_read(&mm->context.active_cpus) > 1)

		return false;

	return cpumask_test_cpu(smp_processor_id(), mm_cpumask(mm));

}

#else /* CONFIG_PPC_BOOK3S_64 */

static inline int mm_is_thread_local(struct mm_struct *mm)

{

	return cpumask_equal(mm_cpumask(mm),

			      cpumask_of(smp_processor_id()));

}

#endif /* !CONFIG_PPC_BOOK3S_64 */



#else

#else /* CONFIG_SMP */

static inline int mm_is_core_local(struct mm_struct *mm)

{

	return 1;

arch/powerpc/mm/mmu_context_book3s64.c

+2 −0
Original line number Diff line number Diff line
@@ -170,6 +170,8 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm) 
#ifdef CONFIG_SPAPR_TCE_IOMMU

	mm_iommu_init(mm);

#endif

	atomic_set(&mm->context.active_cpus, 0);



	return 0;

}