Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e40f2166 authored by Aneesh Kumar K.V's avatar Aneesh Kumar K.V Committed by Greg Kroah-Hartman
Browse files

powerpc/mm: Fixup tlbie vs store ordering issue on POWER9 



commit a5d4b589 upstream.

On POWER9, under some circumstances, a broadcast TLB invalidation
might complete before all previous stores have drained, potentially
allowing stale stores from becoming visible after the invalidation.
This works around it by doubling up those TLB invalidations which was
verified by HW to be sufficient to close the risk window.

This will be documented in a yet-to-be-published errata.

Cc: stable@vger.kernel.org # v4.14
Fixes: 1a472c9d ("powerpc/mm/radix: Add tlbflush routines")
Signed-off-by: default avatarAneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
[mpe: Enable the feature in the DT CPU features code for all Power9,
      rename the feature to CPU_FTR_P9_TLBIE_BUG per benh.]
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20180323045627.16800-3-aneesh.kumar@linux.vnet.ibm.com/


[sandipan: Backported to v4.14]
Signed-off-by: default avatarSandipan Das <sandipan@linux.ibm.com>
Signed-off-by: default avatarGreg Kroah-Hartman <gregkh@linuxfoundation.org>
parent bf31532e

arch/powerpc/include/asm/cputable.h

+3 −1
Original line number Diff line number Diff line
@@ -215,6 +215,7 @@ enum { 
#define CPU_FTR_DAWR			LONG_ASM_CONST(0x0400000000000000)

#define CPU_FTR_DABRX			LONG_ASM_CONST(0x0800000000000000)

#define CPU_FTR_PMAO_BUG		LONG_ASM_CONST(0x1000000000000000)

#define CPU_FTR_P9_TLBIE_BUG		LONG_ASM_CONST(0x2000000000000000)

#define CPU_FTR_POWER9_DD1		LONG_ASM_CONST(0x4000000000000000)



#ifndef __ASSEMBLY__
@@ -475,7 +476,8 @@ enum { 
	    CPU_FTR_STCX_CHECKS_ADDRESS | CPU_FTR_POPCNTB | CPU_FTR_POPCNTD | \

	    CPU_FTR_CFAR | CPU_FTR_HVMODE | CPU_FTR_VMX_COPY | \

	    CPU_FTR_DBELL | CPU_FTR_HAS_PPR | CPU_FTR_DAWR | \

	    CPU_FTR_ARCH_207S | CPU_FTR_TM_COMP | CPU_FTR_ARCH_300)

	    CPU_FTR_ARCH_207S | CPU_FTR_TM_COMP | CPU_FTR_ARCH_300 | \

	    CPU_FTR_P9_TLBIE_BUG)

#define CPU_FTRS_POWER9_DD1 ((CPU_FTRS_POWER9 | CPU_FTR_POWER9_DD1) & \

			     (~CPU_FTR_SAO))

#define CPU_FTRS_CELL	(CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \

arch/powerpc/kernel/dt_cpu_ftrs.c

+3 −0
Original line number Diff line number Diff line
@@ -742,6 +742,9 @@ static __init void cpufeatures_cpu_quirks(void) 
	 */

	if ((version & 0xffffff00) == 0x004e0100)

		cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD1;



	if ((version & 0xffff0000) == 0x004e0000)

		cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_BUG;

}



static void __init cpufeatures_setup_finished(void)

arch/powerpc/kvm/book3s_64_mmu_radix.c

+3 −0
Original line number Diff line number Diff line
@@ -160,6 +160,9 @@ static void kvmppc_radix_tlbie_page(struct kvm *kvm, unsigned long addr, 
	asm volatile("ptesync": : :"memory");

	asm volatile(PPC_TLBIE_5(%0, %1, 0, 0, 1)

		     : : "r" (addr), "r" (kvm->arch.lpid) : "memory");

	if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG))

		asm volatile(PPC_TLBIE_5(%0, %1, 0, 0, 1)

			     : : "r" (addr), "r" (kvm->arch.lpid) : "memory");

	asm volatile("ptesync": : :"memory");

}

arch/powerpc/kvm/book3s_hv_rm_mmu.c

+11 −0
Original line number Diff line number Diff line
@@ -448,6 +448,17 @@ static void do_tlbies(struct kvm *kvm, unsigned long *rbvalues, 
			asm volatile(PPC_TLBIE_5(%0,%1,0,0,0) : :

				     "r" (rbvalues[i]), "r" (kvm->arch.lpid));

		}



		if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG)) {

			/*

			 * Need the extra ptesync to make sure we don't

			 * re-order the tlbie

			 */

			asm volatile("ptesync": : :"memory");

			asm volatile(PPC_TLBIE_5(%0,%1,0,0,0) : :

				     "r" (rbvalues[0]), "r" (kvm->arch.lpid));

		}



		asm volatile("eieio; tlbsync; ptesync" : : : "memory");

		kvm->arch.tlbie_lock = 0;

	} else {

arch/powerpc/mm/hash_native_64.c

+15 −1
Original line number Diff line number Diff line
@@ -104,6 +104,15 @@ static inline unsigned long ___tlbie(unsigned long vpn, int psize, 
	return va;

}



static inline void fixup_tlbie(unsigned long vpn, int psize, int apsize, int ssize)

{

	if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG)) {

		/* Need the extra ptesync to ensure we don't reorder tlbie*/

		asm volatile("ptesync": : :"memory");

		___tlbie(vpn, psize, apsize, ssize);

	}

}



static inline void __tlbie(unsigned long vpn, int psize, int apsize, int ssize)

{

	unsigned long rb;
@@ -181,6 +190,7 @@ static inline void tlbie(unsigned long vpn, int psize, int apsize, 
		asm volatile("ptesync": : :"memory");

	} else {

		__tlbie(vpn, psize, apsize, ssize);

		fixup_tlbie(vpn, psize, apsize, ssize);

		asm volatile("eieio; tlbsync; ptesync": : :"memory");

	}

	if (lock_tlbie && !use_local)
@@ -674,7 +684,7 @@ static void native_hpte_clear(void) 
 */

static void native_flush_hash_range(unsigned long number, int local)

{

	unsigned long vpn;

	unsigned long vpn = 0;

	unsigned long hash, index, hidx, shift, slot;

	struct hash_pte *hptep;

	unsigned long hpte_v;
@@ -746,6 +756,10 @@ static void native_flush_hash_range(unsigned long number, int local) 
				__tlbie(vpn, psize, psize, ssize);

			} pte_iterate_hashed_end();

		}

		/*

		 * Just do one more with the last used values.

		 */

		fixup_tlbie(vpn, psize, psize, ssize);

		asm volatile("eieio; tlbsync; ptesync":::"memory");



		if (lock_tlbie)