powerpc/mm: handle hugepage size correctly when invalidating hpte entries (db3d8534) · Commits · e / devices / android_kernel_xiaomi_nabu

arch/powerpc/include/asm/machdep.h

+4 −4

Original line number	Original line	Diff line number	Diff line
	@@ -36,13 +36,13 @@ struct machdep_calls {
	#ifdef CONFIG_PPC64		#ifdef CONFIG_PPC64
	void (*hpte_invalidate)(unsigned long slot,		void (*hpte_invalidate)(unsigned long slot,
	unsigned long vpn,		unsigned long vpn,
	int psize, int ssize,		int bpsize, int apsize,
	int local);		int ssize, int local);
	long (*hpte_updatepp)(unsigned long slot,		long (*hpte_updatepp)(unsigned long slot,
	unsigned long newpp,		unsigned long newpp,
	unsigned long vpn,		unsigned long vpn,
	int psize, int ssize,		int bpsize, int apsize,
	int local);		int ssize, int local);
	void (*hpte_updateboltedpp)(unsigned long newpp,		void (*hpte_updateboltedpp)(unsigned long newpp,
	unsigned long ea,		unsigned long ea,
	int psize, int ssize);		int psize, int ssize);

arch/powerpc/kvm/book3s_64_mmu_host.c

+1 −1

Original line number	Original line	Diff line number	Diff line
	@@ -34,7 +34,7 @@
	void kvmppc_mmu_invalidate_pte(struct kvm_vcpu vcpu, struct hpte_cache pte)		void kvmppc_mmu_invalidate_pte(struct kvm_vcpu vcpu, struct hpte_cache pte)
	{		{
	ppc_md.hpte_invalidate(pte->slot, pte->host_vpn,		ppc_md.hpte_invalidate(pte->slot, pte->host_vpn,
	MMU_PAGE_4K, MMU_SEGSIZE_256M,		MMU_PAGE_4K, MMU_PAGE_4K, MMU_SEGSIZE_256M,
	false);		false);
	}		}

arch/powerpc/mm/hash_low_64.S

+12 −9

Original line number	Original line	Diff line number	Diff line
	@@ -289,9 +289,10 @@ htab_modify_pte:

	/* Call ppc_md.hpte_updatepp */		/* Call ppc_md.hpte_updatepp */
	mr r5,r29 /* vpn */		mr r5,r29 /* vpn */
	li r6,MMU_PAGE_4K /* page size */		li r6,MMU_PAGE_4K /* base page size */
	ld r7,STK_PARAM(R9)(r1) /* segment size */		li r7,MMU_PAGE_4K /* actual page size */
	ld r8,STK_PARAM(R8)(r1) /* get "local" param */		ld r8,STK_PARAM(R9)(r1) /* segment size */
			ld r9,STK_PARAM(R8)(r1) /* get "local" param */
	_GLOBAL(htab_call_hpte_updatepp)		_GLOBAL(htab_call_hpte_updatepp)
	bl . /* Patched by htab_finish_init() */		bl . /* Patched by htab_finish_init() */

	@@ -649,9 +650,10 @@ htab_modify_pte:

	/* Call ppc_md.hpte_updatepp */		/* Call ppc_md.hpte_updatepp */
	mr r5,r29 /* vpn */		mr r5,r29 /* vpn */
	li r6,MMU_PAGE_4K /* page size */		li r6,MMU_PAGE_4K /* base page size */
	ld r7,STK_PARAM(R9)(r1) /* segment size */		li r7,MMU_PAGE_4K /* actual page size */
	ld r8,STK_PARAM(R8)(r1) /* get "local" param */		ld r8,STK_PARAM(R9)(r1) /* segment size */
			ld r9,STK_PARAM(R8)(r1) /* get "local" param */
	_GLOBAL(htab_call_hpte_updatepp)		_GLOBAL(htab_call_hpte_updatepp)
	bl . /* patched by htab_finish_init() */		bl . /* patched by htab_finish_init() */

	@@ -937,9 +939,10 @@ ht64_modify_pte:

	/* Call ppc_md.hpte_updatepp */		/* Call ppc_md.hpte_updatepp */
	mr r5,r29 /* vpn */		mr r5,r29 /* vpn */
	li r6,MMU_PAGE_64K		li r6,MMU_PAGE_64K /* base page size */
	ld r7,STK_PARAM(R9)(r1) /* segment size */		li r7,MMU_PAGE_64K /* actual page size */
	ld r8,STK_PARAM(R8)(r1) /* get "local" param */		ld r8,STK_PARAM(R9)(r1) /* segment size */
			ld r9,STK_PARAM(R8)(r1) /* get "local" param */
	_GLOBAL(ht64_call_hpte_updatepp)		_GLOBAL(ht64_call_hpte_updatepp)
	bl . /* patched by htab_finish_init() */		bl . /* patched by htab_finish_init() */

arch/powerpc/mm/hash_native_64.c

+45 −77

Original line number	Original line	Diff line number	Diff line
	@@ -273,61 +273,15 @@ static long native_hpte_remove(unsigned long hpte_group)
	return i;		return i;
	}		}

	static inline int __hpte_actual_psize(unsigned int lp, int psize)
	{
	int i, shift;
	unsigned int mask;

	/* start from 1 ignoring MMU_PAGE_4K */
	for (i = 1; i < MMU_PAGE_COUNT; i++) {

	/* invalid penc */
	if (mmu_psize_defs[psize].penc[i] == -1)
	continue;
	/*
	* encoding bits per actual page size
	* PTE LP actual page size
	* rrrr rrrz >=8KB
	* rrrr rrzz >=16KB
	* rrrr rzzz >=32KB
	* rrrr zzzz >=64KB
	* .......
	*/
	shift = mmu_psize_defs[i].shift - LP_SHIFT;
	if (shift > LP_BITS)
	shift = LP_BITS;
	mask = (1 << shift) - 1;
	if ((lp & mask) == mmu_psize_defs[psize].penc[i])
	return i;
	}
	return -1;
	}

	static inline int hpte_actual_psize(struct hash_pte *hptep, int psize)
	{
	/* Look at the 8 bit LP value */
	unsigned int lp = (hptep->r >> LP_SHIFT) & ((1 << LP_BITS) - 1);

	if (!(hptep->v & HPTE_V_VALID))
	return -1;

	/* First check if it is large page */
	if (!(hptep->v & HPTE_V_LARGE))
	return MMU_PAGE_4K;

	return __hpte_actual_psize(lp, psize);
	}

	static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,		static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
	unsigned long vpn, int psize, int ssize,		unsigned long vpn, int bpsize,
	int local)		int apsize, int ssize, int local)
	{		{
	struct hash_pte *hptep = htab_address + slot;		struct hash_pte *hptep = htab_address + slot;
	unsigned long hpte_v, want_v;		unsigned long hpte_v, want_v;
	int ret = 0;		int ret = 0;
	int actual_psize;

	want_v = hpte_encode_avpn(vpn, psize, ssize);		want_v = hpte_encode_avpn(vpn, bpsize, ssize);

	DBG_LOW(" update(vpn=%016lx, avpnv=%016lx, group=%lx, newpp=%lx)",		DBG_LOW(" update(vpn=%016lx, avpnv=%016lx, group=%lx, newpp=%lx)",
	vpn, want_v & HPTE_V_AVPN, slot, newpp);		vpn, want_v & HPTE_V_AVPN, slot, newpp);
	@@ -335,7 +289,6 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
	native_lock_hpte(hptep);		native_lock_hpte(hptep);

	hpte_v = hptep->v;		hpte_v = hptep->v;
	actual_psize = hpte_actual_psize(hptep, psize);
	/*		/*
	* We need to invalidate the TLB always because hpte_remove doesn't do		* We need to invalidate the TLB always because hpte_remove doesn't do
	* a tlb invalidate. If a hash bucket gets full, we "evict" a more/less		* a tlb invalidate. If a hash bucket gets full, we "evict" a more/less
	@@ -343,12 +296,7 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
	* (hpte_remove) because we assume the old translation is still		* (hpte_remove) because we assume the old translation is still
	* technically "valid".		* technically "valid".
	*/		*/
	if (actual_psize < 0) {		if (!HPTE_V_COMPARE(hpte_v, want_v) \|\| !(hpte_v & HPTE_V_VALID)) {
	actual_psize = psize;
	ret = -1;
	goto err_out;
	}
	if (!HPTE_V_COMPARE(hpte_v, want_v)) {
	DBG_LOW(" -> miss\n");		DBG_LOW(" -> miss\n");
	ret = -1;		ret = -1;
	} else {		} else {
	@@ -357,11 +305,10 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
	hptep->r = (hptep->r & ~(HPTE_R_PP \| HPTE_R_N)) \|		hptep->r = (hptep->r & ~(HPTE_R_PP \| HPTE_R_N)) \|
	(newpp & (HPTE_R_PP \| HPTE_R_N \| HPTE_R_C));		(newpp & (HPTE_R_PP \| HPTE_R_N \| HPTE_R_C));
	}		}
	err_out:
	native_unlock_hpte(hptep);		native_unlock_hpte(hptep);

	/* Ensure it is out of the tlb too. */		/* Ensure it is out of the tlb too. */
	tlbie(vpn, psize, actual_psize, ssize, local);		tlbie(vpn, bpsize, apsize, ssize, local);

	return ret;		return ret;
	}		}
	@@ -402,7 +349,6 @@ static long native_hpte_find(unsigned long vpn, int psize, int ssize)
	static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,		static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,
	int psize, int ssize)		int psize, int ssize)
	{		{
	int actual_psize;
	unsigned long vpn;		unsigned long vpn;
	unsigned long vsid;		unsigned long vsid;
	long slot;		long slot;
	@@ -415,36 +361,33 @@ static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,
	if (slot == -1)		if (slot == -1)
	panic("could not find page to bolt\n");		panic("could not find page to bolt\n");
	hptep = htab_address + slot;		hptep = htab_address + slot;
	actual_psize = hpte_actual_psize(hptep, psize);
	if (actual_psize < 0)
	actual_psize = psize;

	/* Update the HPTE */		/* Update the HPTE */
	hptep->r = (hptep->r & ~(HPTE_R_PP \| HPTE_R_N)) \|		hptep->r = (hptep->r & ~(HPTE_R_PP \| HPTE_R_N)) \|
	(newpp & (HPTE_R_PP \| HPTE_R_N));		(newpp & (HPTE_R_PP \| HPTE_R_N));
			/*
	/* Ensure it is out of the tlb too. */		* Ensure it is out of the tlb too. Bolted entries base and
	tlbie(vpn, psize, actual_psize, ssize, 0);		* actual page size will be same.
			*/
			tlbie(vpn, psize, psize, ssize, 0);
	}		}

	static void native_hpte_invalidate(unsigned long slot, unsigned long vpn,		static void native_hpte_invalidate(unsigned long slot, unsigned long vpn,
	int psize, int ssize, int local)		int bpsize, int apsize, int ssize, int local)
	{		{
	struct hash_pte *hptep = htab_address + slot;		struct hash_pte *hptep = htab_address + slot;
	unsigned long hpte_v;		unsigned long hpte_v;
	unsigned long want_v;		unsigned long want_v;
	unsigned long flags;		unsigned long flags;
	int actual_psize;

	local_irq_save(flags);		local_irq_save(flags);

	DBG_LOW(" invalidate(vpn=%016lx, hash: %lx)\n", vpn, slot);		DBG_LOW(" invalidate(vpn=%016lx, hash: %lx)\n", vpn, slot);

	want_v = hpte_encode_avpn(vpn, psize, ssize);		want_v = hpte_encode_avpn(vpn, bpsize, ssize);
	native_lock_hpte(hptep);		native_lock_hpte(hptep);
	hpte_v = hptep->v;		hpte_v = hptep->v;

	actual_psize = hpte_actual_psize(hptep, psize);
	/*		/*
	* We need to invalidate the TLB always because hpte_remove doesn't do		* We need to invalidate the TLB always because hpte_remove doesn't do
	* a tlb invalidate. If a hash bucket gets full, we "evict" a more/less		* a tlb invalidate. If a hash bucket gets full, we "evict" a more/less
	@@ -452,23 +395,48 @@ static void native_hpte_invalidate(unsigned long slot, unsigned long vpn,
	* (hpte_remove) because we assume the old translation is still		* (hpte_remove) because we assume the old translation is still
	* technically "valid".		* technically "valid".
	*/		*/
	if (actual_psize < 0) {		if (!HPTE_V_COMPARE(hpte_v, want_v) \|\| !(hpte_v & HPTE_V_VALID))
	actual_psize = psize;
	native_unlock_hpte(hptep);
	goto err_out;
	}
	if (!HPTE_V_COMPARE(hpte_v, want_v))
	native_unlock_hpte(hptep);		native_unlock_hpte(hptep);
	else		else
	/* Invalidate the hpte. NOTE: this also unlocks it */		/* Invalidate the hpte. NOTE: this also unlocks it */
	hptep->v = 0;		hptep->v = 0;

	err_out:
	/* Invalidate the TLB */		/* Invalidate the TLB */
	tlbie(vpn, psize, actual_psize, ssize, local);		tlbie(vpn, bpsize, apsize, ssize, local);

	local_irq_restore(flags);		local_irq_restore(flags);
	}		}

			static inline int __hpte_actual_psize(unsigned int lp, int psize)
			{
			int i, shift;
			unsigned int mask;

			/* start from 1 ignoring MMU_PAGE_4K */
			for (i = 1; i < MMU_PAGE_COUNT; i++) {

			/* invalid penc */
			if (mmu_psize_defs[psize].penc[i] == -1)
			continue;
			/*
			* encoding bits per actual page size
			* PTE LP actual page size
			* rrrr rrrz >=8KB
			* rrrr rrzz >=16KB
			* rrrr rzzz >=32KB
			* rrrr zzzz >=64KB
			* .......
			*/
			shift = mmu_psize_defs[i].shift - LP_SHIFT;
			if (shift > LP_BITS)
			shift = LP_BITS;
			mask = (1 << shift) - 1;
			if ((lp & mask) == mmu_psize_defs[psize].penc[i])
			return i;
			}
			return -1;
			}

	static void hpte_decode(struct hash_pte *hpte, unsigned long slot,		static void hpte_decode(struct hash_pte *hpte, unsigned long slot,
	int psize, int apsize, int ssize, unsigned long vpn)		int psize, int apsize, int ssize, unsigned long vpn)
	{		{

arch/powerpc/mm/hash_utils_64.c

+7 −2

Original line number	Original line	Diff line number	Diff line
	@@ -1232,7 +1232,11 @@ void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize, int ssize,
	slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;		slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
	slot += hidx & _PTEIDX_GROUP_IX;		slot += hidx & _PTEIDX_GROUP_IX;
	DBG_LOW(" sub %ld: hash=%lx, hidx=%lx\n", index, slot, hidx);		DBG_LOW(" sub %ld: hash=%lx, hidx=%lx\n", index, slot, hidx);
	ppc_md.hpte_invalidate(slot, vpn, psize, ssize, local);		/*
			* We use same base page size and actual psize, because we don't
			* use these functions for hugepage
			*/
			ppc_md.hpte_invalidate(slot, vpn, psize, psize, ssize, local);
	} pte_iterate_hashed_end();		} pte_iterate_hashed_end();

	#ifdef CONFIG_PPC_TRANSACTIONAL_MEM		#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
	@@ -1365,7 +1369,8 @@ static void kernel_unmap_linear_page(unsigned long vaddr, unsigned long lmi)
	hash = ~hash;		hash = ~hash;
	slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;		slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
	slot += hidx & _PTEIDX_GROUP_IX;		slot += hidx & _PTEIDX_GROUP_IX;
	ppc_md.hpte_invalidate(slot, vpn, mmu_linear_psize, mmu_kernel_ssize, 0);		ppc_md.hpte_invalidate(slot, vpn, mmu_linear_psize, mmu_linear_psize,
			mmu_kernel_ssize, 0);
	}		}

	void kernel_map_pages(struct page *page, int numpages, int enable)		void kernel_map_pages(struct page *page, int numpages, int enable)