[POWERPC] Use 1TB segments (1189be65) · Commits · e / devices / android_kernel_fairphone_FP4

arch/powerpc/kernel/entry_64.S

+13 −1

Original line number	Original line	Diff line number	Diff line
	@@ -372,9 +372,17 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
	std r6,PACACURRENT(r13) /* Set new 'current' */		std r6,PACACURRENT(r13) /* Set new 'current' */

	ld r8,KSP(r4) /* new stack pointer */		ld r8,KSP(r4) /* new stack pointer */
			BEGIN_FTR_SECTION
			b 2f
			END_FTR_SECTION_IFCLR(CPU_FTR_SLB)
	BEGIN_FTR_SECTION		BEGIN_FTR_SECTION
	clrrdi r6,r8,28 /* get its ESID */		clrrdi r6,r8,28 /* get its ESID */
	clrrdi r9,r1,28 /* get current sp ESID */		clrrdi r9,r1,28 /* get current sp ESID */
			END_FTR_SECTION_IFCLR(CPU_FTR_1T_SEGMENT)
			BEGIN_FTR_SECTION
			clrrdi r6,r8,40 /* get its 1T ESID */
			clrrdi r9,r1,40 /* get current sp 1T ESID */
			END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
	clrldi. r0,r6,2 /* is new ESID c00000000? */		clrldi. r0,r6,2 /* is new ESID c00000000? */
	cmpd cr1,r6,r9 /* or is new ESID the same as current ESID? */		cmpd cr1,r6,r9 /* or is new ESID the same as current ESID? */
	cror eq,4*cr1+eq,eq		cror eq,4*cr1+eq,eq
	@@ -384,6 +392,11 @@ BEGIN_FTR_SECTION
	ld r7,KSP_VSID(r4) /* Get new stack's VSID */		ld r7,KSP_VSID(r4) /* Get new stack's VSID */
	oris r0,r6,(SLB_ESID_V)@h		oris r0,r6,(SLB_ESID_V)@h
	ori r0,r0,(SLB_NUM_BOLTED-1)@l		ori r0,r0,(SLB_NUM_BOLTED-1)@l
			BEGIN_FTR_SECTION
			li r9,MMU_SEGSIZE_1T /* insert B field */
			oris r6,r6,(MMU_SEGSIZE_1T << SLBIE_SSIZE_SHIFT)@h
			rldimi r7,r9,SLB_VSID_SSIZE_SHIFT,0
			END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)

	/* Update the last bolted SLB. No write barriers are needed		/* Update the last bolted SLB. No write barriers are needed
	* here, provided we only update the current CPU's SLB shadow		* here, provided we only update the current CPU's SLB shadow
	@@ -401,7 +414,6 @@ BEGIN_FTR_SECTION
	isync		isync

	2:		2:
	END_FTR_SECTION_IFSET(CPU_FTR_SLB)
	clrrdi r7,r8,THREAD_SHIFT /* base of new stack */		clrrdi r7,r8,THREAD_SHIFT /* base of new stack */
	/* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE		/* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE
	because we don't need to leave the 288-byte ABI gap at the		because we don't need to leave the 288-byte ABI gap at the

arch/powerpc/kernel/head_64.S

+1 −1

Original line number	Original line	Diff line number	Diff line
	@@ -935,7 +935,7 @@ _GLOBAL(do_stab_bolted)

	/* Calculate VSID */		/* Calculate VSID */
	/* This is a kernel address, so protovsid = ESID */		/* This is a kernel address, so protovsid = ESID */
	ASM_VSID_SCRAMBLE(r11, r9)		ASM_VSID_SCRAMBLE(r11, r9, 256M)
	rldic r9,r11,12,16 /* r9 = vsid << 12 */		rldic r9,r11,12,16 /* r9 = vsid << 12 */

	/* Search the primary group for a free entry */		/* Search the primary group for a free entry */

arch/powerpc/kernel/process.c

+7 −2

Original line number	Original line	Diff line number	Diff line
	@@ -564,10 +564,15 @@ int copy_thread(int nr, unsigned long clone_flags, unsigned long usp,

	#ifdef CONFIG_PPC64		#ifdef CONFIG_PPC64
	if (cpu_has_feature(CPU_FTR_SLB)) {		if (cpu_has_feature(CPU_FTR_SLB)) {
	unsigned long sp_vsid = get_kernel_vsid(sp);		unsigned long sp_vsid;
	unsigned long llp = mmu_psize_defs[mmu_linear_psize].sllp;		unsigned long llp = mmu_psize_defs[mmu_linear_psize].sllp;

	sp_vsid <<= SLB_VSID_SHIFT;		if (cpu_has_feature(CPU_FTR_1T_SEGMENT))
			sp_vsid = get_kernel_vsid(sp, MMU_SEGSIZE_1T)
			<< SLB_VSID_SHIFT_1T;
			else
			sp_vsid = get_kernel_vsid(sp, MMU_SEGSIZE_256M)
			<< SLB_VSID_SHIFT;
	sp_vsid \|= SLB_VSID_KERNEL \| llp;		sp_vsid \|= SLB_VSID_KERNEL \| llp;
	p->thread.ksp_vsid = sp_vsid;		p->thread.ksp_vsid = sp_vsid;
	}		}

arch/powerpc/mm/hash_low_64.S

+65 −8

Original line number	Original line	Diff line number	Diff line
	@@ -54,7 +54,7 @@

	/*		/*
	* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,		* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
	* pte_t *ptep, unsigned long trap, int local)		* pte_t *ptep, unsigned long trap, int local, int ssize)
	*		*
	* Adds a 4K page to the hash table in a segment of 4K pages only		* Adds a 4K page to the hash table in a segment of 4K pages only
	*/		*/
	@@ -66,6 +66,7 @@ _GLOBAL(__hash_page_4K)
	/* Save all params that we need after a function call */		/* Save all params that we need after a function call */
	std r6,STK_PARM(r6)(r1)		std r6,STK_PARM(r6)(r1)
	std r8,STK_PARM(r8)(r1)		std r8,STK_PARM(r8)(r1)
			std r9,STK_PARM(r9)(r1)

	/* Add _PAGE_PRESENT to access */		/* Add _PAGE_PRESENT to access */
	ori r4,r4,_PAGE_PRESENT		ori r4,r4,_PAGE_PRESENT
	@@ -117,6 +118,10 @@ _GLOBAL(__hash_page_4K)
	* r4 (access) is re-useable, we use it for the new HPTE flags		* r4 (access) is re-useable, we use it for the new HPTE flags
	*/		*/

			BEGIN_FTR_SECTION
			cmpdi r9,0 /* check segment size */
			bne 3f
			END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
	/* Calc va and put it in r29 */		/* Calc va and put it in r29 */
	rldicr r29,r5,28,63-28		rldicr r29,r5,28,63-28
	rldicl r3,r3,0,36		rldicl r3,r3,0,36
	@@ -126,9 +131,20 @@ _GLOBAL(__hash_page_4K)
	rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */		rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
	rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */		rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */
	xor r28,r5,r0		xor r28,r5,r0
			b 4f

			3: /* Calc VA and hash in r29 and r28 for 1T segment */
			sldi r29,r5,40 /* vsid << 40 */
			clrldi r3,r3,24 /* ea & 0xffffffffff */
			rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */
			clrldi r5,r5,40 /* vsid & 0xffffff */
			rldicl r0,r3,64-12,36 /* (ea >> 12) & 0xfffffff */
			xor r28,r28,r5
			or r29,r3,r29 /* VA */
			xor r28,r28,r0 /* hash */

	/* Convert linux PTE bits into HW equivalents */		/* Convert linux PTE bits into HW equivalents */
	andi. r3,r30,0x1fe /* Get basic set of flags */		4: andi. r3,r30,0x1fe /* Get basic set of flags */
	xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */		xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
	rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */		rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
	rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */		rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
	@@ -183,6 +199,7 @@ htab_insert_pte:
	mr r4,r29 /* Retreive va */		mr r4,r29 /* Retreive va */
	li r7,0 /* !bolted, !secondary */		li r7,0 /* !bolted, !secondary */
	li r8,MMU_PAGE_4K /* page size */		li r8,MMU_PAGE_4K /* page size */
			ld r9,STK_PARM(r9)(r1) /* segment size */
	_GLOBAL(htab_call_hpte_insert1)		_GLOBAL(htab_call_hpte_insert1)
	bl . /* Patched by htab_finish_init() */		bl . /* Patched by htab_finish_init() */
	cmpdi 0,r3,0		cmpdi 0,r3,0
	@@ -205,6 +222,7 @@ _GLOBAL(htab_call_hpte_insert1)
	mr r4,r29 /* Retreive va */		mr r4,r29 /* Retreive va */
	li r7,HPTE_V_SECONDARY /* !bolted, secondary */		li r7,HPTE_V_SECONDARY /* !bolted, secondary */
	li r8,MMU_PAGE_4K /* page size */		li r8,MMU_PAGE_4K /* page size */
			ld r9,STK_PARM(r9)(r1) /* segment size */
	_GLOBAL(htab_call_hpte_insert2)		_GLOBAL(htab_call_hpte_insert2)
	bl . /* Patched by htab_finish_init() */		bl . /* Patched by htab_finish_init() */
	cmpdi 0,r3,0		cmpdi 0,r3,0
	@@ -273,7 +291,8 @@ htab_modify_pte:
	/* Call ppc_md.hpte_updatepp */		/* Call ppc_md.hpte_updatepp */
	mr r5,r29 /* va */		mr r5,r29 /* va */
	li r6,MMU_PAGE_4K /* page size */		li r6,MMU_PAGE_4K /* page size */
	ld r7,STK_PARM(r8)(r1) /* get "local" param */		ld r7,STK_PARM(r9)(r1) /* segment size */
			ld r8,STK_PARM(r8)(r1) /* get "local" param */
	_GLOBAL(htab_call_hpte_updatepp)		_GLOBAL(htab_call_hpte_updatepp)
	bl . /* Patched by htab_finish_init() */		bl . /* Patched by htab_finish_init() */

	@@ -325,6 +344,7 @@ _GLOBAL(__hash_page_4K)
	/* Save all params that we need after a function call */		/* Save all params that we need after a function call */
	std r6,STK_PARM(r6)(r1)		std r6,STK_PARM(r6)(r1)
	std r8,STK_PARM(r8)(r1)		std r8,STK_PARM(r8)(r1)
			std r9,STK_PARM(r9)(r1)

	/* Add _PAGE_PRESENT to access */		/* Add _PAGE_PRESENT to access */
	ori r4,r4,_PAGE_PRESENT		ori r4,r4,_PAGE_PRESENT
	@@ -383,18 +403,33 @@ _GLOBAL(__hash_page_4K)
	/* Load the hidx index */		/* Load the hidx index */
	rldicl r25,r3,64-12,60		rldicl r25,r3,64-12,60

			BEGIN_FTR_SECTION
			cmpdi r9,0 /* check segment size */
			bne 3f
			END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
	/* Calc va and put it in r29 */		/* Calc va and put it in r29 */
	rldicr r29,r5,28,63-28 /* r29 = (vsid << 28) */		rldicr r29,r5,28,63-28 /* r29 = (vsid << 28) */
	rldicl r3,r3,0,36 /* r3 = (ea & 0x0fffffff) */		rldicl r3,r3,0,36 /* r3 = (ea & 0x0fffffff) */
	or r29,r3,r29 /* r29 = va		or r29,r3,r29 /* r29 = va */

	/* Calculate hash value for primary slot and store it in r28 */		/* Calculate hash value for primary slot and store it in r28 */
	rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */		rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
	rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */		rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */
	xor r28,r5,r0		xor r28,r5,r0
			b 4f

			3: /* Calc VA and hash in r29 and r28 for 1T segment */
			sldi r29,r5,40 /* vsid << 40 */
			clrldi r3,r3,24 /* ea & 0xffffffffff */
			rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */
			clrldi r5,r5,40 /* vsid & 0xffffff */
			rldicl r0,r3,64-12,36 /* (ea >> 12) & 0xfffffff */
			xor r28,r28,r5
			or r29,r3,r29 /* VA */
			xor r28,r28,r0 /* hash */

	/* Convert linux PTE bits into HW equivalents */		/* Convert linux PTE bits into HW equivalents */
	andi. r3,r30,0x1fe /* Get basic set of flags */		4: andi. r3,r30,0x1fe /* Get basic set of flags */
	xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */		xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
	rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */		rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
	rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */		rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
	@@ -462,6 +497,7 @@ htab_special_pfn:
	mr r4,r29 /* Retreive va */		mr r4,r29 /* Retreive va */
	li r7,0 /* !bolted, !secondary */		li r7,0 /* !bolted, !secondary */
	li r8,MMU_PAGE_4K /* page size */		li r8,MMU_PAGE_4K /* page size */
			ld r9,STK_PARM(r9)(r1) /* segment size */
	_GLOBAL(htab_call_hpte_insert1)		_GLOBAL(htab_call_hpte_insert1)
	bl . /* patched by htab_finish_init() */		bl . /* patched by htab_finish_init() */
	cmpdi 0,r3,0		cmpdi 0,r3,0
	@@ -488,6 +524,7 @@ _GLOBAL(htab_call_hpte_insert1)
	mr r4,r29 /* Retreive va */		mr r4,r29 /* Retreive va */
	li r7,HPTE_V_SECONDARY /* !bolted, secondary */		li r7,HPTE_V_SECONDARY /* !bolted, secondary */
	li r8,MMU_PAGE_4K /* page size */		li r8,MMU_PAGE_4K /* page size */
			ld r9,STK_PARM(r9)(r1) /* segment size */
	_GLOBAL(htab_call_hpte_insert2)		_GLOBAL(htab_call_hpte_insert2)
	bl . /* patched by htab_finish_init() */		bl . /* patched by htab_finish_init() */
	cmpdi 0,r3,0		cmpdi 0,r3,0
	@@ -586,7 +623,8 @@ htab_modify_pte:
	/* Call ppc_md.hpte_updatepp */		/* Call ppc_md.hpte_updatepp */
	mr r5,r29 /* va */		mr r5,r29 /* va */
	li r6,MMU_PAGE_4K /* page size */		li r6,MMU_PAGE_4K /* page size */
	ld r7,STK_PARM(r8)(r1) /* get "local" param */		ld r7,STK_PARM(r9)(r1) /* segment size */
			ld r8,STK_PARM(r8)(r1) /* get "local" param */
	_GLOBAL(htab_call_hpte_updatepp)		_GLOBAL(htab_call_hpte_updatepp)
	bl . /* patched by htab_finish_init() */		bl . /* patched by htab_finish_init() */

	@@ -634,6 +672,7 @@ _GLOBAL(__hash_page_64K)
	/* Save all params that we need after a function call */		/* Save all params that we need after a function call */
	std r6,STK_PARM(r6)(r1)		std r6,STK_PARM(r6)(r1)
	std r8,STK_PARM(r8)(r1)		std r8,STK_PARM(r8)(r1)
			std r9,STK_PARM(r9)(r1)

	/* Add _PAGE_PRESENT to access */		/* Add _PAGE_PRESENT to access */
	ori r4,r4,_PAGE_PRESENT		ori r4,r4,_PAGE_PRESENT
	@@ -690,6 +729,10 @@ END_FTR_SECTION_IFCLR(CPU_FTR_CI_LARGE_PAGE)
	* r4 (access) is re-useable, we use it for the new HPTE flags		* r4 (access) is re-useable, we use it for the new HPTE flags
	*/		*/

			BEGIN_FTR_SECTION
			cmpdi r9,0 /* check segment size */
			bne 3f
			END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
	/* Calc va and put it in r29 */		/* Calc va and put it in r29 */
	rldicr r29,r5,28,63-28		rldicr r29,r5,28,63-28
	rldicl r3,r3,0,36		rldicl r3,r3,0,36
	@@ -699,9 +742,20 @@ END_FTR_SECTION_IFCLR(CPU_FTR_CI_LARGE_PAGE)
	rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */		rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
	rldicl r0,r3,64-16,52 /* (ea >> 16) & 0xfff */		rldicl r0,r3,64-16,52 /* (ea >> 16) & 0xfff */
	xor r28,r5,r0		xor r28,r5,r0
			b 4f

			3: /* Calc VA and hash in r29 and r28 for 1T segment */
			sldi r29,r5,40 /* vsid << 40 */
			clrldi r3,r3,24 /* ea & 0xffffffffff */
			rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */
			clrldi r5,r5,40 /* vsid & 0xffffff */
			rldicl r0,r3,64-16,40 /* (ea >> 16) & 0xffffff */
			xor r28,r28,r5
			or r29,r3,r29 /* VA */
			xor r28,r28,r0 /* hash */

	/* Convert linux PTE bits into HW equivalents */		/* Convert linux PTE bits into HW equivalents */
	andi. r3,r30,0x1fe /* Get basic set of flags */		4: andi. r3,r30,0x1fe /* Get basic set of flags */
	xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */		xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
	rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */		rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
	rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */		rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
	@@ -756,6 +810,7 @@ ht64_insert_pte:
	mr r4,r29 /* Retreive va */		mr r4,r29 /* Retreive va */
	li r7,0 /* !bolted, !secondary */		li r7,0 /* !bolted, !secondary */
	li r8,MMU_PAGE_64K		li r8,MMU_PAGE_64K
			ld r9,STK_PARM(r9)(r1) /* segment size */
	_GLOBAL(ht64_call_hpte_insert1)		_GLOBAL(ht64_call_hpte_insert1)
	bl . /* patched by htab_finish_init() */		bl . /* patched by htab_finish_init() */
	cmpdi 0,r3,0		cmpdi 0,r3,0
	@@ -778,6 +833,7 @@ _GLOBAL(ht64_call_hpte_insert1)
	mr r4,r29 /* Retreive va */		mr r4,r29 /* Retreive va */
	li r7,HPTE_V_SECONDARY /* !bolted, secondary */		li r7,HPTE_V_SECONDARY /* !bolted, secondary */
	li r8,MMU_PAGE_64K		li r8,MMU_PAGE_64K
			ld r9,STK_PARM(r9)(r1) /* segment size */
	_GLOBAL(ht64_call_hpte_insert2)		_GLOBAL(ht64_call_hpte_insert2)
	bl . /* patched by htab_finish_init() */		bl . /* patched by htab_finish_init() */
	cmpdi 0,r3,0		cmpdi 0,r3,0
	@@ -846,7 +902,8 @@ ht64_modify_pte:
	/* Call ppc_md.hpte_updatepp */		/* Call ppc_md.hpte_updatepp */
	mr r5,r29 /* va */		mr r5,r29 /* va */
	li r6,MMU_PAGE_64K		li r6,MMU_PAGE_64K
	ld r7,STK_PARM(r8)(r1) /* get "local" param */		ld r7,STK_PARM(r9)(r1) /* segment size */
			ld r8,STK_PARM(r8)(r1) /* get "local" param */
	_GLOBAL(ht64_call_hpte_updatepp)		_GLOBAL(ht64_call_hpte_updatepp)
	bl . /* patched by htab_finish_init() */		bl . /* patched by htab_finish_init() */

arch/powerpc/mm/hash_native_64.c

+46 −46

Original line number	Original line	Diff line number	Diff line
	@@ -38,7 +38,7 @@

	static DEFINE_SPINLOCK(native_tlbie_lock);		static DEFINE_SPINLOCK(native_tlbie_lock);

	static inline void __tlbie(unsigned long va, unsigned int psize)		static inline void __tlbie(unsigned long va, int psize, int ssize)
	{		{
	unsigned int penc;		unsigned int penc;

	@@ -48,18 +48,20 @@ static inline void __tlbie(unsigned long va, unsigned int psize)
	switch (psize) {		switch (psize) {
	case MMU_PAGE_4K:		case MMU_PAGE_4K:
	va &= ~0xffful;		va &= ~0xffful;
			va \|= ssize << 8;
	asm volatile("tlbie %0,0" : : "r" (va) : "memory");		asm volatile("tlbie %0,0" : : "r" (va) : "memory");
	break;		break;
	default:		default:
	penc = mmu_psize_defs[psize].penc;		penc = mmu_psize_defs[psize].penc;
	va &= ~((1ul << mmu_psize_defs[psize].shift) - 1);		va &= ~((1ul << mmu_psize_defs[psize].shift) - 1);
	va \|= penc << 12;		va \|= penc << 12;
			va \|= ssize << 8;
	asm volatile("tlbie %0,1" : : "r" (va) : "memory");		asm volatile("tlbie %0,1" : : "r" (va) : "memory");
	break;		break;
	}		}
	}		}

	static inline void __tlbiel(unsigned long va, unsigned int psize)		static inline void __tlbiel(unsigned long va, int psize, int ssize)
	{		{
	unsigned int penc;		unsigned int penc;

	@@ -69,6 +71,7 @@ static inline void __tlbiel(unsigned long va, unsigned int psize)
	switch (psize) {		switch (psize) {
	case MMU_PAGE_4K:		case MMU_PAGE_4K:
	va &= ~0xffful;		va &= ~0xffful;
			va \|= ssize << 8;
	asm volatile(".long 0x7c000224 \| (%0 << 11) \| (0 << 21)"		asm volatile(".long 0x7c000224 \| (%0 << 11) \| (0 << 21)"
	: : "r"(va) : "memory");		: : "r"(va) : "memory");
	break;		break;
	@@ -76,6 +79,7 @@ static inline void __tlbiel(unsigned long va, unsigned int psize)
	penc = mmu_psize_defs[psize].penc;		penc = mmu_psize_defs[psize].penc;
	va &= ~((1ul << mmu_psize_defs[psize].shift) - 1);		va &= ~((1ul << mmu_psize_defs[psize].shift) - 1);
	va \|= penc << 12;		va \|= penc << 12;
			va \|= ssize << 8;
	asm volatile(".long 0x7c000224 \| (%0 << 11) \| (1 << 21)"		asm volatile(".long 0x7c000224 \| (%0 << 11) \| (1 << 21)"
	: : "r"(va) : "memory");		: : "r"(va) : "memory");
	break;		break;
	@@ -83,7 +87,7 @@ static inline void __tlbiel(unsigned long va, unsigned int psize)

	}		}

	static inline void tlbie(unsigned long va, int psize, int local)		static inline void tlbie(unsigned long va, int psize, int ssize, int local)
	{		{
	unsigned int use_local = local && cpu_has_feature(CPU_FTR_TLBIEL);		unsigned int use_local = local && cpu_has_feature(CPU_FTR_TLBIEL);
	int lock_tlbie = !cpu_has_feature(CPU_FTR_LOCKLESS_TLBIE);		int lock_tlbie = !cpu_has_feature(CPU_FTR_LOCKLESS_TLBIE);
	@@ -94,10 +98,10 @@ static inline void tlbie(unsigned long va, int psize, int local)
	spin_lock(&native_tlbie_lock);		spin_lock(&native_tlbie_lock);
	asm volatile("ptesync": : :"memory");		asm volatile("ptesync": : :"memory");
	if (use_local) {		if (use_local) {
	__tlbiel(va, psize);		__tlbiel(va, psize, ssize);
	asm volatile("ptesync": : :"memory");		asm volatile("ptesync": : :"memory");
	} else {		} else {
	__tlbie(va, psize);		__tlbie(va, psize, ssize);
	asm volatile("eieio; tlbsync; ptesync": : :"memory");		asm volatile("eieio; tlbsync; ptesync": : :"memory");
	}		}
	if (lock_tlbie && !use_local)		if (lock_tlbie && !use_local)
	@@ -126,7 +130,7 @@ static inline void native_unlock_hpte(struct hash_pte *hptep)

	static long native_hpte_insert(unsigned long hpte_group, unsigned long va,		static long native_hpte_insert(unsigned long hpte_group, unsigned long va,
	unsigned long pa, unsigned long rflags,		unsigned long pa, unsigned long rflags,
	unsigned long vflags, int psize)		unsigned long vflags, int psize, int ssize)
	{		{
	struct hash_pte *hptep = htab_address + hpte_group;		struct hash_pte *hptep = htab_address + hpte_group;
	unsigned long hpte_v, hpte_r;		unsigned long hpte_v, hpte_r;
	@@ -153,7 +157,7 @@ static long native_hpte_insert(unsigned long hpte_group, unsigned long va,
	if (i == HPTES_PER_GROUP)		if (i == HPTES_PER_GROUP)
	return -1;		return -1;

	hpte_v = hpte_encode_v(va, psize) \| vflags \| HPTE_V_VALID;		hpte_v = hpte_encode_v(va, psize, ssize) \| vflags \| HPTE_V_VALID;
	hpte_r = hpte_encode_r(pa, psize) \| rflags;		hpte_r = hpte_encode_r(pa, psize) \| rflags;

	if (!(vflags & HPTE_V_BOLTED)) {		if (!(vflags & HPTE_V_BOLTED)) {
	@@ -215,13 +219,14 @@ static long native_hpte_remove(unsigned long hpte_group)
	}		}

	static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,		static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
	unsigned long va, int psize, int local)		unsigned long va, int psize, int ssize,
			int local)
	{		{
	struct hash_pte *hptep = htab_address + slot;		struct hash_pte *hptep = htab_address + slot;
	unsigned long hpte_v, want_v;		unsigned long hpte_v, want_v;
	int ret = 0;		int ret = 0;

	want_v = hpte_encode_v(va, psize);		want_v = hpte_encode_v(va, psize, ssize);

	DBG_LOW(" update(va=%016lx, avpnv=%016lx, hash=%016lx, newpp=%x)",		DBG_LOW(" update(va=%016lx, avpnv=%016lx, hash=%016lx, newpp=%x)",
	va, want_v & HPTE_V_AVPN, slot, newpp);		va, want_v & HPTE_V_AVPN, slot, newpp);
	@@ -243,40 +248,33 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
	native_unlock_hpte(hptep);		native_unlock_hpte(hptep);

	/* Ensure it is out of the tlb too. */		/* Ensure it is out of the tlb too. */
	tlbie(va, psize, local);		tlbie(va, psize, ssize, local);

	return ret;		return ret;
	}		}

	static long native_hpte_find(unsigned long va, int psize)		static long native_hpte_find(unsigned long va, int psize, int ssize)
	{		{
	struct hash_pte *hptep;		struct hash_pte *hptep;
	unsigned long hash;		unsigned long hash;
	unsigned long i, j;		unsigned long i;
	long slot;		long slot;
	unsigned long want_v, hpte_v;		unsigned long want_v, hpte_v;

	hash = hpt_hash(va, mmu_psize_defs[psize].shift);		hash = hpt_hash(va, mmu_psize_defs[psize].shift, ssize);
	want_v = hpte_encode_v(va, psize);		want_v = hpte_encode_v(va, psize, ssize);

	for (j = 0; j < 2; j++) {		/* Bolted mappings are only ever in the primary group */
	slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;		slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
	for (i = 0; i < HPTES_PER_GROUP; i++) {		for (i = 0; i < HPTES_PER_GROUP; i++) {
	hptep = htab_address + slot;		hptep = htab_address + slot;
	hpte_v = hptep->v;		hpte_v = hptep->v;

	if (HPTE_V_COMPARE(hpte_v, want_v)		if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID))
	&& (hpte_v & HPTE_V_VALID)
	&& ( !!(hpte_v & HPTE_V_SECONDARY) == j)) {
	/* HPTE matches */		/* HPTE matches */
	if (j)
	slot = -slot;
	return slot;		return slot;
	}
	++slot;		++slot;
	}		}
	hash = ~hash;
	}

	return -1;		return -1;
	}		}
	@@ -289,16 +287,16 @@ static long native_hpte_find(unsigned long va, int psize)
	* No need to lock here because we should be the only user.		* No need to lock here because we should be the only user.
	*/		*/
	static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,		static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,
	int psize)		int psize, int ssize)
	{		{
	unsigned long vsid, va;		unsigned long vsid, va;
	long slot;		long slot;
	struct hash_pte *hptep;		struct hash_pte *hptep;

	vsid = get_kernel_vsid(ea);		vsid = get_kernel_vsid(ea, ssize);
	va = (vsid << 28) \| (ea & 0x0fffffff);		va = hpt_va(ea, vsid, ssize);

	slot = native_hpte_find(va, psize);		slot = native_hpte_find(va, psize, ssize);
	if (slot == -1)		if (slot == -1)
	panic("could not find page to bolt\n");		panic("could not find page to bolt\n");
	hptep = htab_address + slot;		hptep = htab_address + slot;
	@@ -308,11 +306,11 @@ static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,
	(newpp & (HPTE_R_PP \| HPTE_R_N));		(newpp & (HPTE_R_PP \| HPTE_R_N));

	/* Ensure it is out of the tlb too. */		/* Ensure it is out of the tlb too. */
	tlbie(va, psize, 0);		tlbie(va, psize, ssize, 0);
	}		}

	static void native_hpte_invalidate(unsigned long slot, unsigned long va,		static void native_hpte_invalidate(unsigned long slot, unsigned long va,
	int psize, int local)		int psize, int ssize, int local)
	{		{
	struct hash_pte *hptep = htab_address + slot;		struct hash_pte *hptep = htab_address + slot;
	unsigned long hpte_v;		unsigned long hpte_v;
	@@ -323,7 +321,7 @@ static void native_hpte_invalidate(unsigned long slot, unsigned long va,

	DBG_LOW(" invalidate(va=%016lx, hash: %x)\n", va, slot);		DBG_LOW(" invalidate(va=%016lx, hash: %x)\n", va, slot);

	want_v = hpte_encode_v(va, psize);		want_v = hpte_encode_v(va, psize, ssize);
	native_lock_hpte(hptep);		native_lock_hpte(hptep);
	hpte_v = hptep->v;		hpte_v = hptep->v;

	@@ -335,7 +333,7 @@ static void native_hpte_invalidate(unsigned long slot, unsigned long va,
	hptep->v = 0;		hptep->v = 0;

	/* Invalidate the TLB */		/* Invalidate the TLB */
	tlbie(va, psize, local);		tlbie(va, psize, ssize, local);

	local_irq_restore(flags);		local_irq_restore(flags);
	}		}
	@@ -345,7 +343,7 @@ static void native_hpte_invalidate(unsigned long slot, unsigned long va,
	#define LP_MASK(i) ((0xFF >> (i)) << LP_SHIFT)		#define LP_MASK(i) ((0xFF >> (i)) << LP_SHIFT)

	static void hpte_decode(struct hash_pte *hpte, unsigned long slot,		static void hpte_decode(struct hash_pte *hpte, unsigned long slot,
	int psize, unsigned long va)		int psize, int ssize, unsigned long *va)
	{		{
	unsigned long hpte_r = hpte->r;		unsigned long hpte_r = hpte->r;
	unsigned long hpte_v = hpte->v;		unsigned long hpte_v = hpte->v;
	@@ -401,6 +399,7 @@ static void hpte_decode(struct hash_pte *hpte, unsigned long slot,

	*va = avpn;		*va = avpn;
	*psize = size;		*psize = size;
			*ssize = hpte_v >> HPTE_V_SSIZE_SHIFT;
	}		}

	/*		/*
	@@ -417,7 +416,7 @@ static void native_hpte_clear(void)
	struct hash_pte *hptep = htab_address;		struct hash_pte *hptep = htab_address;
	unsigned long hpte_v, va;		unsigned long hpte_v, va;
	unsigned long pteg_count;		unsigned long pteg_count;
	int psize;		int psize, ssize;

	pteg_count = htab_hash_mask + 1;		pteg_count = htab_hash_mask + 1;

	@@ -443,9 +442,9 @@ static void native_hpte_clear(void)
	* already hold the native_tlbie_lock.		* already hold the native_tlbie_lock.
	*/		*/
	if (hpte_v & HPTE_V_VALID) {		if (hpte_v & HPTE_V_VALID) {
	hpte_decode(hptep, slot, &psize, &va);		hpte_decode(hptep, slot, &psize, &ssize, &va);
	hptep->v = 0;		hptep->v = 0;
	__tlbie(va, psize);		__tlbie(va, psize, ssize);
	}		}
	}		}

	@@ -468,6 +467,7 @@ static void native_flush_hash_range(unsigned long number, int local)
	real_pte_t pte;		real_pte_t pte;
	struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch);		struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch);
	unsigned long psize = batch->psize;		unsigned long psize = batch->psize;
			int ssize = batch->ssize;
	int i;		int i;

	local_irq_save(flags);		local_irq_save(flags);
	@@ -477,14 +477,14 @@ static void native_flush_hash_range(unsigned long number, int local)
	pte = batch->pte[i];		pte = batch->pte[i];

	pte_iterate_hashed_subpages(pte, psize, va, index, shift) {		pte_iterate_hashed_subpages(pte, psize, va, index, shift) {
	hash = hpt_hash(va, shift);		hash = hpt_hash(va, shift, ssize);
	hidx = __rpte_to_hidx(pte, index);		hidx = __rpte_to_hidx(pte, index);
	if (hidx & _PTEIDX_SECONDARY)		if (hidx & _PTEIDX_SECONDARY)
	hash = ~hash;		hash = ~hash;
	slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;		slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
	slot += hidx & _PTEIDX_GROUP_IX;		slot += hidx & _PTEIDX_GROUP_IX;
	hptep = htab_address + slot;		hptep = htab_address + slot;
	want_v = hpte_encode_v(va, psize);		want_v = hpte_encode_v(va, psize, ssize);
	native_lock_hpte(hptep);		native_lock_hpte(hptep);
	hpte_v = hptep->v;		hpte_v = hptep->v;
	if (!HPTE_V_COMPARE(hpte_v, want_v) \|\|		if (!HPTE_V_COMPARE(hpte_v, want_v) \|\|
	@@ -504,7 +504,7 @@ static void native_flush_hash_range(unsigned long number, int local)

	pte_iterate_hashed_subpages(pte, psize, va, index,		pte_iterate_hashed_subpages(pte, psize, va, index,
	shift) {		shift) {
	__tlbiel(va, psize);		__tlbiel(va, psize, ssize);
	} pte_iterate_hashed_end();		} pte_iterate_hashed_end();
	}		}
	asm volatile("ptesync":::"memory");		asm volatile("ptesync":::"memory");
	@@ -521,7 +521,7 @@ static void native_flush_hash_range(unsigned long number, int local)

	pte_iterate_hashed_subpages(pte, psize, va, index,		pte_iterate_hashed_subpages(pte, psize, va, index,
	shift) {		shift) {
	__tlbie(va, psize);		__tlbie(va, psize, ssize);
	} pte_iterate_hashed_end();		} pte_iterate_hashed_end();
	}		}
	asm volatile("eieio; tlbsync; ptesync":::"memory");		asm volatile("eieio; tlbsync; ptesync":::"memory");