Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit bb7f3808 authored by Christophe Leroy's avatar Christophe Leroy Committed by Scott Wood
Browse files

powerpc/8xx: Don't use page table for linear memory space 



Instead of using the first level page table to define mappings for
the linear memory space, we can use direct mapping from the TLB
handling routines. This has several advantages:
* No need to read the tables at each TLB miss
* No issue in 16k pages mode where the 1st level table maps 64 Mbytes

The size of the available linear space is known at system startup.
In order to avoid data access at each TLB miss to know the memory
size, the TLB routine is patched at startup with the proper size

This patch provides a 10%-15% improvment of TLB miss handling for
kernel addresses

Signed-off-by: default avatarChristophe Leroy <christophe.leroy@c-s.fr>
Signed-off-by: default avatarScott Wood <oss@buserror.net>
parent 6264dbb9

arch/powerpc/kernel/head_8xx.S

+37 −34
Original line number Original line Diff line number Diff line
@@ -389,52 +389,52 @@ InstructionTLBMiss: 
 * not enough space in the DataStoreTLBMiss area

 * not enough space in the DataStoreTLBMiss area

 */

 */

DTLBMissIMMR:

DTLBMissIMMR:

	mtcr	r3

	mtcr	r10

	/* Set 512k byte guarded page and mark it valid */

	/* Set 512k byte guarded page and mark it valid */

	li	r10, MD_PS512K | MD_GUARDED | MD_SVALID

	li	r10, MD_PS512K | MD_GUARDED | MD_SVALID

	MTSPR_CPU6(SPRN_MD_TWC, r10, r3)

	MTSPR_CPU6(SPRN_MD_TWC, r10, r11)

	mfspr	r10, SPRN_IMMR			/* Get current IMMR */

	mfspr	r10, SPRN_IMMR			/* Get current IMMR */

	rlwinm	r10, r10, 0, 0xfff80000		/* Get 512 kbytes boundary */

	rlwinm	r10, r10, 0, 0xfff80000		/* Get 512 kbytes boundary */

	ori	r10, r10, 0xf0 | MD_SPS16K | _PAGE_SHARED | _PAGE_DIRTY	| \

	ori	r10, r10, 0xf0 | MD_SPS16K | _PAGE_SHARED | _PAGE_DIRTY	| \

			  _PAGE_PRESENT | _PAGE_NO_CACHE

			  _PAGE_PRESENT | _PAGE_NO_CACHE

	MTSPR_CPU6(SPRN_MD_RPN, r10, r3)	/* Update TLB entry */

	MTSPR_CPU6(SPRN_MD_RPN, r10, r11)	/* Update TLB entry */





	li	r11, RPN_PATTERN

	li	r11, RPN_PATTERN

	mfspr	r3, SPRN_SPRG_SCRATCH2

	mtspr	SPRN_DAR, r11	/* Tag DAR */

	mtspr	SPRN_DAR, r11	/* Tag DAR */

	EXCEPTION_EPILOG_0

	EXCEPTION_EPILOG_0

	rfi

	rfi





	. = 0x1200

	. = 0x1200

DataStoreTLBMiss:

DataStoreTLBMiss:

	mtspr	SPRN_SPRG_SCRATCH2, r3

	EXCEPTION_PROLOG_0

	EXCEPTION_PROLOG_0

	mfcr	r3

	mfcr	r10





	/* If we are faulting a kernel address, we have to use the

	/* If we are faulting a kernel address, we have to use the

	 * kernel page tables.

	 * kernel page tables.

	 */

	 */

	mfspr	r10, SPRN_MD_EPN

	mfspr	r11, SPRN_MD_EPN

	IS_KERNEL(r11, r10)

	rlwinm	r11, r11, 16, 0xfff8

	mfspr	r11, SPRN_M_TW	/* Get level 1 table */

	BRANCH_UNLESS_KERNEL(3f)



	rlwinm	r11, r10, 16, 0xfff8

#ifndef CONFIG_PIN_TLB

#ifndef CONFIG_PIN_TLB

	cmpli	cr0, r11, VIRT_IMMR_BASE@h

	cmpli	cr0, r11, VIRT_IMMR_BASE@h

#endif

	cmpli	cr7, r11, PAGE_OFFSET@h

#ifndef CONFIG_PIN_TLB

_ENTRY(DTLBMiss_jmp)

_ENTRY(DTLBMiss_jmp)

	beq-	DTLBMissIMMR

	beq-	DTLBMissIMMR

#endif

#endif

	bge-	cr7, 4f





	lis	r11, (swapper_pg_dir-PAGE_OFFSET)@ha

	mfspr	r11, SPRN_M_TW	/* Get level 1 table */

3:

3:

	mtcr	r10

#ifdef CONFIG_8xx_CPU6

	mtspr	SPRN_SPRG_SCRATCH2, r3

#endif

	mfspr	r10, SPRN_MD_EPN





	/* Insert level 1 index */

	/* Insert level 1 index */

	rlwimi	r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29

	rlwimi	r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29

	lwz	r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11)	/* Get the level 1 entry */

	lwz	r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11)	/* Get the level 1 entry */

	mtcr	r11

	bt-	28,DTLBMiss8M		/* bit 28 = Large page (8M) */

	mtcr	r3





	/* We have a pte table, so load fetch the pte from the table.

	/* We have a pte table, so load fetch the pte from the table.

	 */

	 */
@@ -482,29 +482,30 @@ _ENTRY(DTLBMiss_jmp) 
	MTSPR_CPU6(SPRN_MD_RPN, r10, r3)	/* Update TLB entry */

	MTSPR_CPU6(SPRN_MD_RPN, r10, r3)	/* Update TLB entry */





	/* Restore registers */

	/* Restore registers */

#ifdef CONFIG_8xx_CPU6

	mfspr	r3, SPRN_SPRG_SCRATCH2

	mfspr	r3, SPRN_SPRG_SCRATCH2

#endif

	mtspr	SPRN_DAR, r11	/* Tag DAR */

	mtspr	SPRN_DAR, r11	/* Tag DAR */

	EXCEPTION_EPILOG_0

	EXCEPTION_EPILOG_0

	rfi

	rfi





DTLBMiss8M:

4:

	mtcr	r3

_ENTRY(DTLBMiss_cmp)

	ori	r11, r11, MD_SVALID

	cmpli	cr0, r11, PAGE_OFFSET@h

	MTSPR_CPU6(SPRN_MD_TWC, r11, r3)

	lis	r11, (swapper_pg_dir-PAGE_OFFSET)@ha

#ifdef CONFIG_PPC_16K_PAGES

	bge-	3b

	/*



	 * In 16k pages mode, each PGD entry defines a 64M block.

	mtcr	r10

	 * Here we select the 8M page within the block.

	/* Set 8M byte page and mark it valid */

	 */

	li	r10, MD_PS8MEG | MD_SVALID

	rlwimi	r11, r10, 0, 0x03800000

	MTSPR_CPU6(SPRN_MD_TWC, r10, r11)

#endif

	mfspr	r10, SPRN_MD_EPN

	rlwinm	r10, r11, 0, 0xff800000

	rlwinm	r10, r10, 0, 0x0f800000		/* 8xx supports max 256Mb RAM */

	ori	r10, r10, 0xf0 | MD_SPS16K | _PAGE_SHARED | _PAGE_DIRTY	| \

	ori	r10, r10, 0xf0 | MD_SPS16K | _PAGE_SHARED | _PAGE_DIRTY	| \

			  _PAGE_PRESENT

			  _PAGE_PRESENT

	MTSPR_CPU6(SPRN_MD_RPN, r10, r3)	/* Update TLB entry */

	MTSPR_CPU6(SPRN_MD_RPN, r10, r11)	/* Update TLB entry */





	li	r11, RPN_PATTERN

	li	r11, RPN_PATTERN

	mfspr	r3, SPRN_SPRG_SCRATCH2

	mtspr	SPRN_DAR, r11	/* Tag DAR */

	mtspr	SPRN_DAR, r11	/* Tag DAR */

	EXCEPTION_EPILOG_0

	EXCEPTION_EPILOG_0

	rfi

	rfi
@@ -583,12 +584,14 @@ FixupDAR:/* Entry point for dcbx workaround. */ 
	IS_KERNEL(r11, r10)

	IS_KERNEL(r11, r10)

	mfspr	r11, SPRN_M_TW	/* Get level 1 table */

	mfspr	r11, SPRN_M_TW	/* Get level 1 table */

	BRANCH_UNLESS_KERNEL(3f)

	BRANCH_UNLESS_KERNEL(3f)

	rlwinm	r11, r10, 16, 0xfff8

_ENTRY(FixupDAR_cmp)

	cmpli	cr7, r11, PAGE_OFFSET@h

	blt-	cr7, 200f

	lis	r11, (swapper_pg_dir-PAGE_OFFSET)@ha

	lis	r11, (swapper_pg_dir-PAGE_OFFSET)@ha

	/* Insert level 1 index */

	/* Insert level 1 index */

3:	rlwimi	r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29

3:	rlwimi	r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29

	lwz	r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11)	/* Get the level 1 entry */

	lwz	r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11)	/* Get the level 1 entry */

	mtcr	r11

	bt	28,200f		/* bit 28 = Large page (8M) */

	rlwinm	r11, r11,0,0,19	/* Extract page descriptor page address */

	rlwinm	r11, r11,0,0,19	/* Extract page descriptor page address */

	/* Insert level 2 index */

	/* Insert level 2 index */

	rlwimi	r11, r10, 32 - (PAGE_SHIFT - 2), 32 - PAGE_SHIFT, 29

	rlwimi	r11, r10, 32 - (PAGE_SHIFT - 2), 32 - PAGE_SHIFT, 29
@@ -614,8 +617,8 @@ FixupDAR:/* Entry point for dcbx workaround. */ 
141:	mfspr	r10,SPRN_SPRG_SCRATCH2

141:	mfspr	r10,SPRN_SPRG_SCRATCH2

	b	DARFixed	/* Nope, go back to normal TLB processing */

	b	DARFixed	/* Nope, go back to normal TLB processing */





	/* concat physical page address(r11) and page offset(r10) */

	/* create physical page address from effective address */

200:	rlwimi	r11, r10, 0, 32 - (PAGE_SHIFT << 1), 31

200:	tophys(r11, r10)

	b	201b

	b	201b





144:	mfspr	r10, SPRN_DSISR

144:	mfspr	r10, SPRN_DSISR

arch/powerpc/mm/8xx_mmu.c

+18 −38
Original line number Original line Diff line number Diff line
@@ -58,9 +58,7 @@ void __init MMU_init_hw(void) 
	/* Nothing to do for the time being but keep it similar to other PPC */

	/* Nothing to do for the time being but keep it similar to other PPC */

}

}





#define LARGE_PAGE_SIZE_4M	(1<<22)

#define LARGE_PAGE_SIZE_8M	(1<<23)

#define LARGE_PAGE_SIZE_8M	(1<<23)

#define LARGE_PAGE_SIZE_64M	(1<<26)





static void mmu_mapin_immr(void)

static void mmu_mapin_immr(void)

{

{
@@ -77,52 +75,33 @@ static void mmu_mapin_immr(void) 
#ifndef CONFIG_PIN_TLB

#ifndef CONFIG_PIN_TLB

extern unsigned int DTLBMiss_jmp;

extern unsigned int DTLBMiss_jmp;

#endif

#endif

extern unsigned int DTLBMiss_cmp, FixupDAR_cmp;





unsigned long __init mmu_mapin_ram(unsigned long top)

void mmu_patch_cmp_limit(unsigned int *addr, unsigned long mapped)

{

{

	unsigned long v, s, mapped;

	unsigned int instr = *addr;

	phys_addr_t p;





	v = KERNELBASE;

	instr &= 0xffff0000;

	p = 0;

	instr |= (unsigned long)__va(mapped) >> 16;

	s = top;

	patch_instruction(addr, instr);

}



unsigned long __init mmu_mapin_ram(unsigned long top)

{

	unsigned long mapped;





	if (__map_without_ltlbs) {

	if (__map_without_ltlbs) {

		mapped = 0;

		mmu_mapin_immr();

		mmu_mapin_immr();

#ifndef CONFIG_PIN_TLB

#ifndef CONFIG_PIN_TLB

		patch_instruction(&DTLBMiss_jmp, PPC_INST_NOP);

		patch_instruction(&DTLBMiss_jmp, PPC_INST_NOP);

#endif

#endif

		return 0;

	} else {

	}

		mapped = top & ~(LARGE_PAGE_SIZE_8M - 1);



#ifdef CONFIG_PPC_4K_PAGES

	while (s >= LARGE_PAGE_SIZE_8M) {

		pmd_t *pmdp;

		unsigned long val = p | MD_PS8MEG;



		pmdp = pmd_offset(pud_offset(pgd_offset_k(v), v), v);

		*pmdp++ = __pmd(val);

		*pmdp++ = __pmd(val + LARGE_PAGE_SIZE_4M);



		v += LARGE_PAGE_SIZE_8M;

		p += LARGE_PAGE_SIZE_8M;

		s -= LARGE_PAGE_SIZE_8M;

	}

	}

#else /* CONFIG_PPC_16K_PAGES */

	while (s >= LARGE_PAGE_SIZE_64M) {

		pmd_t *pmdp;

		unsigned long val = p | MD_PS8MEG;



		pmdp = pmd_offset(pud_offset(pgd_offset_k(v), v), v);

		*pmdp++ = __pmd(val);



		v += LARGE_PAGE_SIZE_64M;

		p += LARGE_PAGE_SIZE_64M;

		s -= LARGE_PAGE_SIZE_64M;

	}

#endif





	mapped = top - s;

	mmu_patch_cmp_limit(&DTLBMiss_cmp, mapped);

	mmu_patch_cmp_limit(&FixupDAR_cmp, mapped);





	/* If the size of RAM is not an exact power of two, we may not

	/* If the size of RAM is not an exact power of two, we may not

	 * have covered RAM in its entirety with 8 MiB

	 * have covered RAM in its entirety with 8 MiB
@@ -131,6 +110,7 @@ unsigned long __init mmu_mapin_ram(unsigned long top) 
	 * coverage with normal-sized pages (or other reasons) do not

	 * coverage with normal-sized pages (or other reasons) do not

	 * attempt to allocate outside the allowed range.

	 * attempt to allocate outside the allowed range.

	 */

	 */

	if (mapped)

		memblock_set_current_limit(mapped);

		memblock_set_current_limit(mapped);





	return mapped;

	return mapped;