Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 6b88a32c authored by Catalin Marinas's avatar Catalin Marinas
Browse files

arm64: kpti: Fix the interaction between ASID switching and software PAN 



With ARM64_SW_TTBR0_PAN enabled, the exception entry code checks the
active ASID to decide whether user access was enabled (non-zero ASID)
when the exception was taken. On return from exception, if user access
was previously disabled, it re-instates TTBR0_EL1 from the per-thread
saved value (updated in switch_mm() or efi_set_pgd()).

Commit 7655abb9 ("arm64: mm: Move ASID from TTBR0 to TTBR1") makes a
TTBR0_EL1 + ASID switching non-atomic. Subsequently, commit 27a921e7
("arm64: mm: Fix and re-enable ARM64_SW_TTBR0_PAN") changes the
__uaccess_ttbr0_disable() function and asm macro to first write the
reserved TTBR0_EL1 followed by the ASID=0 update in TTBR1_EL1. If an
exception occurs between these two, the exception return code will
re-instate a valid TTBR0_EL1. Similar scenario can happen in
cpu_switch_mm() between setting the reserved TTBR0_EL1 and the ASID
update in cpu_do_switch_mm().

This patch reverts the entry.S check for ASID == 0 to TTBR0_EL1 and
disables the interrupts around the TTBR0_EL1 and ASID switching code in
__uaccess_ttbr0_disable(). It also ensures that, when returning from the
EFI runtime services, efi_set_pgd() doesn't leave a non-zero ASID in
TTBR1_EL1 by using uaccess_ttbr0_{enable,disable}.

The accesses to current_thread_info()->ttbr0 are updated to use
READ_ONCE/WRITE_ONCE.

As a safety measure, __uaccess_ttbr0_enable() always masks out any
existing non-zero ASID TTBR1_EL1 before writing in the new ASID.

Fixes: 27a921e7 ("arm64: mm: Fix and re-enable ARM64_SW_TTBR0_PAN")
Acked-by: default avatarWill Deacon <will.deacon@arm.com>
Reported-by: default avatarArd Biesheuvel <ard.biesheuvel@linaro.org>
Tested-by: default avatarArd Biesheuvel <ard.biesheuvel@linaro.org>
Reviewed-by: default avatarJames Morse <james.morse@arm.com>
Tested-by: default avatarJames Morse <james.morse@arm.com>
Co-developed-by: default avatarMarc Zyngier <marc.zyngier@arm.com>
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent 558daf69

arch/arm64/include/asm/asm-uaccess.h

+7 −5
Original line number Diff line number Diff line
@@ -14,11 +14,11 @@ 
#ifdef CONFIG_ARM64_SW_TTBR0_PAN

	.macro	__uaccess_ttbr0_disable, tmp1

	mrs	\tmp1, ttbr1_el1			// swapper_pg_dir

	bic	\tmp1, \tmp1, #TTBR_ASID_MASK

	sub	\tmp1, \tmp1, #RESERVED_TTBR0_SIZE	// reserved_ttbr0 just before swapper_pg_dir

	msr	ttbr0_el1, \tmp1			// set reserved TTBR0_EL1

	isb

	add	\tmp1, \tmp1, #RESERVED_TTBR0_SIZE

	bic	\tmp1, \tmp1, #TTBR_ASID_MASK

	msr	ttbr1_el1, \tmp1		// set reserved ASID

	isb

	.endm
@@ -35,9 +35,11 @@ 
	isb

	.endm



	.macro	uaccess_ttbr0_disable, tmp1

	.macro	uaccess_ttbr0_disable, tmp1, tmp2

alternative_if_not ARM64_HAS_PAN

	save_and_disable_irq \tmp2		// avoid preemption

	__uaccess_ttbr0_disable \tmp1

	restore_irq \tmp2

alternative_else_nop_endif

	.endm
@@ -49,7 +51,7 @@ alternative_if_not ARM64_HAS_PAN 
alternative_else_nop_endif

	.endm

#else

	.macro	uaccess_ttbr0_disable, tmp1

	.macro	uaccess_ttbr0_disable, tmp1, tmp2

	.endm



	.macro	uaccess_ttbr0_enable, tmp1, tmp2, tmp3
@@ -59,8 +61,8 @@ alternative_else_nop_endif 
/*

 * These macros are no-ops when UAO is present.

 */

	.macro	uaccess_disable_not_uao, tmp1

	uaccess_ttbr0_disable \tmp1

	.macro	uaccess_disable_not_uao, tmp1, tmp2

	uaccess_ttbr0_disable \tmp1, \tmp2

alternative_if ARM64_ALT_PAN_NOT_UAO

	SET_PSTATE_PAN(1)

alternative_else_nop_endif

arch/arm64/include/asm/efi.h

+7 −5
Original line number Diff line number Diff line
@@ -121,19 +121,21 @@ static inline void efi_set_pgd(struct mm_struct *mm) 
		if (mm != current->active_mm) {

			/*

			 * Update the current thread's saved ttbr0 since it is

			 * restored as part of a return from exception. Set

			 * the hardware TTBR0_EL1 using cpu_switch_mm()

			 * directly to enable potential errata workarounds.

			 * restored as part of a return from exception. Enable

			 * access to the valid TTBR0_EL1 and invoke the errata

			 * workaround directly since there is no return from

			 * exception when invoking the EFI run-time services.

			 */

			update_saved_ttbr0(current, mm);

			cpu_switch_mm(mm->pgd, mm);

			uaccess_ttbr0_enable();

			post_ttbr_update_workaround();

		} else {

			/*

			 * Defer the switch to the current thread's TTBR0_EL1

			 * until uaccess_enable(). Restore the current

			 * thread's saved ttbr0 corresponding to its active_mm

			 */

			cpu_set_reserved_ttbr0();

			uaccess_ttbr0_disable();

			update_saved_ttbr0(current, current->active_mm);

		}

	}

arch/arm64/include/asm/mmu_context.h

+2 −1
Original line number Diff line number Diff line
@@ -184,7 +184,7 @@ static inline void update_saved_ttbr0(struct task_struct *tsk, 
	else

		ttbr = virt_to_phys(mm->pgd) | ASID(mm) << 48;



	task_thread_info(tsk)->ttbr0 = ttbr;

	WRITE_ONCE(task_thread_info(tsk)->ttbr0, ttbr);

}

#else

static inline void update_saved_ttbr0(struct task_struct *tsk,
@@ -239,6 +239,7 @@ switch_mm(struct mm_struct *prev, struct mm_struct *next, 
#define activate_mm(prev,next)	switch_mm(prev, next, current)



void verify_cpu_asid_bits(void);

void post_ttbr_update_workaround(void);



#endif /* !__ASSEMBLY__ */

arch/arm64/include/asm/uaccess.h

+6 −3
Original line number Diff line number Diff line
@@ -105,16 +105,18 @@ static inline void set_fs(mm_segment_t fs) 
#ifdef CONFIG_ARM64_SW_TTBR0_PAN

static inline void __uaccess_ttbr0_disable(void)

{

	unsigned long ttbr;

	unsigned long flags, ttbr;



	local_irq_save(flags);

	ttbr = read_sysreg(ttbr1_el1);

	ttbr &= ~TTBR_ASID_MASK;

	/* reserved_ttbr0 placed before swapper_pg_dir */

	write_sysreg(ttbr - RESERVED_TTBR0_SIZE, ttbr0_el1);

	isb();

	/* Set reserved ASID */

	ttbr &= ~TTBR_ASID_MASK;

	write_sysreg(ttbr, ttbr1_el1);

	isb();

	local_irq_restore(flags);

}



static inline void __uaccess_ttbr0_enable(void)
@@ -127,10 +129,11 @@ static inline void __uaccess_ttbr0_enable(void) 
	 * roll-over and an update of 'ttbr0'.

	 */

	local_irq_save(flags);

	ttbr0 = current_thread_info()->ttbr0;

	ttbr0 = READ_ONCE(current_thread_info()->ttbr0);



	/* Restore active ASID */

	ttbr1 = read_sysreg(ttbr1_el1);

	ttbr1 &= ~TTBR_ASID_MASK;		/* safety measure */

	ttbr1 |= ttbr0 & TTBR_ASID_MASK;

	write_sysreg(ttbr1, ttbr1_el1);

	isb();

arch/arm64/kernel/entry.S

+1 −1
Original line number Diff line number Diff line
@@ -204,7 +204,7 @@ alternative_if ARM64_HAS_PAN 
alternative_else_nop_endif



	.if	\el != 0

	mrs	x21, ttbr1_el1

	mrs	x21, ttbr0_el1

	tst	x21, #TTBR_ASID_MASK		// Check for the reserved ASID

	orr	x23, x23, #PSR_PAN_BIT		// Set the emulated PAN in the saved SPSR

	b.eq	1f				// TTBR0 access already disabled