Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit d19bcc64 authored by Catalin Marinas's avatar Catalin Marinas Committed by Gerrit - the friendly Code Review server
Browse files

FROMLIST: arm64: kpti: Fix the interaction between ASID switching and software PAN 



With ARM64_SW_TTBR0_PAN enabled, the exception entry code checks the
active ASID to decide whether user access was enabled (non-zero ASID)
when the exception was taken. On return from exception, if user access
was previously disabled, it re-instates TTBR0_EL1 from the per-thread
saved value (updated in switch_mm() or efi_set_pgd()).

Commit 7655abb95386 ("arm64: mm: Move ASID from TTBR0 to TTBR1") makes a
TTBR0_EL1 + ASID switching non-atomic. Subsequently, commit 27a921e75711
("arm64: mm: Fix and re-enable ARM64_SW_TTBR0_PAN") changes the
__uaccess_ttbr0_disable() function and asm macro to first write the
reserved TTBR0_EL1 followed by the ASID=0 update in TTBR1_EL1. If an
exception occurs between these two, the exception return code will
re-instate a valid TTBR0_EL1. Similar scenario can happen in
cpu_switch_mm() between setting the reserved TTBR0_EL1 and the ASID
update in cpu_do_switch_mm().

This patch reverts the entry.S check for ASID == 0 to TTBR0_EL1 and
disables the interrupts around the TTBR0_EL1 and ASID switching code in
__uaccess_ttbr0_disable(). It also ensures that, when returning from the
EFI runtime services, efi_set_pgd() doesn't leave a non-zero ASID in
TTBR1_EL1 by using uaccess_ttbr0_{enable,disable}.

The accesses to current_thread_info()->ttbr0 are updated to use
READ_ONCE/WRITE_ONCE.

As a safety measure, __uaccess_ttbr0_enable() always masks out any
existing non-zero ASID TTBR1_EL1 before writing in the new ASID.

Fixes: 27a921e75711 ("arm64: mm: Fix and re-enable ARM64_SW_TTBR0_PAN")
Acked-by: default avatarWill Deacon <will.deacon@arm.com>
Reported-by: default avatarArd Biesheuvel <ard.biesheuvel@linaro.org>
Tested-by: default avatarArd Biesheuvel <ard.biesheuvel@linaro.org>
Reviewed-by: default avatarJames Morse <james.morse@arm.com>
Tested-by: default avatarJames Morse <james.morse@arm.com>
Co-developed-by: default avatarMarc Zyngier <marc.zyngier@arm.com>
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
(cherry picked from git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux.git


 commit 6b88a32c7af68895134872cdec3b6bfdb532d94e)

Change-Id: I1597fe926e4d7fc0f2c19dc63efbd359b5033796
[ghackmann@google.com:
 - adjust context
 - apply asm-uaccess.h changes to uaccess.h, and efi.h changes to efi.c]
Signed-off-by: default avatarGreg Hackmann <ghackmann@google.com>
Git-Commit: cf43f203
Git-repo: git://android.googlesource.com/kernel/common.git


[vinmenon@codeaurora.org: changes to mmu_context.h and efi.c
due to extra SW PAN fixes on this branch]
Signed-off-by: default avatarVinayak Menon <vinmenon@codeaurora.org>
parent 5e329e47

arch/arm64/include/asm/mmu_context.h

+3 −1
Original line number Diff line number Diff line
@@ -130,7 +130,7 @@ static inline void update_saved_ttbr0(struct task_struct *tsk, 
	else

		ttbr = virt_to_phys(mm->pgd) | ASID(mm) << 48;



	task_thread_info(tsk)->ttbr0 = ttbr;

	WRITE_ONCE(task_thread_info(tsk)->ttbr0, ttbr);

}

#else

static inline void update_saved_ttbr0(struct task_struct *tsk,
@@ -184,4 +184,6 @@ switch_mm(struct mm_struct *prev, struct mm_struct *next, 
#define deactivate_mm(tsk,mm)	do { } while (0)

#define activate_mm(prev,next)	switch_mm(prev, next, current)



void post_ttbr_update_workaround(void);



#endif

arch/arm64/include/asm/uaccess.h

+13 −8
Original line number Diff line number Diff line
@@ -135,16 +135,18 @@ static inline void set_fs(mm_segment_t fs) 
#ifdef CONFIG_ARM64_SW_TTBR0_PAN

static inline void __uaccess_ttbr0_disable(void)

{

	unsigned long ttbr;

	unsigned long flags, ttbr;



	local_irq_save(flags);

	ttbr = read_sysreg(ttbr1_el1);

	ttbr &= ~TTBR_ASID_MASK;

	/* reserved_ttbr0 placed at the end of swapper_pg_dir */

	write_sysreg(ttbr + SWAPPER_DIR_SIZE, ttbr0_el1);

	isb();

	/* Set reserved ASID */

	ttbr &= ~TTBR_ASID_MASK;

	write_sysreg(ttbr, ttbr1_el1);

	isb();

	local_irq_restore(flags);

}



static inline void __uaccess_ttbr0_enable(void)
@@ -157,10 +159,11 @@ static inline void __uaccess_ttbr0_enable(void) 
	 * roll-over and an update of 'ttbr0'.

	 */

	local_irq_save(flags);

	ttbr0 = current_thread_info()->ttbr0;

	ttbr0 = READ_ONCE(current_thread_info()->ttbr0);



	/* Restore active ASID */

	ttbr1 = read_sysreg(ttbr1_el1);

	ttbr1 &= ~TTBR_ASID_MASK;		/* safety measure */

	ttbr1 |= ttbr0 & TTBR_ASID_MASK;

	write_sysreg(ttbr1, ttbr1_el1);

	isb();
@@ -451,11 +454,11 @@ extern __must_check long strnlen_user(const char __user *str, long n); 
#ifdef CONFIG_ARM64_SW_TTBR0_PAN

	.macro	__uaccess_ttbr0_disable, tmp1

	mrs	\tmp1, ttbr1_el1		// swapper_pg_dir

	bic	\tmp1, \tmp1, #TTBR_ASID_MASK

	add	\tmp1, \tmp1, #SWAPPER_DIR_SIZE	// reserved_ttbr0 at the end of swapper_pg_dir

	msr	ttbr0_el1, \tmp1		// set reserved TTBR0_EL1

	isb

	sub	\tmp1, \tmp1, #SWAPPER_DIR_SIZE

	bic	\tmp1, \tmp1, #TTBR_ASID_MASK

	msr	ttbr1_el1, \tmp1		// set reserved ASID

	isb

	.endm
@@ -472,9 +475,11 @@ extern __must_check long strnlen_user(const char __user *str, long n); 
	isb

	.endm



	.macro	uaccess_ttbr0_disable, tmp1

	.macro	uaccess_ttbr0_disable, tmp1, tmp2

alternative_if_not ARM64_HAS_PAN

	save_and_disable_irq \tmp2		// avoid preemption

	__uaccess_ttbr0_disable \tmp1

	restore_irq \tmp2

alternative_else_nop_endif

	.endm
@@ -486,7 +491,7 @@ alternative_if_not ARM64_HAS_PAN 
alternative_else_nop_endif

	.endm

#else

	.macro	uaccess_ttbr0_disable, tmp1

	.macro	uaccess_ttbr0_disable, tmp1, tmp2

	.endm



	.macro	uaccess_ttbr0_enable, tmp1, tmp2, tmp3
@@ -496,8 +501,8 @@ alternative_else_nop_endif 
/*

 * These macros are no-ops when UAO is present.

 */

	.macro	uaccess_disable_not_uao, tmp1

	uaccess_ttbr0_disable \tmp1

	.macro	uaccess_disable_not_uao, tmp1, tmp2

	uaccess_ttbr0_disable \tmp1, \tmp2

alternative_if ARM64_ALT_PAN_NOT_UAO

	SET_PSTATE_PAN(1)

alternative_else_nop_endif

arch/arm64/kernel/efi.c

+7 −5
Original line number Diff line number Diff line
@@ -345,19 +345,21 @@ static void efi_set_pgd(struct mm_struct *mm) 
		if (mm != current->active_mm) {

			/*

			 * Update the current thread's saved ttbr0 since it is

			 * restored as part of a return from exception. Set

			 * the hardware TTBR0_EL1 using cpu_switch_mm()

			 * directly to enable potential errata workarounds.

			 * restored as part of a return from exception. Enable

			 * access to the valid TTBR0_EL1 and invoke the errata

			 * workaround directly since there is no return from

			 * exception when invoking the EFI run-time services.

			 */

			update_saved_ttbr0(current, mm);

			cpu_switch_mm(mm->pgd, mm);

			uaccess_ttbr0_enable();

			post_ttbr_update_workaround();

		} else {

			/*

			 * Defer the switch to the current thread's TTBR0_EL1

			 * until uaccess_enable(). Restore the current

			 * thread's saved ttbr0 corresponding to its active_mm

			 */

			cpu_set_reserved_ttbr0();

			uaccess_ttbr0_disable();

			update_saved_ttbr0(current, current->active_mm);

		}

	}

arch/arm64/kernel/entry.S

+1 −1
Original line number Diff line number Diff line
@@ -148,7 +148,7 @@ alternative_if ARM64_HAS_PAN 
alternative_else_nop_endif



	.if	\el != 0

	mrs	x21, ttbr1_el1

	mrs	x21, ttbr0_el1

	tst	x21, #TTBR_ASID_MASK		// Check for the reserved ASID

	orr	x23, x23, #PSR_PAN_BIT		// Set the emulated PAN in the saved SPSR

	b.eq	1f				// TTBR0 access already disabled

arch/arm64/lib/clear_user.S

+1 −1
Original line number Diff line number Diff line
@@ -50,7 +50,7 @@ uao_user_alternative 9f, strh, sttrh, wzr, x0, 2 
	b.mi	5f

uao_user_alternative 9f, strb, sttrb, wzr, x0, 0

5:	mov	x0, #0

	uaccess_disable_not_uao x2

	uaccess_disable_not_uao x2, x3

	ret

ENDPROC(__clear_user)