Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 8a055d7f authored by Andy Lutomirski's avatar Andy Lutomirski Committed by Ingo Molnar
Browse files

x86/entry/64: Move SWAPGS into the common IRET-to-usermode path 



All of the code paths that ended up doing IRET to usermode did
SWAPGS immediately beforehand.  Move the SWAPGS into the common
code.

Signed-off-by: default avatarAndy Lutomirski <luto@kernel.org>
Cc: Borislav Petkov <bpetkov@suse.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/27fd6f45b7cd640de38fb9066fd0349bcd11f8e1.1509609304.git.luto@kernel.org


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 26c4ef9c

arch/x86/entry/entry_64.S

+14 −18
Original line number Diff line number Diff line
@@ -249,12 +249,14 @@ return_from_SYSCALL_64: 


	/*

	 * Try to use SYSRET instead of IRET if we're returning to

	 * a completely clean 64-bit userspace context.

	 * a completely clean 64-bit userspace context.  If we're not,

	 * go to the slow exit path.

	 */

	movq	RCX(%rsp), %rcx

	movq	RIP(%rsp), %r11

	cmpq	%rcx, %r11			/* RCX == RIP */

	jne	opportunistic_sysret_failed



	cmpq	%rcx, %r11	/* SYSRET requires RCX == RIP */

	jne	swapgs_restore_regs_and_return_to_usermode



	/*

	 * On Intel CPUs, SYSRET with non-canonical RCX/RIP will #GP
@@ -272,14 +274,14 @@ return_from_SYSCALL_64: 


	/* If this changed %rcx, it was not canonical */

	cmpq	%rcx, %r11

	jne	opportunistic_sysret_failed

	jne	swapgs_restore_regs_and_return_to_usermode



	cmpq	$__USER_CS, CS(%rsp)		/* CS must match SYSRET */

	jne	opportunistic_sysret_failed

	jne	swapgs_restore_regs_and_return_to_usermode



	movq	R11(%rsp), %r11

	cmpq	%r11, EFLAGS(%rsp)		/* R11 == RFLAGS */

	jne	opportunistic_sysret_failed

	jne	swapgs_restore_regs_and_return_to_usermode



	/*

	 * SYSCALL clears RF when it saves RFLAGS in R11 and SYSRET cannot
@@ -300,12 +302,12 @@ return_from_SYSCALL_64: 
	 * would never get past 'stuck_here'.

	 */

	testq	$(X86_EFLAGS_RF|X86_EFLAGS_TF), %r11

	jnz	opportunistic_sysret_failed

	jnz	swapgs_restore_regs_and_return_to_usermode



	/* nothing to check for RSP */



	cmpq	$__USER_DS, SS(%rsp)		/* SS must match SYSRET */

	jne	opportunistic_sysret_failed

	jne	swapgs_restore_regs_and_return_to_usermode



	/*

	 * We win! This label is here just for ease of understanding
@@ -318,10 +320,6 @@ syscall_return_via_sysret: 
	movq	RSP(%rsp), %rsp

	UNWIND_HINT_EMPTY

	USERGS_SYSRET64



opportunistic_sysret_failed:

	SWAPGS

	jmp	restore_regs_and_return_to_usermode

END(entry_SYSCALL_64)



ENTRY(stub_ptregs_64)
@@ -422,8 +420,7 @@ ENTRY(ret_from_fork) 
	movq	%rsp, %rdi

	call	syscall_return_slowpath	/* returns with IRQs disabled */

	TRACE_IRQS_ON			/* user mode is traced as IRQS on */

	SWAPGS

	jmp	restore_regs_and_return_to_usermode

	jmp	swapgs_restore_regs_and_return_to_usermode



1:

	/* kernel thread */
@@ -611,9 +608,8 @@ GLOBAL(retint_user) 
	mov	%rsp,%rdi

	call	prepare_exit_to_usermode

	TRACE_IRQS_IRETQ

	SWAPGS



GLOBAL(restore_regs_and_return_to_usermode)

GLOBAL(swapgs_restore_regs_and_return_to_usermode)

#ifdef CONFIG_DEBUG_ENTRY

	/* Assert that pt_regs indicates user mode. */

	testl	$3, CS(%rsp)
@@ -621,6 +617,7 @@ GLOBAL(restore_regs_and_return_to_usermode) 
	ud2

1:

#endif

	SWAPGS

	RESTORE_EXTRA_REGS

	RESTORE_C_REGS

	REMOVE_PT_GPREGS_FROM_STACK 8
@@ -1342,8 +1339,7 @@ ENTRY(nmi) 
	 * Return back to user mode.  We must *not* do the normal exit

	 * work, because we don't want to enable interrupts.

	 */

	SWAPGS

	jmp	restore_regs_and_return_to_usermode

	jmp	swapgs_restore_regs_and_return_to_usermode



.Lnmi_from_kernel:

	/*

arch/x86/entry/entry_64_compat.S

+1 −2
Original line number Diff line number Diff line
@@ -336,8 +336,7 @@ ENTRY(entry_INT80_compat) 


	/* Go back to user mode. */

	TRACE_IRQS_ON

	SWAPGS

	jmp	restore_regs_and_return_to_usermode

	jmp	swapgs_restore_regs_and_return_to_usermode

END(entry_INT80_compat)



ENTRY(stub32_clone)