Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit d4667ca1 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge branch 'x86-pti-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull x86 PTI and Spectre related fixes and updates from Ingo Molnar:
 "Here's the latest set of Spectre and PTI related fixes and updates:

  Spectre:
   - Add entry code register clearing to reduce the Spectre attack
     surface
   - Update the Spectre microcode blacklist
   - Inline the KVM Spectre helpers to get close to v4.14 performance
     again.
   - Fix indirect_branch_prediction_barrier()
   - Fix/improve Spectre related kernel messages
   - Fix array_index_nospec_mask() asm constraint
   - KVM: fix two MSR handling bugs

  PTI:
   - Fix a paranoid entry PTI CR3 handling bug
   - Fix comments

  objtool:
   - Fix paranoid_entry() frame pointer warning
   - Annotate WARN()-related UD2 as reachable
   - Various fixes
   - Add Add Peter Zijlstra as objtool co-maintainer

  Misc:
   - Various x86 entry code self-test fixes
   - Improve/simplify entry code stack frame generation and handling
     after recent heavy-handed PTI and Spectre changes. (There's two
     more WIP improvements expected here.)
   - Type fix for cache entries

  There's also some low risk non-fix changes I've included in this
  branch to reduce backporting conflicts:

   - rename a confusing x86_cpu field name
   - de-obfuscate the naming of single-TLB flushing primitives"

* 'x86-pti-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (41 commits)
  x86/entry/64: Fix CR3 restore in paranoid_exit()
  x86/cpu: Change type of x86_cache_size variable to unsigned int
  x86/spectre: Fix an error message
  x86/cpu: Rename cpu_data.x86_mask to cpu_data.x86_stepping
  selftests/x86/mpx: Fix incorrect bounds with old _sigfault
  x86/mm: Rename flush_tlb_single() and flush_tlb_one() to __flush_tlb_one_[user|kernel]()
  x86/speculation: Add <asm/msr-index.h> dependency
  nospec: Move array_index_nospec() parameter checking into separate macro
  x86/speculation: Fix up array_index_nospec_mask() asm constraint
  x86/debug: Use UD2 for WARN()
  x86/debug, objtool: Annotate WARN()-related UD2 as reachable
  objtool: Fix segfault in ignore_unreachable_insn()
  selftests/x86: Disable tests requiring 32-bit support on pure 64-bit systems
  selftests/x86: Do not rely on "int $0x80" in single_step_syscall.c
  selftests/x86: Do not rely on "int $0x80" in test_mremap_vdso.c
  selftests/x86: Fix build bug caused by the 5lvl test which has been moved to the VM directory
  selftests/x86/pkeys: Remove unused functions
  selftests/x86: Clean up and document sscanf() usage
  selftests/x86: Fix vDSO selftest segfault for vsyscall=none
  x86/entry/64: Remove the unused 'icebp' macro
  ...
parents 6556677a e4865757

MAINTAINERS

+1 −0
Original line number Diff line number Diff line
@@ -9946,6 +9946,7 @@ F: drivers/nfc/nxp-nci 


OBJTOOL

M:	Josh Poimboeuf <jpoimboe@redhat.com>

M:	Peter Zijlstra <peterz@infradead.org>

S:	Supported

F:	tools/objtool/

arch/x86/entry/calling.h

+48 −59
Original line number Diff line number Diff line
@@ -97,79 +97,68 @@ For 32-bit we have the following conventions - kernel is built with 


#define SIZEOF_PTREGS	21*8



	.macro ALLOC_PT_GPREGS_ON_STACK

	addq	$-(15*8), %rsp

	.endm



	.macro SAVE_C_REGS_HELPER offset=0 rax=1 rcx=1 r8910=1 r11=1

	.if \r11

	movq %r11, 6*8+\offset(%rsp)

	.endif

	.if \r8910

	movq %r10, 7*8+\offset(%rsp)

	movq %r9,  8*8+\offset(%rsp)

	movq %r8,  9*8+\offset(%rsp)

	.endif

	.if \rax

	movq %rax, 10*8+\offset(%rsp)

	.endif

	.if \rcx

	movq %rcx, 11*8+\offset(%rsp)

	.endif

	movq %rdx, 12*8+\offset(%rsp)

	movq %rsi, 13*8+\offset(%rsp)

	movq %rdi, 14*8+\offset(%rsp)

	UNWIND_HINT_REGS offset=\offset extra=0

	.endm

	.macro SAVE_C_REGS offset=0

	SAVE_C_REGS_HELPER \offset, 1, 1, 1, 1

	.endm

	.macro SAVE_C_REGS_EXCEPT_RAX_RCX offset=0

	SAVE_C_REGS_HELPER \offset, 0, 0, 1, 1

	.endm

	.macro SAVE_C_REGS_EXCEPT_R891011

	SAVE_C_REGS_HELPER 0, 1, 1, 0, 0

	.endm

	.macro SAVE_C_REGS_EXCEPT_RCX_R891011

	SAVE_C_REGS_HELPER 0, 1, 0, 0, 0

	.endm

	.macro SAVE_C_REGS_EXCEPT_RAX_RCX_R11

	SAVE_C_REGS_HELPER 0, 0, 0, 1, 0

	.endm



	.macro SAVE_EXTRA_REGS offset=0

	movq %r15, 0*8+\offset(%rsp)

	movq %r14, 1*8+\offset(%rsp)

	movq %r13, 2*8+\offset(%rsp)

	movq %r12, 3*8+\offset(%rsp)

	movq %rbp, 4*8+\offset(%rsp)

	movq %rbx, 5*8+\offset(%rsp)

	UNWIND_HINT_REGS offset=\offset

.macro PUSH_AND_CLEAR_REGS rdx=%rdx rax=%rax

	/*

	 * Push registers and sanitize registers of values that a

	 * speculation attack might otherwise want to exploit. The

	 * lower registers are likely clobbered well before they

	 * could be put to use in a speculative execution gadget.

	 * Interleave XOR with PUSH for better uop scheduling:

	 */

	pushq   %rdi		/* pt_regs->di */

	pushq   %rsi		/* pt_regs->si */

	pushq	\rdx		/* pt_regs->dx */

	pushq   %rcx		/* pt_regs->cx */

	pushq   \rax		/* pt_regs->ax */

	pushq   %r8		/* pt_regs->r8 */

	xorq    %r8, %r8	/* nospec   r8 */

	pushq   %r9		/* pt_regs->r9 */

	xorq    %r9, %r9	/* nospec   r9 */

	pushq   %r10		/* pt_regs->r10 */

	xorq    %r10, %r10	/* nospec   r10 */

	pushq   %r11		/* pt_regs->r11 */

	xorq    %r11, %r11	/* nospec   r11*/

	pushq	%rbx		/* pt_regs->rbx */

	xorl    %ebx, %ebx	/* nospec   rbx*/

	pushq	%rbp		/* pt_regs->rbp */

	xorl    %ebp, %ebp	/* nospec   rbp*/

	pushq	%r12		/* pt_regs->r12 */

	xorq    %r12, %r12	/* nospec   r12*/

	pushq	%r13		/* pt_regs->r13 */

	xorq    %r13, %r13	/* nospec   r13*/

	pushq	%r14		/* pt_regs->r14 */

	xorq    %r14, %r14	/* nospec   r14*/

	pushq	%r15		/* pt_regs->r15 */

	xorq    %r15, %r15	/* nospec   r15*/

	UNWIND_HINT_REGS

.endm



	.macro POP_EXTRA_REGS

.macro POP_REGS pop_rdi=1 skip_r11rcx=0

	popq %r15

	popq %r14

	popq %r13

	popq %r12

	popq %rbp

	popq %rbx

	.endm



	.macro POP_C_REGS

	.if \skip_r11rcx

	popq %rsi

	.else

	popq %r11

	.endif

	popq %r10

	popq %r9

	popq %r8

	popq %rax

	.if \skip_r11rcx

	popq %rsi

	.else

	popq %rcx

	.endif

	popq %rdx

	popq %rsi

	.if \pop_rdi

	popq %rdi

	.endm



	.macro icebp

	.byte 0xf1

	.endif

.endm



/*
@@ -178,7 +167,7 @@ For 32-bit we have the following conventions - kernel is built with 
 * is just setting the LSB, which makes it an invalid stack address and is also

 * a signal to the unwinder that it's a pt_regs pointer in disguise.

 *

 * NOTE: This macro must be used *after* SAVE_EXTRA_REGS because it corrupts

 * NOTE: This macro must be used *after* PUSH_AND_CLEAR_REGS because it corrupts

 * the original rbp.

 */

.macro ENCODE_FRAME_POINTER ptregs_offset=0

arch/x86/entry/entry_64.S

+19 −73
Original line number Diff line number Diff line
@@ -213,7 +213,7 @@ ENTRY(entry_SYSCALL_64) 


	swapgs

	/*

	 * This path is not taken when PAGE_TABLE_ISOLATION is disabled so it

	 * This path is only taken when PAGE_TABLE_ISOLATION is disabled so it

	 * is not required to switch CR3.

	 */

	movq	%rsp, PER_CPU_VAR(rsp_scratch)
@@ -227,22 +227,8 @@ ENTRY(entry_SYSCALL_64) 
	pushq	%rcx				/* pt_regs->ip */

GLOBAL(entry_SYSCALL_64_after_hwframe)

	pushq	%rax				/* pt_regs->orig_ax */

	pushq	%rdi				/* pt_regs->di */

	pushq	%rsi				/* pt_regs->si */

	pushq	%rdx				/* pt_regs->dx */

	pushq	%rcx				/* pt_regs->cx */

	pushq	$-ENOSYS			/* pt_regs->ax */

	pushq	%r8				/* pt_regs->r8 */

	pushq	%r9				/* pt_regs->r9 */

	pushq	%r10				/* pt_regs->r10 */

	pushq	%r11				/* pt_regs->r11 */

	pushq	%rbx				/* pt_regs->rbx */

	pushq	%rbp				/* pt_regs->rbp */

	pushq	%r12				/* pt_regs->r12 */

	pushq	%r13				/* pt_regs->r13 */

	pushq	%r14				/* pt_regs->r14 */

	pushq	%r15				/* pt_regs->r15 */

	UNWIND_HINT_REGS



	PUSH_AND_CLEAR_REGS rax=$-ENOSYS



	TRACE_IRQS_OFF
@@ -321,15 +307,7 @@ GLOBAL(entry_SYSCALL_64_after_hwframe) 
syscall_return_via_sysret:

	/* rcx and r11 are already restored (see code above) */

	UNWIND_HINT_EMPTY

	POP_EXTRA_REGS

	popq	%rsi	/* skip r11 */

	popq	%r10

	popq	%r9

	popq	%r8

	popq	%rax

	popq	%rsi	/* skip rcx */

	popq	%rdx

	popq	%rsi

	POP_REGS pop_rdi=0 skip_r11rcx=1



	/*

	 * Now all regs are restored except RSP and RDI.
@@ -559,9 +537,7 @@ END(irq_entries_start) 
	call	switch_to_thread_stack

1:



	ALLOC_PT_GPREGS_ON_STACK

	SAVE_C_REGS

	SAVE_EXTRA_REGS

	PUSH_AND_CLEAR_REGS

	ENCODE_FRAME_POINTER



	testb	$3, CS(%rsp)
@@ -622,15 +598,7 @@ GLOBAL(swapgs_restore_regs_and_return_to_usermode) 
	ud2

1:

#endif

	POP_EXTRA_REGS

	popq	%r11

	popq	%r10

	popq	%r9

	popq	%r8

	popq	%rax

	popq	%rcx

	popq	%rdx

	popq	%rsi

	POP_REGS pop_rdi=0



	/*

	 * The stack is now user RDI, orig_ax, RIP, CS, EFLAGS, RSP, SS.
@@ -688,8 +656,7 @@ GLOBAL(restore_regs_and_return_to_kernel) 
	ud2

1:

#endif

	POP_EXTRA_REGS

	POP_C_REGS

	POP_REGS

	addq	$8, %rsp	/* skip regs->orig_ax */

	/*

	 * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization
@@ -908,7 +875,9 @@ ENTRY(\sym) 
	pushq	$-1				/* ORIG_RAX: no syscall to restart */

	.endif



	ALLOC_PT_GPREGS_ON_STACK

	/* Save all registers in pt_regs */

	PUSH_AND_CLEAR_REGS

	ENCODE_FRAME_POINTER



	.if \paranoid < 2

	testb	$3, CS(%rsp)			/* If coming from userspace, switch stacks */
@@ -1121,9 +1090,7 @@ ENTRY(xen_failsafe_callback) 
	addq	$0x30, %rsp

	UNWIND_HINT_IRET_REGS

	pushq	$-1 /* orig_ax = -1 => not a system call */

	ALLOC_PT_GPREGS_ON_STACK

	SAVE_C_REGS

	SAVE_EXTRA_REGS

	PUSH_AND_CLEAR_REGS

	ENCODE_FRAME_POINTER

	jmp	error_exit

END(xen_failsafe_callback)
@@ -1163,16 +1130,13 @@ idtentry machine_check do_mce has_error_code=0 paranoid=1 
#endif



/*

 * Save all registers in pt_regs, and switch gs if needed.

 * Switch gs if needed.

 * Use slow, but surefire "are we in kernel?" check.

 * Return: ebx=0: need swapgs on exit, ebx=1: otherwise

 */

ENTRY(paranoid_entry)

	UNWIND_HINT_FUNC

	cld

	SAVE_C_REGS 8

	SAVE_EXTRA_REGS 8

	ENCODE_FRAME_POINTER 8

	movl	$1, %ebx

	movl	$MSR_GS_BASE, %ecx

	rdmsr
@@ -1211,21 +1175,18 @@ ENTRY(paranoid_exit) 
	jmp	.Lparanoid_exit_restore

.Lparanoid_exit_no_swapgs:

	TRACE_IRQS_IRETQ_DEBUG

	RESTORE_CR3	scratch_reg=%rbx save_reg=%r14

.Lparanoid_exit_restore:

	jmp restore_regs_and_return_to_kernel

END(paranoid_exit)



/*

 * Save all registers in pt_regs, and switch gs if needed.

 * Switch gs if needed.

 * Return: EBX=0: came from user mode; EBX=1: otherwise

 */

ENTRY(error_entry)

	UNWIND_HINT_FUNC

	UNWIND_HINT_REGS offset=8

	cld

	SAVE_C_REGS 8

	SAVE_EXTRA_REGS 8

	ENCODE_FRAME_POINTER 8

	xorl	%ebx, %ebx

	testb	$3, CS+8(%rsp)

	jz	.Lerror_kernelspace
@@ -1406,22 +1367,7 @@ ENTRY(nmi) 
	pushq	1*8(%rdx)	/* pt_regs->rip */

	UNWIND_HINT_IRET_REGS

	pushq   $-1		/* pt_regs->orig_ax */

	pushq   %rdi		/* pt_regs->di */

	pushq   %rsi		/* pt_regs->si */

	pushq   (%rdx)		/* pt_regs->dx */

	pushq   %rcx		/* pt_regs->cx */

	pushq   %rax		/* pt_regs->ax */

	pushq   %r8		/* pt_regs->r8 */

	pushq   %r9		/* pt_regs->r9 */

	pushq   %r10		/* pt_regs->r10 */

	pushq   %r11		/* pt_regs->r11 */

	pushq	%rbx		/* pt_regs->rbx */

	pushq	%rbp		/* pt_regs->rbp */

	pushq	%r12		/* pt_regs->r12 */

	pushq	%r13		/* pt_regs->r13 */

	pushq	%r14		/* pt_regs->r14 */

	pushq	%r15		/* pt_regs->r15 */

	UNWIND_HINT_REGS

	PUSH_AND_CLEAR_REGS rdx=(%rdx)

	ENCODE_FRAME_POINTER



	/*
@@ -1631,7 +1577,8 @@ end_repeat_nmi: 
	 * frame to point back to repeat_nmi.

	 */

	pushq	$-1				/* ORIG_RAX: no syscall to restart */

	ALLOC_PT_GPREGS_ON_STACK

	PUSH_AND_CLEAR_REGS

	ENCODE_FRAME_POINTER



	/*

	 * Use paranoid_entry to handle SWAPGS, but no need to use paranoid_exit
@@ -1655,8 +1602,7 @@ end_repeat_nmi: 
nmi_swapgs:

	SWAPGS_UNSAFE_STACK

nmi_restore:

	POP_EXTRA_REGS

	POP_C_REGS

	POP_REGS



	/*

	 * Skip orig_ax and the "outermost" frame to point RSP at the "iret"

arch/x86/entry/entry_64_compat.S

+30 −0
Original line number Diff line number Diff line
@@ -85,15 +85,25 @@ ENTRY(entry_SYSENTER_compat) 
	pushq	%rcx			/* pt_regs->cx */

	pushq	$-ENOSYS		/* pt_regs->ax */

	pushq   $0			/* pt_regs->r8  = 0 */

	xorq	%r8, %r8		/* nospec   r8 */

	pushq   $0			/* pt_regs->r9  = 0 */

	xorq	%r9, %r9		/* nospec   r9 */

	pushq   $0			/* pt_regs->r10 = 0 */

	xorq	%r10, %r10		/* nospec   r10 */

	pushq   $0			/* pt_regs->r11 = 0 */

	xorq	%r11, %r11		/* nospec   r11 */

	pushq   %rbx                    /* pt_regs->rbx */

	xorl	%ebx, %ebx		/* nospec   rbx */

	pushq   %rbp                    /* pt_regs->rbp (will be overwritten) */

	xorl	%ebp, %ebp		/* nospec   rbp */

	pushq   $0			/* pt_regs->r12 = 0 */

	xorq	%r12, %r12		/* nospec   r12 */

	pushq   $0			/* pt_regs->r13 = 0 */

	xorq	%r13, %r13		/* nospec   r13 */

	pushq   $0			/* pt_regs->r14 = 0 */

	xorq	%r14, %r14		/* nospec   r14 */

	pushq   $0			/* pt_regs->r15 = 0 */

	xorq	%r15, %r15		/* nospec   r15 */

	cld



	/*
@@ -214,15 +224,25 @@ GLOBAL(entry_SYSCALL_compat_after_hwframe) 
	pushq	%rbp			/* pt_regs->cx (stashed in bp) */

	pushq	$-ENOSYS		/* pt_regs->ax */

	pushq   $0			/* pt_regs->r8  = 0 */

	xorq	%r8, %r8		/* nospec   r8 */

	pushq   $0			/* pt_regs->r9  = 0 */

	xorq	%r9, %r9		/* nospec   r9 */

	pushq   $0			/* pt_regs->r10 = 0 */

	xorq	%r10, %r10		/* nospec   r10 */

	pushq   $0			/* pt_regs->r11 = 0 */

	xorq	%r11, %r11		/* nospec   r11 */

	pushq   %rbx                    /* pt_regs->rbx */

	xorl	%ebx, %ebx		/* nospec   rbx */

	pushq   %rbp                    /* pt_regs->rbp (will be overwritten) */

	xorl	%ebp, %ebp		/* nospec   rbp */

	pushq   $0			/* pt_regs->r12 = 0 */

	xorq	%r12, %r12		/* nospec   r12 */

	pushq   $0			/* pt_regs->r13 = 0 */

	xorq	%r13, %r13		/* nospec   r13 */

	pushq   $0			/* pt_regs->r14 = 0 */

	xorq	%r14, %r14		/* nospec   r14 */

	pushq   $0			/* pt_regs->r15 = 0 */

	xorq	%r15, %r15		/* nospec   r15 */



	/*

	 * User mode is traced as though IRQs are on, and SYSENTER
@@ -338,15 +358,25 @@ ENTRY(entry_INT80_compat) 
	pushq	%rcx			/* pt_regs->cx */

	pushq	$-ENOSYS		/* pt_regs->ax */

	pushq   $0			/* pt_regs->r8  = 0 */

	xorq	%r8, %r8		/* nospec   r8 */

	pushq   $0			/* pt_regs->r9  = 0 */

	xorq	%r9, %r9		/* nospec   r9 */

	pushq   $0			/* pt_regs->r10 = 0 */

	xorq	%r10, %r10		/* nospec   r10 */

	pushq   $0			/* pt_regs->r11 = 0 */

	xorq	%r11, %r11		/* nospec   r11 */

	pushq   %rbx                    /* pt_regs->rbx */

	xorl	%ebx, %ebx		/* nospec   rbx */

	pushq   %rbp                    /* pt_regs->rbp */

	xorl	%ebp, %ebp		/* nospec   rbp */

	pushq   %r12                    /* pt_regs->r12 */

	xorq	%r12, %r12		/* nospec   r12 */

	pushq   %r13                    /* pt_regs->r13 */

	xorq	%r13, %r13		/* nospec   r13 */

	pushq   %r14                    /* pt_regs->r14 */

	xorq	%r14, %r14		/* nospec   r14 */

	pushq   %r15                    /* pt_regs->r15 */

	xorq	%r15, %r15		/* nospec   r15 */

	cld



	/*

arch/x86/events/intel/core.c

+1 −1
Original line number Diff line number Diff line
@@ -3559,7 +3559,7 @@ static int intel_snb_pebs_broken(int cpu) 
		break;



	case INTEL_FAM6_SANDYBRIDGE_X:

		switch (cpu_data(cpu).x86_mask) {

		switch (cpu_data(cpu).x86_stepping) {

		case 6: rev = 0x618; break;

		case 7: rev = 0x70c; break;

		}