Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 2684b12a authored by Hugh Dickins's avatar Hugh Dickins Committed by Greg Kroah-Hartman
Browse files

kaiser: enhanced by kernel and user PCIDs 




Merged performance improvements to Kaiser, using distinct kernel
and user Process Context Identifiers to minimize the TLB flushing.

[This work actually all from Dave Hansen 2017-08-30:
still omitting trackswitch mods, and KAISER_REAL_SWITCH deleted.]

Signed-off-by: default avatarDave Hansen <dave.hansen@linux.intel.com>
Signed-off-by: default avatarHugh Dickins <hughd@google.com>
Signed-off-by: default avatarGreg Kroah-Hartman <gregkh@linuxfoundation.org>
parent 1972bb9d

arch/x86/entry/entry_64.S

+8 −2
Original line number Diff line number Diff line
@@ -1317,7 +1317,10 @@ ENTRY(nmi) 
	/* %rax is saved above, so OK to clobber here */

	movq	%cr3, %rax

	pushq	%rax

	andq	$(~KAISER_SHADOW_PGD_OFFSET), %rax

	/* mask off "user" bit of pgd address and 12 PCID bits: */

	andq	$(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), %rax

	/* Add back kernel PCID and "no flush" bit */

	orq	X86_CR3_PCID_KERN_VAR, %rax

	movq	%rax, %cr3

#endif

	call	do_nmi
@@ -1558,7 +1561,10 @@ end_repeat_nmi: 
	/* %rax is saved above, so OK to clobber here */

	movq	%cr3, %rax

	pushq	%rax

	andq	$(~KAISER_SHADOW_PGD_OFFSET), %rax

	/* mask off "user" bit of pgd address and 12 PCID bits: */

	andq	$(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), %rax

	/* Add back kernel PCID and "no flush" bit */

	orq	X86_CR3_PCID_KERN_VAR, %rax

	movq	%rax, %cr3

#endif

arch/x86/entry/entry_64_compat.S

+1 −0
Original line number Diff line number Diff line
@@ -13,6 +13,7 @@ 
#include <asm/irqflags.h>

#include <asm/asm.h>

#include <asm/smap.h>

#include <asm/pgtable_types.h>

#include <asm/kaiser.h>

#include <linux/linkage.h>

#include <linux/err.h>

arch/x86/include/asm/cpufeatures.h

+1 −0
Original line number Diff line number Diff line
@@ -189,6 +189,7 @@ 


#define X86_FEATURE_CPB		( 7*32+ 2) /* AMD Core Performance Boost */

#define X86_FEATURE_EPB		( 7*32+ 3) /* IA32_ENERGY_PERF_BIAS support */

#define X86_FEATURE_INVPCID_SINGLE ( 7*32+ 4) /* Effectively INVPCID && CR4.PCIDE=1 */



#define X86_FEATURE_HW_PSTATE	( 7*32+ 8) /* AMD HW-PState */

#define X86_FEATURE_PROC_FEEDBACK ( 7*32+ 9) /* AMD ProcFeedbackInterface */

arch/x86/include/asm/kaiser.h

+13 −2
Original line number Diff line number Diff line 
#ifndef _ASM_X86_KAISER_H

#define _ASM_X86_KAISER_H



#include <uapi/asm/processor-flags.h> /* For PCID constants */



/*

 * This file includes the definitions for the KAISER feature.

 * KAISER is a counter measure against x86_64 side channel attacks on
@@ -21,12 +24,20 @@ 


.macro _SWITCH_TO_KERNEL_CR3 reg

movq %cr3, \reg

andq $(~KAISER_SHADOW_PGD_OFFSET), \reg

andq $(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), \reg

orq  X86_CR3_PCID_KERN_VAR, \reg

movq \reg, %cr3

.endm



.macro _SWITCH_TO_USER_CR3 reg

movq %cr3, \reg

andq $(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), \reg

/*

 * This can obviously be one instruction by putting the

 * KAISER_SHADOW_PGD_OFFSET bit in the X86_CR3_PCID_USER_VAR.

 * But, just leave it now for simplicity.

 */

orq  X86_CR3_PCID_USER_VAR, \reg

orq  $(KAISER_SHADOW_PGD_OFFSET), \reg

movq \reg, %cr3

.endm

arch/x86/include/asm/pgtable_types.h

+26 −0
Original line number Diff line number Diff line
@@ -141,6 +141,32 @@ 
			 _PAGE_SOFT_DIRTY)

#define _HPAGE_CHG_MASK (_PAGE_CHG_MASK | _PAGE_PSE)



/* The ASID is the lower 12 bits of CR3 */

#define X86_CR3_PCID_ASID_MASK  (_AC((1<<12)-1,UL))



/* Mask for all the PCID-related bits in CR3: */

#define X86_CR3_PCID_MASK       (X86_CR3_PCID_NOFLUSH | X86_CR3_PCID_ASID_MASK)

#if defined(CONFIG_KAISER) && defined(CONFIG_X86_64)

#define X86_CR3_PCID_ASID_KERN  (_AC(0x4,UL))

#define X86_CR3_PCID_ASID_USER  (_AC(0x6,UL))



#define X86_CR3_PCID_KERN_FLUSH		(X86_CR3_PCID_ASID_KERN)

#define X86_CR3_PCID_USER_FLUSH		(X86_CR3_PCID_ASID_USER)

#define X86_CR3_PCID_KERN_NOFLUSH	(X86_CR3_PCID_NOFLUSH | X86_CR3_PCID_ASID_KERN)

#define X86_CR3_PCID_USER_NOFLUSH	(X86_CR3_PCID_NOFLUSH | X86_CR3_PCID_ASID_USER)

#else

#define X86_CR3_PCID_ASID_KERN  (_AC(0x0,UL))

#define X86_CR3_PCID_ASID_USER  (_AC(0x0,UL))

/*

 * PCIDs are unsupported on 32-bit and none of these bits can be

 * set in CR3:

 */

#define X86_CR3_PCID_KERN_FLUSH		(0)

#define X86_CR3_PCID_USER_FLUSH		(0)

#define X86_CR3_PCID_KERN_NOFLUSH	(0)

#define X86_CR3_PCID_USER_NOFLUSH	(0)

#endif



/*

 * The cache modes defined here are used to translate between pure SW usage

 * and the HW defined cache mode bits and/or PAT entries.