Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit d0142ceb authored by Hugh Dickins's avatar Hugh Dickins Committed by Greg Kroah-Hartman
Browse files

kaiser: x86_cr3_pcid_noflush and x86_cr3_pcid_user 




Mostly this commit is just unshouting X86_CR3_PCID_KERN_VAR and
X86_CR3_PCID_USER_VAR: we usually name variables in lower-case.

But why does x86_cr3_pcid_noflush need to be __aligned(PAGE_SIZE)?
Ah, it's a leftover from when kaiser_add_user_map() once complained
about mapping the same page twice.  Make it __read_mostly instead.
(I'm a little uneasy about all the unrelated data which shares its
page getting user-mapped too, but that was so before, and not a big
deal: though we call it user-mapped, it's not mapped with _PAGE_USER.)

And there is a little change around the two calls to do_nmi().
Previously they set the NOFLUSH bit (if PCID supported) when
forcing to kernel context before do_nmi(); now they also have the
NOFLUSH bit set (if PCID supported) when restoring context after:
nothing done in do_nmi() should require a TLB to be flushed here.

Signed-off-by: default avatarHugh Dickins <hughd@google.com>
Signed-off-by: default avatarGreg Kroah-Hartman <gregkh@linuxfoundation.org>
parent 6a2b4117

arch/x86/entry/entry_64.S

+4 −4
Original line number Diff line number Diff line
@@ -1316,11 +1316,11 @@ ENTRY(nmi) 
	/* Unconditionally use kernel CR3 for do_nmi() */

	/* %rax is saved above, so OK to clobber here */

	movq	%cr3, %rax

	/* If PCID enabled, NOFLUSH now and NOFLUSH on return */

	orq	x86_cr3_pcid_noflush, %rax

	pushq	%rax

	/* mask off "user" bit of pgd address and 12 PCID bits: */

	andq	$(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), %rax

	/* Add back kernel PCID and "no flush" bit */

	orq	X86_CR3_PCID_KERN_VAR, %rax

	movq	%rax, %cr3

#endif

	call	do_nmi
@@ -1560,11 +1560,11 @@ end_repeat_nmi: 
	/* Unconditionally use kernel CR3 for do_nmi() */

	/* %rax is saved above, so OK to clobber here */

	movq	%cr3, %rax

	/* If PCID enabled, NOFLUSH now and NOFLUSH on return */

	orq	x86_cr3_pcid_noflush, %rax

	pushq	%rax

	/* mask off "user" bit of pgd address and 12 PCID bits: */

	andq	$(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), %rax

	/* Add back kernel PCID and "no flush" bit */

	orq	X86_CR3_PCID_KERN_VAR, %rax

	movq	%rax, %cr3

#endif

arch/x86/include/asm/kaiser.h

+5 −6
Original line number Diff line number Diff line
@@ -25,7 +25,7 @@ 
.macro _SWITCH_TO_KERNEL_CR3 reg

movq %cr3, \reg

andq $(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), \reg

orq  X86_CR3_PCID_KERN_VAR, \reg

orq  x86_cr3_pcid_noflush, \reg

movq \reg, %cr3

.endm
@@ -37,11 +37,10 @@ movq \reg, %cr3 
 * not enabled): so that the one register can update both memory and cr3.

 */

movq %cr3, \reg

andq $(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), \reg

orq  PER_CPU_VAR(X86_CR3_PCID_USER_VAR), \reg

orq  PER_CPU_VAR(x86_cr3_pcid_user), \reg

js   9f

/* FLUSH this time, reset to NOFLUSH for next time (if PCID enabled) */

movb \regb, PER_CPU_VAR(X86_CR3_PCID_USER_VAR+7)

movb \regb, PER_CPU_VAR(x86_cr3_pcid_user+7)

9:

movq \reg, %cr3

.endm
@@ -94,8 +93,8 @@ movq PER_CPU_VAR(unsafe_stack_register_backup), %rax 
*/

DECLARE_PER_CPU_USER_MAPPED(unsigned long, unsafe_stack_register_backup);



extern unsigned long X86_CR3_PCID_KERN_VAR;

DECLARE_PER_CPU(unsigned long, X86_CR3_PCID_USER_VAR);

extern unsigned long x86_cr3_pcid_noflush;

DECLARE_PER_CPU(unsigned long, x86_cr3_pcid_user);



extern char __per_cpu_user_mapped_start[], __per_cpu_user_mapped_end[];

arch/x86/mm/kaiser.c

+7 −6
Original line number Diff line number Diff line
@@ -28,8 +28,8 @@ DEFINE_PER_CPU_USER_MAPPED(unsigned long, unsafe_stack_register_backup); 
 * This is also handy because systems that do not support PCIDs

 * just end up or'ing a 0 into their CR3, which does no harm.

 */

__aligned(PAGE_SIZE) unsigned long X86_CR3_PCID_KERN_VAR;

DEFINE_PER_CPU(unsigned long, X86_CR3_PCID_USER_VAR);

unsigned long x86_cr3_pcid_noflush __read_mostly;

DEFINE_PER_CPU(unsigned long, x86_cr3_pcid_user);



/*

 * At runtime, the only things we map are some things for CPU
@@ -303,7 +303,8 @@ void __init kaiser_init(void) 
				  sizeof(gate_desc) * NR_VECTORS,

				  __PAGE_KERNEL);



	kaiser_add_user_map_early(&X86_CR3_PCID_KERN_VAR, PAGE_SIZE,

	kaiser_add_user_map_early(&x86_cr3_pcid_noflush,

				  sizeof(x86_cr3_pcid_noflush),

				  __PAGE_KERNEL);

}
@@ -381,8 +382,8 @@ void kaiser_setup_pcid(void) 
	 * These variables are used by the entry/exit

	 * code to change PCID and pgd and TLB flushing.

	 */

	X86_CR3_PCID_KERN_VAR = kern_cr3;

	this_cpu_write(X86_CR3_PCID_USER_VAR, user_cr3);

	x86_cr3_pcid_noflush = kern_cr3;

	this_cpu_write(x86_cr3_pcid_user, user_cr3);

}



/*
@@ -392,7 +393,7 @@ void kaiser_setup_pcid(void) 
 */

void kaiser_flush_tlb_on_return_to_user(void)

{

	this_cpu_write(X86_CR3_PCID_USER_VAR,

	this_cpu_write(x86_cr3_pcid_user,

			X86_CR3_PCID_USER_FLUSH | KAISER_SHADOW_PGD_OFFSET);

}

EXPORT_SYMBOL(kaiser_flush_tlb_on_return_to_user);