Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 6c690ee1 authored by Andy Lutomirski's avatar Andy Lutomirski Committed by Ingo Molnar
Browse files

x86/mm: Split read_cr3() into read_cr3_pa() and __read_cr3() 



The kernel has several code paths that read CR3.  Most of them assume that
CR3 contains the PGD's physical address, whereas some of them awkwardly
use PHYSICAL_PAGE_MASK to mask off low bits.

Add explicit mask macros for CR3 and convert all of the CR3 readers.
This will keep them from breaking when PCID is enabled.

Signed-off-by: default avatarAndy Lutomirski <luto@kernel.org>
Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Tom Lendacky <thomas.lendacky@amd.com>
Cc: xen-devel <xen-devel@lists.xen.org>
Link: http://lkml.kernel.org/r/883f8fb121f4616c1c1427ad87350bb2f5ffeca1.1497288170.git.luto@kernel.org


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 3f365cf3

arch/x86/boot/compressed/pagetable.c

+1 −1
Original line number Diff line number Diff line
@@ -92,7 +92,7 @@ void initialize_identity_maps(void) 
	 * and we must append to the existing area instead of entirely

	 * overwriting it.

	 */

	level4p = read_cr3();

	level4p = read_cr3_pa();

	if (level4p == (unsigned long)_pgtable) {

		debug_putstr("booted via startup_32()\n");

		pgt_data.pgt_buf = _pgtable + BOOT_INIT_PGT_SIZE;

arch/x86/include/asm/efi.h

+1 −1
Original line number Diff line number Diff line
@@ -74,7 +74,7 @@ struct efi_scratch { 
	__kernel_fpu_begin();						\

									\

	if (efi_scratch.use_pgd) {					\

		efi_scratch.prev_cr3 = read_cr3();			\

		efi_scratch.prev_cr3 = __read_cr3();			\

		write_cr3((unsigned long)efi_scratch.efi_pgt);		\

		__flush_tlb_all();					\

	}								\

arch/x86/include/asm/mmu_context.h

+2 −2
Original line number Diff line number Diff line
@@ -269,7 +269,7 @@ static inline bool arch_vma_access_permitted(struct vm_area_struct *vma, 


/*

 * This can be used from process context to figure out what the value of

 * CR3 is without needing to do a (slow) read_cr3().

 * CR3 is without needing to do a (slow) __read_cr3().

 *

 * It's intended to be used for code like KVM that sneakily changes CR3

 * and needs to restore it.  It needs to be used very carefully.
@@ -281,7 +281,7 @@ static inline unsigned long __get_current_cr3_fast(void) 
	/* For now, be very restrictive about when this can be called. */

	VM_WARN_ON(in_nmi() || !in_atomic());



	VM_BUG_ON(cr3 != read_cr3());

	VM_BUG_ON(cr3 != __read_cr3());

	return cr3;

}

arch/x86/include/asm/paravirt.h

+1 −1
Original line number Diff line number Diff line
@@ -61,7 +61,7 @@ static inline void write_cr2(unsigned long x) 
	PVOP_VCALL1(pv_mmu_ops.write_cr2, x);

}



static inline unsigned long read_cr3(void)

static inline unsigned long __read_cr3(void)

{

	return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr3);

}

arch/x86/include/asm/processor-flags.h

+36 −0
Original line number Diff line number Diff line
@@ -8,4 +8,40 @@ 
#else

#define X86_VM_MASK	0 /* No VM86 support */

#endif



/*

 * CR3's layout varies depending on several things.

 *

 * If CR4.PCIDE is set (64-bit only), then CR3[11:0] is the address space ID.

 * If PAE is enabled, then CR3[11:5] is part of the PDPT address

 * (i.e. it's 32-byte aligned, not page-aligned) and CR3[4:0] is ignored.

 * Otherwise (non-PAE, non-PCID), CR3[3] is PWT, CR3[4] is PCD, and

 * CR3[2:0] and CR3[11:5] are ignored.

 *

 * In all cases, Linux puts zeros in the low ignored bits and in PWT and PCD.

 *

 * CR3[63] is always read as zero.  If CR4.PCIDE is set, then CR3[63] may be

 * written as 1 to prevent the write to CR3 from flushing the TLB.

 *

 * On systems with SME, one bit (in a variable position!) is stolen to indicate

 * that the top-level paging structure is encrypted.

 *

 * All of the remaining bits indicate the physical address of the top-level

 * paging structure.

 *

 * CR3_ADDR_MASK is the mask used by read_cr3_pa().

 */

#ifdef CONFIG_X86_64

/* Mask off the address space ID bits. */

#define CR3_ADDR_MASK 0x7FFFFFFFFFFFF000ull

#define CR3_PCID_MASK 0xFFFull

#else

/*

 * CR3_ADDR_MASK needs at least bits 31:5 set on PAE systems, and we save

 * a tiny bit of code size by setting all the bits.

 */

#define CR3_ADDR_MASK 0xFFFFFFFFull

#define CR3_PCID_MASK 0ull

#endif



#endif /* _ASM_X86_PROCESSOR_FLAGS_H */