Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 5868f365 authored by Tom Lendacky's avatar Tom Lendacky Committed by Ingo Molnar
Browse files

x86/mm: Add support to enable SME in early boot processing 



Add support to the early boot code to use Secure Memory Encryption (SME).
Since the kernel has been loaded into memory in a decrypted state, encrypt
the kernel in place and update the early pagetables with the memory
encryption mask so that new pagetable entries will use memory encryption.

The routines to set the encryption mask and perform the encryption are
stub routines for now with functionality to be added in a later patch.

Signed-off-by: default avatarTom Lendacky <thomas.lendacky@amd.com>
Reviewed-by: default avatarThomas Gleixner <tglx@linutronix.de>
Cc: Alexander Potapenko <glider@google.com>
Cc: Andrey Ryabinin <aryabinin@virtuozzo.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brijesh Singh <brijesh.singh@amd.com>
Cc: Dave Young <dyoung@redhat.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Matt Fleming <matt@codeblueprint.co.uk>
Cc: Michael S. Tsirkin <mst@redhat.com>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Radim Krčmář <rkrcmar@redhat.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Toshimitsu Kani <toshi.kani@hpe.com>
Cc: kasan-dev@googlegroups.com
Cc: kvm@vger.kernel.org
Cc: linux-arch@vger.kernel.org
Cc: linux-doc@vger.kernel.org
Cc: linux-efi@vger.kernel.org
Cc: linux-mm@kvack.org
Link: http://lkml.kernel.org/r/e52ad781f085224bf835b3caff9aa3aee6febccb.1500319216.git.thomas.lendacky@amd.com


Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 33c2b803

arch/x86/include/asm/mem_encrypt.h

+8 −0
Original line number Diff line number Diff line
@@ -15,14 +15,22 @@ 


#ifndef __ASSEMBLY__



#include <linux/init.h>



#ifdef CONFIG_AMD_MEM_ENCRYPT



extern unsigned long sme_me_mask;



void __init sme_encrypt_kernel(void);

void __init sme_enable(void);



#else	/* !CONFIG_AMD_MEM_ENCRYPT */



#define sme_me_mask	0UL



static inline void __init sme_encrypt_kernel(void) { }

static inline void __init sme_enable(void) { }



#endif	/* CONFIG_AMD_MEM_ENCRYPT */



#endif	/* __ASSEMBLY__ */

arch/x86/kernel/head64.c

+42 −11
Original line number Diff line number Diff line
@@ -14,6 +14,7 @@ 
#include <linux/start_kernel.h>

#include <linux/io.h>

#include <linux/memblock.h>

#include <linux/mem_encrypt.h>



#include <asm/processor.h>

#include <asm/proto.h>
@@ -45,9 +46,10 @@ static void __head *fixup_pointer(void *ptr, unsigned long physaddr) 
	return ptr - (void *)_text + (void *)physaddr;

}



void __head __startup_64(unsigned long physaddr)

unsigned long __head __startup_64(unsigned long physaddr)

{

	unsigned long load_delta, *p;

	unsigned long pgtable_flags;

	pgdval_t *pgd;

	p4dval_t *p4d;

	pudval_t *pud;
@@ -68,6 +70,12 @@ void __head __startup_64(unsigned long physaddr) 
	if (load_delta & ~PMD_PAGE_MASK)

		for (;;);



	/* Activate Secure Memory Encryption (SME) if supported and enabled */

	sme_enable();



	/* Include the SME encryption mask in the fixup value */

	load_delta += sme_get_me_mask();



	/* Fixup the physical addresses in the page table */



	pgd = fixup_pointer(&early_top_pgt, physaddr);
@@ -94,28 +102,30 @@ void __head __startup_64(unsigned long physaddr) 


	pud = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr);

	pmd = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr);

	pgtable_flags = _KERNPG_TABLE + sme_get_me_mask();



	if (IS_ENABLED(CONFIG_X86_5LEVEL)) {

		p4d = fixup_pointer(early_dynamic_pgts[next_early_pgt++], physaddr);



		i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;

		pgd[i + 0] = (pgdval_t)p4d + _KERNPG_TABLE;

		pgd[i + 1] = (pgdval_t)p4d + _KERNPG_TABLE;

		pgd[i + 0] = (pgdval_t)p4d + pgtable_flags;

		pgd[i + 1] = (pgdval_t)p4d + pgtable_flags;



		i = (physaddr >> P4D_SHIFT) % PTRS_PER_P4D;

		p4d[i + 0] = (pgdval_t)pud + _KERNPG_TABLE;

		p4d[i + 1] = (pgdval_t)pud + _KERNPG_TABLE;

		p4d[i + 0] = (pgdval_t)pud + pgtable_flags;

		p4d[i + 1] = (pgdval_t)pud + pgtable_flags;

	} else {

		i = (physaddr >> PGDIR_SHIFT) % PTRS_PER_PGD;

		pgd[i + 0] = (pgdval_t)pud + _KERNPG_TABLE;

		pgd[i + 1] = (pgdval_t)pud + _KERNPG_TABLE;

		pgd[i + 0] = (pgdval_t)pud + pgtable_flags;

		pgd[i + 1] = (pgdval_t)pud + pgtable_flags;

	}



	i = (physaddr >> PUD_SHIFT) % PTRS_PER_PUD;

	pud[i + 0] = (pudval_t)pmd + _KERNPG_TABLE;

	pud[i + 1] = (pudval_t)pmd + _KERNPG_TABLE;

	pud[i + 0] = (pudval_t)pmd + pgtable_flags;

	pud[i + 1] = (pudval_t)pmd + pgtable_flags;



	pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL;

	pmd_entry += sme_get_me_mask();

	pmd_entry +=  physaddr;



	for (i = 0; i < DIV_ROUND_UP(_end - _text, PMD_SIZE); i++) {
@@ -136,9 +146,30 @@ void __head __startup_64(unsigned long physaddr) 
			pmd[i] += load_delta;

	}



	/* Fixup phys_base */

	/*

	 * Fixup phys_base - remove the memory encryption mask to obtain

	 * the true physical address.

	 */

	p = fixup_pointer(&phys_base, physaddr);

	*p += load_delta;

	*p += load_delta - sme_get_me_mask();



	/* Encrypt the kernel (if SME is active) */

	sme_encrypt_kernel();



	/*

	 * Return the SME encryption mask (if SME is active) to be used as a

	 * modifier for the initial pgdir entry programmed into CR3.

	 */

	return sme_get_me_mask();

}



unsigned long __startup_secondary_64(void)

{

	/*

	 * Return the SME encryption mask (if SME is active) to be used as a

	 * modifier for the initial pgdir entry programmed into CR3.

	 */

	return sme_get_me_mask();

}



/* Wipe all early page tables except for the kernel symbol map */

arch/x86/kernel/head_64.S

+18 −2
Original line number Diff line number Diff line
@@ -73,12 +73,19 @@ startup_64: 
	/* Sanitize CPU configuration */

	call verify_cpu



	/*

	 * Perform pagetable fixups. Additionally, if SME is active, encrypt

	 * the kernel and retrieve the modifier (SME encryption mask if SME

	 * is active) to be added to the initial pgdir entry that will be

	 * programmed into CR3.

	 */

	leaq	_text(%rip), %rdi

	pushq	%rsi

	call	__startup_64

	popq	%rsi



	movq	$(early_top_pgt - __START_KERNEL_map), %rax

	/* Form the CR3 value being sure to include the CR3 modifier */

	addq	$(early_top_pgt - __START_KERNEL_map), %rax

	jmp 1f

ENTRY(secondary_startup_64)

	/*
@@ -98,7 +105,16 @@ ENTRY(secondary_startup_64) 
	/* Sanitize CPU configuration */

	call verify_cpu



	movq	$(init_top_pgt - __START_KERNEL_map), %rax

	/*

	 * Retrieve the modifier (SME encryption mask if SME is active) to be

	 * added to the initial pgdir entry that will be programmed into CR3.

	 */

	pushq	%rsi

	call	__startup_secondary_64

	popq	%rsi



	/* Form the CR3 value being sure to include the CR3 modifier */

	addq	$(init_top_pgt - __START_KERNEL_map), %rax

1:



	/* Enable PAE mode, PGE and LA57 */

arch/x86/mm/mem_encrypt.c

+9 −0
Original line number Diff line number Diff line
@@ -11,6 +11,7 @@ 
 */



#include <linux/linkage.h>

#include <linux/init.h>



/*

 * Since SME related variables are set early in the boot process they must
@@ -19,3 +20,11 @@ 
 */

unsigned long sme_me_mask __section(.data) = 0;

EXPORT_SYMBOL_GPL(sme_me_mask);



void __init sme_encrypt_kernel(void)

{

}



void __init sme_enable(void)

{

}

include/linux/mem_encrypt.h

+5 −0
Original line number Diff line number Diff line
@@ -30,6 +30,11 @@ static inline bool sme_active(void) 
	return !!sme_me_mask;

}



static inline unsigned long sme_get_me_mask(void)

{

	return sme_me_mask;

}



#endif	/* __ASSEMBLY__ */



#endif	/* __MEM_ENCRYPT_H__ */