Merge branch 'x86/paravirt' into x86/apic (eca217b3) · Commits · e / devices / android_kernel_oneplus_sm7250

arch/x86/include/asm/paravirt.h

+301 −134

Original line number	Diff line number	Diff line
		@@ -12,21 +12,38 @@
		#define CLBR_EAX (1 << 0)
		#define CLBR_ECX (1 << 1)
		#define CLBR_EDX (1 << 2)
		#define CLBR_EDI (1 << 3)

		#ifdef CONFIG_X86_64
		#define CLBR_RSI (1 << 3)
		#define CLBR_RDI (1 << 4)
		#ifdef CONFIG_X86_32
		/* CLBR_ANY should match all regs platform has. For i386, that's just it */
		#define CLBR_ANY ((1 << 4) - 1)

		#define CLBR_ARG_REGS (CLBR_EAX \| CLBR_EDX \| CLBR_ECX)
		#define CLBR_RET_REG (CLBR_EAX \| CLBR_EDX)
		#define CLBR_SCRATCH (0)
		#else
		#define CLBR_RAX CLBR_EAX
		#define CLBR_RCX CLBR_ECX
		#define CLBR_RDX CLBR_EDX
		#define CLBR_RDI CLBR_EDI
		#define CLBR_RSI (1 << 4)
		#define CLBR_R8 (1 << 5)
		#define CLBR_R9 (1 << 6)
		#define CLBR_R10 (1 << 7)
		#define CLBR_R11 (1 << 8)

		#define CLBR_ANY ((1 << 9) - 1)

		#define CLBR_ARG_REGS (CLBR_RDI \| CLBR_RSI \| CLBR_RDX \| \
		CLBR_RCX \| CLBR_R8 \| CLBR_R9)
		#define CLBR_RET_REG (CLBR_RAX)
		#define CLBR_SCRATCH (CLBR_R10 \| CLBR_R11)

		#include <asm/desc_defs.h>
		#else
		/* CLBR_ANY should match all regs platform has. For i386, that's just it */
		#define CLBR_ANY ((1 << 3) - 1)
		#endif /* X86_64 */

		#define CLBR_CALLEE_SAVE ((CLBR_ARG_REGS \| CLBR_SCRATCH) & ~CLBR_RET_REG)

		#ifndef __ASSEMBLY__
		#include <linux/types.h>
		#include <linux/cpumask.h>
		@@ -40,6 +57,14 @@ struct tss_struct;
		struct mm_struct;
		struct desc_struct;

		/*
		* Wrapper type for pointers to code which uses the non-standard
		* calling convention. See PV_CALL_SAVE_REGS_THUNK below.
		*/
		struct paravirt_callee_save {
		void *func;
		};

		/* general info */
		struct pv_info {
		unsigned int kernel_rpl;
		@@ -189,11 +214,15 @@ struct pv_irq_ops {
		* expected to use X86_EFLAGS_IF; all other bits
		* returned from save_fl are undefined, and may be ignored by
		* restore_fl.
		*
		* NOTE: These functions callers expect the callee to preserve
		* more registers than the standard C calling convention.
		*/
		unsigned long (*save_fl)(void);
		void (*restore_fl)(unsigned long);
		void (*irq_disable)(void);
		void (*irq_enable)(void);
		struct paravirt_callee_save save_fl;
		struct paravirt_callee_save restore_fl;
		struct paravirt_callee_save irq_disable;
		struct paravirt_callee_save irq_enable;

		void (*safe_halt)(void);
		void (*halt)(void);

		@@ -279,11 +308,11 @@ struct pv_mmu_ops {
		void (ptep_modify_prot_commit)(struct mm_struct mm, unsigned long addr,
		pte_t *ptep, pte_t pte);

		pteval_t (*pte_val)(pte_t);
		pte_t (*make_pte)(pteval_t pte);
		struct paravirt_callee_save pte_val;
		struct paravirt_callee_save make_pte;

		pgdval_t (*pgd_val)(pgd_t);
		pgd_t (*make_pgd)(pgdval_t pgd);
		struct paravirt_callee_save pgd_val;
		struct paravirt_callee_save make_pgd;

		#if PAGETABLE_LEVELS >= 3
		#ifdef CONFIG_X86_PAE
		@@ -298,12 +327,12 @@ struct pv_mmu_ops {

		void (set_pud)(pud_t pudp, pud_t pudval);

		pmdval_t (*pmd_val)(pmd_t);
		pmd_t (*make_pmd)(pmdval_t pmd);
		struct paravirt_callee_save pmd_val;
		struct paravirt_callee_save make_pmd;

		#if PAGETABLE_LEVELS == 4
		pudval_t (*pud_val)(pud_t);
		pud_t (*make_pud)(pudval_t pud);
		struct paravirt_callee_save pud_val;
		struct paravirt_callee_save make_pud;

		void (set_pgd)(pgd_t pudp, pgd_t pgdval);
		#endif /* PAGETABLE_LEVELS == 4 */
		@@ -388,6 +417,8 @@ extern struct pv_lock_ops pv_lock_ops;
		asm("start_" #ops "_" #name ": " code "; end_" #ops "_" #name ":")

		unsigned paravirt_patch_nop(void);
		unsigned paravirt_patch_ident_32(void *insnbuf, unsigned len);
		unsigned paravirt_patch_ident_64(void *insnbuf, unsigned len);
		unsigned paravirt_patch_ignore(unsigned len);
		unsigned paravirt_patch_call(void *insnbuf,
		const void *target, u16 tgt_clobbers,
		@@ -479,25 +510,45 @@ int paravirt_disable_iospace(void);
		* makes sure the incoming and outgoing types are always correct.
		*/
		#ifdef CONFIG_X86_32
		#define PVOP_VCALL_ARGS unsigned long __eax, __edx, __ecx
		#define PVOP_VCALL_ARGS \
		unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx
		#define PVOP_CALL_ARGS PVOP_VCALL_ARGS

		#define PVOP_CALL_ARG1(x) "a" ((unsigned long)(x))
		#define PVOP_CALL_ARG2(x) "d" ((unsigned long)(x))
		#define PVOP_CALL_ARG3(x) "c" ((unsigned long)(x))

		#define PVOP_VCALL_CLOBBERS "=a" (__eax), "=d" (__edx), \
		"=c" (__ecx)
		#define PVOP_CALL_CLOBBERS PVOP_VCALL_CLOBBERS

		#define PVOP_VCALLEE_CLOBBERS "=a" (__eax), "=d" (__edx)
		#define PVOP_CALLEE_CLOBBERS PVOP_VCALLEE_CLOBBERS

		#define EXTRA_CLOBBERS
		#define VEXTRA_CLOBBERS
		#else
		#define PVOP_VCALL_ARGS unsigned long __edi, __esi, __edx, __ecx
		#else /* CONFIG_X86_64 */
		#define PVOP_VCALL_ARGS \
		unsigned long __edi = __edi, __esi = __esi, \
		__edx = __edx, __ecx = __ecx
		#define PVOP_CALL_ARGS PVOP_VCALL_ARGS, __eax

		#define PVOP_CALL_ARG1(x) "D" ((unsigned long)(x))
		#define PVOP_CALL_ARG2(x) "S" ((unsigned long)(x))
		#define PVOP_CALL_ARG3(x) "d" ((unsigned long)(x))
		#define PVOP_CALL_ARG4(x) "c" ((unsigned long)(x))

		#define PVOP_VCALL_CLOBBERS "=D" (__edi), \
		"=S" (__esi), "=d" (__edx), \
		"=c" (__ecx)

		#define PVOP_CALL_CLOBBERS PVOP_VCALL_CLOBBERS, "=a" (__eax)

		#define PVOP_VCALLEE_CLOBBERS "=a" (__eax)
		#define PVOP_CALLEE_CLOBBERS PVOP_VCALLEE_CLOBBERS

		#define EXTRA_CLOBBERS , "r8", "r9", "r10", "r11"
		#define VEXTRA_CLOBBERS , "rax", "r8", "r9", "r10", "r11"
		#endif
		#endif /* CONFIG_X86_32 */

		#ifdef CONFIG_PARAVIRT_DEBUG
		#define PVOP_TEST_NULL(op) BUG_ON(op == NULL)
		@@ -505,7 +556,8 @@ int paravirt_disable_iospace(void);
		#define PVOP_TEST_NULL(op) ((void)op)
		#endif

		#define __PVOP_CALL(rettype, op, pre, post, ...) \
		#define ____PVOP_CALL(rettype, op, clbr, call_clbr, extra_clbr, \
		pre, post, ...) \
		({ \
		rettype __ret; \
		PVOP_CALL_ARGS; \
		@@ -516,70 +568,113 @@ int paravirt_disable_iospace(void);
		asm volatile(pre \
		paravirt_alt(PARAVIRT_CALL) \
		post \
		: PVOP_CALL_CLOBBERS \
		: call_clbr \
		: paravirt_type(op), \
		paravirt_clobber(CLBR_ANY), \
		paravirt_clobber(clbr), \
		##__VA_ARGS__ \
		: "memory", "cc" EXTRA_CLOBBERS); \
		: "memory", "cc" extra_clbr); \
		__ret = (rettype)((((u64)__edx) << 32) \| __eax); \
		} else { \
		asm volatile(pre \
		paravirt_alt(PARAVIRT_CALL) \
		post \
		: PVOP_CALL_CLOBBERS \
		: call_clbr \
		: paravirt_type(op), \
		paravirt_clobber(CLBR_ANY), \
		paravirt_clobber(clbr), \
		##__VA_ARGS__ \
		: "memory", "cc" EXTRA_CLOBBERS); \
		: "memory", "cc" extra_clbr); \
		__ret = (rettype)__eax; \
		} \
		__ret; \
		})
		#define __PVOP_VCALL(op, pre, post, ...) \

		#define __PVOP_CALL(rettype, op, pre, post, ...) \
		____PVOP_CALL(rettype, op, CLBR_ANY, PVOP_CALL_CLOBBERS, \
		EXTRA_CLOBBERS, pre, post, ##__VA_ARGS__)

		#define __PVOP_CALLEESAVE(rettype, op, pre, post, ...) \
		____PVOP_CALL(rettype, op.func, CLBR_RET_REG, \
		PVOP_CALLEE_CLOBBERS, , \
		pre, post, ##__VA_ARGS__)


		#define ____PVOP_VCALL(op, clbr, call_clbr, extra_clbr, pre, post, ...) \
		({ \
		PVOP_VCALL_ARGS; \
		PVOP_TEST_NULL(op); \
		asm volatile(pre \
		paravirt_alt(PARAVIRT_CALL) \
		post \
		: PVOP_VCALL_CLOBBERS \
		: call_clbr \
		: paravirt_type(op), \
		paravirt_clobber(CLBR_ANY), \
		paravirt_clobber(clbr), \
		##__VA_ARGS__ \
		: "memory", "cc" VEXTRA_CLOBBERS); \
		: "memory", "cc" extra_clbr); \
		})

		#define __PVOP_VCALL(op, pre, post, ...) \
		____PVOP_VCALL(op, CLBR_ANY, PVOP_VCALL_CLOBBERS, \
		VEXTRA_CLOBBERS, \
		pre, post, ##__VA_ARGS__)

		#define __PVOP_VCALLEESAVE(rettype, op, pre, post, ...) \
		____PVOP_CALL(rettype, op.func, CLBR_RET_REG, \
		PVOP_VCALLEE_CLOBBERS, , \
		pre, post, ##__VA_ARGS__)



		#define PVOP_CALL0(rettype, op) \
		__PVOP_CALL(rettype, op, "", "")
		#define PVOP_VCALL0(op) \
		__PVOP_VCALL(op, "", "")

		#define PVOP_CALLEE0(rettype, op) \
		__PVOP_CALLEESAVE(rettype, op, "", "")
		#define PVOP_VCALLEE0(op) \
		__PVOP_VCALLEESAVE(op, "", "")


		#define PVOP_CALL1(rettype, op, arg1) \
		__PVOP_CALL(rettype, op, "", "", "0" ((unsigned long)(arg1)))
		__PVOP_CALL(rettype, op, "", "", PVOP_CALL_ARG1(arg1))
		#define PVOP_VCALL1(op, arg1) \
		__PVOP_VCALL(op, "", "", "0" ((unsigned long)(arg1)))
		__PVOP_VCALL(op, "", "", PVOP_CALL_ARG1(arg1))

		#define PVOP_CALLEE1(rettype, op, arg1) \
		__PVOP_CALLEESAVE(rettype, op, "", "", PVOP_CALL_ARG1(arg1))
		#define PVOP_VCALLEE1(op, arg1) \
		__PVOP_VCALLEESAVE(op, "", "", PVOP_CALL_ARG1(arg1))


		#define PVOP_CALL2(rettype, op, arg1, arg2) \
		__PVOP_CALL(rettype, op, "", "", "0" ((unsigned long)(arg1)), \
		"1" ((unsigned long)(arg2)))
		__PVOP_CALL(rettype, op, "", "", PVOP_CALL_ARG1(arg1), \
		PVOP_CALL_ARG2(arg2))
		#define PVOP_VCALL2(op, arg1, arg2) \
		__PVOP_VCALL(op, "", "", "0" ((unsigned long)(arg1)), \
		"1" ((unsigned long)(arg2)))
		__PVOP_VCALL(op, "", "", PVOP_CALL_ARG1(arg1), \
		PVOP_CALL_ARG2(arg2))

		#define PVOP_CALLEE2(rettype, op, arg1, arg2) \
		__PVOP_CALLEESAVE(rettype, op, "", "", PVOP_CALL_ARG1(arg1), \
		PVOP_CALL_ARG2(arg2))
		#define PVOP_VCALLEE2(op, arg1, arg2) \
		__PVOP_VCALLEESAVE(op, "", "", PVOP_CALL_ARG1(arg1), \
		PVOP_CALL_ARG2(arg2))


		#define PVOP_CALL3(rettype, op, arg1, arg2, arg3) \
		__PVOP_CALL(rettype, op, "", "", "0" ((unsigned long)(arg1)), \
		"1"((unsigned long)(arg2)), "2"((unsigned long)(arg3)))
		__PVOP_CALL(rettype, op, "", "", PVOP_CALL_ARG1(arg1), \
		PVOP_CALL_ARG2(arg2), PVOP_CALL_ARG3(arg3))
		#define PVOP_VCALL3(op, arg1, arg2, arg3) \
		__PVOP_VCALL(op, "", "", "0" ((unsigned long)(arg1)), \
		"1"((unsigned long)(arg2)), "2"((unsigned long)(arg3)))
		__PVOP_VCALL(op, "", "", PVOP_CALL_ARG1(arg1), \
		PVOP_CALL_ARG2(arg2), PVOP_CALL_ARG3(arg3))

		/* This is the only difference in x86_64. We can make it much simpler */
		#ifdef CONFIG_X86_32
		#define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4) \
		__PVOP_CALL(rettype, op, \
		"push %[_arg4];", "lea 4(%%esp),%%esp;", \
		"0" ((u32)(arg1)), "1" ((u32)(arg2)), \
		"2" ((u32)(arg3)), [_arg4] "mr" ((u32)(arg4)))
		PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2), \
		PVOP_CALL_ARG3(arg3), [_arg4] "mr" ((u32)(arg4)))
		#define PVOP_VCALL4(op, arg1, arg2, arg3, arg4) \
		__PVOP_VCALL(op, \
		"push %[_arg4];", "lea 4(%%esp),%%esp;", \
		@@ -587,13 +682,13 @@ int paravirt_disable_iospace(void);
		"2" ((u32)(arg3)), [_arg4] "mr" ((u32)(arg4)))
		#else
		#define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4) \
		__PVOP_CALL(rettype, op, "", "", "0" ((unsigned long)(arg1)), \
		"1"((unsigned long)(arg2)), "2"((unsigned long)(arg3)), \
		"3"((unsigned long)(arg4)))
		__PVOP_CALL(rettype, op, "", "", \
		PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2), \
		PVOP_CALL_ARG3(arg3), PVOP_CALL_ARG4(arg4))
		#define PVOP_VCALL4(op, arg1, arg2, arg3, arg4) \
		__PVOP_VCALL(op, "", "", "0" ((unsigned long)(arg1)), \
		"1"((unsigned long)(arg2)), "2"((unsigned long)(arg3)), \
		"3"((unsigned long)(arg4)))
		__PVOP_VCALL(op, "", "", \
		PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2), \
		PVOP_CALL_ARG3(arg3), PVOP_CALL_ARG4(arg4))
		#endif

		static inline int paravirt_enabled(void)
		@@ -1060,11 +1155,11 @@ static inline pte_t __pte(pteval_t val)
		pteval_t ret;

		if (sizeof(pteval_t) > sizeof(long))
		ret = PVOP_CALL2(pteval_t,
		ret = PVOP_CALLEE2(pteval_t,
		pv_mmu_ops.make_pte,
		val, (u64)val >> 32);
		else
		ret = PVOP_CALL1(pteval_t,
		ret = PVOP_CALLEE1(pteval_t,
		pv_mmu_ops.make_pte,
		val);

		@@ -1076,10 +1171,10 @@ static inline pteval_t pte_val(pte_t pte)
		pteval_t ret;

		if (sizeof(pteval_t) > sizeof(long))
		ret = PVOP_CALL2(pteval_t, pv_mmu_ops.pte_val,
		ret = PVOP_CALLEE2(pteval_t, pv_mmu_ops.pte_val,
		pte.pte, (u64)pte.pte >> 32);
		else
		ret = PVOP_CALL1(pteval_t, pv_mmu_ops.pte_val,
		ret = PVOP_CALLEE1(pteval_t, pv_mmu_ops.pte_val,
		pte.pte);

		return ret;
		@@ -1090,10 +1185,10 @@ static inline pgd_t __pgd(pgdval_t val)
		pgdval_t ret;

		if (sizeof(pgdval_t) > sizeof(long))
		ret = PVOP_CALL2(pgdval_t, pv_mmu_ops.make_pgd,
		ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.make_pgd,
		val, (u64)val >> 32);
		else
		ret = PVOP_CALL1(pgdval_t, pv_mmu_ops.make_pgd,
		ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.make_pgd,
		val);

		return (pgd_t) { ret };
		@@ -1104,10 +1199,10 @@ static inline pgdval_t pgd_val(pgd_t pgd)
		pgdval_t ret;

		if (sizeof(pgdval_t) > sizeof(long))
		ret = PVOP_CALL2(pgdval_t, pv_mmu_ops.pgd_val,
		ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.pgd_val,
		pgd.pgd, (u64)pgd.pgd >> 32);
		else
		ret = PVOP_CALL1(pgdval_t, pv_mmu_ops.pgd_val,
		ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.pgd_val,
		pgd.pgd);

		return ret;
		@@ -1172,10 +1267,10 @@ static inline pmd_t __pmd(pmdval_t val)
		pmdval_t ret;

		if (sizeof(pmdval_t) > sizeof(long))
		ret = PVOP_CALL2(pmdval_t, pv_mmu_ops.make_pmd,
		ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.make_pmd,
		val, (u64)val >> 32);
		else
		ret = PVOP_CALL1(pmdval_t, pv_mmu_ops.make_pmd,
		ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.make_pmd,
		val);

		return (pmd_t) { ret };
		@@ -1186,10 +1281,10 @@ static inline pmdval_t pmd_val(pmd_t pmd)
		pmdval_t ret;

		if (sizeof(pmdval_t) > sizeof(long))
		ret = PVOP_CALL2(pmdval_t, pv_mmu_ops.pmd_val,
		ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.pmd_val,
		pmd.pmd, (u64)pmd.pmd >> 32);
		else
		ret = PVOP_CALL1(pmdval_t, pv_mmu_ops.pmd_val,
		ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.pmd_val,
		pmd.pmd);

		return ret;
		@@ -1212,10 +1307,10 @@ static inline pud_t __pud(pudval_t val)
		pudval_t ret;

		if (sizeof(pudval_t) > sizeof(long))
		ret = PVOP_CALL2(pudval_t, pv_mmu_ops.make_pud,
		ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.make_pud,
		val, (u64)val >> 32);
		else
		ret = PVOP_CALL1(pudval_t, pv_mmu_ops.make_pud,
		ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.make_pud,
		val);

		return (pud_t) { ret };
		@@ -1226,10 +1321,10 @@ static inline pudval_t pud_val(pud_t pud)
		pudval_t ret;

		if (sizeof(pudval_t) > sizeof(long))
		ret = PVOP_CALL2(pudval_t, pv_mmu_ops.pud_val,
		ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.pud_val,
		pud.pud, (u64)pud.pud >> 32);
		else
		ret = PVOP_CALL1(pudval_t, pv_mmu_ops.pud_val,
		ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.pud_val,
		pud.pud);

		return ret;
		@@ -1371,6 +1466,9 @@ static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx,
		}

		void _paravirt_nop(void);
		u32 _paravirt_ident_32(u32);
		u64 _paravirt_ident_64(u64);

		#define paravirt_nop ((void *)_paravirt_nop)

		#ifdef CONFIG_SMP
		@@ -1420,12 +1518,37 @@ extern struct paravirt_patch_site __parainstructions[],
		__parainstructions_end[];

		#ifdef CONFIG_X86_32
		#define PV_SAVE_REGS "pushl %%ecx; pushl %%edx;"
		#define PV_RESTORE_REGS "popl %%edx; popl %%ecx"
		#define PV_SAVE_REGS "pushl %ecx; pushl %edx;"
		#define PV_RESTORE_REGS "popl %edx; popl %ecx;"

		/* save and restore all caller-save registers, except return value */
		#define PV_SAVE_ALL_CALLER_REGS "pushl %ecx;"
		#define PV_RESTORE_ALL_CALLER_REGS "popl %ecx;"

		#define PV_FLAGS_ARG "0"
		#define PV_EXTRA_CLOBBERS
		#define PV_VEXTRA_CLOBBERS
		#else
		/* save and restore all caller-save registers, except return value */
		#define PV_SAVE_ALL_CALLER_REGS \
		"push %rcx;" \
		"push %rdx;" \
		"push %rsi;" \
		"push %rdi;" \
		"push %r8;" \
		"push %r9;" \
		"push %r10;" \
		"push %r11;"
		#define PV_RESTORE_ALL_CALLER_REGS \
		"pop %r11;" \
		"pop %r10;" \
		"pop %r9;" \
		"pop %r8;" \
		"pop %rdi;" \
		"pop %rsi;" \
		"pop %rdx;" \
		"pop %rcx;"

		/* We save some registers, but all of them, that's too much. We clobber all
		* caller saved registers but the argument parameter */
		#define PV_SAVE_REGS "pushq %%rdi;"
		@@ -1435,52 +1558,76 @@ extern struct paravirt_patch_site __parainstructions[],
		#define PV_FLAGS_ARG "D"
		#endif

		/*
		* Generate a thunk around a function which saves all caller-save
		* registers except for the return value. This allows C functions to
		* be called from assembler code where fewer than normal registers are
		* available. It may also help code generation around calls from C
		* code if the common case doesn't use many registers.
		*
		* When a callee is wrapped in a thunk, the caller can assume that all
		* arg regs and all scratch registers are preserved across the
		* call. The return value in rax/eax will not be saved, even for void
		* functions.
		*/
		#define PV_CALLEE_SAVE_REGS_THUNK(func) \
		extern typeof(func) __raw_callee_save_##func; \
		static void *__##func##__ __used = func; \
		\
		asm(".pushsection .text;" \
		"__raw_callee_save_" #func ": " \
		PV_SAVE_ALL_CALLER_REGS \
		"call " #func ";" \
		PV_RESTORE_ALL_CALLER_REGS \
		"ret;" \
		".popsection")

		/* Get a reference to a callee-save function */
		#define PV_CALLEE_SAVE(func) \
		((struct paravirt_callee_save) { __raw_callee_save_##func })

		/* Promise that "func" already uses the right calling convention */
		#define __PV_IS_CALLEE_SAVE(func) \
		((struct paravirt_callee_save) { func })

		static inline unsigned long __raw_local_save_flags(void)
		{
		unsigned long f;

		asm volatile(paravirt_alt(PV_SAVE_REGS
		PARAVIRT_CALL
		PV_RESTORE_REGS)
		asm volatile(paravirt_alt(PARAVIRT_CALL)
		: "=a"(f)
		: paravirt_type(pv_irq_ops.save_fl),
		paravirt_clobber(CLBR_EAX)
		: "memory", "cc" PV_VEXTRA_CLOBBERS);
		: "memory", "cc");
		return f;
		}

		static inline void raw_local_irq_restore(unsigned long f)
		{
		asm volatile(paravirt_alt(PV_SAVE_REGS
		PARAVIRT_CALL
		PV_RESTORE_REGS)
		asm volatile(paravirt_alt(PARAVIRT_CALL)
		: "=a"(f)
		: PV_FLAGS_ARG(f),
		paravirt_type(pv_irq_ops.restore_fl),
		paravirt_clobber(CLBR_EAX)
		: "memory", "cc" PV_EXTRA_CLOBBERS);
		: "memory", "cc");
		}

		static inline void raw_local_irq_disable(void)
		{
		asm volatile(paravirt_alt(PV_SAVE_REGS
		PARAVIRT_CALL
		PV_RESTORE_REGS)
		asm volatile(paravirt_alt(PARAVIRT_CALL)
		:
		: paravirt_type(pv_irq_ops.irq_disable),
		paravirt_clobber(CLBR_EAX)
		: "memory", "eax", "cc" PV_EXTRA_CLOBBERS);
		: "memory", "eax", "cc");
		}

		static inline void raw_local_irq_enable(void)
		{
		asm volatile(paravirt_alt(PV_SAVE_REGS
		PARAVIRT_CALL
		PV_RESTORE_REGS)
		asm volatile(paravirt_alt(PARAVIRT_CALL)
		:
		: paravirt_type(pv_irq_ops.irq_enable),
		paravirt_clobber(CLBR_EAX)
		: "memory", "eax", "cc" PV_EXTRA_CLOBBERS);
		: "memory", "eax", "cc");
		}

		static inline unsigned long __raw_local_irq_save(void)
		@@ -1523,33 +1670,49 @@ static inline unsigned long __raw_local_irq_save(void)
		.popsection


		#define COND_PUSH(set, mask, reg) \
		.if ((~(set)) & mask); push %reg; .endif
		#define COND_POP(set, mask, reg) \
		.if ((~(set)) & mask); pop %reg; .endif

		#ifdef CONFIG_X86_64
		#define PV_SAVE_REGS \
		push %rax; \
		push %rcx; \
		push %rdx; \
		push %rsi; \
		push %rdi; \
		push %r8; \
		push %r9; \
		push %r10; \
		push %r11
		#define PV_RESTORE_REGS \
		pop %r11; \
		pop %r10; \
		pop %r9; \
		pop %r8; \
		pop %rdi; \
		pop %rsi; \
		pop %rdx; \
		pop %rcx; \
		pop %rax

		#define PV_SAVE_REGS(set) \
		COND_PUSH(set, CLBR_RAX, rax); \
		COND_PUSH(set, CLBR_RCX, rcx); \
		COND_PUSH(set, CLBR_RDX, rdx); \
		COND_PUSH(set, CLBR_RSI, rsi); \
		COND_PUSH(set, CLBR_RDI, rdi); \
		COND_PUSH(set, CLBR_R8, r8); \
		COND_PUSH(set, CLBR_R9, r9); \
		COND_PUSH(set, CLBR_R10, r10); \
		COND_PUSH(set, CLBR_R11, r11)
		#define PV_RESTORE_REGS(set) \
		COND_POP(set, CLBR_R11, r11); \
		COND_POP(set, CLBR_R10, r10); \
		COND_POP(set, CLBR_R9, r9); \
		COND_POP(set, CLBR_R8, r8); \
		COND_POP(set, CLBR_RDI, rdi); \
		COND_POP(set, CLBR_RSI, rsi); \
		COND_POP(set, CLBR_RDX, rdx); \
		COND_POP(set, CLBR_RCX, rcx); \
		COND_POP(set, CLBR_RAX, rax)

		#define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 8)
		#define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .quad, 8)
		#define PARA_INDIRECT(addr) *addr(%rip)
		#else
		#define PV_SAVE_REGS pushl %eax; pushl %edi; pushl %ecx; pushl %edx
		#define PV_RESTORE_REGS popl %edx; popl %ecx; popl %edi; popl %eax
		#define PV_SAVE_REGS(set) \
		COND_PUSH(set, CLBR_EAX, eax); \
		COND_PUSH(set, CLBR_EDI, edi); \
		COND_PUSH(set, CLBR_ECX, ecx); \
		COND_PUSH(set, CLBR_EDX, edx)
		#define PV_RESTORE_REGS(set) \
		COND_POP(set, CLBR_EDX, edx); \
		COND_POP(set, CLBR_ECX, ecx); \
		COND_POP(set, CLBR_EDI, edi); \
		COND_POP(set, CLBR_EAX, eax)

		#define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 4)
		#define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .long, 4)
		#define PARA_INDIRECT(addr) *%cs:addr
		@@ -1561,15 +1724,15 @@ static inline unsigned long __raw_local_irq_save(void)

		#define DISABLE_INTERRUPTS(clobbers) \
		PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_disable), clobbers, \
		PV_SAVE_REGS; \
		PV_SAVE_REGS(clobbers \| CLBR_CALLEE_SAVE); \
		call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_disable); \
		PV_RESTORE_REGS;) \
		PV_RESTORE_REGS(clobbers \| CLBR_CALLEE_SAVE);)

		#define ENABLE_INTERRUPTS(clobbers) \
		PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_enable), clobbers, \
		PV_SAVE_REGS; \
		PV_SAVE_REGS(clobbers \| CLBR_CALLEE_SAVE); \
		call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_enable); \
		PV_RESTORE_REGS;)
		PV_RESTORE_REGS(clobbers \| CLBR_CALLEE_SAVE);)

		#define USERGS_SYSRET32 \
		PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret32), \
		@@ -1599,11 +1762,15 @@ static inline unsigned long __raw_local_irq_save(void)
		PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE, \
		swapgs)

		/*
		* Note: swapgs is very special, and in practise is either going to be
		* implemented with a single "swapgs" instruction or something very
		* special. Either way, we don't need to save any registers for
		* it.
		*/
		#define SWAPGS \
		PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE, \
		PV_SAVE_REGS; \
		call PARA_INDIRECT(pv_cpu_ops+PV_CPU_swapgs); \
		PV_RESTORE_REGS \
		call PARA_INDIRECT(pv_cpu_ops+PV_CPU_swapgs) \
		)

		#define GET_CR2_INTO_RCX \

arch/x86/include/asm/processor.h

+2 −1

Original line number	Diff line number	Diff line
		@@ -768,7 +768,8 @@ extern int sysenter_setup(void);
		extern struct desc_ptr early_gdt_descr;

		extern void cpu_set_gdt(int);
		extern void switch_to_new_gdt(void);
		extern void switch_to_new_gdt(int);
		extern void load_percpu_segment(int);
		extern void cpu_init(void);

		static inline unsigned long get_debugctlmsr(void)

arch/x86/kernel/cpu/common.c

+15 −10

Original line number	Diff line number	Diff line
		@@ -296,23 +296,28 @@ static char __cpuinit table_lookup_model(struct cpuinfo_x86 c)

		__u32 cleared_cpu_caps[NCAPINTS] __cpuinitdata;

		void load_percpu_segment(int cpu)
		{
		#ifdef CONFIG_X86_32
		loadsegment(fs, __KERNEL_PERCPU);
		#else
		loadsegment(gs, 0);
		wrmsrl(MSR_GS_BASE, (unsigned long)per_cpu(irq_stack_union.gs_base, cpu));
		#endif
		}

		/* Current gdt points %fs at the "master" per-cpu area: after this,
		* it's on the real one. */
		void switch_to_new_gdt(void)
		void switch_to_new_gdt(int cpu)
		{
		struct desc_ptr gdt_descr;
		int cpu = smp_processor_id();

		gdt_descr.address = (long)get_cpu_gdt_table(cpu);
		gdt_descr.size = GDT_SIZE - 1;
		load_gdt(&gdt_descr);
		/* Reload the per-cpu base */
		#ifdef CONFIG_X86_32
		loadsegment(fs, __KERNEL_PERCPU);
		#else
		loadsegment(gs, 0);
		wrmsrl(MSR_GS_BASE, (unsigned long)per_cpu(irq_stack_union.gs_base, cpu));
		#endif

		load_percpu_segment(cpu);
		}

		static struct cpu_dev *cpu_devs[X86_VENDOR_NUM] = {};
		@@ -1029,7 +1034,7 @@ void __cpuinit cpu_init(void)
		* and set up the GDT descriptor:
		*/

		switch_to_new_gdt();
		switch_to_new_gdt(cpu);
		loadsegment(fs, 0);

		load_idt((const struct desc_ptr *)&idt_descr);
		@@ -1131,7 +1136,7 @@ void __cpuinit cpu_init(void)
		clear_in_cr4(X86_CR4_VME\|X86_CR4_PVI\|X86_CR4_TSD\|X86_CR4_DE);

		load_idt(&idt_descr);
		switch_to_new_gdt();
		switch_to_new_gdt(cpu);

		/*
		* Set up and load the per-CPU TSS and LDT

arch/x86/kernel/entry_64.S

+1 −1

Original line number	Diff line number	Diff line
		@@ -1143,7 +1143,7 @@ ENTRY(native_load_gs_index)
		CFI_STARTPROC
		pushf
		CFI_ADJUST_CFA_OFFSET 8
		DISABLE_INTERRUPTS(CLBR_ANY \| ~(CLBR_RDI))
		DISABLE_INTERRUPTS(CLBR_ANY & ~CLBR_RDI)
		SWAPGS
		gs_change:
		movl %edi,%gs

arch/x86/kernel/paravirt.c

+41 −13

File changed.

Preview size limit exceeded, changes collapsed.