Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit b359e8a4 authored by Suresh Siddha's avatar Suresh Siddha Committed by Ingo Molnar
Browse files

x86, xsave: context switch support using xsave/xrstor 



Uses xsave/xrstor (instead of traditional fxsave/fxrstor) in context switch
when available.

Introduces TS_XSAVE flag, which determine the need to use xsave/xrstor
instructions during context switch instead of the legacy fxsave/fxrstor
instructions. Thread-synchronous status word is already in L1 cache during
this code patch and thus minimizes the performance penality compared to
(cpu_has_xsave) checks.

Signed-off-by: default avatarSuresh Siddha <suresh.b.siddha@intel.com>
Signed-off-by: default avatarH. Peter Anvin <hpa@zytor.com>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent dc1e35c6

arch/x86/kernel/cpu/common.c

+4 −1
Original line number Original line Diff line number Diff line
@@ -709,6 +709,9 @@ void __cpuinit cpu_init(void) 
	/*

	/*

	 * Force FPU initialization:

	 * Force FPU initialization:

	 */

	 */

	if (cpu_has_xsave)

		current_thread_info()->status = TS_XSAVE;

	else

		current_thread_info()->status = 0;

		current_thread_info()->status = 0;

	clear_used_math();

	clear_used_math();

	mxcsr_feature_mask_init();

	mxcsr_feature_mask_init();

arch/x86/kernel/i387.c

+4 −1
Original line number Original line Diff line number Diff line
@@ -97,6 +97,9 @@ void __cpuinit fpu_init(void) 




	mxcsr_feature_mask_init();

	mxcsr_feature_mask_init();

	/* clean state in init */

	/* clean state in init */

	if (cpu_has_xsave)

		current_thread_info()->status = TS_XSAVE;

	else

		current_thread_info()->status = 0;

		current_thread_info()->status = 0;

	clear_used_math();

	clear_used_math();

}

}

arch/x86/kernel/traps_64.c

+1 −1
Original line number Original line Diff line number Diff line
@@ -1134,7 +1134,7 @@ asmlinkage void math_state_restore(void) 
	/*

	/*

	 * Paranoid restore. send a SIGSEGV if we fail to restore the state.

	 * Paranoid restore. send a SIGSEGV if we fail to restore the state.

	 */

	 */

	if (unlikely(restore_fpu_checking(&me->thread.xstate->fxsave))) {

	if (unlikely(restore_fpu_checking(me))) {

		stts();

		stts();

		force_sig(SIGSEGV, me);

		force_sig(SIGSEGV, me);

		return;

		return;

include/asm-x86/i387.h

+59 −5
Original line number Original line Diff line number Diff line
@@ -36,6 +36,8 @@ extern int save_i387_ia32(struct _fpstate_ia32 __user *buf); 
extern int restore_i387_ia32(struct _fpstate_ia32 __user *buf);

extern int restore_i387_ia32(struct _fpstate_ia32 __user *buf);

#endif

#endif





#define X87_FSW_ES (1 << 7)	/* Exception Summary */



#ifdef CONFIG_X86_64

#ifdef CONFIG_X86_64





/* Ignore delayed exceptions from user space */

/* Ignore delayed exceptions from user space */
@@ -46,7 +48,7 @@ static inline void tolerant_fwait(void) 
		     _ASM_EXTABLE(1b, 2b));

		     _ASM_EXTABLE(1b, 2b));

}

}





static inline int restore_fpu_checking(struct i387_fxsave_struct *fx)

static inline int fxrstor_checking(struct i387_fxsave_struct *fx)

{

{

	int err;

	int err;
@@ -66,15 +68,31 @@ static inline int restore_fpu_checking(struct i387_fxsave_struct *fx) 
	return err;

	return err;

}

}





#define X87_FSW_ES (1 << 7)	/* Exception Summary */

static inline int restore_fpu_checking(struct task_struct *tsk)

{

	if (task_thread_info(tsk)->status & TS_XSAVE)

		return xrstor_checking(&tsk->thread.xstate->xsave);

	else

		return fxrstor_checking(&tsk->thread.xstate->fxsave);

}





/* AMD CPUs don't save/restore FDP/FIP/FOP unless an exception

/* AMD CPUs don't save/restore FDP/FIP/FOP unless an exception

   is pending. Clear the x87 state here by setting it to fixed

   is pending. Clear the x87 state here by setting it to fixed

   values. The kernel data segment can be sometimes 0 and sometimes

   values. The kernel data segment can be sometimes 0 and sometimes

   new user value. Both should be ok.

   new user value. Both should be ok.

   Use the PDA as safe address because it should be already in L1. */

   Use the PDA as safe address because it should be already in L1. */

static inline void clear_fpu_state(struct i387_fxsave_struct *fx)

static inline void clear_fpu_state(struct task_struct *tsk)

{

{

	struct xsave_struct *xstate = &tsk->thread.xstate->xsave;

	struct i387_fxsave_struct *fx = &tsk->thread.xstate->fxsave;



	/*

	 * xsave header may indicate the init state of the FP.

	 */

	if ((task_thread_info(tsk)->status & TS_XSAVE) &&

	    !(xstate->xsave_hdr.xstate_bv & XSTATE_FP))

		return;



	if (unlikely(fx->swd & X87_FSW_ES))

	if (unlikely(fx->swd & X87_FSW_ES))

		asm volatile("fnclex");

		asm volatile("fnclex");

	alternative_input(ASM_NOP8 ASM_NOP2,

	alternative_input(ASM_NOP8 ASM_NOP2,
@@ -107,7 +125,7 @@ static inline int save_i387_checking(struct i387_fxsave_struct __user *fx) 
	return err;

	return err;

}

}





static inline void __save_init_fpu(struct task_struct *tsk)

static inline void fxsave(struct task_struct *tsk)

{

{

	/* Using "rex64; fxsave %0" is broken because, if the memory operand

	/* Using "rex64; fxsave %0" is broken because, if the memory operand

	   uses any extended registers for addressing, a second REX prefix

	   uses any extended registers for addressing, a second REX prefix
@@ -132,7 +150,16 @@ static inline void __save_init_fpu(struct task_struct *tsk) 
			     : "=m" (tsk->thread.xstate->fxsave)

			     : "=m" (tsk->thread.xstate->fxsave)

			     : "cdaSDb" (&tsk->thread.xstate->fxsave));

			     : "cdaSDb" (&tsk->thread.xstate->fxsave));

#endif

#endif

	clear_fpu_state(&tsk->thread.xstate->fxsave);

}



static inline void __save_init_fpu(struct task_struct *tsk)

{

	if (task_thread_info(tsk)->status & TS_XSAVE)

		xsave(tsk);

	else

		fxsave(tsk);



	clear_fpu_state(tsk);

	task_thread_info(tsk)->status &= ~TS_USEDFPU;

	task_thread_info(tsk)->status &= ~TS_USEDFPU;

}

}
@@ -147,6 +174,10 @@ static inline void tolerant_fwait(void) 




static inline void restore_fpu(struct task_struct *tsk)

static inline void restore_fpu(struct task_struct *tsk)

{

{

	if (task_thread_info(tsk)->status & TS_XSAVE) {

		xrstor_checking(&tsk->thread.xstate->xsave);

		return;

	}

	/*

	/*

	 * The "nop" is needed to make the instructions the same

	 * The "nop" is needed to make the instructions the same

	 * length.

	 * length.
@@ -172,6 +203,27 @@ static inline void restore_fpu(struct task_struct *tsk) 
 */

 */

static inline void __save_init_fpu(struct task_struct *tsk)

static inline void __save_init_fpu(struct task_struct *tsk)

{

{

	if (task_thread_info(tsk)->status & TS_XSAVE) {

		struct xsave_struct *xstate = &tsk->thread.xstate->xsave;

		struct i387_fxsave_struct *fx = &tsk->thread.xstate->fxsave;



		xsave(tsk);



		/*

		 * xsave header may indicate the init state of the FP.

		 */

		if (!(xstate->xsave_hdr.xstate_bv & XSTATE_FP))

			goto end;



		if (unlikely(fx->swd & X87_FSW_ES))

			asm volatile("fnclex");



		/*

		 * we can do a simple return here or be paranoid :)

		 */

		goto clear_state;

	}



	/* Use more nops than strictly needed in case the compiler

	/* Use more nops than strictly needed in case the compiler

	   varies code */

	   varies code */

	alternative_input(

	alternative_input(
@@ -181,6 +233,7 @@ static inline void __save_init_fpu(struct task_struct *tsk) 
		X86_FEATURE_FXSR,

		X86_FEATURE_FXSR,

		[fx] "m" (tsk->thread.xstate->fxsave),

		[fx] "m" (tsk->thread.xstate->fxsave),

		[fsw] "m" (tsk->thread.xstate->fxsave.swd) : "memory");

		[fsw] "m" (tsk->thread.xstate->fxsave.swd) : "memory");

clear_state:

	/* AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception

	/* AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception

	   is pending.  Clear the x87 state here by setting it to fixed

	   is pending.  Clear the x87 state here by setting it to fixed

	   values. safe_address is a random variable that should be in L1 */

	   values. safe_address is a random variable that should be in L1 */
@@ -190,6 +243,7 @@ static inline void __save_init_fpu(struct task_struct *tsk) 
		"fildl %[addr]", 	/* set F?P to defined value */

		"fildl %[addr]", 	/* set F?P to defined value */

		X86_FEATURE_FXSAVE_LEAK,

		X86_FEATURE_FXSAVE_LEAK,

		[addr] "m" (safe_address));

		[addr] "m" (safe_address));

end:

	task_thread_info(tsk)->status &= ~TS_USEDFPU;

	task_thread_info(tsk)->status &= ~TS_USEDFPU;

}

}

include/asm-x86/processor.h

+1 −0
Original line number Original line Diff line number Diff line
@@ -362,6 +362,7 @@ union thread_xstate { 
	struct i387_fsave_struct	fsave;

	struct i387_fsave_struct	fsave;

	struct i387_fxsave_struct	fxsave;

	struct i387_fxsave_struct	fxsave;

	struct i387_soft_struct		soft;

	struct i387_soft_struct		soft;

	struct xsave_struct		xsave;

};

};





#ifdef CONFIG_X86_64

#ifdef CONFIG_X86_64