Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 947e76cd authored by Brian Gerst's avatar Brian Gerst Committed by Tejun Heo
Browse files

x86: move stack_canary into irq_stack 



Impact: x86_64 percpu area layout change, irq_stack now at the beginning

Now that the PDA is empty except for the stack canary, it can be removed.
The irqstack is moved to the start of the per-cpu section.  If the stack
protector is enabled, the canary overlaps the bottom 48 bytes of the irqstack.

tj: * updated subject
    * dropped asm relocation of irq_stack_ptr
    * updated comments a bit
    * rebased on top of stack canary changes

Signed-off-by: default avatarBrian Gerst <brgerst@gmail.com>
Signed-off-by: default avatarTejun Heo <tj@kernel.org>
parent 8c7e58e6

arch/x86/include/asm/pda.h

+0 −3
Original line number Original line Diff line number Diff line
@@ -17,9 +17,6 @@ struct x8664_pda { 
	unsigned long unused4;

	unsigned long unused4;

	int unused5;

	int unused5;

	unsigned int unused6;		/* 36 was cpunumber */

	unsigned int unused6;		/* 36 was cpunumber */

	unsigned long stack_canary;	/* 40 stack canary value */

					/* gcc-ABI: this canary MUST be at

					   offset 40!!! */

	short in_bootmem;		/* pda lives in bootmem */

	short in_bootmem;		/* pda lives in bootmem */

} ____cacheline_aligned_in_smp;

} ____cacheline_aligned_in_smp;

arch/x86/include/asm/percpu.h

+0 −6
Original line number Original line Diff line number Diff line
@@ -133,12 +133,6 @@ do { \ 
/* We can use this directly for local CPU (faster). */

/* We can use this directly for local CPU (faster). */

DECLARE_PER_CPU(unsigned long, this_cpu_off);

DECLARE_PER_CPU(unsigned long, this_cpu_off);





#ifdef CONFIG_X86_64

extern void load_pda_offset(int cpu);

#else

static inline void load_pda_offset(int cpu) { }

#endif



#endif /* !__ASSEMBLY__ */

#endif /* !__ASSEMBLY__ */





#ifdef CONFIG_SMP

#ifdef CONFIG_SMP

arch/x86/include/asm/processor.h

+22 −1
Original line number Original line Diff line number Diff line
@@ -379,8 +379,29 @@ union thread_xstate { 
#ifdef CONFIG_X86_64

#ifdef CONFIG_X86_64

DECLARE_PER_CPU(struct orig_ist, orig_ist);

DECLARE_PER_CPU(struct orig_ist, orig_ist);





DECLARE_PER_CPU(char[IRQ_STACK_SIZE], irq_stack);

union irq_stack_union {

	char irq_stack[IRQ_STACK_SIZE];

	/*

	 * GCC hardcodes the stack canary as %gs:40.  Since the

	 * irq_stack is the object at %gs:0, we reserve the bottom

	 * 48 bytes of the irq stack for the canary.

	 */

	struct {

		char gs_base[40];

		unsigned long stack_canary;

	};

};



DECLARE_PER_CPU(union irq_stack_union, irq_stack_union);

DECLARE_PER_CPU(char *, irq_stack_ptr);

DECLARE_PER_CPU(char *, irq_stack_ptr);



static inline void load_gs_base(int cpu)

{

	/* Memory clobbers used to order pda/percpu accesses */

	mb();

	wrmsrl(MSR_GS_BASE, (unsigned long)per_cpu(irq_stack_union.gs_base, cpu));

	mb();

}

#endif

#endif





extern void print_cpu_info(struct cpuinfo_x86 *);

extern void print_cpu_info(struct cpuinfo_x86 *);

arch/x86/include/asm/stackprotector.h

+3 −3
Original line number Original line Diff line number Diff line
@@ -2,7 +2,7 @@ 
#define _ASM_STACKPROTECTOR_H 1

#define _ASM_STACKPROTECTOR_H 1





#include <asm/tsc.h>

#include <asm/tsc.h>

#include <asm/pda.h>

#include <asm/processor.h>





/*

/*

 * Initialize the stackprotector canary value.

 * Initialize the stackprotector canary value.
@@ -19,7 +19,7 @@ static __always_inline void boot_init_stack_canary(void) 
	 * Build time only check to make sure the stack_canary is at

	 * Build time only check to make sure the stack_canary is at

	 * offset 40 in the pda; this is a gcc ABI requirement

	 * offset 40 in the pda; this is a gcc ABI requirement

	 */

	 */

	BUILD_BUG_ON(offsetof(struct x8664_pda, stack_canary) != 40);

	BUILD_BUG_ON(offsetof(union irq_stack_union, stack_canary) != 40);





	/*

	/*

	 * We both use the random pool and the current TSC as a source

	 * We both use the random pool and the current TSC as a source
@@ -32,7 +32,7 @@ static __always_inline void boot_init_stack_canary(void) 
	canary += tsc + (tsc << 32UL);

	canary += tsc + (tsc << 32UL);





	current->stack_canary = canary;

	current->stack_canary = canary;

	write_pda(stack_canary, canary);

	percpu_write(irq_stack_union.stack_canary, canary);

}

}





#endif
 
#endif

arch/x86/include/asm/system.h

+2 −2
Original line number Original line Diff line number Diff line
@@ -89,10 +89,10 @@ do { \ 
#ifdef CONFIG_CC_STACKPROTECTOR

#ifdef CONFIG_CC_STACKPROTECTOR

#define __switch_canary							  \

#define __switch_canary							  \

	"movq %P[task_canary](%%rsi),%%r8\n\t"				  \

	"movq %P[task_canary](%%rsi),%%r8\n\t"				  \

	"movq %%r8,%%gs:%P[pda_canary]\n\t"

	"movq %%r8,%%gs:%P[gs_canary]\n\t"

#define __switch_canary_param						  \

#define __switch_canary_param						  \

	, [task_canary] "i" (offsetof(struct task_struct, stack_canary))  \

	, [task_canary] "i" (offsetof(struct task_struct, stack_canary))  \

	, [pda_canary] "i" (offsetof(struct x8664_pda, stack_canary))

	, [gs_canary] "i" (offsetof(union irq_stack_union, stack_canary))

#else	/* CC_STACKPROTECTOR */

#else	/* CC_STACKPROTECTOR */

#define __switch_canary

#define __switch_canary

#define __switch_canary_param

#define __switch_canary_param