Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit bdf977b3 authored by Tejun Heo's avatar Tejun Heo
Browse files

x86, percpu: Collect hot percpu variables into one cacheline 



On x86_64, percpu variables current_task and kernel_stack are used for
get_current() and current_thread_info() respectively and thus are
often used close to each other.  Move definition of current_task to
kernel/cpu/common.c right above kernel_stack definition and align it
to cacheline so that they always fall into the same cacheline.  Two
percpu variables defined there together - irq_stack_ptr and irq_count
- are also pretty hot and will benefit from sharing the cacheline.

For consistency, current_task definition for x86_32 is also moved to
kernel/cpu/common.c.

Putting current_task and kernel_stack into the same cacheline was
suggested by Linus Torvalds.

Signed-off-by: default avatarTejun Heo <tj@kernel.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Ingo Molnar <mingo@elte.hu>
Signed-off-by: default avatarH. Peter Anvin <hpa@zytor.com>
parent 3e352aa8

arch/x86/kernel/cpu/common.c

+13 −2
Original line number Diff line number Diff line
@@ -987,13 +987,21 @@ struct desc_ptr idt_descr = { 256 * 16 - 1, (unsigned long) idt_table }; 
DEFINE_PER_CPU_FIRST(union irq_stack_union,

		     irq_stack_union) __aligned(PAGE_SIZE);



DEFINE_PER_CPU(char *, irq_stack_ptr) =

	init_per_cpu_var(irq_stack_union.irq_stack) + IRQ_STACK_SIZE - 64;

/*

 * The following four percpu variables are hot.  Align current_task to

 * cacheline size such that all four fall in the same cacheline.

 */

DEFINE_PER_CPU(struct task_struct *, current_task) ____cacheline_aligned =

	&init_task;

EXPORT_PER_CPU_SYMBOL(current_task);



DEFINE_PER_CPU(unsigned long, kernel_stack) =

	(unsigned long)&init_thread_union - KERNEL_STACK_OFFSET + THREAD_SIZE;

EXPORT_PER_CPU_SYMBOL(kernel_stack);



DEFINE_PER_CPU(char *, irq_stack_ptr) =

	init_per_cpu_var(irq_stack_union.irq_stack) + IRQ_STACK_SIZE - 64;



DEFINE_PER_CPU(unsigned int, irq_count) = -1;



/*
@@ -1041,6 +1049,9 @@ DEFINE_PER_CPU(struct orig_ist, orig_ist); 


#else	/* CONFIG_X86_64 */



DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;

EXPORT_PER_CPU_SYMBOL(current_task);



#ifdef CONFIG_CC_STACKPROTECTOR

DEFINE_PER_CPU(unsigned long, stack_canary);

#endif

arch/x86/kernel/process_32.c

+0 −3
Original line number Diff line number Diff line
@@ -61,9 +61,6 @@ 


asmlinkage void ret_from_fork(void) __asm__("ret_from_fork");



DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;

EXPORT_PER_CPU_SYMBOL(current_task);



/*

 * Return saved PC of a blocked thread.

 */

arch/x86/kernel/process_64.c

+0 −3
Original line number Diff line number Diff line
@@ -55,9 +55,6 @@ 


asmlinkage extern void ret_from_fork(void);



DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;

EXPORT_PER_CPU_SYMBOL(current_task);



DEFINE_PER_CPU(unsigned long, old_rsp);

static DEFINE_PER_CPU(unsigned char, is_idle);