Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit db7829c6 authored by Brian Gerst's avatar Brian Gerst Committed by Tejun Heo
Browse files

x86, percpu: Optimize this_cpu_ptr 



Allow arches to implement __this_cpu_ptr, and provide an x86 version.

Before:
	movq $foo, %rax
	movq %gs:this_cpu_off, %rdx
	addq %rdx, %rax

After:
	movq $foo, %rax
	addq %gs:this_cpu_off, %rax

The benefit is doing it in one less instruction and not clobbering
a temporary register.

tj: * Beefed up the comment a bit and renamed in-macro temp variable
      to match neighboring macros.

    * Folded fix for const pointer case found in linux-next.

    * Fixed sparse notation.

Signed-off-by: default avatarBrian Gerst <brgerst@gmail.com>
Signed-off-by: default avatarTejun Heo <tj@kernel.org>
parent fc1481a9

arch/x86/include/asm/percpu.h

+14 −0
Original line number Diff line number Diff line
@@ -47,6 +47,20 @@ 
#ifdef CONFIG_SMP

#define __percpu_arg(x)		"%%"__stringify(__percpu_seg)":%P" #x

#define __my_cpu_offset		percpu_read(this_cpu_off)



/*

 * Compared to the generic __my_cpu_offset version, the following

 * saves one instruction and avoids clobbering a temp register.

 */

#define __this_cpu_ptr(ptr)				\

({							\

	unsigned long tcp_ptr__;			\

	__verify_pcpu_ptr(ptr);				\

	asm volatile("add " __percpu_arg(1) ", %0"	\

		     : "=r" (tcp_ptr__)			\

		     : "m" (this_cpu_off), "0" (ptr));	\

	(typeof(*(ptr)) __kernel __force *)tcp_ptr__;	\

})

#else

#define __percpu_arg(x)		"%P" #x

#endif

include/asm-generic/percpu.h

+7 −2
Original line number Diff line number Diff line
@@ -60,9 +60,14 @@ extern unsigned long __per_cpu_offset[NR_CPUS]; 
#define __raw_get_cpu_var(var) \

	(*SHIFT_PERCPU_PTR(&(var), __my_cpu_offset))



#define this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, my_cpu_offset)

#ifndef __this_cpu_ptr

#define __this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)



#endif

#ifdef CONFIG_DEBUG_PREEMPT

#define this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, my_cpu_offset)

#else

#define this_cpu_ptr(ptr) __this_cpu_ptr(ptr)

#endif



#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA

extern void setup_per_cpu_areas(void);