Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 40304775 authored by Tejun Heo's avatar Tejun Heo
Browse files

percpu,x86: relocate this_cpu_add_return() and friends 



- include/linux/percpu.h: this_cpu_add_return() and friends were
  located next to __this_cpu_add_return().  However, the overall
  organization is to first group by preemption safeness.  Relocate
  this_cpu_add_return() and friends to preemption-safe area.

- arch/x86/include/asm/percpu.h: Relocate percpu_add_return_op() after
  other more basic operations.  Relocate [__]this_cpu_add_return_8()
  so that they're first grouped by preemption safeness.

Signed-off-by: default avatarTejun Heo <tj@kernel.org>
Cc: Christoph Lameter <cl@linux.com>
parent 8f1d97c7

arch/x86/include/asm/percpu.h

+35 −36
Original line number Diff line number Diff line
@@ -177,39 +177,6 @@ do { \ 
	}								\

} while (0)



/*

 * Add return operation

 */

#define percpu_add_return_op(var, val)					\

({									\

	typeof(var) paro_ret__ = val;					\

	switch (sizeof(var)) {						\

	case 1:								\

		asm("xaddb %0, "__percpu_arg(1)				\

			    : "+q" (paro_ret__), "+m" (var)		\

			    : : "memory");				\

		break;							\

	case 2:								\

		asm("xaddw %0, "__percpu_arg(1)				\

			    : "+r" (paro_ret__), "+m" (var)		\

			    : : "memory");				\

		break;							\

	case 4:								\

		asm("xaddl %0, "__percpu_arg(1)				\

			    : "+r" (paro_ret__), "+m" (var)		\

			    : : "memory");				\

		break;							\

	case 8:								\

		asm("xaddq %0, "__percpu_arg(1)				\

			    : "+re" (paro_ret__), "+m" (var)		\

			    : : "memory");				\

		break;							\

	default: __bad_percpu_size();					\

	}								\

	paro_ret__ += val;						\

	paro_ret__;							\

})



#define percpu_from_op(op, var, constraint)		\

({							\

	typeof(var) pfo_ret__;				\
@@ -262,6 +229,39 @@ do { \ 
	}						\

})



/*

 * Add return operation

 */

#define percpu_add_return_op(var, val)					\

({									\

	typeof(var) paro_ret__ = val;					\

	switch (sizeof(var)) {						\

	case 1:								\

		asm("xaddb %0, "__percpu_arg(1)				\

			    : "+q" (paro_ret__), "+m" (var)		\

			    : : "memory");				\

		break;							\

	case 2:								\

		asm("xaddw %0, "__percpu_arg(1)				\

			    : "+r" (paro_ret__), "+m" (var)		\

			    : : "memory");				\

		break;							\

	case 4:								\

		asm("xaddl %0, "__percpu_arg(1)				\

			    : "+r" (paro_ret__), "+m" (var)		\

			    : : "memory");				\

		break;							\

	case 8:								\

		asm("xaddq %0, "__percpu_arg(1)				\

			    : "+re" (paro_ret__), "+m" (var)		\

			    : : "memory");				\

		break;							\

	default: __bad_percpu_size();					\

	}								\

	paro_ret__ += val;						\

	paro_ret__;							\

})



/*

 * percpu_read() makes gcc load the percpu variable every time it is

 * accessed while percpu_read_stable() allows the value to be cached.
@@ -352,6 +352,7 @@ do { \ 
#define __this_cpu_and_8(pcp, val)	percpu_to_op("and", (pcp), val)

#define __this_cpu_or_8(pcp, val)	percpu_to_op("or", (pcp), val)

#define __this_cpu_xor_8(pcp, val)	percpu_to_op("xor", (pcp), val)

#define __this_cpu_add_return_8(pcp, val) percpu_add_return_op(pcp, val)



#define this_cpu_read_8(pcp)		percpu_from_op("mov", (pcp), "m"(pcp))

#define this_cpu_write_8(pcp, val)	percpu_to_op("mov", (pcp), val)
@@ -359,14 +360,12 @@ do { \ 
#define this_cpu_and_8(pcp, val)	percpu_to_op("and", (pcp), val)

#define this_cpu_or_8(pcp, val)		percpu_to_op("or", (pcp), val)

#define this_cpu_xor_8(pcp, val)	percpu_to_op("xor", (pcp), val)

#define this_cpu_add_return_8(pcp, val)	percpu_add_return_op(pcp, val)



#define irqsafe_cpu_add_8(pcp, val)	percpu_add_op((pcp), val)

#define irqsafe_cpu_and_8(pcp, val)	percpu_to_op("and", (pcp), val)

#define irqsafe_cpu_or_8(pcp, val)	percpu_to_op("or", (pcp), val)

#define irqsafe_cpu_xor_8(pcp, val)	percpu_to_op("xor", (pcp), val)



#define __this_cpu_add_return_8(pcp, val) percpu_add_return_op(pcp, val)

#define this_cpu_add_return_8(pcp, val)	percpu_add_return_op(pcp, val)

#endif



/* This is not atomic against other CPUs -- CPU preemption needs to be off */

include/linux/percpu.h

+30 −30
Original line number Diff line number Diff line
@@ -417,6 +417,36 @@ do { \ 
# define this_cpu_xor(pcp, val)		__pcpu_size_call(this_cpu_or_, (pcp), (val))

#endif



#define _this_cpu_generic_add_return(pcp, val)				\

({									\

	typeof(pcp) ret__;						\

	preempt_disable();						\

	__this_cpu_add(pcp, val);					\

	ret__ = __this_cpu_read(pcp);					\

	preempt_enable();						\

	ret__;								\

})



#ifndef this_cpu_add_return

# ifndef this_cpu_add_return_1

#  define this_cpu_add_return_1(pcp, val)	_this_cpu_generic_add_return(pcp, val)

# endif

# ifndef this_cpu_add_return_2

#  define this_cpu_add_return_2(pcp, val)	_this_cpu_generic_add_return(pcp, val)

# endif

# ifndef this_cpu_add_return_4

#  define this_cpu_add_return_4(pcp, val)	_this_cpu_generic_add_return(pcp, val)

# endif

# ifndef this_cpu_add_return_8

#  define this_cpu_add_return_8(pcp, val)	_this_cpu_generic_add_return(pcp, val)

# endif

# define this_cpu_add_return(pcp, val)	__pcpu_size_call_return2(this_cpu_add_return_, pcp, val)

#endif



#define this_cpu_sub_return(pcp, val)	this_cpu_add_return(pcp, -(val))

#define this_cpu_inc_return(pcp)	this_cpu_add_return(pcp, 1)

#define this_cpu_dec_return(pcp)	this_cpu_add_return(pcp, -1)



/*

 * Generic percpu operations that do not require preemption handling.

 * Either we do not care about races or the caller has the
@@ -544,36 +574,6 @@ do { \ 
# define __this_cpu_xor(pcp, val)	__pcpu_size_call(__this_cpu_xor_, (pcp), (val))

#endif



#define _this_cpu_generic_add_return(pcp, val)				\

({									\

	typeof(pcp) ret__;						\

	preempt_disable();						\

	__this_cpu_add(pcp, val);					\

	ret__ = __this_cpu_read(pcp);					\

	preempt_enable();						\

	ret__;								\

})



#ifndef this_cpu_add_return

# ifndef this_cpu_add_return_1

#  define this_cpu_add_return_1(pcp, val)	_this_cpu_generic_add_return(pcp, val)

# endif

# ifndef this_cpu_add_return_2

#  define this_cpu_add_return_2(pcp, val)	_this_cpu_generic_add_return(pcp, val)

# endif

# ifndef this_cpu_add_return_4

#  define this_cpu_add_return_4(pcp, val)	_this_cpu_generic_add_return(pcp, val)

# endif

# ifndef this_cpu_add_return_8

#  define this_cpu_add_return_8(pcp, val)	_this_cpu_generic_add_return(pcp, val)

# endif

# define this_cpu_add_return(pcp, val)	__pcpu_size_call_return2(this_cpu_add_return_, pcp, val)

#endif



#define this_cpu_sub_return(pcp, val)	this_cpu_add_return(pcp, -(val))

#define this_cpu_inc_return(pcp)	this_cpu_add_return(pcp, 1)

#define this_cpu_dec_return(pcp)	this_cpu_add_return(pcp, -1)



#define __this_cpu_generic_add_return(pcp, val)				\

({									\

	__this_cpu_add(pcp, val);					\