Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 5917dae8 authored by Christoph Lameter's avatar Christoph Lameter Committed by Tejun Heo
Browse files

percpu, x86: Generic inc / dec percpu instructions 



Optimize code generated for percpu access by checking for increment and
decrements.

tj: fix incorrect usage of __builtin_constant_p() and restructure
    percpu_add_op() macro.

Signed-off-by: default avatarChristoph Lameter <cl@linux-foundation.org>
Signed-off-by: default avatarTejun Heo <tj@kernel.org>
parent 79615760

arch/x86/include/asm/percpu.h

+72 −14
Original line number Diff line number Diff line
@@ -103,6 +103,64 @@ do { \ 
	}						\

} while (0)



/*

 * Generate a percpu add to memory instruction and optimize code

 * if a one is added or subtracted.

 */

#define percpu_add_op(var, val)						\

do {									\

	typedef typeof(var) pao_T__;					\

	const int pao_ID__ = (__builtin_constant_p(val) &&		\

			      ((val) == 1 || (val) == -1)) ? (val) : 0;	\

	if (0) {							\

		pao_T__ pao_tmp__;					\

		pao_tmp__ = (val);					\

	}								\

	switch (sizeof(var)) {						\

	case 1:								\

		if (pao_ID__ == 1)					\

			asm("incb "__percpu_arg(0) : "+m" (var));	\

		else if (pao_ID__ == -1)				\

			asm("decb "__percpu_arg(0) : "+m" (var));	\

		else							\

			asm("addb %1, "__percpu_arg(0)			\

			    : "+m" (var)				\

			    : "qi" ((pao_T__)(val)));			\

		break;							\

	case 2:								\

		if (pao_ID__ == 1)					\

			asm("incw "__percpu_arg(0) : "+m" (var));	\

		else if (pao_ID__ == -1)				\

			asm("decw "__percpu_arg(0) : "+m" (var));	\

		else							\

			asm("addw %1, "__percpu_arg(0)			\

			    : "+m" (var)				\

			    : "ri" ((pao_T__)(val)));			\

		break;							\

	case 4:								\

		if (pao_ID__ == 1)					\

			asm("incl "__percpu_arg(0) : "+m" (var));	\

		else if (pao_ID__ == -1)				\

			asm("decl "__percpu_arg(0) : "+m" (var));	\

		else							\

			asm("addl %1, "__percpu_arg(0)			\

			    : "+m" (var)				\

			    : "ri" ((pao_T__)(val)));			\

		break;							\

	case 8:								\

		if (pao_ID__ == 1)					\

			asm("incq "__percpu_arg(0) : "+m" (var));	\

		else if (pao_ID__ == -1)				\

			asm("decq "__percpu_arg(0) : "+m" (var));	\

		else							\

			asm("addq %1, "__percpu_arg(0)			\

			    : "+m" (var)				\

			    : "re" ((pao_T__)(val)));			\

		break;							\

	default: __bad_percpu_size();					\

	}								\

} while (0)



#define percpu_from_op(op, var, constraint)		\

({							\

	typeof(var) pfo_ret__;				\
@@ -144,8 +202,8 @@ do { \ 
#define percpu_read(var)		percpu_from_op("mov", var, "m" (var))

#define percpu_read_stable(var)		percpu_from_op("mov", var, "p" (&(var)))

#define percpu_write(var, val)		percpu_to_op("mov", var, val)

#define percpu_add(var, val)		percpu_to_op("add", var, val)

#define percpu_sub(var, val)		percpu_to_op("sub", var, val)

#define percpu_add(var, val)		percpu_add_op(var, val)

#define percpu_sub(var, val)		percpu_add_op(var, -(val))

#define percpu_and(var, val)		percpu_to_op("and", var, val)

#define percpu_or(var, val)		percpu_to_op("or", var, val)

#define percpu_xor(var, val)		percpu_to_op("xor", var, val)
@@ -157,9 +215,9 @@ do { \ 
#define __this_cpu_write_1(pcp, val)	percpu_to_op("mov", (pcp), val)

#define __this_cpu_write_2(pcp, val)	percpu_to_op("mov", (pcp), val)

#define __this_cpu_write_4(pcp, val)	percpu_to_op("mov", (pcp), val)

#define __this_cpu_add_1(pcp, val)	percpu_to_op("add", (pcp), val)

#define __this_cpu_add_2(pcp, val)	percpu_to_op("add", (pcp), val)

#define __this_cpu_add_4(pcp, val)	percpu_to_op("add", (pcp), val)

#define __this_cpu_add_1(pcp, val)	percpu_add_op((pcp), val)

#define __this_cpu_add_2(pcp, val)	percpu_add_op((pcp), val)

#define __this_cpu_add_4(pcp, val)	percpu_add_op((pcp), val)

#define __this_cpu_and_1(pcp, val)	percpu_to_op("and", (pcp), val)

#define __this_cpu_and_2(pcp, val)	percpu_to_op("and", (pcp), val)

#define __this_cpu_and_4(pcp, val)	percpu_to_op("and", (pcp), val)
@@ -176,9 +234,9 @@ do { \ 
#define this_cpu_write_1(pcp, val)	percpu_to_op("mov", (pcp), val)

#define this_cpu_write_2(pcp, val)	percpu_to_op("mov", (pcp), val)

#define this_cpu_write_4(pcp, val)	percpu_to_op("mov", (pcp), val)

#define this_cpu_add_1(pcp, val)	percpu_to_op("add", (pcp), val)

#define this_cpu_add_2(pcp, val)	percpu_to_op("add", (pcp), val)

#define this_cpu_add_4(pcp, val)	percpu_to_op("add", (pcp), val)

#define this_cpu_add_1(pcp, val)	percpu_add_op((pcp), val)

#define this_cpu_add_2(pcp, val)	percpu_add_op((pcp), val)

#define this_cpu_add_4(pcp, val)	percpu_add_op((pcp), val)

#define this_cpu_and_1(pcp, val)	percpu_to_op("and", (pcp), val)

#define this_cpu_and_2(pcp, val)	percpu_to_op("and", (pcp), val)

#define this_cpu_and_4(pcp, val)	percpu_to_op("and", (pcp), val)
@@ -189,9 +247,9 @@ do { \ 
#define this_cpu_xor_2(pcp, val)	percpu_to_op("xor", (pcp), val)

#define this_cpu_xor_4(pcp, val)	percpu_to_op("xor", (pcp), val)



#define irqsafe_cpu_add_1(pcp, val)	percpu_to_op("add", (pcp), val)

#define irqsafe_cpu_add_2(pcp, val)	percpu_to_op("add", (pcp), val)

#define irqsafe_cpu_add_4(pcp, val)	percpu_to_op("add", (pcp), val)

#define irqsafe_cpu_add_1(pcp, val)	percpu_add_op((pcp), val)

#define irqsafe_cpu_add_2(pcp, val)	percpu_add_op((pcp), val)

#define irqsafe_cpu_add_4(pcp, val)	percpu_add_op((pcp), val)

#define irqsafe_cpu_and_1(pcp, val)	percpu_to_op("and", (pcp), val)

#define irqsafe_cpu_and_2(pcp, val)	percpu_to_op("and", (pcp), val)

#define irqsafe_cpu_and_4(pcp, val)	percpu_to_op("and", (pcp), val)
@@ -209,19 +267,19 @@ do { \ 
#ifdef CONFIG_X86_64

#define __this_cpu_read_8(pcp)		percpu_from_op("mov", (pcp), "m"(pcp))

#define __this_cpu_write_8(pcp, val)	percpu_to_op("mov", (pcp), val)

#define __this_cpu_add_8(pcp, val)	percpu_to_op("add", (pcp), val)

#define __this_cpu_add_8(pcp, val)	percpu_add_op((pcp), val)

#define __this_cpu_and_8(pcp, val)	percpu_to_op("and", (pcp), val)

#define __this_cpu_or_8(pcp, val)	percpu_to_op("or", (pcp), val)

#define __this_cpu_xor_8(pcp, val)	percpu_to_op("xor", (pcp), val)



#define this_cpu_read_8(pcp)		percpu_from_op("mov", (pcp), "m"(pcp))

#define this_cpu_write_8(pcp, val)	percpu_to_op("mov", (pcp), val)

#define this_cpu_add_8(pcp, val)	percpu_to_op("add", (pcp), val)

#define this_cpu_add_8(pcp, val)	percpu_add_op((pcp), val)

#define this_cpu_and_8(pcp, val)	percpu_to_op("and", (pcp), val)

#define this_cpu_or_8(pcp, val)		percpu_to_op("or", (pcp), val)

#define this_cpu_xor_8(pcp, val)	percpu_to_op("xor", (pcp), val)



#define irqsafe_cpu_add_8(pcp, val)	percpu_to_op("add", (pcp), val)

#define irqsafe_cpu_add_8(pcp, val)	percpu_add_op((pcp), val)

#define irqsafe_cpu_and_8(pcp, val)	percpu_to_op("and", (pcp), val)

#define irqsafe_cpu_or_8(pcp, val)	percpu_to_op("or", (pcp), val)

#define irqsafe_cpu_xor_8(pcp, val)	percpu_to_op("xor", (pcp), val)