Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit d430d3d7 authored by Jason Baron's avatar Jason Baron Committed by Steven Rostedt
Browse files

jump label: Introduce static_branch() interface 



Introduce:

static __always_inline bool static_branch(struct jump_label_key *key);

instead of the old JUMP_LABEL(key, label) macro.

In this way, jump labels become really easy to use:

Define:

        struct jump_label_key jump_key;

Can be used as:

        if (static_branch(&jump_key))
                do unlikely code

enable/disale via:

        jump_label_inc(&jump_key);
        jump_label_dec(&jump_key);

that's it!

For the jump labels disabled case, the static_branch() becomes an
atomic_read(), and jump_label_inc()/dec() are simply atomic_inc(),
atomic_dec() operations. We show testing results for this change below.

Thanks to H. Peter Anvin for suggesting the 'static_branch()' construct.

Since we now require a 'struct jump_label_key *key', we can store a pointer into
the jump table addresses. In this way, we can enable/disable jump labels, in
basically constant time. This change allows us to completely remove the previous
hashtable scheme. Thanks to Peter Zijlstra for this re-write.

Testing:

I ran a series of 'tbench 20' runs 5 times (with reboots) for 3
configurations, where tracepoints were disabled.

jump label configured in
avg: 815.6

jump label *not* configured in (using atomic reads)
avg: 800.1

jump label *not* configured in (regular reads)
avg: 803.4

Signed-off-by: default avatarPeter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <20110316212947.GA8792@redhat.com>
Signed-off-by: default avatarJason Baron <jbaron@redhat.com>
Suggested-by: default avatarH. Peter Anvin <hpa@linux.intel.com>
Tested-by: default avatarDavid Daney <ddaney@caviumnetworks.com>
Acked-by: default avatarRalf Baechle <ralf@linux-mips.org>
Acked-by: default avatarDavid S. Miller <davem@davemloft.net>
Acked-by: default avatarMathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: default avatarSteven Rostedt <rostedt@goodmis.org>
parent ee5e51f5

arch/mips/include/asm/jump_label.h

+12 −10
Original line number Diff line number Diff line
@@ -20,16 +20,18 @@ 
#define WORD_INSN ".word"

#endif



#define JUMP_LABEL(key, label)						\

	do {								\

		asm goto("1:\tnop\n\t"					\

			"nop\n\t"					\

			".pushsection __jump_table,  \"a\"\n\t"		\

			WORD_INSN " 1b, %l[" #label "], %0\n\t"		\

			".popsection\n\t"				\

			: :  "i" (key) :  : label);			\

	} while (0)



static __always_inline bool arch_static_branch(struct jump_label_key *key)

{

	asm goto("1:\tnop\n\t"

		"nop\n\t"

		".pushsection __jump_table,  \"aw\"\n\t"

		WORD_INSN " 1b, %l[l_yes], %0\n\t"

		".popsection\n\t"

		: :  "i" (key) : : l_yes);

	return false;

l_yes:

	return true;

}



#endif /* __KERNEL__ */

arch/sparc/include/asm/jump_label.h

+14 −11
Original line number Diff line number Diff line
@@ -7,17 +7,20 @@ 


#define JUMP_LABEL_NOP_SIZE 4



#define JUMP_LABEL(key, label)					\

	do {							\

		asm goto("1:\n\t"				\

			 "nop\n\t"				\

			 "nop\n\t"				\

			 ".pushsection __jump_table,  \"a\"\n\t"\

			 ".align 4\n\t"				\

			 ".word 1b, %l[" #label "], %c0\n\t"	\

			 ".popsection \n\t"			\

			 : :  "i" (key) :  : label);\

	} while (0)

static __always_inline bool arch_static_branch(struct jump_label_key *key)

{

		asm goto("1:\n\t"

			 "nop\n\t"

			 "nop\n\t"

			 ".pushsection __jump_table,  \"aw\"\n\t"

			 ".align 4\n\t"

			 ".word 1b, %l[l_yes], %c0\n\t"

			 ".popsection \n\t"

			 : :  "i" (key) : : l_yes);

	return false;

l_yes:

	return true;

}



#endif /* __KERNEL__ */

arch/x86/include/asm/alternative.h

+1 −2
Original line number Diff line number Diff line
@@ -4,7 +4,6 @@ 
#include <linux/types.h>

#include <linux/stddef.h>

#include <linux/stringify.h>

#include <linux/jump_label.h>

#include <asm/asm.h>



/*
@@ -191,7 +190,7 @@ extern void *text_poke(void *addr, const void *opcode, size_t len); 
extern void *text_poke_smp(void *addr, const void *opcode, size_t len);

extern void text_poke_smp_batch(struct text_poke_param *params, int n);



#if defined(CONFIG_DYNAMIC_FTRACE) || defined(HAVE_JUMP_LABEL)

#if defined(CONFIG_DYNAMIC_FTRACE) || defined(CONFIG_JUMP_LABEL)

#define IDEAL_NOP_SIZE_5 5

extern unsigned char ideal_nop5[IDEAL_NOP_SIZE_5];

extern void arch_init_ideal_nop5(void);

arch/x86/include/asm/jump_label.h

+15 −11
Original line number Diff line number Diff line
@@ -5,20 +5,24 @@ 


#include <linux/types.h>

#include <asm/nops.h>

#include <asm/asm.h>



#define JUMP_LABEL_NOP_SIZE 5



#define JUMP_LABEL_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t"



# define JUMP_LABEL(key, label)					\

	do {							\

		asm goto("1:"					\

			JUMP_LABEL_INITIAL_NOP			\

			".pushsection __jump_table,  \"aw\" \n\t"\

			_ASM_PTR "1b, %l[" #label "], %c0 \n\t" \

			".popsection \n\t"			\

			: :  "i" (key) :  : label);		\

	} while (0)

static __always_inline bool arch_static_branch(struct jump_label_key *key)

{

	asm goto("1:"

		JUMP_LABEL_INITIAL_NOP

		".pushsection __jump_table,  \"aw\" \n\t"

		_ASM_PTR "1b, %l[l_yes], %c0 \n\t"

		".popsection \n\t"

		: :  "i" (key) : : l_yes);

	return false;

l_yes:

	return true;

}



#endif /* __KERNEL__ */

arch/x86/kernel/alternative.c

+1 −1
Original line number Diff line number Diff line
@@ -679,7 +679,7 @@ void __kprobes text_poke_smp_batch(struct text_poke_param *params, int n) 
	__stop_machine(stop_machine_text_poke, (void *)&tpp, NULL);

}



#if defined(CONFIG_DYNAMIC_FTRACE) || defined(HAVE_JUMP_LABEL)

#if defined(CONFIG_DYNAMIC_FTRACE) || defined(CONFIG_JUMP_LABEL)



#ifdef CONFIG_X86_64

unsigned char ideal_nop5[5] = { 0x66, 0x66, 0x66, 0x66, 0x90 };