Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 442e0973 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge branch 'x86/jumplabel' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull x86 jumplabel changes from Peter Anvin:
 "One more x86 tree for this merge window.  This tree improves the
  handling of jump labels, so that most of the time we don't have to do
  a massive initial patching run.

  Furthermore, we will error out of the jump label is not what is
  expected, eg if it has been corrupted or tampered with"

* 'x86/jumplabel' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86/jump-label: Show where and what was wrong on errors
  x86/jump-label: Add safety checks to jump label conversions
  x86/jump-label: Do not bother updating nops if they are correct
  x86/jump-label: Use best default nops for inital jump label calls
parents 8d7551eb fb40d7a8

arch/x86/include/asm/jump_label.h

+7 −2
Original line number Diff line number Diff line
@@ -3,18 +3,23 @@ 


#ifdef __KERNEL__



#include <linux/stringify.h>

#include <linux/types.h>

#include <asm/nops.h>

#include <asm/asm.h>



#define JUMP_LABEL_NOP_SIZE 5



#define STATIC_KEY_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t"

#ifdef CONFIG_X86_64

# define STATIC_KEY_INIT_NOP P6_NOP5_ATOMIC

#else

# define STATIC_KEY_INIT_NOP GENERIC_NOP5_ATOMIC

#endif



static __always_inline bool arch_static_branch(struct static_key *key)

{

	asm goto("1:"

		STATIC_KEY_INITIAL_NOP

		".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t"

		".pushsection __jump_table,  \"aw\" \n\t"

		_ASM_ALIGN "\n\t"

		_ASM_PTR "1b, %l[l_yes], %c0 \n\t"

arch/x86/kernel/jump_label.c

+66 −4
Original line number Diff line number Diff line
@@ -24,18 +24,57 @@ union jump_code_union { 
	} __attribute__((packed));

};



static void bug_at(unsigned char *ip, int line)

{

	/*

	 * The location is not an op that we were expecting.

	 * Something went wrong. Crash the box, as something could be

	 * corrupting the kernel.

	 */

	pr_warning("Unexpected op at %pS [%p] (%02x %02x %02x %02x %02x) %s:%d\n",

	       ip, ip, ip[0], ip[1], ip[2], ip[3], ip[4], __FILE__, line);

	BUG();

}



static void __jump_label_transform(struct jump_entry *entry,

				   enum jump_label_type type,

				   void *(*poker)(void *, const void *, size_t))

				   void *(*poker)(void *, const void *, size_t),

				   int init)

{

	union jump_code_union code;

	const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5];



	if (type == JUMP_LABEL_ENABLE) {

		/*

		 * We are enabling this jump label. If it is not a nop

		 * then something must have gone wrong.

		 */

		if (unlikely(memcmp((void *)entry->code, ideal_nop, 5) != 0))

			bug_at((void *)entry->code, __LINE__);



		code.jump = 0xe9;

		code.offset = entry->target -

				(entry->code + JUMP_LABEL_NOP_SIZE);

	} else

	} else {

		/*

		 * We are disabling this jump label. If it is not what

		 * we think it is, then something must have gone wrong.

		 * If this is the first initialization call, then we

		 * are converting the default nop to the ideal nop.

		 */

		if (init) {

			const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP };

			if (unlikely(memcmp((void *)entry->code, default_nop, 5) != 0))

				bug_at((void *)entry->code, __LINE__);

		} else {

			code.jump = 0xe9;

			code.offset = entry->target -

				(entry->code + JUMP_LABEL_NOP_SIZE);

			if (unlikely(memcmp((void *)entry->code, &code, 5) != 0))

				bug_at((void *)entry->code, __LINE__);

		}

		memcpy(&code, ideal_nops[NOP_ATOMIC5], JUMP_LABEL_NOP_SIZE);

	}



	/*

	 * Make text_poke_bp() a default fallback poker.
@@ -57,15 +96,38 @@ void arch_jump_label_transform(struct jump_entry *entry, 
{

	get_online_cpus();

	mutex_lock(&text_mutex);

	__jump_label_transform(entry, type, NULL);

	__jump_label_transform(entry, type, NULL, 0);

	mutex_unlock(&text_mutex);

	put_online_cpus();

}



static enum {

	JL_STATE_START,

	JL_STATE_NO_UPDATE,

	JL_STATE_UPDATE,

} jlstate __initdata_or_module = JL_STATE_START;



__init_or_module void arch_jump_label_transform_static(struct jump_entry *entry,

				      enum jump_label_type type)

{

	__jump_label_transform(entry, type, text_poke_early);

	/*

	 * This function is called at boot up and when modules are

	 * first loaded. Check if the default nop, the one that is

	 * inserted at compile time, is the ideal nop. If it is, then

	 * we do not need to update the nop, and we can leave it as is.

	 * If it is not, then we need to update the nop to the ideal nop.

	 */

	if (jlstate == JL_STATE_START) {

		const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP };

		const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5];



		if (memcmp(ideal_nop, default_nop, 5) != 0)

			jlstate = JL_STATE_UPDATE;

		else

			jlstate = JL_STATE_NO_UPDATE;

	}

	if (jlstate == JL_STATE_UPDATE)

		__jump_label_transform(entry, type, text_poke_early, 1);

}



#endif