Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 9732cafd authored by Jiang Liu's avatar Jiang Liu Committed by Catalin Marinas
Browse files

arm64, jump label: optimize jump label implementation 



Optimize jump label implementation for ARM64 by dynamically patching
kernel text.

Reviewed-by: default avatarWill Deacon <will.deacon@arm.com>
Signed-off-by: default avatarJiang Liu <liuj97@gmail.com>
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent f3c003f7

arch/arm64/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -25,6 +25,7 @@ config ARM64 
	select GENERIC_STRNLEN_USER

	select GENERIC_TIME_VSYSCALL

	select HARDIRQS_SW_RESEND

	select HAVE_ARCH_JUMP_LABEL

	select HAVE_ARCH_TRACEHOOK

	select HAVE_DEBUG_BUGVERBOSE

	select HAVE_DEBUG_KMEMLEAK

arch/arm64/include/asm/jump_label.h

0 → 100644
+52 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2013 Huawei Ltd.

 * Author: Jiang Liu <liuj97@gmail.com>

 *

 * Based on arch/arm/include/asm/jump_label.h

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License version 2 as

 * published by the Free Software Foundation.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program.  If not, see <http://www.gnu.org/licenses/>.

 */

#ifndef __ASM_JUMP_LABEL_H

#define __ASM_JUMP_LABEL_H

#include <linux/types.h>

#include <asm/insn.h>



#ifdef __KERNEL__



#define JUMP_LABEL_NOP_SIZE		AARCH64_INSN_SIZE



static __always_inline bool arch_static_branch(struct static_key *key)

{

	asm goto("1: nop\n\t"

		 ".pushsection __jump_table,  \"aw\"\n\t"

		 ".align 3\n\t"

		 ".quad 1b, %l[l_yes], %c0\n\t"

		 ".popsection\n\t"

		 :  :  "i"(key) :  : l_yes);



	return false;

l_yes:

	return true;

}



#endif /* __KERNEL__ */



typedef u64 jump_label_t;



struct jump_entry {

	jump_label_t code;

	jump_label_t target;

	jump_label_t key;

};



#endif	/* __ASM_JUMP_LABEL_H */

arch/arm64/kernel/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o 
arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT)+= hw_breakpoint.o

arm64-obj-$(CONFIG_EARLY_PRINTK)	+= early_printk.o

arm64-obj-$(CONFIG_ARM64_CPU_SUSPEND)	+= sleep.o suspend.o

arm64-obj-$(CONFIG_JUMP_LABEL)		+= jump_label.o



obj-y					+= $(arm64-obj-y) vdso/

obj-m					+= $(arm64-obj-m)

arch/arm64/kernel/jump_label.c

0 → 100644
+58 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2013 Huawei Ltd.

 * Author: Jiang Liu <liuj97@gmail.com>

 *

 * Based on arch/arm/kernel/jump_label.c

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License version 2 as

 * published by the Free Software Foundation.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program.  If not, see <http://www.gnu.org/licenses/>.

 */

#include <linux/kernel.h>

#include <linux/jump_label.h>

#include <asm/insn.h>



#ifdef HAVE_JUMP_LABEL



static void __arch_jump_label_transform(struct jump_entry *entry,

					enum jump_label_type type,

					bool is_static)

{

	void *addr = (void *)entry->code;

	u32 insn;



	if (type == JUMP_LABEL_ENABLE) {

		insn = aarch64_insn_gen_branch_imm(entry->code,

						   entry->target,

						   AARCH64_INSN_BRANCH_NOLINK);

	} else {

		insn = aarch64_insn_gen_nop();

	}



	if (is_static)

		aarch64_insn_patch_text_nosync(addr, insn);

	else

		aarch64_insn_patch_text(&addr, &insn, 1);

}



void arch_jump_label_transform(struct jump_entry *entry,

			       enum jump_label_type type)

{

	__arch_jump_label_transform(entry, type, false);

}



void arch_jump_label_transform_static(struct jump_entry *entry,

				      enum jump_label_type type)

{

	__arch_jump_label_transform(entry, type, true);

}



#endif	/* HAVE_JUMP_LABEL */