Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 64711f9a authored by Max Filippov's avatar Max Filippov
Browse files

xtensa: implement jump_label support 



Use 3-byte 'nop' and 'j' instructions that are always present. Don't let
assembler mark a spot right after patchable 'j' instruction as
unreachable and later put literals or padding bytes there. Add separate
implementations of patch_text for SMP and UP cases, avoiding use of
atomics on UP.

Signed-off-by: default avatarMax Filippov <jcmvbkbc@gmail.com>
parent af5395c2

Documentation/features/core/jump-labels/arch-support.txt

+1 −1
Original line number Diff line number Diff line
@@ -29,5 +29,5 @@ 
    |          um: | TODO |

    |   unicore32: | TODO |

    |         x86: |  ok  |

    |      xtensa: | TODO |

    |      xtensa: |  ok  |

    -----------------------

arch/xtensa/Kconfig

+1 −0
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ config XTENSA 
	select GENERIC_PCI_IOMAP

	select GENERIC_SCHED_CLOCK

	select GENERIC_STRNCPY_FROM_USER if KASAN

	select HAVE_ARCH_JUMP_LABEL

	select HAVE_ARCH_KASAN if MMU

	select HAVE_ARCH_TRACEHOOK

	select HAVE_DEBUG_KMEMLEAK

arch/xtensa/include/asm/jump_label.h

0 → 100644
+65 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */

/* Copyright (C) 2018 Cadence Design Systems Inc. */



#ifndef _ASM_XTENSA_JUMP_LABEL_H

#define _ASM_XTENSA_JUMP_LABEL_H



#ifndef __ASSEMBLY__



#include <linux/types.h>



#define JUMP_LABEL_NOP_SIZE 3



static __always_inline bool arch_static_branch(struct static_key *key,

					       bool branch)

{

	asm_volatile_goto("1:\n\t"

			  "_nop\n\t"

			  ".pushsection __jump_table,  \"aw\"\n\t"

			  ".word 1b, %l[l_yes], %c0\n\t"

			  ".popsection\n\t"

			  : :  "i" (&((char *)key)[branch]) :  : l_yes);



	return false;

l_yes:

	return true;

}



static __always_inline bool arch_static_branch_jump(struct static_key *key,

						    bool branch)

{

	/*

	 * Xtensa assembler will mark certain points in the code

	 * as unreachable, so that later assembler or linker relaxation

	 * passes could use them. A spot right after the J instruction

	 * is one such point. Assembler and/or linker may insert padding

	 * or literals here, breaking code flow in case the J instruction

	 * is later replaced with NOP. Put a label right after the J to

	 * make it reachable and wrap both into a no-transform block

	 * to avoid any assembler interference with this.

	 */

	asm_volatile_goto("1:\n\t"

			  ".begin no-transform\n\t"

			  "_j %l[l_yes]\n\t"

			  "2:\n\t"

			  ".end no-transform\n\t"

			  ".pushsection __jump_table,  \"aw\"\n\t"

			  ".word 1b, %l[l_yes], %c0\n\t"

			  ".popsection\n\t"

			  : :  "i" (&((char *)key)[branch]) :  : l_yes);



	return false;

l_yes:

	return true;

}



typedef u32 jump_label_t;



struct jump_entry {

	jump_label_t code;

	jump_label_t target;

	jump_label_t key;

};



#endif  /* __ASSEMBLY__ */

#endif

arch/xtensa/kernel/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -16,6 +16,7 @@ obj-$(CONFIG_SMP) += smp.o mxhead.o 
obj-$(CONFIG_XTENSA_VARIANT_HAVE_PERF_EVENTS) += perf_event.o

obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o

obj-$(CONFIG_S32C1I_SELFTEST) += s32c1i_selftest.o

obj-$(CONFIG_JUMP_LABEL) += jump_label.o



# In the Xtensa architecture, assembly generates literals which must always

# precede the L32R instruction with a relative offset less than 256 kB.

arch/xtensa/kernel/jump_label.c

0 → 100644
+99 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0

// Copyright (C) 2018 Cadence Design Systems Inc.



#include <linux/cpu.h>

#include <linux/jump_label.h>

#include <linux/kernel.h>

#include <linux/memory.h>

#include <linux/stop_machine.h>

#include <linux/types.h>



#include <asm/cacheflush.h>



#ifdef HAVE_JUMP_LABEL



#define J_OFFSET_MASK 0x0003ffff

#define J_SIGN_MASK (~(J_OFFSET_MASK >> 1))



#if defined(__XTENSA_EL__)

#define J_INSN 0x6

#define NOP_INSN 0x0020f0

#elif defined(__XTENSA_EB__)

#define J_INSN 0x60000000

#define NOP_INSN 0x0f020000

#else

#error Unsupported endianness.

#endif



struct patch {

	atomic_t cpu_count;

	unsigned long addr;

	size_t sz;

	const void *data;

};



static void local_patch_text(unsigned long addr, const void *data, size_t sz)

{

	memcpy((void *)addr, data, sz);

	local_flush_icache_range(addr, addr + sz);

}



static int patch_text_stop_machine(void *data)

{

	struct patch *patch = data;



	if (atomic_inc_return(&patch->cpu_count) == 1) {

		local_patch_text(patch->addr, patch->data, patch->sz);

		atomic_inc(&patch->cpu_count);

	} else {

		while (atomic_read(&patch->cpu_count) <= num_online_cpus())

			cpu_relax();

		__invalidate_icache_range(patch->addr, patch->sz);

	}

	return 0;

}



static void patch_text(unsigned long addr, const void *data, size_t sz)

{

	if (IS_ENABLED(CONFIG_SMP)) {

		struct patch patch = {

			.cpu_count = ATOMIC_INIT(0),

			.addr = addr,

			.sz = sz,

			.data = data,

		};

		stop_machine_cpuslocked(patch_text_stop_machine,

					&patch, NULL);

	} else {

		unsigned long flags;



		local_irq_save(flags);

		local_patch_text(addr, data, sz);

		local_irq_restore(flags);

	}

}



void arch_jump_label_transform(struct jump_entry *e,

			       enum jump_label_type type)

{

	u32 d = (jump_entry_target(e) - (jump_entry_code(e) + 4));

	u32 insn;



	/* Jump only works within 128K of the J instruction. */

	BUG_ON(!((d & J_SIGN_MASK) == 0 ||

		 (d & J_SIGN_MASK) == J_SIGN_MASK));



	if (type == JUMP_LABEL_JMP) {

#if defined(__XTENSA_EL__)

		insn = ((d & J_OFFSET_MASK) << 6) | J_INSN;

#elif defined(__XTENSA_EB__)

		insn = ((d & J_OFFSET_MASK) << 8) | J_INSN;

#endif

	} else {

		insn = NOP_INSN;

	}



	patch_text(jump_entry_code(e), &insn, JUMP_LABEL_NOP_SIZE);

}



#endif /* HAVE_JUMP_LABEL */