Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 686140a1 authored by Vasily Gorbik's avatar Vasily Gorbik Committed by Martin Schwidefsky
Browse files

s390: introduce CPU alternatives 



Implement CPU alternatives, which allows to optionally patch newer
instructions at runtime, based on CPU facilities availability.

A new kernel boot parameter "noaltinstr" disables patching.

Current implementation is derived from x86 alternatives. Although
ideal instructions padding (when altinstr is longer then oldinstr)
is added at compile time, and no oldinstr nops optimization has to be
done at runtime. Also couple of compile time sanity checks are done:
1. oldinstr and altinstr must be <= 254 bytes long,
2. oldinstr and altinstr must not have an odd length.

alternative(oldinstr, altinstr, facility);
alternative_2(oldinstr, altinstr1, facility1, altinstr2, facility2);

Both compile time and runtime padding consists of either 6/4/2 bytes nop
or a jump (brcl) + 2 bytes nop filler if padding is longer then 6 bytes.

.altinstructions and .altinstr_replacement sections are part of
__init_begin : __init_end region and are freed after initialization.

Signed-off-by: default avatarVasily Gorbik <gor@linux.vnet.ibm.com>
Signed-off-by: default avatarMartin Schwidefsky <schwidefsky@de.ibm.com>
parent 76b3b62a

Documentation/admin-guide/kernel-parameters.txt

+3 −0
Original line number Original line Diff line number Diff line
@@ -2548,6 +2548,9 @@ 




	noalign		[KNL,ARM]

	noalign		[KNL,ARM]





	noaltinstr	[S390] Disables alternative instructions patching

			(CPU alternatives feature).



	noapic		[SMP,APIC] Tells the kernel to not make use of any

	noapic		[SMP,APIC] Tells the kernel to not make use of any

			IOAPICs that may be present in the system.

			IOAPICs that may be present in the system.

arch/s390/Kconfig

+17 −0
Original line number Original line Diff line number Diff line
@@ -537,6 +537,22 @@ config ARCH_RANDOM 




	  If unsure, say Y.

	  If unsure, say Y.





config ALTERNATIVES

	def_bool y

	prompt "Patch optimized instructions for running CPU type"

	help

	  When enabled the kernel code is compiled with additional

	  alternative instructions blocks optimized for newer CPU types.

	  These alternative instructions blocks are patched at kernel boot

	  time when running CPU supports them. This mechanism is used to

	  optimize some critical code paths (i.e. spinlocks) for newer CPUs

	  even if kernel is build to support older machine generations.



	  This mechanism could be disabled by appending "noaltinstr"

	  option to the kernel command line.



	  If unsure, say Y.



endmenu

endmenu





menu "Memory setup"

menu "Memory setup"
@@ -811,6 +827,7 @@ config PFAULT 
config SHARED_KERNEL

config SHARED_KERNEL

	bool "VM shared kernel support"

	bool "VM shared kernel support"

	depends on !JUMP_LABEL

	depends on !JUMP_LABEL

	depends on !ALTERNATIVES

	help

	help

	  Select this option, if you want to share the text segment of the

	  Select this option, if you want to share the text segment of the

	  Linux kernel between different VM guests. This reduces memory

	  Linux kernel between different VM guests. This reduces memory

arch/s390/include/asm/alternative.h

0 → 100644
+163 −0
Original line number Original line Diff line number Diff line 
#ifndef _ASM_S390_ALTERNATIVE_H

#define _ASM_S390_ALTERNATIVE_H



#ifndef __ASSEMBLY__



#include <linux/types.h>

#include <linux/stddef.h>

#include <linux/stringify.h>



struct alt_instr {

	s32 instr_offset;	/* original instruction */

	s32 repl_offset;	/* offset to replacement instruction */

	u16 facility;		/* facility bit set for replacement */

	u8  instrlen;		/* length of original instruction */

	u8  replacementlen;	/* length of new instruction */

} __packed;



#ifdef CONFIG_ALTERNATIVES

extern void apply_alternative_instructions(void);

extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end);

#else

static inline void apply_alternative_instructions(void) {};

static inline void apply_alternatives(struct alt_instr *start,

				      struct alt_instr *end) {};

#endif

/*

 * |661:       |662:	  |6620      |663:

 * +-----------+---------------------+

 * | oldinstr  | oldinstr_padding    |

 * |	       +----------+----------+

 * |	       |	  |	     |

 * |	       | >6 bytes |6/4/2 nops|

 * |	       |6 bytes jg----------->

 * +-----------+---------------------+

 *		 ^^ static padding ^^

 *

 * .altinstr_replacement section

 * +---------------------+-----------+

 * |6641:			     |6651:

 * | alternative instr 1	     |

 * +-----------+---------+- - - - - -+

 * |6642:		 |6652:      |

 * | alternative instr 2 | padding

 * +---------------------+- - - - - -+

 *			  ^ runtime ^

 *

 * .altinstructions section

 * +---------------------------------+

 * | alt_instr entries for each      |

 * | alternative instr		     |

 * +---------------------------------+

 */



#define b_altinstr(num)	"664"#num

#define e_altinstr(num)	"665"#num



#define e_oldinstr_pad_end	"663"

#define oldinstr_len		"662b-661b"

#define oldinstr_total_len	e_oldinstr_pad_end"b-661b"

#define altinstr_len(num)	e_altinstr(num)"b-"b_altinstr(num)"b"

#define oldinstr_pad_len(num) \

	"-(((" altinstr_len(num) ")-(" oldinstr_len ")) > 0) * " \

	"((" altinstr_len(num) ")-(" oldinstr_len "))"



#define INSTR_LEN_SANITY_CHECK(len)					\

	".if " len " > 254\n"						\

	"\t.error \"cpu alternatives does not support instructions "	\

		"blocks > 254 bytes\"\n"				\

	".endif\n"							\

	".if (" len ") %% 2\n"						\

	"\t.error \"cpu alternatives instructions length is odd\"\n"	\

	".endif\n"



#define OLDINSTR_PADDING(oldinstr, num)					\

	".if " oldinstr_pad_len(num) " > 6\n"				\

	"\tjg " e_oldinstr_pad_end "f\n"				\

	"6620:\n"							\

	"\t.fill (" oldinstr_pad_len(num) " - (6620b-662b)) / 2, 2, 0x0700\n" \

	".else\n"							\

	"\t.fill " oldinstr_pad_len(num) " / 6, 6, 0xc0040000\n"	\

	"\t.fill " oldinstr_pad_len(num) " %% 6 / 4, 4, 0x47000000\n"	\

	"\t.fill " oldinstr_pad_len(num) " %% 6 %% 4 / 2, 2, 0x0700\n"	\

	".endif\n"



#define OLDINSTR(oldinstr, num)						\

	"661:\n\t" oldinstr "\n662:\n"					\

	OLDINSTR_PADDING(oldinstr, num)					\

	e_oldinstr_pad_end ":\n"					\

	INSTR_LEN_SANITY_CHECK(oldinstr_len)



#define OLDINSTR_2(oldinstr, num1, num2)				\

	"661:\n\t" oldinstr "\n662:\n"					\

	".if " altinstr_len(num1) " < " altinstr_len(num2) "\n"		\

	OLDINSTR_PADDING(oldinstr, num2)				\

	".else\n"							\

	OLDINSTR_PADDING(oldinstr, num1)				\

	".endif\n"							\

	e_oldinstr_pad_end ":\n"					\

	INSTR_LEN_SANITY_CHECK(oldinstr_len)



#define ALTINSTR_ENTRY(facility, num)					\

	"\t.long 661b - .\n"			/* old instruction */	\

	"\t.long " b_altinstr(num)"b - .\n"	/* alt instruction */	\

	"\t.word " __stringify(facility) "\n"	/* facility bit    */	\

	"\t.byte " oldinstr_total_len "\n"	/* source len	   */	\

	"\t.byte " altinstr_len(num) "\n"	/* alt instruction len */



#define ALTINSTR_REPLACEMENT(altinstr, num)	/* replacement */	\

	b_altinstr(num)":\n\t" altinstr "\n" e_altinstr(num) ":\n"	\

	INSTR_LEN_SANITY_CHECK(altinstr_len(num))



#ifdef CONFIG_ALTERNATIVES

/* alternative assembly primitive: */

#define ALTERNATIVE(oldinstr, altinstr, facility) \

	".pushsection .altinstr_replacement, \"ax\"\n"			\

	ALTINSTR_REPLACEMENT(altinstr, 1)				\

	".popsection\n"							\

	OLDINSTR(oldinstr, 1)						\

	".pushsection .altinstructions,\"a\"\n"				\

	ALTINSTR_ENTRY(facility, 1)					\

	".popsection\n"



#define ALTERNATIVE_2(oldinstr, altinstr1, facility1, altinstr2, facility2)\

	".pushsection .altinstr_replacement, \"ax\"\n"			\

	ALTINSTR_REPLACEMENT(altinstr1, 1)				\

	ALTINSTR_REPLACEMENT(altinstr2, 2)				\

	".popsection\n"							\

	OLDINSTR_2(oldinstr, 1, 2)					\

	".pushsection .altinstructions,\"a\"\n"				\

	ALTINSTR_ENTRY(facility1, 1)					\

	ALTINSTR_ENTRY(facility2, 2)					\

	".popsection\n"

#else

/* Alternative instructions are disabled, let's put just oldinstr in */

#define ALTERNATIVE(oldinstr, altinstr, facility) \

	oldinstr "\n"



#define ALTERNATIVE_2(oldinstr, altinstr1, facility1, altinstr2, facility2) \

	oldinstr "\n"

#endif



/*

 * Alternative instructions for different CPU types or capabilities.

 *

 * This allows to use optimized instructions even on generic binary

 * kernels.

 *

 * oldinstr is padded with jump and nops at compile time if altinstr is

 * longer. altinstr is padded with jump and nops at run-time during patching.

 *

 * For non barrier like inlines please define new variants

 * without volatile and memory clobber.

 */

#define alternative(oldinstr, altinstr, facility)			\

	asm volatile(ALTERNATIVE(oldinstr, altinstr, facility) : : : "memory")



#define alternative_2(oldinstr, altinstr1, facility1, altinstr2, facility2) \

	asm volatile(ALTERNATIVE_2(oldinstr, altinstr1, facility1,	    \

				   altinstr2, facility2) ::: "memory")



#endif /* __ASSEMBLY__ */



#endif /* _ASM_S390_ALTERNATIVE_H */

arch/s390/kernel/Makefile

+1 −0
Original line number Original line Diff line number Diff line
@@ -76,6 +76,7 @@ obj-$(CONFIG_KPROBES) += kprobes.o 
obj-$(CONFIG_FUNCTION_TRACER)	+= mcount.o ftrace.o

obj-$(CONFIG_FUNCTION_TRACER)	+= mcount.o ftrace.o

obj-$(CONFIG_CRASH_DUMP)	+= crash_dump.o

obj-$(CONFIG_CRASH_DUMP)	+= crash_dump.o

obj-$(CONFIG_UPROBES)		+= uprobes.o

obj-$(CONFIG_UPROBES)		+= uprobes.o

obj-$(CONFIG_ALTERNATIVES)	+= alternative.o





obj-$(CONFIG_PERF_EVENTS)	+= perf_event.o perf_cpum_cf.o perf_cpum_sf.o

obj-$(CONFIG_PERF_EVENTS)	+= perf_event.o perf_cpum_cf.o perf_cpum_sf.o

obj-$(CONFIG_PERF_EVENTS)	+= perf_cpum_cf_events.o

obj-$(CONFIG_PERF_EVENTS)	+= perf_cpum_cf_events.o

arch/s390/kernel/alternative.c

0 → 100644
+110 −0
Original line number Original line Diff line number Diff line 
#include <linux/module.h>

#include <asm/alternative.h>

#include <asm/facility.h>



#define MAX_PATCH_LEN (255 - 1)



static int __initdata_or_module alt_instr_disabled;



static int __init disable_alternative_instructions(char *str)

{

	alt_instr_disabled = 1;

	return 0;

}



early_param("noaltinstr", disable_alternative_instructions);



struct brcl_insn {

	u16 opc;

	s32 disp;

} __packed;



static u16 __initdata_or_module nop16 = 0x0700;

static u32 __initdata_or_module nop32 = 0x47000000;

static struct brcl_insn __initdata_or_module nop48 = {

	0xc004, 0

};



static const void *nops[] __initdata_or_module = {

	&nop16,

	&nop32,

	&nop48

};



static void __init_or_module add_jump_padding(void *insns, unsigned int len)

{

	struct brcl_insn brcl = {

		0xc0f4,

		len / 2

	};



	memcpy(insns, &brcl, sizeof(brcl));

	insns += sizeof(brcl);

	len -= sizeof(brcl);



	while (len > 0) {

		memcpy(insns, &nop16, 2);

		insns += 2;

		len -= 2;

	}

}



static void __init_or_module add_padding(void *insns, unsigned int len)

{

	if (len > 6)

		add_jump_padding(insns, len);

	else if (len >= 2)

		memcpy(insns, nops[len / 2 - 1], len);

}



static void __init_or_module __apply_alternatives(struct alt_instr *start,

						  struct alt_instr *end)

{

	struct alt_instr *a;

	u8 *instr, *replacement;

	u8 insnbuf[MAX_PATCH_LEN];



	/*

	 * The scan order should be from start to end. A later scanned

	 * alternative code can overwrite previously scanned alternative code.

	 */

	for (a = start; a < end; a++) {

		int insnbuf_sz = 0;



		instr = (u8 *)&a->instr_offset + a->instr_offset;

		replacement = (u8 *)&a->repl_offset + a->repl_offset;



		if (!test_facility(a->facility))

			continue;



		if (unlikely(a->instrlen % 2 || a->replacementlen % 2)) {

			WARN_ONCE(1, "cpu alternatives instructions length is "

				     "odd, skipping patching\n");

			continue;

		}



		memcpy(insnbuf, replacement, a->replacementlen);

		insnbuf_sz = a->replacementlen;



		if (a->instrlen > a->replacementlen) {

			add_padding(insnbuf + a->replacementlen,

				    a->instrlen - a->replacementlen);

			insnbuf_sz += a->instrlen - a->replacementlen;

		}



		s390_kernel_write(instr, insnbuf, insnbuf_sz);

	}

}



void __init_or_module apply_alternatives(struct alt_instr *start,

					 struct alt_instr *end)

{

	if (!alt_instr_disabled)

		__apply_alternatives(start, end);

}



extern struct alt_instr __alt_instructions[], __alt_instructions_end[];

void __init apply_alternative_instructions(void)

{

	apply_alternatives(__alt_instructions, __alt_instructions_end);

}