bpf, s390x: remove ld_abs/ld_ind (e1cf4bef) · Commits · e / devices / android_kernel_fairphone_FP5

arch/s390/net/Makefile

+1 −1

Original line number	Original line	Diff line number	Diff line
	@@ -2,4 +2,4 @@
	#		#
	# Arch-specific network modules		# Arch-specific network modules
	#		#
	obj-$(CONFIG_BPF_JIT) += bpf_jit.o bpf_jit_comp.o		obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o

arch/s390/net/bpf_jit.S

deleted100644 → 0

+0 −116

Original line number	Original line	Diff line number	Diff line
	/* SPDX-License-Identifier: GPL-2.0 */
	/*
	* BPF Jit compiler for s390, help functions.
	*
	* Copyright IBM Corp. 2012,2015
	*
	* Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
	* Michael Holzheu <holzheu@linux.vnet.ibm.com>
	*/

	#include <linux/linkage.h>
	#include "bpf_jit.h"

	/*
	* Calling convention:
	* registers %r7-%r10, %r11,%r13, and %r15 are call saved
	*
	* Input (64 bit):
	* %r3 (%b2) = offset into skb data
	* %r6 (%b5) = return address
	* %r7 (%b6) = skb pointer
	* %r12 = skb data pointer
	*
	* Output:
	* %r14= %b0 = return value (read skb value)
	*
	* Work registers: %r2,%r4,%r5,%r14
	*
	* skb_copy_bits takes 4 parameters:
	* %r2 = skb pointer
	* %r3 = offset into skb data
	* %r4 = pointer to temp buffer
	* %r5 = length to copy
	* Return value in %r2: 0 = ok
	*
	* bpf_internal_load_pointer_neg_helper takes 3 parameters:
	* %r2 = skb pointer
	* %r3 = offset into data
	* %r4 = length to copy
	* Return value in %r2: Pointer to data
	*/

	#define SKF_MAX_NEG_OFF -0x200000 /* SKF_LL_OFF from filter.h */

	/*
	* Load SIZE bytes from SKB
	*/
	#define sk_load_common(NAME, SIZE, LOAD) \
	ENTRY(sk_load_##NAME); \
	ltgr %r3,%r3; /* Is offset negative? */ \
	jl sk_load_##NAME##_slow_neg; \
	ENTRY(sk_load_##NAME##_pos); \
	aghi %r3,SIZE; /* Offset + SIZE */ \
	clg %r3,STK_OFF_HLEN(%r15); /* Offset + SIZE > hlen? */ \
	jh sk_load_##NAME##_slow; \
	LOAD %r14,-SIZE(%r3,%r12); /* Get data from skb */ \
	b OFF_OK(%r6); /* Return */ \
	\
	sk_load_##NAME##_slow:; \
	lgr %r2,%r7; /* Arg1 = skb pointer */ \
	aghi %r3,-SIZE; /* Arg2 = offset */ \
	la %r4,STK_OFF_TMP(%r15); /* Arg3 = temp bufffer */ \
	lghi %r5,SIZE; /* Arg4 = size */ \
	brasl %r14,skb_copy_bits; /* Get data from skb */ \
	LOAD %r14,STK_OFF_TMP(%r15); /* Load from temp bufffer */ \
	ltgr %r2,%r2; /* Set cc to (%r2 != 0) */ \
	br %r6; /* Return */

	sk_load_common(word, 4, llgf) /* r14 = (u32 ) (skb->data+offset) */
	sk_load_common(half, 2, llgh) /* r14 = (u16 ) (skb->data+offset) */

	/*
	* Load 1 byte from SKB (optimized version)
	*/
	/* r14 = (u8 ) (skb->data+offset) */
	ENTRY(sk_load_byte)
	ltgr %r3,%r3 # Is offset negative?
	jl sk_load_byte_slow_neg
	ENTRY(sk_load_byte_pos)
	clg %r3,STK_OFF_HLEN(%r15) # Offset >= hlen?
	jnl sk_load_byte_slow
	llgc %r14,0(%r3,%r12) # Get byte from skb
	b OFF_OK(%r6) # Return OK

	sk_load_byte_slow:
	lgr %r2,%r7 # Arg1 = skb pointer
	# Arg2 = offset
	la %r4,STK_OFF_TMP(%r15) # Arg3 = pointer to temp buffer
	lghi %r5,1 # Arg4 = size (1 byte)
	brasl %r14,skb_copy_bits # Get data from skb
	llgc %r14,STK_OFF_TMP(%r15) # Load result from temp buffer
	ltgr %r2,%r2 # Set cc to (%r2 != 0)
	br %r6 # Return cc

	#define sk_negative_common(NAME, SIZE, LOAD) \
	sk_load_##NAME##_slow_neg:; \
	cgfi %r3,SKF_MAX_NEG_OFF; \
	jl bpf_error; \
	lgr %r2,%r7; /* Arg1 = skb pointer */ \
	/* Arg2 = offset */ \
	lghi %r4,SIZE; /* Arg3 = size */ \
	brasl %r14,bpf_internal_load_pointer_neg_helper; \
	ltgr %r2,%r2; \
	jz bpf_error; \
	LOAD %r14,0(%r2); /* Get data from pointer */ \
	xr %r3,%r3; /* Set cc to zero */ \
	br %r6; /* Return cc */

	sk_negative_common(word, 4, llgf)
	sk_negative_common(half, 2, llgh)
	sk_negative_common(byte, 1, llgc)

	bpf_error:
	# force a return 0 from jit handler
	ltgr %r15,%r15 # Set condition code
	br %r6

arch/s390/net/bpf_jit.h

+1 −19

Original line number	Original line	Diff line number	Diff line
	@@ -16,9 +16,6 @@
	#include <linux/filter.h>		#include <linux/filter.h>
	#include <linux/types.h>		#include <linux/types.h>

	extern u8 sk_load_word_pos[], sk_load_half_pos[], sk_load_byte_pos[];
	extern u8 sk_load_word[], sk_load_half[], sk_load_byte[];

	#endif /* __ASSEMBLY__ */		#endif /* __ASSEMBLY__ */

	/*		/*
	@@ -36,15 +33,6 @@ extern u8 sk_load_word[], sk_load_half[], sk_load_byte[];
	* \| \| \|		* \| \| \|
	* \| BPF stack \| \|		* \| BPF stack \| \|
	* \| \| \|		* \| \| \|
	* +---------------+ \|
	* \| 8 byte skbp \| \|
	* R15+176 -> +---------------+ \|
	* \| 8 byte hlen \| \|
	* R15+168 -> +---------------+ \|
	* \| 4 byte align \| \|
	* +---------------+ \|
	* \| 4 byte temp \| \|
	* \| for bpf_jit.S \| \|
	* R15+160 -> +---------------+ \|		* R15+160 -> +---------------+ \|
	* \| new backchain \| \|		* \| new backchain \| \|
	* R15+152 -> +---------------+ \|		* R15+152 -> +---------------+ \|
	@@ -57,17 +45,11 @@ extern u8 sk_load_word[], sk_load_half[], sk_load_byte[];
	* The stack size used by the BPF program ("BPF stack" above) is passed		* The stack size used by the BPF program ("BPF stack" above) is passed
	* via "aux->stack_depth".		* via "aux->stack_depth".
	*/		*/
	#define STK_SPACE_ADD (8 + 8 + 4 + 4 + 160)		#define STK_SPACE_ADD (160)
	#define STK_160_UNUSED (160 - 12 * 8)		#define STK_160_UNUSED (160 - 12 * 8)
	#define STK_OFF (STK_SPACE_ADD - STK_160_UNUSED)		#define STK_OFF (STK_SPACE_ADD - STK_160_UNUSED)
	#define STK_OFF_TMP 160 /* Offset of tmp buffer on stack */
	#define STK_OFF_HLEN 168 /* Offset of SKB header length on stack */
	#define STK_OFF_SKBP 176 /* Offset of SKB pointer on stack */

	#define STK_OFF_R6 (160 - 11 * 8) /* Offset of r6 on stack */		#define STK_OFF_R6 (160 - 11 * 8) /* Offset of r6 on stack */
	#define STK_OFF_TCCNT (160 - 12 * 8) /* Offset of tail_call_cnt on stack */		#define STK_OFF_TCCNT (160 - 12 * 8) /* Offset of tail_call_cnt on stack */

	/* Offset to skip condition code check */
	#define OFF_OK 4

	#endif /* __ARCH_S390_NET_BPF_JIT_H */		#endif /* __ARCH_S390_NET_BPF_JIT_H */

arch/s390/net/bpf_jit_comp.c

+11 −116

Original line number	Original line	Diff line number	Diff line
	@@ -47,23 +47,21 @@ struct bpf_jit {

	#define BPF_SIZE_MAX 0xffff /* Max size for program (16 bit branches) */		#define BPF_SIZE_MAX 0xffff /* Max size for program (16 bit branches) */

	#define SEEN_SKB 1 /* skb access */		#define SEEN_MEM (1 << 0) /* use mem[] for temporary storage */
	#define SEEN_MEM 2 /* use mem[] for temporary storage */		#define SEEN_RET0 (1 << 1) /* ret0_ip points to a valid return 0 */
	#define SEEN_RET0 4 /* ret0_ip points to a valid return 0 */		#define SEEN_LITERAL (1 << 2) /* code uses literals */
	#define SEEN_LITERAL 8 /* code uses literals */		#define SEEN_FUNC (1 << 3) /* calls C functions */
	#define SEEN_FUNC 16 /* calls C functions */		#define SEEN_TAIL_CALL (1 << 4) /* code uses tail calls */
	#define SEEN_TAIL_CALL 32 /* code uses tail calls */		#define SEEN_REG_AX (1 << 5) /* code uses constant blinding */
	#define SEEN_REG_AX 64 /* code uses constant blinding */		#define SEEN_STACK (SEEN_FUNC \| SEEN_MEM)
	#define SEEN_STACK (SEEN_FUNC \| SEEN_MEM \| SEEN_SKB)

	/*		/*
	* s390 registers		* s390 registers
	*/		*/
	#define REG_W0 (MAX_BPF_JIT_REG + 0) /* Work register 1 (even) */		#define REG_W0 (MAX_BPF_JIT_REG + 0) /* Work register 1 (even) */
	#define REG_W1 (MAX_BPF_JIT_REG + 1) /* Work register 2 (odd) */		#define REG_W1 (MAX_BPF_JIT_REG + 1) /* Work register 2 (odd) */
	#define REG_SKB_DATA (MAX_BPF_JIT_REG + 2) /* SKB data register */		#define REG_L (MAX_BPF_JIT_REG + 2) /* Literal pool register */
	#define REG_L (MAX_BPF_JIT_REG + 3) /* Literal pool register */		#define REG_15 (MAX_BPF_JIT_REG + 3) /* Register 15 */
	#define REG_15 (MAX_BPF_JIT_REG + 4) /* Register 15 */
	#define REG_0 REG_W0 /* Register 0 */		#define REG_0 REG_W0 /* Register 0 */
	#define REG_1 REG_W1 /* Register 1 */		#define REG_1 REG_W1 /* Register 1 */
	#define REG_2 BPF_REG_1 /* Register 2 */		#define REG_2 BPF_REG_1 /* Register 2 */
	@@ -88,10 +86,8 @@ static const int reg2hex[] = {
	[BPF_REG_9] = 10,		[BPF_REG_9] = 10,
	/* BPF stack pointer */		/* BPF stack pointer */
	[BPF_REG_FP] = 13,		[BPF_REG_FP] = 13,
	/* Register for blinding (shared with REG_SKB_DATA) */		/* Register for blinding */
	[BPF_REG_AX] = 12,		[BPF_REG_AX] = 12,
	/* SKB data pointer */
	[REG_SKB_DATA] = 12,
	/* Work registers for s390x backend */		/* Work registers for s390x backend */
	[REG_W0] = 0,		[REG_W0] = 0,
	[REG_W1] = 1,		[REG_W1] = 1,
	@@ -384,27 +380,6 @@ static void save_restore_regs(struct bpf_jit *jit, int op, u32 stack_depth)
	} while (re <= 15);		} while (re <= 15);
	}		}

	/*
	* For SKB access %b1 contains the SKB pointer. For "bpf_jit.S"
	* we store the SKB header length on the stack and the SKB data
	* pointer in REG_SKB_DATA if BPF_REG_AX is not used.
	*/
	static void emit_load_skb_data_hlen(struct bpf_jit *jit)
	{
	/* Header length: llgf %w1,<len>(%b1) */
	EMIT6_DISP_LH(0xe3000000, 0x0016, REG_W1, REG_0, BPF_REG_1,
	offsetof(struct sk_buff, len));
	/* s %w1,<data_len>(%b1) */
	EMIT4_DISP(0x5b000000, REG_W1, BPF_REG_1,
	offsetof(struct sk_buff, data_len));
	/* stg %w1,ST_OFF_HLEN(%r0,%r15) */
	EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0, REG_15, STK_OFF_HLEN);
	if (!(jit->seen & SEEN_REG_AX))
	/* lg %skb_data,data_off(%b1) */
	EMIT6_DISP_LH(0xe3000000, 0x0004, REG_SKB_DATA, REG_0,
	BPF_REG_1, offsetof(struct sk_buff, data));
	}

	/*		/*
	* Emit function prologue		* Emit function prologue
	*		*
	@@ -445,12 +420,6 @@ static void bpf_jit_prologue(struct bpf_jit *jit, u32 stack_depth)
	EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0,		EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0,
	REG_15, 152);		REG_15, 152);
	}		}
	if (jit->seen & SEEN_SKB) {
	emit_load_skb_data_hlen(jit);
	/* stg %b1,ST_OFF_SKBP(%r0,%r15) */
	EMIT6_DISP_LH(0xe3000000, 0x0024, BPF_REG_1, REG_0, REG_15,
	STK_OFF_SKBP);
	}
	}		}

	/*		/*
	@@ -483,12 +452,12 @@ static noinline int bpf_jit_insn(struct bpf_jit jit, struct bpf_prog fp, int i
	{		{
	struct bpf_insn *insn = &fp->insnsi[i];		struct bpf_insn *insn = &fp->insnsi[i];
	int jmp_off, last, insn_count = 1;		int jmp_off, last, insn_count = 1;
	unsigned int func_addr, mask;
	u32 dst_reg = insn->dst_reg;		u32 dst_reg = insn->dst_reg;
	u32 src_reg = insn->src_reg;		u32 src_reg = insn->src_reg;
	u32 *addrs = jit->addrs;		u32 *addrs = jit->addrs;
	s32 imm = insn->imm;		s32 imm = insn->imm;
	s16 off = insn->off;		s16 off = insn->off;
			unsigned int mask;

	if (dst_reg == BPF_REG_AX \|\| src_reg == BPF_REG_AX)		if (dst_reg == BPF_REG_AX \|\| src_reg == BPF_REG_AX)
	jit->seen \|= SEEN_REG_AX;		jit->seen \|= SEEN_REG_AX;
	@@ -970,13 +939,6 @@ static noinline int bpf_jit_insn(struct bpf_jit jit, struct bpf_prog fp, int i
	EMIT2(0x0d00, REG_14, REG_W1);		EMIT2(0x0d00, REG_14, REG_W1);
	/* lgr %b0,%r2: load return value into %b0 */		/* lgr %b0,%r2: load return value into %b0 */
	EMIT4(0xb9040000, BPF_REG_0, REG_2);		EMIT4(0xb9040000, BPF_REG_0, REG_2);
	if ((jit->seen & SEEN_SKB) &&
	bpf_helper_changes_pkt_data((void *)func)) {
	/* lg %b1,ST_OFF_SKBP(%r15) */
	EMIT6_DISP_LH(0xe3000000, 0x0004, BPF_REG_1, REG_0,
	REG_15, STK_OFF_SKBP);
	emit_load_skb_data_hlen(jit);
	}
	break;		break;
	}		}
	case BPF_JMP \| BPF_TAIL_CALL:		case BPF_JMP \| BPF_TAIL_CALL:
	@@ -1176,73 +1138,6 @@ static noinline int bpf_jit_insn(struct bpf_jit jit, struct bpf_prog fp, int i
	jmp_off = addrs[i + off + 1] - (addrs[i + 1] - 4);		jmp_off = addrs[i + off + 1] - (addrs[i + 1] - 4);
	EMIT4_PCREL(0xa7040000 \| mask << 8, jmp_off);		EMIT4_PCREL(0xa7040000 \| mask << 8, jmp_off);
	break;		break;
	/*
	* BPF_LD
	*/
	case BPF_LD \| BPF_ABS \| BPF_B: /* b0 = (u8 ) (skb->data+imm) */
	case BPF_LD \| BPF_IND \| BPF_B: /* b0 = (u8 ) (skb->data+imm+src) */
	if ((BPF_MODE(insn->code) == BPF_ABS) && (imm >= 0))
	func_addr = __pa(sk_load_byte_pos);
	else
	func_addr = __pa(sk_load_byte);
	goto call_fn;
	case BPF_LD \| BPF_ABS \| BPF_H: /* b0 = (u16 ) (skb->data+imm) */
	case BPF_LD \| BPF_IND \| BPF_H: /* b0 = (u16 ) (skb->data+imm+src) */
	if ((BPF_MODE(insn->code) == BPF_ABS) && (imm >= 0))
	func_addr = __pa(sk_load_half_pos);
	else
	func_addr = __pa(sk_load_half);
	goto call_fn;
	case BPF_LD \| BPF_ABS \| BPF_W: /* b0 = (u32 ) (skb->data+imm) */
	case BPF_LD \| BPF_IND \| BPF_W: /* b0 = (u32 ) (skb->data+imm+src) */
	if ((BPF_MODE(insn->code) == BPF_ABS) && (imm >= 0))
	func_addr = __pa(sk_load_word_pos);
	else
	func_addr = __pa(sk_load_word);
	goto call_fn;
	call_fn:
	jit->seen \|= SEEN_SKB \| SEEN_RET0 \| SEEN_FUNC;
	REG_SET_SEEN(REG_14); /* Return address of possible func call */

	/*
	* Implicit input:
	* BPF_REG_6 (R7) : skb pointer
	* REG_SKB_DATA (R12): skb data pointer (if no BPF_REG_AX)
	*
	* Calculated input:
	* BPF_REG_2 (R3) : offset of byte(s) to fetch in skb
	* BPF_REG_5 (R6) : return address
	*
	* Output:
	* BPF_REG_0 (R14): data read from skb
	*
	* Scratch registers (BPF_REG_1-5)
	*/

	/* Call function: llilf %w1,func_addr */
	EMIT6_IMM(0xc00f0000, REG_W1, func_addr);

	/* Offset: lgfi %b2,imm */
	EMIT6_IMM(0xc0010000, BPF_REG_2, imm);
	if (BPF_MODE(insn->code) == BPF_IND)
	/* agfr %b2,%src (%src is s32 here) */
	EMIT4(0xb9180000, BPF_REG_2, src_reg);

	/* Reload REG_SKB_DATA if BPF_REG_AX is used */
	if (jit->seen & SEEN_REG_AX)
	/* lg %skb_data,data_off(%b6) */
	EMIT6_DISP_LH(0xe3000000, 0x0004, REG_SKB_DATA, REG_0,
	BPF_REG_6, offsetof(struct sk_buff, data));
	/* basr %b5,%w1 (%b5 is call saved) */
	EMIT2(0x0d00, BPF_REG_5, REG_W1);

	/*
	* Note: For fast access we jump directly after the
	* jnz instruction from bpf_jit.S
	*/
	/* jnz <ret0> */
	EMIT4_PCREL(0xa7740000, jit->ret0_ip - jit->prg);
	break;
	default: /* too complex, give up */		default: /* too complex, give up */
	pr_err("Unknown opcode %02x\n", insn->code);		pr_err("Unknown opcode %02x\n", insn->code);
	return -1;		return -1;