Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 0462eaac authored by David S. Miller's avatar David S. Miller
Browse files

Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next 



Alexei Starovoitov says:

====================
pull-request: bpf-next 2019-05-31

The following pull-request contains BPF updates for your *net-next* tree.

Lots of exciting new features in the first PR of this developement cycle!
The main changes are:

1) misc verifier improvements, from Alexei.

2) bpftool can now convert btf to valid C, from Andrii.

3) verifier can insert explicit ZEXT insn when requested by 32-bit JITs.
   This feature greatly improves BPF speed on 32-bit architectures. From Jiong.

4) cgroups will now auto-detach bpf programs. This fixes issue of thousands
   bpf programs got stuck in dying cgroups. From Roman.

5) new bpf_send_signal() helper, from Yonghong.

6) cgroup inet skb programs can signal CN to the stack, from Lawrence.

7) miscellaneous cleanups, from many developers.
====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 33aae282 cd538502

Documentation/bpf/bpf_design_QA.rst

+25 −5
Original line number Diff line number Diff line
@@ -172,11 +172,31 @@ registers which makes BPF inefficient virtual machine for 32-bit 
CPU architectures and 32-bit HW accelerators. Can true 32-bit registers

be added to BPF in the future?



A: NO. The first thing to improve performance on 32-bit archs is to teach

LLVM to generate code that uses 32-bit subregisters. Then second step

is to teach verifier to mark operations where zero-ing upper bits

is unnecessary. Then JITs can take advantage of those markings and

drastically reduce size of generated code and improve performance.

A: NO.



But some optimizations on zero-ing the upper 32 bits for BPF registers are

available, and can be leveraged to improve the performance of JITed BPF

programs for 32-bit architectures.



Starting with version 7, LLVM is able to generate instructions that operate

on 32-bit subregisters, provided the option -mattr=+alu32 is passed for

compiling a program. Furthermore, the verifier can now mark the

instructions for which zero-ing the upper bits of the destination register

is required, and insert an explicit zero-extension (zext) instruction

(a mov32 variant). This means that for architectures without zext hardware

support, the JIT back-ends do not need to clear the upper bits for

subregisters written by alu32 instructions or narrow loads. Instead, the

back-ends simply need to support code generation for that mov32 variant,

and to overwrite bpf_jit_needs_zext() to make it return "true" (in order to

enable zext insertion in the verifier).



Note that it is possible for a JIT back-end to have partial hardware

support for zext. In that case, if verifier zext insertion is enabled,

it could lead to the insertion of unnecessary zext instructions. Such

instructions could be removed by creating a simple peephole inside the JIT

back-end: if one instruction has hardware support for zext and if the next

instruction is an explicit zext, then the latter can be skipped when doing

the code generation.



Q: Does BPF have a stable ABI?

------------------------------

arch/arm/net/bpf_jit_32.c

+31 −11
Original line number Diff line number Diff line
@@ -736,6 +736,7 @@ static inline void emit_a32_alu_r64(const bool is64, const s8 dst[], 


		/* ALU operation */

		emit_alu_r(rd[1], rs, true, false, op, ctx);

		if (!ctx->prog->aux->verifier_zext)

			emit_a32_mov_i(rd[0], 0, ctx);

	}
@@ -758,6 +759,7 @@ static inline void emit_a32_mov_r64(const bool is64, const s8 dst[], 
				  struct jit_ctx *ctx) {

	if (!is64) {

		emit_a32_mov_r(dst_lo, src_lo, ctx);

		if (!ctx->prog->aux->verifier_zext)

			/* Zero out high 4 bytes */

			emit_a32_mov_i(dst_hi, 0, ctx);

	} else if (__LINUX_ARM_ARCH__ < 6 &&
@@ -1060,16 +1062,19 @@ static inline void emit_ldx_r(const s8 dst[], const s8 src, 
	case BPF_B:

		/* Load a Byte */

		emit(ARM_LDRB_I(rd[1], rm, off), ctx);

		if (!ctx->prog->aux->verifier_zext)

			emit_a32_mov_i(rd[0], 0, ctx);

		break;

	case BPF_H:

		/* Load a HalfWord */

		emit(ARM_LDRH_I(rd[1], rm, off), ctx);

		if (!ctx->prog->aux->verifier_zext)

			emit_a32_mov_i(rd[0], 0, ctx);

		break;

	case BPF_W:

		/* Load a Word */

		emit(ARM_LDR_I(rd[1], rm, off), ctx);

		if (!ctx->prog->aux->verifier_zext)

			emit_a32_mov_i(rd[0], 0, ctx);

		break;

	case BPF_DW:
@@ -1359,6 +1364,11 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) 
	case BPF_ALU64 | BPF_MOV | BPF_X:

		switch (BPF_SRC(code)) {

		case BPF_X:

			if (imm == 1) {

				/* Special mov32 for zext */

				emit_a32_mov_i(dst_hi, 0, ctx);

				break;

			}

			emit_a32_mov_r64(is64, dst, src, ctx);

			break;

		case BPF_K:
@@ -1438,6 +1448,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) 
		}

		emit_udivmod(rd_lo, rd_lo, rt, ctx, BPF_OP(code));

		arm_bpf_put_reg32(dst_lo, rd_lo, ctx);

		if (!ctx->prog->aux->verifier_zext)

			emit_a32_mov_i(dst_hi, 0, ctx);

		break;

	case BPF_ALU64 | BPF_DIV | BPF_K:
@@ -1453,6 +1464,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) 
			return -EINVAL;

		if (imm)

			emit_a32_alu_i(dst_lo, imm, ctx, BPF_OP(code));

		if (!ctx->prog->aux->verifier_zext)

			emit_a32_mov_i(dst_hi, 0, ctx);

		break;

	/* dst = dst << imm */
@@ -1488,6 +1500,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) 
	/* dst = ~dst */

	case BPF_ALU | BPF_NEG:

		emit_a32_alu_i(dst_lo, 0, ctx, BPF_OP(code));

		if (!ctx->prog->aux->verifier_zext)

			emit_a32_mov_i(dst_hi, 0, ctx);

		break;

	/* dst = ~dst (64 bit) */
@@ -1544,10 +1557,12 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) 
#else /* ARMv6+ */

			emit(ARM_UXTH(rd[1], rd[1]), ctx);

#endif

			if (!ctx->prog->aux->verifier_zext)

				emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx);

			break;

		case 32:

			/* zero-extend 32 bits into 64 bits */

			if (!ctx->prog->aux->verifier_zext)

				emit(ARM_EOR_R(rd[0], rd[0], rd[0]), ctx);

			break;

		case 64:
@@ -1838,6 +1853,11 @@ void bpf_jit_compile(struct bpf_prog *prog) 
	/* Nothing to do here. We support Internal BPF. */

}



bool bpf_jit_needs_zext(void)

{

	return true;

}



struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)

{

	struct bpf_prog *tmp, *orig_prog = prog;

arch/powerpc/net/bpf_jit_comp64.c

+33 −3
Original line number Diff line number Diff line
@@ -504,6 +504,9 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, 
		case BPF_ALU | BPF_LSH | BPF_X: /* (u32) dst <<= (u32) src */

			/* slw clears top 32 bits */

			PPC_SLW(dst_reg, dst_reg, src_reg);

			/* skip zero extension move, but set address map. */

			if (insn_is_zext(&insn[i + 1]))

				addrs[++i] = ctx->idx * 4;

			break;

		case BPF_ALU64 | BPF_LSH | BPF_X: /* dst <<= src; */

			PPC_SLD(dst_reg, dst_reg, src_reg);
@@ -511,6 +514,8 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, 
		case BPF_ALU | BPF_LSH | BPF_K: /* (u32) dst <<== (u32) imm */

			/* with imm 0, we still need to clear top 32 bits */

			PPC_SLWI(dst_reg, dst_reg, imm);

			if (insn_is_zext(&insn[i + 1]))

				addrs[++i] = ctx->idx * 4;

			break;

		case BPF_ALU64 | BPF_LSH | BPF_K: /* dst <<== imm */

			if (imm != 0)
@@ -518,12 +523,16 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, 
			break;

		case BPF_ALU | BPF_RSH | BPF_X: /* (u32) dst >>= (u32) src */

			PPC_SRW(dst_reg, dst_reg, src_reg);

			if (insn_is_zext(&insn[i + 1]))

				addrs[++i] = ctx->idx * 4;

			break;

		case BPF_ALU64 | BPF_RSH | BPF_X: /* dst >>= src */

			PPC_SRD(dst_reg, dst_reg, src_reg);

			break;

		case BPF_ALU | BPF_RSH | BPF_K: /* (u32) dst >>= (u32) imm */

			PPC_SRWI(dst_reg, dst_reg, imm);

			if (insn_is_zext(&insn[i + 1]))

				addrs[++i] = ctx->idx * 4;

			break;

		case BPF_ALU64 | BPF_RSH | BPF_K: /* dst >>= imm */

			if (imm != 0)
@@ -548,6 +557,11 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, 
		 */

		case BPF_ALU | BPF_MOV | BPF_X: /* (u32) dst = src */

		case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */

			if (imm == 1) {

				/* special mov32 for zext */

				PPC_RLWINM(dst_reg, dst_reg, 0, 0, 31);

				break;

			}

			PPC_MR(dst_reg, src_reg);

			goto bpf_alu32_trunc;

		case BPF_ALU | BPF_MOV | BPF_K: /* (u32) dst = imm */
@@ -555,11 +569,13 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, 
			PPC_LI32(dst_reg, imm);

			if (imm < 0)

				goto bpf_alu32_trunc;

			else if (insn_is_zext(&insn[i + 1]))

				addrs[++i] = ctx->idx * 4;

			break;



bpf_alu32_trunc:

		/* Truncate to 32-bits */

		if (BPF_CLASS(code) == BPF_ALU)

		if (BPF_CLASS(code) == BPF_ALU && !fp->aux->verifier_zext)

			PPC_RLWINM(dst_reg, dst_reg, 0, 0, 31);

		break;
@@ -618,8 +634,11 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, 
			case 16:

				/* zero-extend 16 bits into 64 bits */

				PPC_RLDICL(dst_reg, dst_reg, 0, 48);

				if (insn_is_zext(&insn[i + 1]))

					addrs[++i] = ctx->idx * 4;

				break;

			case 32:

				if (!fp->aux->verifier_zext)

					/* zero-extend 32 bits into 64 bits */

					PPC_RLDICL(dst_reg, dst_reg, 0, 32);

				break;
@@ -698,14 +717,20 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, 
		/* dst = *(u8 *)(ul) (src + off) */

		case BPF_LDX | BPF_MEM | BPF_B:

			PPC_LBZ(dst_reg, src_reg, off);

			if (insn_is_zext(&insn[i + 1]))

				addrs[++i] = ctx->idx * 4;

			break;

		/* dst = *(u16 *)(ul) (src + off) */

		case BPF_LDX | BPF_MEM | BPF_H:

			PPC_LHZ(dst_reg, src_reg, off);

			if (insn_is_zext(&insn[i + 1]))

				addrs[++i] = ctx->idx * 4;

			break;

		/* dst = *(u32 *)(ul) (src + off) */

		case BPF_LDX | BPF_MEM | BPF_W:

			PPC_LWZ(dst_reg, src_reg, off);

			if (insn_is_zext(&insn[i + 1]))

				addrs[++i] = ctx->idx * 4;

			break;

		/* dst = *(u64 *)(ul) (src + off) */

		case BPF_LDX | BPF_MEM | BPF_DW:
@@ -1046,6 +1071,11 @@ struct powerpc64_jit_data { 
	struct codegen_context ctx;

};



bool bpf_jit_needs_zext(void)

{

	return true;

}



struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)

{

	u32 proglen;

arch/riscv/net/bpf_jit_comp.c

+30 −13
Original line number Diff line number Diff line
@@ -731,6 +731,7 @@ static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 
{

	bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 ||

		    BPF_CLASS(insn->code) == BPF_JMP;

	struct bpf_prog_aux *aux = ctx->prog->aux;

	int rvoff, i = insn - ctx->prog->insnsi;

	u8 rd = -1, rs = -1, code = insn->code;

	s16 off = insn->off;
@@ -742,8 +743,13 @@ static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 
	/* dst = src */

	case BPF_ALU | BPF_MOV | BPF_X:

	case BPF_ALU64 | BPF_MOV | BPF_X:

		if (imm == 1) {

			/* Special mov32 for zext */

			emit_zext_32(rd, ctx);

			break;

		}

		emit(is64 ? rv_addi(rd, rs, 0) : rv_addiw(rd, rs, 0), ctx);

		if (!is64)

		if (!is64 && !aux->verifier_zext)

			emit_zext_32(rd, ctx);

		break;
@@ -771,19 +777,19 @@ static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 
	case BPF_ALU | BPF_MUL | BPF_X:

	case BPF_ALU64 | BPF_MUL | BPF_X:

		emit(is64 ? rv_mul(rd, rd, rs) : rv_mulw(rd, rd, rs), ctx);

		if (!is64)

		if (!is64 && !aux->verifier_zext)

			emit_zext_32(rd, ctx);

		break;

	case BPF_ALU | BPF_DIV | BPF_X:

	case BPF_ALU64 | BPF_DIV | BPF_X:

		emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx);

		if (!is64)

		if (!is64 && !aux->verifier_zext)

			emit_zext_32(rd, ctx);

		break;

	case BPF_ALU | BPF_MOD | BPF_X:

	case BPF_ALU64 | BPF_MOD | BPF_X:

		emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx);

		if (!is64)

		if (!is64 && !aux->verifier_zext)

			emit_zext_32(rd, ctx);

		break;

	case BPF_ALU | BPF_LSH | BPF_X:
@@ -867,7 +873,7 @@ static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 
	case BPF_ALU | BPF_MOV | BPF_K:

	case BPF_ALU64 | BPF_MOV | BPF_K:

		emit_imm(rd, imm, ctx);

		if (!is64)

		if (!is64 && !aux->verifier_zext)

			emit_zext_32(rd, ctx);

		break;
@@ -882,7 +888,7 @@ static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 
			emit(is64 ? rv_add(rd, rd, RV_REG_T1) :

			     rv_addw(rd, rd, RV_REG_T1), ctx);

		}

		if (!is64)

		if (!is64 && !aux->verifier_zext)

			emit_zext_32(rd, ctx);

		break;

	case BPF_ALU | BPF_SUB | BPF_K:
@@ -895,7 +901,7 @@ static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 
			emit(is64 ? rv_sub(rd, rd, RV_REG_T1) :

			     rv_subw(rd, rd, RV_REG_T1), ctx);

		}

		if (!is64)

		if (!is64 && !aux->verifier_zext)

			emit_zext_32(rd, ctx);

		break;

	case BPF_ALU | BPF_AND | BPF_K:
@@ -906,7 +912,7 @@ static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 
			emit_imm(RV_REG_T1, imm, ctx);

			emit(rv_and(rd, rd, RV_REG_T1), ctx);

		}

		if (!is64)

		if (!is64 && !aux->verifier_zext)

			emit_zext_32(rd, ctx);

		break;

	case BPF_ALU | BPF_OR | BPF_K:
@@ -917,7 +923,7 @@ static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 
			emit_imm(RV_REG_T1, imm, ctx);

			emit(rv_or(rd, rd, RV_REG_T1), ctx);

		}

		if (!is64)

		if (!is64 && !aux->verifier_zext)

			emit_zext_32(rd, ctx);

		break;

	case BPF_ALU | BPF_XOR | BPF_K:
@@ -928,7 +934,7 @@ static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 
			emit_imm(RV_REG_T1, imm, ctx);

			emit(rv_xor(rd, rd, RV_REG_T1), ctx);

		}

		if (!is64)

		if (!is64 && !aux->verifier_zext)

			emit_zext_32(rd, ctx);

		break;

	case BPF_ALU | BPF_MUL | BPF_K:
@@ -936,7 +942,7 @@ static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 
		emit_imm(RV_REG_T1, imm, ctx);

		emit(is64 ? rv_mul(rd, rd, RV_REG_T1) :

		     rv_mulw(rd, rd, RV_REG_T1), ctx);

		if (!is64)

		if (!is64 && !aux->verifier_zext)

			emit_zext_32(rd, ctx);

		break;

	case BPF_ALU | BPF_DIV | BPF_K:
@@ -944,7 +950,7 @@ static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 
		emit_imm(RV_REG_T1, imm, ctx);

		emit(is64 ? rv_divu(rd, rd, RV_REG_T1) :

		     rv_divuw(rd, rd, RV_REG_T1), ctx);

		if (!is64)

		if (!is64 && !aux->verifier_zext)

			emit_zext_32(rd, ctx);

		break;

	case BPF_ALU | BPF_MOD | BPF_K:
@@ -952,7 +958,7 @@ static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 
		emit_imm(RV_REG_T1, imm, ctx);

		emit(is64 ? rv_remu(rd, rd, RV_REG_T1) :

		     rv_remuw(rd, rd, RV_REG_T1), ctx);

		if (!is64)

		if (!is64 && !aux->verifier_zext)

			emit_zext_32(rd, ctx);

		break;

	case BPF_ALU | BPF_LSH | BPF_K:
@@ -1239,6 +1245,8 @@ static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 
		emit_imm(RV_REG_T1, off, ctx);

		emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx);

		emit(rv_lbu(rd, 0, RV_REG_T1), ctx);

		if (insn_is_zext(&insn[1]))

			return 1;

		break;

	case BPF_LDX | BPF_MEM | BPF_H:

		if (is_12b_int(off)) {
@@ -1249,6 +1257,8 @@ static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 
		emit_imm(RV_REG_T1, off, ctx);

		emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx);

		emit(rv_lhu(rd, 0, RV_REG_T1), ctx);

		if (insn_is_zext(&insn[1]))

			return 1;

		break;

	case BPF_LDX | BPF_MEM | BPF_W:

		if (is_12b_int(off)) {
@@ -1259,6 +1269,8 @@ static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 
		emit_imm(RV_REG_T1, off, ctx);

		emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx);

		emit(rv_lwu(rd, 0, RV_REG_T1), ctx);

		if (insn_is_zext(&insn[1]))

			return 1;

		break;

	case BPF_LDX | BPF_MEM | BPF_DW:

		if (is_12b_int(off)) {
@@ -1503,6 +1515,11 @@ static void bpf_flush_icache(void *start, void *end) 
	flush_icache_range((unsigned long)start, (unsigned long)end);

}



bool bpf_jit_needs_zext(void)

{

	return true;

}



struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)

{

	bool tmp_blinded = false, extra_pass = false;

arch/s390/net/bpf_jit_comp.c

+34 −7
Original line number Diff line number Diff line
@@ -299,9 +299,11 @@ static inline void reg_set_seen(struct bpf_jit *jit, u32 b1) 


#define EMIT_ZERO(b1)						\

({								\

	if (!fp->aux->verifier_zext) {				\

		/* llgfr %dst,%dst (zero extend to 64 bit) */	\

		EMIT4(0xb9160000, b1, b1);			\

		REG_SET_SEEN(b1);				\

	}							\

})



/*
@@ -520,6 +522,8 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i 
	case BPF_ALU | BPF_MOV | BPF_X: /* dst = (u32) src */

		/* llgfr %dst,%src */

		EMIT4(0xb9160000, dst_reg, src_reg);

		if (insn_is_zext(&insn[1]))

			insn_count = 2;

		break;

	case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */

		/* lgr %dst,%src */
@@ -528,6 +532,8 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i 
	case BPF_ALU | BPF_MOV | BPF_K: /* dst = (u32) imm */

		/* llilf %dst,imm */

		EMIT6_IMM(0xc00f0000, dst_reg, imm);

		if (insn_is_zext(&insn[1]))

			insn_count = 2;

		break;

	case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = imm */

		/* lgfi %dst,imm */
@@ -639,6 +645,8 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i 
		EMIT4(0xb9970000, REG_W0, src_reg);

		/* llgfr %dst,%rc */

		EMIT4(0xb9160000, dst_reg, rc_reg);

		if (insn_is_zext(&insn[1]))

			insn_count = 2;

		break;

	}

	case BPF_ALU64 | BPF_DIV | BPF_X: /* dst = dst / src */
@@ -676,6 +684,8 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i 
			      EMIT_CONST_U32(imm));

		/* llgfr %dst,%rc */

		EMIT4(0xb9160000, dst_reg, rc_reg);

		if (insn_is_zext(&insn[1]))

			insn_count = 2;

		break;

	}

	case BPF_ALU64 | BPF_DIV | BPF_K: /* dst = dst / imm */
@@ -864,8 +874,11 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i 
		case 16: /* dst = (u16) cpu_to_be16(dst) */

			/* llghr %dst,%dst */

			EMIT4(0xb9850000, dst_reg, dst_reg);

			if (insn_is_zext(&insn[1]))

				insn_count = 2;

			break;

		case 32: /* dst = (u32) cpu_to_be32(dst) */

			if (!fp->aux->verifier_zext)

				/* llgfr %dst,%dst */

				EMIT4(0xb9160000, dst_reg, dst_reg);

			break;
@@ -882,10 +895,13 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i 
			EMIT4_DISP(0x88000000, dst_reg, REG_0, 16);

			/* llghr %dst,%dst */

			EMIT4(0xb9850000, dst_reg, dst_reg);

			if (insn_is_zext(&insn[1]))

				insn_count = 2;

			break;

		case 32: /* dst = (u32) cpu_to_le32(dst) */

			/* lrvr %dst,%dst */

			EMIT4(0xb91f0000, dst_reg, dst_reg);

			if (!fp->aux->verifier_zext)

				/* llgfr %dst,%dst */

				EMIT4(0xb9160000, dst_reg, dst_reg);

			break;
@@ -968,16 +984,22 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i 
		/* llgc %dst,0(off,%src) */

		EMIT6_DISP_LH(0xe3000000, 0x0090, dst_reg, src_reg, REG_0, off);

		jit->seen |= SEEN_MEM;

		if (insn_is_zext(&insn[1]))

			insn_count = 2;

		break;

	case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */

		/* llgh %dst,0(off,%src) */

		EMIT6_DISP_LH(0xe3000000, 0x0091, dst_reg, src_reg, REG_0, off);

		jit->seen |= SEEN_MEM;

		if (insn_is_zext(&insn[1]))

			insn_count = 2;

		break;

	case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */

		/* llgf %dst,off(%src) */

		jit->seen |= SEEN_MEM;

		EMIT6_DISP_LH(0xe3000000, 0x0016, dst_reg, src_reg, REG_0, off);

		if (insn_is_zext(&insn[1]))

			insn_count = 2;

		break;

	case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */

		/* lg %dst,0(off,%src) */
@@ -1282,6 +1304,11 @@ static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp) 
	return 0;

}



bool bpf_jit_needs_zext(void)

{

	return true;

}



/*

 * Compile eBPF program "fp"

 */