Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 626a5f66 authored by Jiong Wang's avatar Jiong Wang Committed by Alexei Starovoitov
Browse files

s390: bpf: implement jitting of JMP32 



This patch implements code-gen for new JMP32 instructions on s390.

Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: default avatarJiong Wang <jiong.wang@netronome.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent 5f645996

arch/s390/net/bpf_jit_comp.c

+52 −14
Original line number Original line Diff line number Diff line
@@ -1110,103 +1110,141 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i 
		mask = 0xf000; /* j */

		mask = 0xf000; /* j */

		goto branch_oc;

		goto branch_oc;

	case BPF_JMP | BPF_JSGT | BPF_K: /* ((s64) dst > (s64) imm) */

	case BPF_JMP | BPF_JSGT | BPF_K: /* ((s64) dst > (s64) imm) */

	case BPF_JMP32 | BPF_JSGT | BPF_K: /* ((s32) dst > (s32) imm) */

		mask = 0x2000; /* jh */

		mask = 0x2000; /* jh */

		goto branch_ks;

		goto branch_ks;

	case BPF_JMP | BPF_JSLT | BPF_K: /* ((s64) dst < (s64) imm) */

	case BPF_JMP | BPF_JSLT | BPF_K: /* ((s64) dst < (s64) imm) */

	case BPF_JMP32 | BPF_JSLT | BPF_K: /* ((s32) dst < (s32) imm) */

		mask = 0x4000; /* jl */

		mask = 0x4000; /* jl */

		goto branch_ks;

		goto branch_ks;

	case BPF_JMP | BPF_JSGE | BPF_K: /* ((s64) dst >= (s64) imm) */

	case BPF_JMP | BPF_JSGE | BPF_K: /* ((s64) dst >= (s64) imm) */

	case BPF_JMP32 | BPF_JSGE | BPF_K: /* ((s32) dst >= (s32) imm) */

		mask = 0xa000; /* jhe */

		mask = 0xa000; /* jhe */

		goto branch_ks;

		goto branch_ks;

	case BPF_JMP | BPF_JSLE | BPF_K: /* ((s64) dst <= (s64) imm) */

	case BPF_JMP | BPF_JSLE | BPF_K: /* ((s64) dst <= (s64) imm) */

	case BPF_JMP32 | BPF_JSLE | BPF_K: /* ((s32) dst <= (s32) imm) */

		mask = 0xc000; /* jle */

		mask = 0xc000; /* jle */

		goto branch_ks;

		goto branch_ks;

	case BPF_JMP | BPF_JGT | BPF_K: /* (dst_reg > imm) */

	case BPF_JMP | BPF_JGT | BPF_K: /* (dst_reg > imm) */

	case BPF_JMP32 | BPF_JGT | BPF_K: /* ((u32) dst_reg > (u32) imm) */

		mask = 0x2000; /* jh */

		mask = 0x2000; /* jh */

		goto branch_ku;

		goto branch_ku;

	case BPF_JMP | BPF_JLT | BPF_K: /* (dst_reg < imm) */

	case BPF_JMP | BPF_JLT | BPF_K: /* (dst_reg < imm) */

	case BPF_JMP32 | BPF_JLT | BPF_K: /* ((u32) dst_reg < (u32) imm) */

		mask = 0x4000; /* jl */

		mask = 0x4000; /* jl */

		goto branch_ku;

		goto branch_ku;

	case BPF_JMP | BPF_JGE | BPF_K: /* (dst_reg >= imm) */

	case BPF_JMP | BPF_JGE | BPF_K: /* (dst_reg >= imm) */

	case BPF_JMP32 | BPF_JGE | BPF_K: /* ((u32) dst_reg >= (u32) imm) */

		mask = 0xa000; /* jhe */

		mask = 0xa000; /* jhe */

		goto branch_ku;

		goto branch_ku;

	case BPF_JMP | BPF_JLE | BPF_K: /* (dst_reg <= imm) */

	case BPF_JMP | BPF_JLE | BPF_K: /* (dst_reg <= imm) */

	case BPF_JMP32 | BPF_JLE | BPF_K: /* ((u32) dst_reg <= (u32) imm) */

		mask = 0xc000; /* jle */

		mask = 0xc000; /* jle */

		goto branch_ku;

		goto branch_ku;

	case BPF_JMP | BPF_JNE | BPF_K: /* (dst_reg != imm) */

	case BPF_JMP | BPF_JNE | BPF_K: /* (dst_reg != imm) */

	case BPF_JMP32 | BPF_JNE | BPF_K: /* ((u32) dst_reg != (u32) imm) */

		mask = 0x7000; /* jne */

		mask = 0x7000; /* jne */

		goto branch_ku;

		goto branch_ku;

	case BPF_JMP | BPF_JEQ | BPF_K: /* (dst_reg == imm) */

	case BPF_JMP | BPF_JEQ | BPF_K: /* (dst_reg == imm) */

	case BPF_JMP32 | BPF_JEQ | BPF_K: /* ((u32) dst_reg == (u32) imm) */

		mask = 0x8000; /* je */

		mask = 0x8000; /* je */

		goto branch_ku;

		goto branch_ku;

	case BPF_JMP | BPF_JSET | BPF_K: /* (dst_reg & imm) */

	case BPF_JMP | BPF_JSET | BPF_K: /* (dst_reg & imm) */

	case BPF_JMP32 | BPF_JSET | BPF_K: /* ((u32) dst_reg & (u32) imm) */

		mask = 0x7000; /* jnz */

		mask = 0x7000; /* jnz */

		if (BPF_CLASS(insn->code) == BPF_JMP32) {

			/* llilf %w1,imm (load zero extend imm) */

			EMIT6_IMM(0xc0010000, REG_W1, imm);

			/* nr %w1,%dst */

			EMIT2(0x1400, REG_W1, dst_reg);

		} else {

			/* lgfi %w1,imm (load sign extend imm) */

			/* lgfi %w1,imm (load sign extend imm) */

			EMIT6_IMM(0xc0010000, REG_W1, imm);

			EMIT6_IMM(0xc0010000, REG_W1, imm);

			/* ngr %w1,%dst */

			/* ngr %w1,%dst */

			EMIT4(0xb9800000, REG_W1, dst_reg);

			EMIT4(0xb9800000, REG_W1, dst_reg);

		}

		goto branch_oc;

		goto branch_oc;





	case BPF_JMP | BPF_JSGT | BPF_X: /* ((s64) dst > (s64) src) */

	case BPF_JMP | BPF_JSGT | BPF_X: /* ((s64) dst > (s64) src) */

	case BPF_JMP32 | BPF_JSGT | BPF_X: /* ((s32) dst > (s32) src) */

		mask = 0x2000; /* jh */

		mask = 0x2000; /* jh */

		goto branch_xs;

		goto branch_xs;

	case BPF_JMP | BPF_JSLT | BPF_X: /* ((s64) dst < (s64) src) */

	case BPF_JMP | BPF_JSLT | BPF_X: /* ((s64) dst < (s64) src) */

	case BPF_JMP32 | BPF_JSLT | BPF_X: /* ((s32) dst < (s32) src) */

		mask = 0x4000; /* jl */

		mask = 0x4000; /* jl */

		goto branch_xs;

		goto branch_xs;

	case BPF_JMP | BPF_JSGE | BPF_X: /* ((s64) dst >= (s64) src) */

	case BPF_JMP | BPF_JSGE | BPF_X: /* ((s64) dst >= (s64) src) */

	case BPF_JMP32 | BPF_JSGE | BPF_X: /* ((s32) dst >= (s32) src) */

		mask = 0xa000; /* jhe */

		mask = 0xa000; /* jhe */

		goto branch_xs;

		goto branch_xs;

	case BPF_JMP | BPF_JSLE | BPF_X: /* ((s64) dst <= (s64) src) */

	case BPF_JMP | BPF_JSLE | BPF_X: /* ((s64) dst <= (s64) src) */

	case BPF_JMP32 | BPF_JSLE | BPF_X: /* ((s32) dst <= (s32) src) */

		mask = 0xc000; /* jle */

		mask = 0xc000; /* jle */

		goto branch_xs;

		goto branch_xs;

	case BPF_JMP | BPF_JGT | BPF_X: /* (dst > src) */

	case BPF_JMP | BPF_JGT | BPF_X: /* (dst > src) */

	case BPF_JMP32 | BPF_JGT | BPF_X: /* ((u32) dst > (u32) src) */

		mask = 0x2000; /* jh */

		mask = 0x2000; /* jh */

		goto branch_xu;

		goto branch_xu;

	case BPF_JMP | BPF_JLT | BPF_X: /* (dst < src) */

	case BPF_JMP | BPF_JLT | BPF_X: /* (dst < src) */

	case BPF_JMP32 | BPF_JLT | BPF_X: /* ((u32) dst < (u32) src) */

		mask = 0x4000; /* jl */

		mask = 0x4000; /* jl */

		goto branch_xu;

		goto branch_xu;

	case BPF_JMP | BPF_JGE | BPF_X: /* (dst >= src) */

	case BPF_JMP | BPF_JGE | BPF_X: /* (dst >= src) */

	case BPF_JMP32 | BPF_JGE | BPF_X: /* ((u32) dst >= (u32) src) */

		mask = 0xa000; /* jhe */

		mask = 0xa000; /* jhe */

		goto branch_xu;

		goto branch_xu;

	case BPF_JMP | BPF_JLE | BPF_X: /* (dst <= src) */

	case BPF_JMP | BPF_JLE | BPF_X: /* (dst <= src) */

	case BPF_JMP32 | BPF_JLE | BPF_X: /* ((u32) dst <= (u32) src) */

		mask = 0xc000; /* jle */

		mask = 0xc000; /* jle */

		goto branch_xu;

		goto branch_xu;

	case BPF_JMP | BPF_JNE | BPF_X: /* (dst != src) */

	case BPF_JMP | BPF_JNE | BPF_X: /* (dst != src) */

	case BPF_JMP32 | BPF_JNE | BPF_X: /* ((u32) dst != (u32) src) */

		mask = 0x7000; /* jne */

		mask = 0x7000; /* jne */

		goto branch_xu;

		goto branch_xu;

	case BPF_JMP | BPF_JEQ | BPF_X: /* (dst == src) */

	case BPF_JMP | BPF_JEQ | BPF_X: /* (dst == src) */

	case BPF_JMP32 | BPF_JEQ | BPF_X: /* ((u32) dst == (u32) src) */

		mask = 0x8000; /* je */

		mask = 0x8000; /* je */

		goto branch_xu;

		goto branch_xu;

	case BPF_JMP | BPF_JSET | BPF_X: /* (dst & src) */

	case BPF_JMP | BPF_JSET | BPF_X: /* (dst & src) */

	case BPF_JMP32 | BPF_JSET | BPF_X: /* ((u32) dst & (u32) src) */

	{

		bool is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;



		mask = 0x7000; /* jnz */

		mask = 0x7000; /* jnz */

		/* ngrk %w1,%dst,%src */

		/* nrk or ngrk %w1,%dst,%src */

		EMIT4_RRF(0xb9e40000, REG_W1, dst_reg, src_reg);

		EMIT4_RRF((is_jmp32 ? 0xb9f40000 : 0xb9e40000),

			  REG_W1, dst_reg, src_reg);

		goto branch_oc;

		goto branch_oc;

branch_ks:

branch_ks:

		/* lgfi %w1,imm (load sign extend imm) */

		/* lgfi %w1,imm (load sign extend imm) */

		EMIT6_IMM(0xc0010000, REG_W1, imm);

		EMIT6_IMM(0xc0010000, REG_W1, imm);

		/* cgrj %dst,%w1,mask,off */

		/* crj or cgrj %dst,%w1,mask,off */

		EMIT6_PCREL(0xec000000, 0x0064, dst_reg, REG_W1, i, off, mask);

		EMIT6_PCREL(0xec000000, (is_jmp32 ? 0x0076 : 0x0064),

			    dst_reg, REG_W1, i, off, mask);

		break;

		break;

branch_ku:

branch_ku:

		/* lgfi %w1,imm (load sign extend imm) */

		/* lgfi %w1,imm (load sign extend imm) */

		EMIT6_IMM(0xc0010000, REG_W1, imm);

		EMIT6_IMM(0xc0010000, REG_W1, imm);

		/* clgrj %dst,%w1,mask,off */

		/* clrj or clgrj %dst,%w1,mask,off */

		EMIT6_PCREL(0xec000000, 0x0065, dst_reg, REG_W1, i, off, mask);

		EMIT6_PCREL(0xec000000, (is_jmp32 ? 0x0077 : 0x0065),

			    dst_reg, REG_W1, i, off, mask);

		break;

		break;

branch_xs:

branch_xs:

		/* cgrj %dst,%src,mask,off */

		/* crj or cgrj %dst,%src,mask,off */

		EMIT6_PCREL(0xec000000, 0x0064, dst_reg, src_reg, i, off, mask);

		EMIT6_PCREL(0xec000000, (is_jmp32 ? 0x0076 : 0x0064),

			    dst_reg, src_reg, i, off, mask);

		break;

		break;

branch_xu:

branch_xu:

		/* clgrj %dst,%src,mask,off */

		/* clrj or clgrj %dst,%src,mask,off */

		EMIT6_PCREL(0xec000000, 0x0065, dst_reg, src_reg, i, off, mask);

		EMIT6_PCREL(0xec000000, (is_jmp32 ? 0x0077 : 0x0065),

			    dst_reg, src_reg, i, off, mask);

		break;

		break;

branch_oc:

branch_oc:

		/* brc mask,jmp_off (branch instruction needs 4 bytes) */

		/* brc mask,jmp_off (branch instruction needs 4 bytes) */

		jmp_off = addrs[i + off + 1] - (addrs[i + 1] - 4);

		jmp_off = addrs[i + off + 1] - (addrs[i + 1] - 4);

		EMIT4_PCREL(0xa7040000 | mask << 8, jmp_off);

		EMIT4_PCREL(0xa7040000 | mask << 8, jmp_off);

		break;

		break;

	}

	default: /* too complex, give up */

	default: /* too complex, give up */

		pr_err("Unknown opcode %02x\n", insn->code);

		pr_err("Unknown opcode %02x\n", insn->code);

		return -1;

		return -1;