Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 632130ed authored by Daniel Borkmann's avatar Daniel Borkmann
Browse files

Merge branch 'bpf-nfp-misc-improvements' 



Jakub Kicinski says:

====================
This series starts with a fix to Jesper's recent work, somehow I forgot
about control rings during review.  Second patch is cleaning up a vNIC
header, in kdoc we should not use @ for #define constants.  Aligning of
the top of the stack as well as bottom (last bytes will be unused) helps
the performance.  We should check offload datapath's max MTU when program
is loaded and we can allow TC hw offload flag to be changed freely while
XDP offload is active.

Next group of patches adds more fully featured relocation support.  Due
to limited amount of code space we only load the image to NIC's memory
when program is attached.  Since we can't predict which programs are
loaded later, we should translate as if image was to be loaded at offset
zero and only apply relocations at load time.  Many more advanced features
(eg. tail class, subprograms, dynamic allocation of program space and
sharing it between ports) will depend on this.

Nic adds support for signed comparison instructions.

Quentin makes use of the verifier log in our driver, the verifier print
function (verbose()) has to be renamed and exported.

v2:
 - replace #define by function aliasing for verbose() in patch 13
====================

Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
parents 148989d8 ff627e3d

drivers/net/ethernet/netronome/nfp/bpf/jit.c

+167 −81
Original line number Diff line number Diff line
@@ -85,7 +85,7 @@ static void nfp_prog_push(struct nfp_prog *nfp_prog, u64 insn) 


static unsigned int nfp_prog_current_offset(struct nfp_prog *nfp_prog)

{

	return nfp_prog->start_off + nfp_prog->prog_len;

	return nfp_prog->prog_len;

}



static bool
@@ -100,12 +100,6 @@ nfp_prog_confirm_current_offset(struct nfp_prog *nfp_prog, unsigned int off) 
	return !WARN_ON_ONCE(nfp_prog_current_offset(nfp_prog) != off);

}



static unsigned int

nfp_prog_offset_to_index(struct nfp_prog *nfp_prog, unsigned int offset)

{

	return offset - nfp_prog->start_off;

}



/* --- Emitters --- */

static void

__emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op,
@@ -195,22 +189,28 @@ __emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, enum br_ev_pip ev_pip, 
	nfp_prog_push(nfp_prog, insn);

}



static void emit_br_def(struct nfp_prog *nfp_prog, u16 addr, u8 defer)

static void

emit_br_relo(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer,

	     enum nfp_relo_type relo)

{

	if (defer > 2) {

	if (mask == BR_UNC && defer > 2) {

		pr_err("BUG: branch defer out of bounds %d\n", defer);

		nfp_prog->error = -EFAULT;

		return;

	}

	__emit_br(nfp_prog, BR_UNC, BR_EV_PIP_UNCOND, BR_CSS_NONE, addr, defer);



	__emit_br(nfp_prog, mask,

		  mask != BR_UNC ? BR_EV_PIP_COND : BR_EV_PIP_UNCOND,

		  BR_CSS_NONE, addr, defer);



	nfp_prog->prog[nfp_prog->prog_len - 1] |=

		FIELD_PREP(OP_RELO_TYPE, relo);

}



static void

emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer)

{

	__emit_br(nfp_prog, mask,

		  mask != BR_UNC ? BR_EV_PIP_COND : BR_EV_PIP_UNCOND,

		  BR_CSS_NONE, addr, defer);

	emit_br_relo(nfp_prog, mask, addr, defer, RELO_BR_REL);

}



static void
@@ -515,16 +515,6 @@ static void wrp_nops(struct nfp_prog *nfp_prog, unsigned int count) 
		emit_nop(nfp_prog);

}



static void

wrp_br_special(struct nfp_prog *nfp_prog, enum br_mask mask,

	       enum br_special special)

{

	emit_br(nfp_prog, mask, 0, 0);



	nfp_prog->prog[nfp_prog->prog_len - 1] |=

		FIELD_PREP(OP_BR_SPECIAL, special);

}



static void wrp_mov(struct nfp_prog *nfp_prog, swreg dst, swreg src)

{

	emit_alu(nfp_prog, dst, reg_none(), ALU_OP_NONE, src);
@@ -749,7 +739,7 @@ construct_data_ind_ld(struct nfp_prog *nfp_prog, u16 offset, u16 src, u8 size) 
		 imm_a(nfp_prog), ALU_OP_ADD, reg_imm(size));

	emit_alu(nfp_prog, reg_none(),

		 plen_reg(nfp_prog), ALU_OP_SUB, imm_a(nfp_prog));

	wrp_br_special(nfp_prog, BR_BLO, OP_BR_GO_ABORT);

	emit_br_relo(nfp_prog, BR_BLO, BR_OFF_RELO, 0, RELO_BR_GO_ABORT);



	/* Load data */

	return data_ld(nfp_prog, imm_b(nfp_prog), 0, size);
@@ -762,7 +752,7 @@ static int construct_data_ld(struct nfp_prog *nfp_prog, u16 offset, u8 size) 
	/* Check packet length */

	tmp_reg = ur_load_imm_any(nfp_prog, offset + size, imm_a(nfp_prog));

	emit_alu(nfp_prog, reg_none(), plen_reg(nfp_prog), ALU_OP_SUB, tmp_reg);

	wrp_br_special(nfp_prog, BR_BLO, OP_BR_GO_ABORT);

	emit_br_relo(nfp_prog, BR_BLO, BR_OFF_RELO, 0, RELO_BR_GO_ABORT);



	/* Load data */

	tmp_reg = re_load_imm_any(nfp_prog, offset, imm_b(nfp_prog));
@@ -1269,7 +1259,7 @@ static int adjust_head(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 
	emit_ld_field(nfp_prog, pptr_reg(nfp_prog), 0x3, tmp, SHF_SC_NONE, 0);



	/* Skip over the -EINVAL ret code (defer 2) */

	emit_br_def(nfp_prog, end, 2);

	emit_br(nfp_prog, BR_UNC, end, 2);



	emit_alu(nfp_prog, plen_reg(nfp_prog),

		 plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2));
@@ -1924,6 +1914,26 @@ static int jle_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 
	return wrp_cmp_imm(nfp_prog, meta, BR_BHS, true);

}



static int jsgt_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)

{

	return wrp_cmp_imm(nfp_prog, meta, BR_BLT, true);

}



static int jsge_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)

{

	return wrp_cmp_imm(nfp_prog, meta, BR_BGE, false);

}



static int jslt_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)

{

	return wrp_cmp_imm(nfp_prog, meta, BR_BLT, false);

}



static int jsle_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)

{

	return wrp_cmp_imm(nfp_prog, meta, BR_BGE, true);

}



static int jset_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)

{

	const struct bpf_insn *insn = &meta->insn;
@@ -2013,6 +2023,26 @@ static int jle_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 
	return wrp_cmp_reg(nfp_prog, meta, BR_BHS, true);

}



static int jsgt_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)

{

	return wrp_cmp_reg(nfp_prog, meta, BR_BLT, true);

}



static int jsge_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)

{

	return wrp_cmp_reg(nfp_prog, meta, BR_BGE, false);

}



static int jslt_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)

{

	return wrp_cmp_reg(nfp_prog, meta, BR_BLT, false);

}



static int jsle_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)

{

	return wrp_cmp_reg(nfp_prog, meta, BR_BGE, true);

}



static int jset_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)

{

	return wrp_test_reg(nfp_prog, meta, ALU_OP_AND, BR_BNE);
@@ -2036,7 +2066,7 @@ static int call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) 


static int goto_out(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)

{

	wrp_br_special(nfp_prog, BR_UNC, OP_BR_GO_OUT);

	emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 0, RELO_BR_GO_OUT);



	return 0;

}
@@ -2097,6 +2127,10 @@ static const instr_cb_t instr_cb[256] = { 
	[BPF_JMP | BPF_JGE | BPF_K] =	jge_imm,

	[BPF_JMP | BPF_JLT | BPF_K] =	jlt_imm,

	[BPF_JMP | BPF_JLE | BPF_K] =	jle_imm,

	[BPF_JMP | BPF_JSGT | BPF_K] =  jsgt_imm,

	[BPF_JMP | BPF_JSGE | BPF_K] =  jsge_imm,

	[BPF_JMP | BPF_JSLT | BPF_K] =  jslt_imm,

	[BPF_JMP | BPF_JSLE | BPF_K] =  jsle_imm,

	[BPF_JMP | BPF_JSET | BPF_K] =	jset_imm,

	[BPF_JMP | BPF_JNE | BPF_K] =	jne_imm,

	[BPF_JMP | BPF_JEQ | BPF_X] =	jeq_reg,
@@ -2104,24 +2138,16 @@ static const instr_cb_t instr_cb[256] = { 
	[BPF_JMP | BPF_JGE | BPF_X] =	jge_reg,

	[BPF_JMP | BPF_JLT | BPF_X] =	jlt_reg,

	[BPF_JMP | BPF_JLE | BPF_X] =	jle_reg,

	[BPF_JMP | BPF_JSGT | BPF_X] =  jsgt_reg,

	[BPF_JMP | BPF_JSGE | BPF_X] =  jsge_reg,

	[BPF_JMP | BPF_JSLT | BPF_X] =  jslt_reg,

	[BPF_JMP | BPF_JSLE | BPF_X] =  jsle_reg,

	[BPF_JMP | BPF_JSET | BPF_X] =	jset_reg,

	[BPF_JMP | BPF_JNE | BPF_X] =	jne_reg,

	[BPF_JMP | BPF_CALL] =		call,

	[BPF_JMP | BPF_EXIT] =		goto_out,

};



/* --- Misc code --- */

static void br_set_offset(u64 *instr, u16 offset)

{

	u16 addr_lo, addr_hi;



	addr_lo = offset & (OP_BR_ADDR_LO >> __bf_shf(OP_BR_ADDR_LO));

	addr_hi = offset != addr_lo;

	*instr &= ~(OP_BR_ADDR_HI | OP_BR_ADDR_LO);

	*instr |= FIELD_PREP(OP_BR_ADDR_HI, addr_hi);

	*instr |= FIELD_PREP(OP_BR_ADDR_LO, addr_lo);

}



/* --- Assembler logic --- */

static int nfp_fixup_branches(struct nfp_prog *nfp_prog)

{
@@ -2137,11 +2163,9 @@ static int nfp_fixup_branches(struct nfp_prog *nfp_prog) 
			continue;



		if (list_is_last(&meta->l, &nfp_prog->insns))

			idx = nfp_prog->last_bpf_off;

			br_idx = nfp_prog->last_bpf_off;

		else

			idx = list_next_entry(meta, l)->off - 1;



		br_idx = nfp_prog_offset_to_index(nfp_prog, idx);

			br_idx = list_next_entry(meta, l)->off - 1;



		if (!nfp_is_br(nfp_prog->prog[br_idx])) {

			pr_err("Fixup found block not ending in branch %d %02x %016llx!!\n",
@@ -2149,7 +2173,8 @@ static int nfp_fixup_branches(struct nfp_prog *nfp_prog) 
			return -ELOOP;

		}

		/* Leave special branches for later */

		if (FIELD_GET(OP_BR_SPECIAL, nfp_prog->prog[br_idx]))

		if (FIELD_GET(OP_RELO_TYPE, nfp_prog->prog[br_idx]) !=

		    RELO_BR_REL)

			continue;



		if (!meta->jmp_dst) {
@@ -2164,38 +2189,13 @@ static int nfp_fixup_branches(struct nfp_prog *nfp_prog) 
			return -ELOOP;

		}



		for (idx = nfp_prog_offset_to_index(nfp_prog, meta->off);

		     idx <= br_idx; idx++) {

		for (idx = meta->off; idx <= br_idx; idx++) {

			if (!nfp_is_br(nfp_prog->prog[idx]))

				continue;

			br_set_offset(&nfp_prog->prog[idx], jmp_dst->off);

		}

	}



	/* Fixup 'goto out's separately, they can be scattered around */

	for (br_idx = 0; br_idx < nfp_prog->prog_len; br_idx++) {

		enum br_special special;



		if ((nfp_prog->prog[br_idx] & OP_BR_BASE_MASK) != OP_BR_BASE)

			continue;



		special = FIELD_GET(OP_BR_SPECIAL, nfp_prog->prog[br_idx]);

		switch (special) {

		case OP_BR_NORMAL:

			break;

		case OP_BR_GO_OUT:

			br_set_offset(&nfp_prog->prog[br_idx],

				      nfp_prog->tgt_out);

			break;

		case OP_BR_GO_ABORT:

			br_set_offset(&nfp_prog->prog[br_idx],

				      nfp_prog->tgt_abort);

			break;

		}



		nfp_prog->prog[br_idx] &= ~OP_BR_SPECIAL;

	}



	return 0;

}
@@ -2223,7 +2223,7 @@ static void nfp_outro_tc_da(struct nfp_prog *nfp_prog) 
	/* Target for aborts */

	nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog);



	emit_br_def(nfp_prog, nfp_prog->tgt_done, 2);

	emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT);



	wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);

	emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x11), SHF_SC_L_SHF, 16);
@@ -2250,7 +2250,7 @@ static void nfp_outro_tc_da(struct nfp_prog *nfp_prog) 
	emit_shf(nfp_prog, reg_b(2),

		 reg_imm(0xf), SHF_OP_AND, reg_b(3), SHF_SC_R_SHF, 0);



	emit_br_def(nfp_prog, nfp_prog->tgt_done, 2);

	emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT);



	emit_shf(nfp_prog, reg_b(2),

		 reg_a(2), SHF_OP_OR, reg_b(2), SHF_SC_L_SHF, 4);
@@ -2269,7 +2269,7 @@ static void nfp_outro_xdp(struct nfp_prog *nfp_prog) 
	/* Target for aborts */

	nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog);



	emit_br_def(nfp_prog, nfp_prog->tgt_done, 2);

	emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT);



	wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);

	emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x82), SHF_SC_L_SHF, 16);
@@ -2290,7 +2290,7 @@ static void nfp_outro_xdp(struct nfp_prog *nfp_prog) 
	emit_shf(nfp_prog, reg_b(2),

		 reg_imm(0xff), SHF_OP_AND, reg_b(2), SHF_SC_R_SHF, 0);



	emit_br_def(nfp_prog, nfp_prog->tgt_done, 2);

	emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT);



	wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);

	emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16);
@@ -2706,25 +2706,38 @@ static int nfp_bpf_optimize(struct nfp_prog *nfp_prog) 
	return 0;

}



static int nfp_bpf_ustore_calc(struct nfp_prog *nfp_prog, __le64 *ustore)

static int nfp_bpf_ustore_calc(u64 *prog, unsigned int len)

{

	__le64 *ustore = (__force __le64 *)prog;

	int i;



	for (i = 0; i < nfp_prog->prog_len; i++) {

	for (i = 0; i < len; i++) {

		int err;



		err = nfp_ustore_check_valid_no_ecc(nfp_prog->prog[i]);

		err = nfp_ustore_check_valid_no_ecc(prog[i]);

		if (err)

			return err;



		nfp_prog->prog[i] = nfp_ustore_calc_ecc_insn(nfp_prog->prog[i]);



		ustore[i] = cpu_to_le64(nfp_prog->prog[i]);

		ustore[i] = cpu_to_le64(nfp_ustore_calc_ecc_insn(prog[i]));

	}



	return 0;

}



static void nfp_bpf_prog_trim(struct nfp_prog *nfp_prog)

{

	void *prog;



	prog = kvmalloc_array(nfp_prog->prog_len, sizeof(u64), GFP_KERNEL);

	if (!prog)

		return;



	nfp_prog->__prog_alloc_len = nfp_prog->prog_len * sizeof(u64);

	memcpy(prog, nfp_prog->prog, nfp_prog->__prog_alloc_len);

	kvfree(nfp_prog->prog);

	nfp_prog->prog = prog;

}



int nfp_bpf_jit(struct nfp_prog *nfp_prog)

{

	int ret;
@@ -2740,5 +2753,78 @@ int nfp_bpf_jit(struct nfp_prog *nfp_prog) 
		return -EINVAL;

	}



	return nfp_bpf_ustore_calc(nfp_prog, (__force __le64 *)nfp_prog->prog);

	nfp_bpf_prog_trim(nfp_prog);



	return ret;

}



void nfp_bpf_jit_prepare(struct nfp_prog *nfp_prog, unsigned int cnt)

{

	struct nfp_insn_meta *meta;



	/* Another pass to record jump information. */

	list_for_each_entry(meta, &nfp_prog->insns, l) {

		u64 code = meta->insn.code;



		if (BPF_CLASS(code) == BPF_JMP && BPF_OP(code) != BPF_EXIT &&

		    BPF_OP(code) != BPF_CALL) {

			struct nfp_insn_meta *dst_meta;

			unsigned short dst_indx;



			dst_indx = meta->n + 1 + meta->insn.off;

			dst_meta = nfp_bpf_goto_meta(nfp_prog, meta, dst_indx,

						     cnt);



			meta->jmp_dst = dst_meta;

			dst_meta->flags |= FLAG_INSN_IS_JUMP_DST;

		}

	}

}



void *nfp_bpf_relo_for_vnic(struct nfp_prog *nfp_prog, struct nfp_bpf_vnic *bv)

{

	unsigned int i;

	u64 *prog;

	int err;



	prog = kmemdup(nfp_prog->prog, nfp_prog->prog_len * sizeof(u64),

		       GFP_KERNEL);

	if (!prog)

		return ERR_PTR(-ENOMEM);



	for (i = 0; i < nfp_prog->prog_len; i++) {

		enum nfp_relo_type special;



		special = FIELD_GET(OP_RELO_TYPE, prog[i]);

		switch (special) {

		case RELO_NONE:

			continue;

		case RELO_BR_REL:

			br_add_offset(&prog[i], bv->start_off);

			break;

		case RELO_BR_GO_OUT:

			br_set_offset(&prog[i],

				      nfp_prog->tgt_out + bv->start_off);

			break;

		case RELO_BR_GO_ABORT:

			br_set_offset(&prog[i],

				      nfp_prog->tgt_abort + bv->start_off);

			break;

		case RELO_BR_NEXT_PKT:

			br_set_offset(&prog[i], bv->tgt_done);

			break;

		}



		prog[i] &= ~OP_RELO_TYPE;

	}



	err = nfp_bpf_ustore_calc(prog, nfp_prog->prog_len);

	if (err)

		goto err_free_prog;



	return prog;



err_free_prog:

	kfree(prog);

	return ERR_PTR(err);

}

drivers/net/ethernet/netronome/nfp/bpf/main.c

+31 −7
Original line number Diff line number Diff line
@@ -87,16 +87,21 @@ static const char *nfp_bpf_extra_cap(struct nfp_app *app, struct nfp_net *nn) 
static int

nfp_bpf_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id)

{

	struct nfp_bpf_vnic *bv;

	int err;



	nn->app_priv = kzalloc(sizeof(struct nfp_bpf_vnic), GFP_KERNEL);

	if (!nn->app_priv)

	bv = kzalloc(sizeof(*bv), GFP_KERNEL);

	if (!bv)

		return -ENOMEM;

	nn->app_priv = bv;



	err = nfp_app_nic_vnic_alloc(app, nn, id);

	if (err)

		goto err_free_priv;



	bv->start_off = nn_readw(nn, NFP_NET_CFG_BPF_START);

	bv->tgt_done = nn_readw(nn, NFP_NET_CFG_BPF_DONE);



	return 0;

err_free_priv:

	kfree(nn->app_priv);
@@ -191,7 +196,27 @@ static int nfp_bpf_setup_tc(struct nfp_app *app, struct net_device *netdev, 


static bool nfp_bpf_tc_busy(struct nfp_app *app, struct nfp_net *nn)

{

	return nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF;

	struct nfp_bpf_vnic *bv = nn->app_priv;



	return !!bv->tc_prog;

}



static int

nfp_bpf_change_mtu(struct nfp_app *app, struct net_device *netdev, int new_mtu)

{

	struct nfp_net *nn = netdev_priv(netdev);

	unsigned int max_mtu;



	if (~nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)

		return 0;



	max_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32;

	if (new_mtu > max_mtu) {

		nn_info(nn, "BPF offload active, MTU over %u not supported\n",

			max_mtu);

		return -EBUSY;

	}

	return 0;

}



static int
@@ -311,6 +336,8 @@ const struct nfp_app_type app_bpf = { 
	.init		= nfp_bpf_init,

	.clean		= nfp_bpf_clean,



	.change_mtu	= nfp_bpf_change_mtu,



	.extra_cap	= nfp_bpf_extra_cap,



	.vnic_alloc	= nfp_bpf_vnic_alloc,
@@ -318,9 +345,6 @@ const struct nfp_app_type app_bpf = { 


	.setup_tc	= nfp_bpf_setup_tc,

	.tc_busy	= nfp_bpf_tc_busy,

	.bpf		= nfp_ndo_bpf,

	.xdp_offload	= nfp_bpf_xdp_offload,



	.bpf_verifier_prep	= nfp_bpf_verifier_prep,

	.bpf_translate		= nfp_bpf_translate,

	.bpf_destroy		= nfp_bpf_destroy,

};

drivers/net/ethernet/netronome/nfp/bpf/main.h

+27 −17
Original line number Diff line number Diff line
@@ -42,17 +42,28 @@ 


#include "../nfp_asm.h"



/* For branch fixup logic use up-most byte of branch instruction as scratch

/* For relocation logic use up-most byte of branch instruction as scratch

 * area.  Remember to clear this before sending instructions to HW!

 */

#define OP_BR_SPECIAL	0xff00000000000000ULL



enum br_special {

	OP_BR_NORMAL = 0,

	OP_BR_GO_OUT,

	OP_BR_GO_ABORT,

#define OP_RELO_TYPE	0xff00000000000000ULL



enum nfp_relo_type {

	RELO_NONE = 0,

	/* standard internal jumps */

	RELO_BR_REL,

	/* internal jumps to parts of the outro */

	RELO_BR_GO_OUT,

	RELO_BR_GO_ABORT,

	/* external jumps to fixed addresses */

	RELO_BR_NEXT_PKT,

};



/* To make absolute relocated branches (branches other than RELO_BR_REL)

 * distinguishable in user space dumps from normal jumps, add a large offset

 * to them.

 */

#define BR_OFF_RELO		15000



enum static_regs {

	STATIC_REG_IMM		= 21, /* Bank AB */

	STATIC_REG_STACK	= 22, /* Bank A */
@@ -191,11 +202,9 @@ static inline bool is_mbpf_store(const struct nfp_insn_meta *meta) 
 * @__prog_alloc_len: alloc size of @prog array

 * @verifier_meta: temporary storage for verifier's insn meta

 * @type: BPF program type

 * @start_off: address of the first instruction in the memory

 * @last_bpf_off: address of the last instruction translated from BPF

 * @tgt_out: jump target for normal exit

 * @tgt_abort: jump target for abort (e.g. access outside of packet buffer)

 * @tgt_done: jump target to get the next packet

 * @n_translated: number of successfully translated instructions (for errors)

 * @error: error code if something went wrong

 * @stack_depth: max stack depth from the verifier
@@ -213,11 +222,9 @@ struct nfp_prog { 


	enum bpf_prog_type type;



	unsigned int start_off;

	unsigned int last_bpf_off;

	unsigned int tgt_out;

	unsigned int tgt_abort;

	unsigned int tgt_done;



	unsigned int n_translated;

	int error;
@@ -231,11 +238,16 @@ struct nfp_prog { 
/**

 * struct nfp_bpf_vnic - per-vNIC BPF priv structure

 * @tc_prog:	currently loaded cls_bpf program

 * @start_off:	address of the first instruction in the memory

 * @tgt_done:	jump target to get the next packet

 */

struct nfp_bpf_vnic {

	struct bpf_prog *tc_prog;

	unsigned int start_off;

	unsigned int tgt_done;

};



void nfp_bpf_jit_prepare(struct nfp_prog *nfp_prog, unsigned int cnt);

int nfp_bpf_jit(struct nfp_prog *prog);



extern const struct bpf_prog_offload_ops nfp_bpf_analyzer_ops;
@@ -244,16 +256,14 @@ struct netdev_bpf; 
struct nfp_app;

struct nfp_net;



int nfp_ndo_bpf(struct nfp_app *app, struct nfp_net *nn,

		struct netdev_bpf *bpf);

int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog,

			bool old_prog);



int nfp_bpf_verifier_prep(struct nfp_app *app, struct nfp_net *nn,

			  struct netdev_bpf *bpf);

int nfp_bpf_translate(struct nfp_app *app, struct nfp_net *nn,

		      struct bpf_prog *prog);

int nfp_bpf_destroy(struct nfp_app *app, struct nfp_net *nn,

		    struct bpf_prog *prog);

struct nfp_insn_meta *

nfp_bpf_goto_meta(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,

		  unsigned int insn_idx, unsigned int n_insns);



void *nfp_bpf_relo_for_vnic(struct nfp_prog *nfp_prog, struct nfp_bpf_vnic *bv);

#endif

drivers/net/ethernet/netronome/nfp/bpf/offload.c

+34 −31
Original line number Diff line number Diff line
@@ -42,6 +42,7 @@ 
#include <linux/jiffies.h>

#include <linux/timer.h>

#include <linux/list.h>

#include <linux/mm.h>



#include <net/pkt_cls.h>

#include <net/tc_act/tc_gact.h>
@@ -70,23 +71,7 @@ nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog, 
		list_add_tail(&meta->l, &nfp_prog->insns);

	}



	/* Another pass to record jump information. */

	list_for_each_entry(meta, &nfp_prog->insns, l) {

		u64 code = meta->insn.code;



		if (BPF_CLASS(code) == BPF_JMP && BPF_OP(code) != BPF_EXIT &&

		    BPF_OP(code) != BPF_CALL) {

			struct nfp_insn_meta *dst_meta;

			unsigned short dst_indx;



			dst_indx = meta->n + 1 + meta->insn.off;

			dst_meta = nfp_bpf_goto_meta(nfp_prog, meta, dst_indx,

						     cnt);



			meta->jmp_dst = dst_meta;

			dst_meta->flags |= FLAG_INSN_IS_JUMP_DST;

		}

	}

	nfp_bpf_jit_prepare(nfp_prog, cnt);



	return 0;

}
@@ -102,7 +87,8 @@ static void nfp_prog_free(struct nfp_prog *nfp_prog) 
	kfree(nfp_prog);

}



int nfp_bpf_verifier_prep(struct nfp_app *app, struct nfp_net *nn,

static int

nfp_bpf_verifier_prep(struct nfp_app *app, struct nfp_net *nn,

		      struct netdev_bpf *bpf)

{

	struct bpf_prog *prog = bpf->verifier.prog;
@@ -133,8 +119,7 @@ int nfp_bpf_verifier_prep(struct nfp_app *app, struct nfp_net *nn, 
	return ret;

}



int nfp_bpf_translate(struct nfp_app *app, struct nfp_net *nn,

		      struct bpf_prog *prog)

static int nfp_bpf_translate(struct nfp_net *nn, struct bpf_prog *prog)

{

	struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;

	unsigned int stack_size;
@@ -146,37 +131,48 @@ int nfp_bpf_translate(struct nfp_app *app, struct nfp_net *nn, 
			prog->aux->stack_depth, stack_size);

		return -EOPNOTSUPP;

	}



	nfp_prog->stack_depth = prog->aux->stack_depth;

	nfp_prog->start_off = nn_readw(nn, NFP_NET_CFG_BPF_START);

	nfp_prog->tgt_done = nn_readw(nn, NFP_NET_CFG_BPF_DONE);

	nfp_prog->stack_depth = round_up(prog->aux->stack_depth, 4);



	max_instr = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN);

	nfp_prog->__prog_alloc_len = max_instr * sizeof(u64);



	nfp_prog->prog = kmalloc(nfp_prog->__prog_alloc_len, GFP_KERNEL);

	nfp_prog->prog = kvmalloc(nfp_prog->__prog_alloc_len, GFP_KERNEL);

	if (!nfp_prog->prog)

		return -ENOMEM;



	return nfp_bpf_jit(nfp_prog);

}



int nfp_bpf_destroy(struct nfp_app *app, struct nfp_net *nn,

		    struct bpf_prog *prog)

static int nfp_bpf_destroy(struct nfp_net *nn, struct bpf_prog *prog)

{

	struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;



	kfree(nfp_prog->prog);

	kvfree(nfp_prog->prog);

	nfp_prog_free(nfp_prog);



	return 0;

}



int nfp_ndo_bpf(struct nfp_app *app, struct nfp_net *nn, struct netdev_bpf *bpf)

{

	switch (bpf->command) {

	case BPF_OFFLOAD_VERIFIER_PREP:

		return nfp_bpf_verifier_prep(app, nn, bpf);

	case BPF_OFFLOAD_TRANSLATE:

		return nfp_bpf_translate(nn, bpf->offload.prog);

	case BPF_OFFLOAD_DESTROY:

		return nfp_bpf_destroy(nn, bpf->offload.prog);

	default:

		return -EINVAL;

	}

}



static int nfp_net_bpf_load(struct nfp_net *nn, struct bpf_prog *prog)

{

	struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;

	unsigned int max_mtu;

	dma_addr_t dma_addr;

	void *img;

	int err;



	max_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32;
@@ -185,11 +181,17 @@ static int nfp_net_bpf_load(struct nfp_net *nn, struct bpf_prog *prog) 
		return -EOPNOTSUPP;

	}



	dma_addr = dma_map_single(nn->dp.dev, nfp_prog->prog,

	img = nfp_bpf_relo_for_vnic(nfp_prog, nn->app_priv);

	if (IS_ERR(img))

		return PTR_ERR(img);



	dma_addr = dma_map_single(nn->dp.dev, img,

				  nfp_prog->prog_len * sizeof(u64),

				  DMA_TO_DEVICE);

	if (dma_mapping_error(nn->dp.dev, dma_addr))

	if (dma_mapping_error(nn->dp.dev, dma_addr)) {

		kfree(img);

		return -ENOMEM;

	}



	nn_writew(nn, NFP_NET_CFG_BPF_SIZE, nfp_prog->prog_len);

	nn_writeq(nn, NFP_NET_CFG_BPF_ADDR, dma_addr);
@@ -201,6 +203,7 @@ static int nfp_net_bpf_load(struct nfp_net *nn, struct bpf_prog *prog) 


	dma_unmap_single(nn->dp.dev, dma_addr, nfp_prog->prog_len * sizeof(u64),

			 DMA_TO_DEVICE);

	kfree(img);



	return err;

}

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

+16 −14
Original line number Diff line number Diff line
@@ -31,8 +31,6 @@ 
 * SOFTWARE.

 */



#define pr_fmt(fmt)	"NFP net bpf: " fmt



#include <linux/bpf.h>

#include <linux/bpf_verifier.h>

#include <linux/kernel.h>
@@ -41,6 +39,9 @@ 
#include "fw.h"

#include "main.h"



#define pr_vlog(env, fmt, ...)	\

	bpf_verifier_log_write(env, "[nfp] " fmt, ##__VA_ARGS__)



struct nfp_insn_meta *

nfp_bpf_goto_meta(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,

		  unsigned int insn_idx, unsigned int n_insns)
@@ -116,18 +117,18 @@ nfp_bpf_check_call(struct nfp_prog *nfp_prog, struct bpf_verifier_env *env, 
	switch (func_id) {

	case BPF_FUNC_xdp_adjust_head:

		if (!bpf->adjust_head.off_max) {

			pr_warn("adjust_head not supported by FW\n");

			pr_vlog(env, "adjust_head not supported by FW\n");

			return -EOPNOTSUPP;

		}

		if (!(bpf->adjust_head.flags & NFP_BPF_ADJUST_HEAD_NO_META)) {

			pr_warn("adjust_head: FW requires shifting metadata, not supported by the driver\n");

			pr_vlog(env, "adjust_head: FW requires shifting metadata, not supported by the driver\n");

			return -EOPNOTSUPP;

		}



		nfp_record_adjust_head(bpf, nfp_prog, meta, reg2);

		break;

	default:

		pr_warn("unsupported function id: %d\n", func_id);

		pr_vlog(env, "unsupported function id: %d\n", func_id);

		return -EOPNOTSUPP;

	}
@@ -150,7 +151,7 @@ nfp_bpf_check_exit(struct nfp_prog *nfp_prog, 
		char tn_buf[48];



		tnum_strn(tn_buf, sizeof(tn_buf), reg0->var_off);

		pr_info("unsupported exit state: %d, var_off: %s\n",

		pr_vlog(env, "unsupported exit state: %d, var_off: %s\n",

			reg0->type, tn_buf);

		return -EINVAL;

	}
@@ -160,7 +161,7 @@ nfp_bpf_check_exit(struct nfp_prog *nfp_prog, 
	    imm <= TC_ACT_REDIRECT &&

	    imm != TC_ACT_SHOT && imm != TC_ACT_STOLEN &&

	    imm != TC_ACT_QUEUED) {

		pr_info("unsupported exit state: %d, imm: %llx\n",

		pr_vlog(env, "unsupported exit state: %d, imm: %llx\n",

			reg0->type, imm);

		return -EINVAL;

	}
@@ -171,12 +172,13 @@ nfp_bpf_check_exit(struct nfp_prog *nfp_prog, 
static int

nfp_bpf_check_stack_access(struct nfp_prog *nfp_prog,

			   struct nfp_insn_meta *meta,

			   const struct bpf_reg_state *reg)

			   const struct bpf_reg_state *reg,

			   struct bpf_verifier_env *env)

{

	s32 old_off, new_off;



	if (!tnum_is_const(reg->var_off)) {

		pr_info("variable ptr stack access\n");

		pr_vlog(env, "variable ptr stack access\n");

		return -EINVAL;

	}
@@ -194,7 +196,7 @@ nfp_bpf_check_stack_access(struct nfp_prog *nfp_prog, 
	if (old_off % 4 == new_off % 4)

		return 0;



	pr_info("stack access changed location was:%d is:%d\n",

	pr_vlog(env, "stack access changed location was:%d is:%d\n",

		old_off, new_off);

	return -EINVAL;

}
@@ -209,18 +211,18 @@ nfp_bpf_check_ptr(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, 
	if (reg->type != PTR_TO_CTX &&

	    reg->type != PTR_TO_STACK &&

	    reg->type != PTR_TO_PACKET) {

		pr_info("unsupported ptr type: %d\n", reg->type);

		pr_vlog(env, "unsupported ptr type: %d\n", reg->type);

		return -EINVAL;

	}



	if (reg->type == PTR_TO_STACK) {

		err = nfp_bpf_check_stack_access(nfp_prog, meta, reg);

		err = nfp_bpf_check_stack_access(nfp_prog, meta, reg, env);

		if (err)

			return err;

	}



	if (meta->ptr.type != NOT_INIT && meta->ptr.type != reg->type) {

		pr_info("ptr type changed for instruction %d -> %d\n",

		pr_vlog(env, "ptr type changed for instruction %d -> %d\n",

			meta->ptr.type, reg->type);

		return -EINVAL;

	}
@@ -241,7 +243,7 @@ nfp_verify_insn(struct bpf_verifier_env *env, int insn_idx, int prev_insn_idx) 


	if (meta->insn.src_reg >= MAX_BPF_REG ||

	    meta->insn.dst_reg >= MAX_BPF_REG) {

		pr_err("program uses extended registers - jit hardening?\n");

		pr_vlog(env, "program uses extended registers - jit hardening?\n");

		return -EINVAL;

	}