Merge branch 'bpf-nfp-jit-adjust-head-support' (1c45597e) · Commits · e / devices / android_kernel_teracube_emerald

drivers/net/ethernet/netronome/nfp/bpf/fw.h

0 → 100644

+54 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2017 Netronome Systems, Inc.
		*
		* This software is dual licensed under the GNU General License Version 2,
		* June 1991 as shown in the file COPYING in the top-level directory of this
		* source tree or the BSD 2-Clause License provided below. You have the
		* option to license this software under the complete terms of either license.
		*
		* The BSD 2-Clause License:
		*
		* Redistribution and use in source and binary forms, with or
		* without modification, are permitted provided that the following
		* conditions are met:
		*
		* 1. Redistributions of source code must retain the above
		* copyright notice, this list of conditions and the following
		* disclaimer.
		*
		* 2. Redistributions in binary form must reproduce the above
		* copyright notice, this list of conditions and the following
		* disclaimer in the documentation and/or other materials
		* provided with the distribution.
		*
		* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
		* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
		* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
		* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
		* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
		* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
		* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
		* SOFTWARE.
		*/

		#ifndef NFP_BPF_FW_H
		#define NFP_BPF_FW_H 1

		#include <linux/bitops.h>
		#include <linux/types.h>

		enum bpf_cap_tlv_type {
		NFP_BPF_CAP_TYPE_ADJUST_HEAD = 2,
		};

		struct nfp_bpf_cap_tlv_adjust_head {
		__le32 flags;
		__le32 off_min;
		__le32 off_max;
		__le32 guaranteed_sub;
		__le32 guaranteed_add;
		};

		#define NFP_BPF_ADJUST_HEAD_NO_META BIT(0)

		#endif

drivers/net/ethernet/netronome/nfp/bpf/jit.c

+107 −0

Original line number	Diff line number	Diff line
		@@ -33,6 +33,7 @@

		#define pr_fmt(fmt) "NFP net bpf: " fmt

		#include <linux/bug.h>
		#include <linux/kernel.h>
		#include <linux/bpf.h>
		#include <linux/filter.h>
		@@ -87,6 +88,18 @@ static unsigned int nfp_prog_current_offset(struct nfp_prog *nfp_prog)
		return nfp_prog->start_off + nfp_prog->prog_len;
		}

		static bool
		nfp_prog_confirm_current_offset(struct nfp_prog *nfp_prog, unsigned int off)
		{
		/* If there is a recorded error we may have dropped instructions;
		* that doesn't have to be due to translator bug, and the translation
		* will fail anyway, so just return OK.
		*/
		if (nfp_prog->error)
		return true;
		return !WARN_ON_ONCE(nfp_prog_current_offset(nfp_prog) != off);
		}

		static unsigned int
		nfp_prog_offset_to_index(struct nfp_prog *nfp_prog, unsigned int offset)
		{
		@@ -1196,6 +1209,86 @@ static void wrp_end32(struct nfp_prog *nfp_prog, swreg reg_in, u8 gpr_out)
		SHF_SC_R_ROT, 16);
		}

		static int adjust_head(struct nfp_prog nfp_prog, struct nfp_insn_meta meta)
		{
		swreg tmp = imm_a(nfp_prog), tmp_len = imm_b(nfp_prog);
		struct nfp_bpf_cap_adjust_head *adjust_head;
		u32 ret_einval, end;

		adjust_head = &nfp_prog->bpf->adjust_head;

		/* Optimized version - 5 vs 14 cycles */
		if (nfp_prog->adjust_head_location != UINT_MAX) {
		if (WARN_ON_ONCE(nfp_prog->adjust_head_location != meta->n))
		return -EINVAL;

		emit_alu(nfp_prog, pptr_reg(nfp_prog),
		reg_a(2 * 2), ALU_OP_ADD, pptr_reg(nfp_prog));
		emit_alu(nfp_prog, plen_reg(nfp_prog),
		plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2));
		emit_alu(nfp_prog, pv_len(nfp_prog),
		pv_len(nfp_prog), ALU_OP_SUB, reg_a(2 * 2));

		wrp_immed(nfp_prog, reg_both(0), 0);
		wrp_immed(nfp_prog, reg_both(1), 0);

		/* TODO: when adjust head is guaranteed to succeed we can
		* also eliminate the following if (r0 == 0) branch.
		*/

		return 0;
		}

		ret_einval = nfp_prog_current_offset(nfp_prog) + 14;
		end = ret_einval + 2;

		/* We need to use a temp because offset is just a part of the pkt ptr */
		emit_alu(nfp_prog, tmp,
		reg_a(2 * 2), ALU_OP_ADD_2B, pptr_reg(nfp_prog));

		/* Validate result will fit within FW datapath constraints */
		emit_alu(nfp_prog, reg_none(),
		tmp, ALU_OP_SUB, reg_imm(adjust_head->off_min));
		emit_br(nfp_prog, BR_BLO, ret_einval, 0);
		emit_alu(nfp_prog, reg_none(),
		reg_imm(adjust_head->off_max), ALU_OP_SUB, tmp);
		emit_br(nfp_prog, BR_BLO, ret_einval, 0);

		/* Validate the length is at least ETH_HLEN */
		emit_alu(nfp_prog, tmp_len,
		plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2));
		emit_alu(nfp_prog, reg_none(),
		tmp_len, ALU_OP_SUB, reg_imm(ETH_HLEN));
		emit_br(nfp_prog, BR_BMI, ret_einval, 0);

		/* Load the ret code */
		wrp_immed(nfp_prog, reg_both(0), 0);
		wrp_immed(nfp_prog, reg_both(1), 0);

		/* Modify the packet metadata */
		emit_ld_field(nfp_prog, pptr_reg(nfp_prog), 0x3, tmp, SHF_SC_NONE, 0);

		/* Skip over the -EINVAL ret code (defer 2) */
		emit_br_def(nfp_prog, end, 2);

		emit_alu(nfp_prog, plen_reg(nfp_prog),
		plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2));
		emit_alu(nfp_prog, pv_len(nfp_prog),
		pv_len(nfp_prog), ALU_OP_SUB, reg_a(2 * 2));

		/* return -EINVAL target */
		if (!nfp_prog_confirm_current_offset(nfp_prog, ret_einval))
		return -EINVAL;

		wrp_immed(nfp_prog, reg_both(0), -22);
		wrp_immed(nfp_prog, reg_both(1), ~0);

		if (!nfp_prog_confirm_current_offset(nfp_prog, end))
		return -EINVAL;

		return 0;
		}

		/* --- Callbacks --- */
		static int mov_reg64(struct nfp_prog nfp_prog, struct nfp_insn_meta meta)
		{
		@@ -1930,6 +2023,17 @@ static int jne_reg(struct nfp_prog nfp_prog, struct nfp_insn_meta meta)
		return wrp_test_reg(nfp_prog, meta, ALU_OP_XOR, BR_BNE);
		}

		static int call(struct nfp_prog nfp_prog, struct nfp_insn_meta meta)
		{
		switch (meta->insn.imm) {
		case BPF_FUNC_xdp_adjust_head:
		return adjust_head(nfp_prog, meta);
		default:
		WARN_ONCE(1, "verifier allowed unsupported function\n");
		return -EOPNOTSUPP;
		}
		}

		static int goto_out(struct nfp_prog nfp_prog, struct nfp_insn_meta meta)
		{
		wrp_br_special(nfp_prog, BR_UNC, OP_BR_GO_OUT);
		@@ -2002,6 +2106,7 @@ static const instr_cb_t instr_cb[256] = {
		[BPF_JMP \| BPF_JLE \| BPF_X] = jle_reg,
		[BPF_JMP \| BPF_JSET \| BPF_X] = jset_reg,
		[BPF_JMP \| BPF_JNE \| BPF_X] = jne_reg,
		[BPF_JMP \| BPF_CALL] = call,
		[BPF_JMP \| BPF_EXIT] = goto_out,
		};

		@@ -2026,6 +2131,8 @@ static int nfp_fixup_branches(struct nfp_prog *nfp_prog)
		list_for_each_entry(meta, &nfp_prog->insns, l) {
		if (meta->skip)
		continue;
		if (meta->insn.code == (BPF_JMP \| BPF_CALL))
		continue;
		if (BPF_CLASS(meta->insn.code) != BPF_JMP)
		continue;

drivers/net/ethernet/netronome/nfp/bpf/main.c

+115 −0

Original line number	Diff line number	Diff line
		@@ -34,10 +34,12 @@
		#include <net/pkt_cls.h>

		#include "../nfpcore/nfp_cpp.h"
		#include "../nfpcore/nfp_nffw.h"
		#include "../nfp_app.h"
		#include "../nfp_main.h"
		#include "../nfp_net.h"
		#include "../nfp_port.h"
		#include "fw.h"
		#include "main.h"

		static bool nfp_net_ebpf_capable(struct nfp_net *nn)
		@@ -155,10 +157,123 @@ static bool nfp_bpf_tc_busy(struct nfp_app app, struct nfp_net nn)
		return nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF;
		}

		static int
		nfp_bpf_parse_cap_adjust_head(struct nfp_app_bpf bpf, void __iomem value,
		u32 length)
		{
		struct nfp_bpf_cap_tlv_adjust_head __iomem *cap = value;
		struct nfp_cpp *cpp = bpf->app->pf->cpp;

		if (length < sizeof(*cap)) {
		nfp_err(cpp, "truncated adjust_head TLV: %d\n", length);
		return -EINVAL;
		}

		bpf->adjust_head.flags = readl(&cap->flags);
		bpf->adjust_head.off_min = readl(&cap->off_min);
		bpf->adjust_head.off_max = readl(&cap->off_max);
		bpf->adjust_head.guaranteed_sub = readl(&cap->guaranteed_sub);
		bpf->adjust_head.guaranteed_add = readl(&cap->guaranteed_add);

		if (bpf->adjust_head.off_min > bpf->adjust_head.off_max) {
		nfp_err(cpp, "invalid adjust_head TLV: min > max\n");
		return -EINVAL;
		}
		if (!FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_min) \|\|
		!FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_max)) {
		nfp_warn(cpp, "disabling adjust_head - driver expects min/max to fit in as immediates\n");
		memset(&bpf->adjust_head, 0, sizeof(bpf->adjust_head));
		return 0;
		}

		return 0;
		}

		static int nfp_bpf_parse_capabilities(struct nfp_app *app)
		{
		struct nfp_cpp *cpp = app->pf->cpp;
		struct nfp_cpp_area *area;
		u8 __iomem mem, start;

		mem = nfp_rtsym_map(app->pf->rtbl, "_abi_bpf_capabilities", "bpf.cap",
		8, &area);
		if (IS_ERR(mem))
		return PTR_ERR(mem) == -ENOENT ? 0 : PTR_ERR(mem);

		start = mem;
		while (mem - start + 8 < nfp_cpp_area_size(area)) {
		u8 __iomem *value;
		u32 type, length;

		type = readl(mem);
		length = readl(mem + 4);
		value = mem + 8;

		mem += 8 + length;
		if (mem - start > nfp_cpp_area_size(area))
		goto err_release_free;

		switch (type) {
		case NFP_BPF_CAP_TYPE_ADJUST_HEAD:
		if (nfp_bpf_parse_cap_adjust_head(app->priv, value,
		length))
		goto err_release_free;
		break;
		default:
		nfp_dbg(cpp, "unknown BPF capability: %d\n", type);
		break;
		}
		}
		if (mem - start != nfp_cpp_area_size(area)) {
		nfp_err(cpp, "BPF capabilities left after parsing, parsed:%lu total length:%lu\n",
		mem - start, nfp_cpp_area_size(area));
		goto err_release_free;
		}

		nfp_cpp_area_release_free(area);

		return 0;

		err_release_free:
		nfp_err(cpp, "invalid BPF capabilities at offset:%ld\n", mem - start);
		nfp_cpp_area_release_free(area);
		return -EINVAL;
		}

		static int nfp_bpf_init(struct nfp_app *app)
		{
		struct nfp_app_bpf *bpf;
		int err;

		bpf = kzalloc(sizeof(*bpf), GFP_KERNEL);
		if (!bpf)
		return -ENOMEM;
		bpf->app = app;
		app->priv = bpf;

		err = nfp_bpf_parse_capabilities(app);
		if (err)
		goto err_free_bpf;

		return 0;

		err_free_bpf:
		kfree(bpf);
		return err;
		}

		static void nfp_bpf_clean(struct nfp_app *app)
		{
		kfree(app->priv);
		}

		const struct nfp_app_type app_bpf = {
		.id = NFP_APP_BPF_NIC,
		.name = "ebpf",

		.init = nfp_bpf_init,
		.clean = nfp_bpf_clean,

		.extra_cap = nfp_bpf_extra_cap,

		.vnic_alloc = nfp_app_nic_vnic_alloc,

drivers/net/ethernet/netronome/nfp/bpf/main.h

+30 −0

Original line number	Diff line number	Diff line
		@@ -78,6 +78,29 @@ enum pkt_vec {
		#define NFP_BPF_ABI_FLAGS reg_imm(0)
		#define NFP_BPF_ABI_FLAG_MARK 1

		/**
		* struct nfp_app_bpf - bpf app priv structure
		* @app: backpointer to the app
		*
		* @adjust_head: adjust head capability
		* @flags: extra flags for adjust head
		* @off_min: minimal packet offset within buffer required
		* @off_max: maximum packet offset within buffer required
		* @guaranteed_sub: amount of negative adjustment guaranteed possible
		* @guaranteed_add: amount of positive adjustment guaranteed possible
		*/
		struct nfp_app_bpf {
		struct nfp_app *app;

		struct nfp_bpf_cap_adjust_head {
		u32 flags;
		int off_min;
		int off_max;
		int guaranteed_sub;
		int guaranteed_add;
		} adjust_head;
		};

		struct nfp_prog;
		struct nfp_insn_meta;
		typedef int (instr_cb_t)(struct nfp_prog , struct nfp_insn_meta *);
		@@ -97,6 +120,7 @@ typedef int (instr_cb_t)(struct nfp_prog , struct nfp_insn_meta *);
		* @ptr: pointer type for memory operations
		* @ldst_gather_len: memcpy length gathered from load/store sequence
		* @paired_st: the paired store insn at the head of the sequence
		* @arg2: arg2 for call instructions
		* @ptr_not_const: pointer is not always constant
		* @jmp_dst: destination info for jump instructions
		* @off: index of first generated machine instruction (in nfp_prog.prog)
		@@ -116,6 +140,7 @@ struct nfp_insn_meta {
		bool ptr_not_const;
		};
		struct nfp_insn_meta *jmp_dst;
		struct bpf_reg_state arg2;
		};
		unsigned int off;
		unsigned short n;
		@@ -160,6 +185,7 @@ static inline bool is_mbpf_store(const struct nfp_insn_meta *meta)

		/**
		* struct nfp_prog - nfp BPF program
		* @bpf: backpointer to the bpf app priv structure
		* @prog: machine code
		* @prog_len: number of valid instructions in @prog array
		* @__prog_alloc_len: alloc size of @prog array
		@@ -173,9 +199,12 @@ static inline bool is_mbpf_store(const struct nfp_insn_meta *meta)
		* @n_translated: number of successfully translated instructions (for errors)
		* @error: error code if something went wrong
		* @stack_depth: max stack depth from the verifier
		* @adjust_head_location: if program has single adjust head call - the insn no.
		* @insns: list of BPF instruction wrappers (struct nfp_insn_meta)
		*/
		struct nfp_prog {
		struct nfp_app_bpf *bpf;

		u64 *prog;
		unsigned int prog_len;
		unsigned int __prog_alloc_len;
		@@ -194,6 +223,7 @@ struct nfp_prog {
		int error;

		unsigned int stack_depth;
		unsigned int adjust_head_location;

		struct list_head insns;
		};

drivers/net/ethernet/netronome/nfp/bpf/offload.c

+2 −0

Original line number	Diff line number	Diff line
		@@ -48,6 +48,7 @@
		#include <net/tc_act/tc_mirred.h>

		#include "main.h"
		#include "../nfp_app.h"
		#include "../nfp_net_ctrl.h"
		#include "../nfp_net.h"

		@@ -115,6 +116,7 @@ int nfp_bpf_verifier_prep(struct nfp_app app, struct nfp_net nn,

		INIT_LIST_HEAD(&nfp_prog->insns);
		nfp_prog->type = prog->type;
		nfp_prog->bpf = app->priv;

		ret = nfp_prog_prepare(nfp_prog, prog->insnsi, prog->len);
		if (ret)