Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 293bfa9b authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'xdp-head-adjustment' 



Martin KaFai Lau says:

====================
Allow head adjustment in XDP prog

This series adds a helper to allow head adjusting in XDP prog.  mlx4
driver has been modified to support this feature.  An example is written
to encapsulate a packet with an IPv4/v6 header and then XDP_TX it
out.

v4:
1. Remove XDP_QUERY_FEATURES command.  Instead, check
   the prog->xdp_adjust_head bit inside the driver itself
   during XDP_SETUP_PROG in patch 1of4.
   Thanks for everybody's ideas.
2. Nit changes on sample code per Jesper

v3:
1. Check if the driver supports head adjustment before
   setting the xdp_prog fd to the device in patch 1of4.
2. Remove the page alignment assumption on the data_hard_start.
   Instead, add data_hard_start to the struct xdp_buff and the
   driver has to fill it if it supports head adjustment.
3. Keep the wire MTU as before in mlx4
4. Set map0_byte_count to PAGE_SIZE in patch 3of4

v2:
1. Make a variable name change in bpf_xdp_adjust_head() in patch 1
2. Ensure no less than ETH_HLEN data in bpf_xdp_adjust_head() in patch 1
3. Some clarifications in commit log messages of patch 2 and 3
====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 8a03cf2c 12d8bb64

arch/powerpc/net/bpf_jit_comp64.c

+2 −2
Original line number Diff line number Diff line
@@ -766,7 +766,7 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, 
			func = (u8 *) __bpf_call_base + imm;



			/* Save skb pointer if we need to re-cache skb data */

			if (bpf_helper_changes_skb_data(func))

			if (bpf_helper_changes_pkt_data(func))

				PPC_BPF_STL(3, 1, bpf_jit_stack_local(ctx));



			bpf_jit_emit_func_call(image, ctx, (u64)func);
@@ -775,7 +775,7 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, 
			PPC_MR(b2p[BPF_REG_0], 3);



			/* refresh skb cache */

			if (bpf_helper_changes_skb_data(func)) {

			if (bpf_helper_changes_pkt_data(func)) {

				/* reload skb pointer to r3 */

				PPC_BPF_LL(3, 1, bpf_jit_stack_local(ctx));

				bpf_jit_emit_skb_loads(image, ctx);

arch/s390/net/bpf_jit_comp.c

+1 −1
Original line number Diff line number Diff line
@@ -981,7 +981,7 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i 
		EMIT2(0x0d00, REG_14, REG_W1);

		/* lgr %b0,%r2: load return value into %b0 */

		EMIT4(0xb9040000, BPF_REG_0, REG_2);

		if (bpf_helper_changes_skb_data((void *)func)) {

		if (bpf_helper_changes_pkt_data((void *)func)) {

			jit->seen |= SEEN_SKB_CHANGE;

			/* lg %b1,ST_OFF_SKBP(%r15) */

			EMIT6_DISP_LH(0xe3000000, 0x0004, BPF_REG_1, REG_0,

arch/x86/net/bpf_jit_comp.c

+1 −1
Original line number Diff line number Diff line
@@ -853,7 +853,7 @@ xadd: if (is_imm8(insn->off)) 
			func = (u8 *) __bpf_call_base + imm32;

			jmp_offset = func - (image + addrs[i]);

			if (seen_ld_abs) {

				reload_skb_data = bpf_helper_changes_skb_data(func);

				reload_skb_data = bpf_helper_changes_pkt_data(func);

				if (reload_skb_data) {

					EMIT1(0x57); /* push %rdi */

					jmp_offset += 22; /* pop, mov, sub, mov */

drivers/net/ethernet/mellanox/mlx4/en_netdev.c

+21 −8
Original line number Diff line number Diff line
@@ -51,6 +51,9 @@ 
#include "mlx4_en.h"

#include "en_port.h"



#define MLX4_EN_MAX_XDP_MTU ((int)(PAGE_SIZE - ETH_HLEN - (2 * VLAN_HLEN) - \

				   XDP_PACKET_HEADROOM))



int mlx4_en_setup_tc(struct net_device *dev, u8 up)

{

	struct mlx4_en_priv *priv = netdev_priv(dev);
@@ -2249,6 +2252,19 @@ void mlx4_en_destroy_netdev(struct net_device *dev) 
	free_netdev(dev);

}



static bool mlx4_en_check_xdp_mtu(struct net_device *dev, int mtu)

{

	struct mlx4_en_priv *priv = netdev_priv(dev);



	if (mtu > MLX4_EN_MAX_XDP_MTU) {

		en_err(priv, "mtu:%d > max:%d when XDP prog is attached\n",

		       mtu, MLX4_EN_MAX_XDP_MTU);

		return false;

	}



	return true;

}



static int mlx4_en_change_mtu(struct net_device *dev, int new_mtu)

{

	struct mlx4_en_priv *priv = netdev_priv(dev);
@@ -2258,11 +2274,10 @@ static int mlx4_en_change_mtu(struct net_device *dev, int new_mtu) 
	en_dbg(DRV, priv, "Change MTU called - current:%d new:%d\n",

		 dev->mtu, new_mtu);



	if (priv->tx_ring_num[TX_XDP] && MLX4_EN_EFF_MTU(new_mtu) > FRAG_SZ0) {

		en_err(priv, "MTU size:%d requires frags but XDP running\n",

		       new_mtu);

		return -EOPNOTSUPP;

	}

	if (priv->tx_ring_num[TX_XDP] &&

	    !mlx4_en_check_xdp_mtu(dev, new_mtu))

		return -ENOTSUPP;



	dev->mtu = new_mtu;



	if (netif_running(dev)) {
@@ -2710,10 +2725,8 @@ static int mlx4_xdp_set(struct net_device *dev, struct bpf_prog *prog) 
		return 0;

	}



	if (priv->num_frags > 1) {

		en_err(priv, "Cannot set XDP if MTU requires multiple frags\n");

	if (!mlx4_en_check_xdp_mtu(dev, dev->mtu))

		return -EOPNOTSUPP;

	}



	tmp = kzalloc(sizeof(*tmp), GFP_KERNEL);

	if (!tmp)

drivers/net/ethernet/mellanox/mlx4/en_rx.c

+42 −28
Original line number Diff line number Diff line
@@ -96,7 +96,6 @@ static int mlx4_en_alloc_frags(struct mlx4_en_priv *priv, 
	struct mlx4_en_rx_alloc page_alloc[MLX4_EN_MAX_RX_FRAGS];

	const struct mlx4_en_frag_info *frag_info;

	struct page *page;

	dma_addr_t dma;

	int i;



	for (i = 0; i < priv->num_frags; i++) {
@@ -115,9 +114,10 @@ static int mlx4_en_alloc_frags(struct mlx4_en_priv *priv, 


	for (i = 0; i < priv->num_frags; i++) {

		frags[i] = ring_alloc[i];

		dma = ring_alloc[i].dma + ring_alloc[i].page_offset;

		frags[i].page_offset += priv->frag_info[i].rx_headroom;

		rx_desc->data[i].addr = cpu_to_be64(frags[i].dma +

						    frags[i].page_offset);

		ring_alloc[i] = page_alloc[i];

		rx_desc->data[i].addr = cpu_to_be64(dma);

	}



	return 0;
@@ -250,7 +250,8 @@ static int mlx4_en_prepare_rx_desc(struct mlx4_en_priv *priv, 


	if (ring->page_cache.index > 0) {

		frags[0] = ring->page_cache.buf[--ring->page_cache.index];

		rx_desc->data[0].addr = cpu_to_be64(frags[0].dma);

		rx_desc->data[0].addr = cpu_to_be64(frags[0].dma +

						    frags[0].page_offset);

		return 0;

	}
@@ -889,6 +890,7 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud 
		if (xdp_prog) {

			struct xdp_buff xdp;

			dma_addr_t dma;

			void *orig_data;

			u32 act;



			dma = be64_to_cpu(rx_desc->data[0].addr);
@@ -896,11 +898,19 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud 
						priv->frag_info[0].frag_size,

						DMA_FROM_DEVICE);



			xdp.data = page_address(frags[0].page) +

							frags[0].page_offset;

			xdp.data_hard_start = page_address(frags[0].page);

			xdp.data = xdp.data_hard_start + frags[0].page_offset;

			xdp.data_end = xdp.data + length;

			orig_data = xdp.data;



			act = bpf_prog_run_xdp(xdp_prog, &xdp);



			if (xdp.data != orig_data) {

				length = xdp.data_end - xdp.data;

				frags[0].page_offset = xdp.data -

					xdp.data_hard_start;

			}



			switch (act) {

			case XDP_PASS:

				break;
@@ -1164,38 +1174,42 @@ static const int frag_sizes[] = { 


void mlx4_en_calc_rx_buf(struct net_device *dev)

{

	enum dma_data_direction dma_dir = PCI_DMA_FROMDEVICE;

	struct mlx4_en_priv *priv = netdev_priv(dev);

	int eff_mtu = MLX4_EN_EFF_MTU(dev->mtu);

	int order = MLX4_EN_ALLOC_PREFER_ORDER;

	u32 align = SMP_CACHE_BYTES;

	int buf_size = 0;

	int i = 0;



	/* bpf requires buffers to be set up as 1 packet per page.

	 * This only works when num_frags == 1.

	 */

	if (priv->tx_ring_num[TX_XDP]) {

		dma_dir = PCI_DMA_BIDIRECTIONAL;

		/* This will gain efficient xdp frame recycling at the expense

		 * of more costly truesize accounting

		priv->frag_info[0].order = 0;

		priv->frag_info[0].frag_size = eff_mtu;

		priv->frag_info[0].frag_prefix_size = 0;

		/* This will gain efficient xdp frame recycling at the

		 * expense of more costly truesize accounting

		 */

		align = PAGE_SIZE;

		order = 0;

	}

		priv->frag_info[0].frag_stride = PAGE_SIZE;

		priv->frag_info[0].dma_dir = PCI_DMA_BIDIRECTIONAL;

		priv->frag_info[0].rx_headroom = XDP_PACKET_HEADROOM;

		i = 1;

	} else {

		int buf_size = 0;



		while (buf_size < eff_mtu) {

		priv->frag_info[i].order = order;

			priv->frag_info[i].order = MLX4_EN_ALLOC_PREFER_ORDER;

			priv->frag_info[i].frag_size =

				(eff_mtu > buf_size + frag_sizes[i]) ?

					frag_sizes[i] : eff_mtu - buf_size;

			priv->frag_info[i].frag_prefix_size = buf_size;

			priv->frag_info[i].frag_stride =

				ALIGN(priv->frag_info[i].frag_size, align);

		priv->frag_info[i].dma_dir = dma_dir;

				ALIGN(priv->frag_info[i].frag_size,

				      SMP_CACHE_BYTES);

			priv->frag_info[i].dma_dir = PCI_DMA_FROMDEVICE;

			priv->frag_info[i].rx_headroom = 0;

			buf_size += priv->frag_info[i].frag_size;

			i++;

		}

	}



	priv->num_frags = i;

	priv->rx_skb_size = eff_mtu;