Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit d0bcacd0 authored by Björn Töpel's avatar Björn Töpel Committed by Jeff Kirsher
Browse files

ixgbe: add AF_XDP zero-copy Rx support 



This patch adds zero-copy Rx support for AF_XDP sockets. Instead of
allocating buffers of type MEM_TYPE_PAGE_SHARED, the Rx frames are
allocated as MEM_TYPE_ZERO_COPY when AF_XDP is enabled for a certain
queue.

All AF_XDP specific functions are added to a new file, ixgbe_xsk.c.

Note that when AF_XDP zero-copy is enabled, the XDP action XDP_PASS
will allocate a new buffer and copy the zero-copy frame prior passing
it to the kernel stack.

Signed-off-by: default avatarBjörn Töpel <bjorn.topel@intel.com>
Tested-by: default avatarWilliam Tu <u9012063@gmail.com>
Tested-by: default avatarAndrew Bowers <andrewx.bowers@intel.com>
Signed-off-by: default avatarJeff Kirsher <jeffrey.t.kirsher@intel.com>
parent 46515fdb

drivers/net/ethernet/intel/ixgbe/Makefile

+2 −1
Original line number Diff line number Diff line
@@ -8,7 +8,8 @@ obj-$(CONFIG_IXGBE) += ixgbe.o 


ixgbe-objs := ixgbe_main.o ixgbe_common.o ixgbe_ethtool.o \

              ixgbe_82599.o ixgbe_82598.o ixgbe_phy.o ixgbe_sriov.o \

              ixgbe_mbx.o ixgbe_x540.o ixgbe_x550.o ixgbe_lib.o ixgbe_ptp.o

              ixgbe_mbx.o ixgbe_x540.o ixgbe_x550.o ixgbe_lib.o ixgbe_ptp.o \

              ixgbe_xsk.o



ixgbe-$(CONFIG_IXGBE_DCB) +=  ixgbe_dcb.o ixgbe_dcb_82598.o \

                              ixgbe_dcb_82599.o ixgbe_dcb_nl.o

drivers/net/ethernet/intel/ixgbe/ixgbe.h

+20 −7
Original line number Diff line number Diff line
@@ -228,14 +228,18 @@ struct ixgbe_tx_buffer { 
struct ixgbe_rx_buffer {

	struct sk_buff *skb;

	dma_addr_t dma;

	union {

		struct {

			struct page *page;

#if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536)

			__u32 page_offset;

#else

	__u16 page_offset;

#endif

			__u16 pagecnt_bias;

		};

		struct {

			void *addr;

			u64 handle;

		};

	};

};



struct ixgbe_queue_stats {

	u64 packets;
@@ -348,6 +352,10 @@ struct ixgbe_ring { 
		struct ixgbe_rx_queue_stats rx_stats;

	};

	struct xdp_rxq_info xdp_rxq;

	struct xdp_umem *xsk_umem;

	struct zero_copy_allocator zca; /* ZC allocator anchor */

	u16 ring_idx;		/* {rx,tx,xdp}_ring back reference idx */

	u16 rx_buf_len;

} ____cacheline_internodealigned_in_smp;



enum ixgbe_ring_f_enum {
@@ -765,6 +773,11 @@ struct ixgbe_adapter { 
#ifdef CONFIG_XFRM_OFFLOAD

	struct ixgbe_ipsec *ipsec;

#endif /* CONFIG_XFRM_OFFLOAD */



	/* AF_XDP zero-copy */

	struct xdp_umem **xsk_umems;

	u16 num_xsk_umems_used;

	u16 num_xsk_umems;

};



static inline u8 ixgbe_max_rss_indices(struct ixgbe_adapter *adapter)

drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c

+16 −1
Original line number Diff line number Diff line
@@ -1055,7 +1055,7 @@ static int ixgbe_alloc_q_vectors(struct ixgbe_adapter *adapter) 
	int txr_remaining = adapter->num_tx_queues;

	int xdp_remaining = adapter->num_xdp_queues;

	int rxr_idx = 0, txr_idx = 0, xdp_idx = 0, v_idx = 0;

	int err;

	int err, i;



	/* only one q_vector if MSI-X is disabled. */

	if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED))
@@ -1097,6 +1097,21 @@ static int ixgbe_alloc_q_vectors(struct ixgbe_adapter *adapter) 
		xdp_idx += xqpv;

	}



	for (i = 0; i < adapter->num_rx_queues; i++) {

		if (adapter->rx_ring[i])

			adapter->rx_ring[i]->ring_idx = i;

	}



	for (i = 0; i < adapter->num_tx_queues; i++) {

		if (adapter->tx_ring[i])

			adapter->tx_ring[i]->ring_idx = i;

	}



	for (i = 0; i < adapter->num_xdp_queues; i++) {

		if (adapter->xdp_ring[i])

			adapter->xdp_ring[i]->ring_idx = i;

	}



	return 0;



err_out:

drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

+66 −12
Original line number Diff line number Diff line
@@ -34,6 +34,7 @@ 
#include <net/tc_act/tc_mirred.h>

#include <net/vxlan.h>

#include <net/mpls.h>

#include <net/xdp_sock.h>



#include "ixgbe.h"

#include "ixgbe_common.h"
@@ -3176,7 +3177,10 @@ int ixgbe_poll(struct napi_struct *napi, int budget) 
		per_ring_budget = budget;



	ixgbe_for_each_ring(ring, q_vector->rx) {

		int cleaned = ixgbe_clean_rx_irq(q_vector, ring,

		int cleaned = ring->xsk_umem ?

			      ixgbe_clean_rx_irq_zc(q_vector, ring,

						    per_ring_budget) :

			      ixgbe_clean_rx_irq(q_vector, ring,

						 per_ring_budget);



		work_done += cleaned;
@@ -3704,10 +3708,27 @@ static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter, 
	srrctl = IXGBE_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT;



	/* configure the packet buffer length */

	if (test_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state))

		srrctl |= IXGBE_RXBUFFER_3K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;

	if (rx_ring->xsk_umem) {

		u32 xsk_buf_len = rx_ring->xsk_umem->chunk_size_nohr -

				  XDP_PACKET_HEADROOM;



		/* If the MAC support setting RXDCTL.RLPML, the

		 * SRRCTL[n].BSIZEPKT is set to PAGE_SIZE and

		 * RXDCTL.RLPML is set to the actual UMEM buffer

		 * size. If not, then we are stuck with a 1k buffer

		 * size resolution. In this case frames larger than

		 * the UMEM buffer size viewed in a 1k resolution will

		 * be dropped.

		 */

		if (hw->mac.type != ixgbe_mac_82599EB)

			srrctl |= PAGE_SIZE >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;

		else

			srrctl |= xsk_buf_len >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;

	} else if (test_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state)) {

		srrctl |= IXGBE_RXBUFFER_3K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;

	} else {

		srrctl |= IXGBE_RXBUFFER_2K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;

	}



	/* configure descriptor type */

	srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
@@ -4030,6 +4051,19 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, 
	u32 rxdctl;

	u8 reg_idx = ring->reg_idx;



	xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq);

	ring->xsk_umem = ixgbe_xsk_umem(adapter, ring);

	if (ring->xsk_umem) {

		ring->zca.free = ixgbe_zca_free;

		WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,

						   MEM_TYPE_ZERO_COPY,

						   &ring->zca));



	} else {

		WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,

						   MEM_TYPE_PAGE_SHARED, NULL));

	}



	/* disable queue to avoid use of these values while updating state */

	rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));

	rxdctl &= ~IXGBE_RXDCTL_ENABLE;
@@ -4079,6 +4113,17 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, 
#endif

	}



	if (ring->xsk_umem && hw->mac.type != ixgbe_mac_82599EB) {

		u32 xsk_buf_len = ring->xsk_umem->chunk_size_nohr -

				  XDP_PACKET_HEADROOM;



		rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK |

			    IXGBE_RXDCTL_RLPML_EN);

		rxdctl |= xsk_buf_len | IXGBE_RXDCTL_RLPML_EN;



		ring->rx_buf_len = xsk_buf_len;

	}



	/* initialize rx_buffer_info */

	memset(ring->rx_buffer_info, 0,

	       sizeof(struct ixgbe_rx_buffer) * ring->count);
@@ -4092,6 +4137,9 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, 
	IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);



	ixgbe_rx_desc_queue_enable(adapter, ring);

	if (ring->xsk_umem)

		ixgbe_alloc_rx_buffers_zc(ring, ixgbe_desc_unused(ring));

	else

		ixgbe_alloc_rx_buffers(ring, ixgbe_desc_unused(ring));

}
@@ -5206,6 +5254,11 @@ static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring) 
	u16 i = rx_ring->next_to_clean;

	struct ixgbe_rx_buffer *rx_buffer = &rx_ring->rx_buffer_info[i];



	if (rx_ring->xsk_umem) {

		ixgbe_xsk_clean_rx_ring(rx_ring);

		goto skip_free;

	}



	/* Free all the Rx ring sk_buffs */

	while (i != rx_ring->next_to_alloc) {

		if (rx_buffer->skb) {
@@ -5244,6 +5297,7 @@ static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring) 
		}

	}



skip_free:

	rx_ring->next_to_alloc = 0;

	rx_ring->next_to_clean = 0;

	rx_ring->next_to_use = 0;
@@ -6439,7 +6493,7 @@ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, 
	struct device *dev = rx_ring->dev;

	int orig_node = dev_to_node(dev);

	int ring_node = -1;

	int size, err;

	int size;



	size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count;
@@ -6476,13 +6530,6 @@ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, 
			     rx_ring->queue_index) < 0)

		goto err;



	err = xdp_rxq_info_reg_mem_model(&rx_ring->xdp_rxq,

					 MEM_TYPE_PAGE_SHARED, NULL);

	if (err) {

		xdp_rxq_info_unreg(&rx_ring->xdp_rxq);

		goto err;

	}



	rx_ring->xdp_prog = adapter->xdp_prog;



	return 0;
@@ -10198,6 +10245,13 @@ static int ixgbe_xdp(struct net_device *dev, struct netdev_bpf *xdp) 
		xdp->prog_id = adapter->xdp_prog ?

			adapter->xdp_prog->aux->id : 0;

		return 0;

	case XDP_QUERY_XSK_UMEM:

		return ixgbe_xsk_umem_query(adapter, &xdp->xsk.umem,

					    xdp->xsk.queue_id);

	case XDP_SETUP_XSK_UMEM:

		return ixgbe_xsk_umem_setup(adapter, xdp->xsk.umem,

					    xdp->xsk.queue_id);



	default:

		return -EINVAL;

	}

drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h

+15 −0
Original line number Diff line number Diff line
@@ -23,4 +23,19 @@ void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector, 
void ixgbe_txrx_ring_disable(struct ixgbe_adapter *adapter, int ring);

void ixgbe_txrx_ring_enable(struct ixgbe_adapter *adapter, int ring);



struct xdp_umem *ixgbe_xsk_umem(struct ixgbe_adapter *adapter,

				struct ixgbe_ring *ring);

int ixgbe_xsk_umem_query(struct ixgbe_adapter *adapter, struct xdp_umem **umem,

			 u16 qid);

int ixgbe_xsk_umem_setup(struct ixgbe_adapter *adapter, struct xdp_umem *umem,

			 u16 qid);



void ixgbe_zca_free(struct zero_copy_allocator *alloc, unsigned long handle);



void ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 cleaned_count);

int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector,

			  struct ixgbe_ring *rx_ring,

			  const int budget);

void ixgbe_xsk_clean_rx_ring(struct ixgbe_ring *rx_ring);



#endif /* #define _IXGBE_TXRX_COMMON_H_ */