Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c0f031bc authored by Jakub Kicinski's avatar Jakub Kicinski Committed by David S. Miller
Browse files

nfp: use alloc_frag() and build_skb() 



Speed up RX processing by moving to the alloc_frag()/build_skb()
paradigm.  Since we're no longer mapping the entire buffer for
DMA add helpers which take care of calculating offsets and
lengths.

Signed-off-by: default avatarJakub Kicinski <jakub.kicinski@netronome.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent e9949aeb

drivers/net/ethernet/netronome/nfp/nfp_net.h

+6 −2
Original line number Diff line number Diff line
@@ -101,6 +101,10 @@ 
/* Offload definitions */

#define NFP_NET_N_VXLAN_PORTS	(NFP_NET_CFG_VXLAN_SZ / sizeof(__be16))



#define NFP_NET_RX_BUF_HEADROOM	(NET_SKB_PAD + NET_IP_ALIGN)

#define NFP_NET_RX_BUF_NON_DATA	(NFP_NET_RX_BUF_HEADROOM +		\

				 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))



/* Forward declarations */

struct nfp_net;

struct nfp_net_r_vector;
@@ -277,11 +281,11 @@ struct nfp_net_rx_hash { 


/**

 * struct nfp_net_rx_buf - software RX buffer descriptor

 * @skb:	sk_buff associated with this buffer

 * @frag:	page fragment buffer

 * @dma_addr:	DMA mapping address of the buffer

 */

struct nfp_net_rx_buf {

	struct sk_buff *skb;

	void *frag;

	dma_addr_t dma_addr;

};

drivers/net/ethernet/netronome/nfp/nfp_net_common.c

+72 −43
Original line number Diff line number Diff line
@@ -50,6 +50,7 @@ 
#include <linux/interrupt.h>

#include <linux/ip.h>

#include <linux/ipv6.h>

#include <linux/page_ref.h>

#include <linux/pci.h>

#include <linux/pci_regs.h>

#include <linux/msi.h>
@@ -80,6 +81,22 @@ void nfp_net_get_fw_version(struct nfp_net_fw_version *fw_ver, 
	put_unaligned_le32(reg, fw_ver);

}



static dma_addr_t

nfp_net_dma_map_rx(struct nfp_net *nn, void *frag, unsigned int bufsz,

		   int direction)

{

	return dma_map_single(&nn->pdev->dev, frag + NFP_NET_RX_BUF_HEADROOM,

			      bufsz - NFP_NET_RX_BUF_NON_DATA, direction);

}



static void

nfp_net_dma_unmap_rx(struct nfp_net *nn, dma_addr_t dma_addr,

		     unsigned int bufsz, int direction)

{

	dma_unmap_single(&nn->pdev->dev, dma_addr,

			 bufsz - NFP_NET_RX_BUF_NON_DATA, direction);

}



/* Firmware reconfig

 *

 * Firmware reconfig may take a while so we have two versions of it -
@@ -1035,64 +1052,67 @@ nfp_net_calc_fl_bufsz(struct nfp_net *nn, unsigned int mtu) 
{

	unsigned int fl_bufsz;



	fl_bufsz = NFP_NET_RX_BUF_HEADROOM;

	if (nn->rx_offset == NFP_NET_CFG_RX_OFFSET_DYNAMIC)

		fl_bufsz = NFP_NET_MAX_PREPEND;

		fl_bufsz += NFP_NET_MAX_PREPEND;

	else

		fl_bufsz = nn->rx_offset;

		fl_bufsz += nn->rx_offset;

	fl_bufsz += ETH_HLEN + VLAN_HLEN * 2 + mtu;



	fl_bufsz = SKB_DATA_ALIGN(fl_bufsz);

	fl_bufsz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));



	return fl_bufsz;

}



/**

 * nfp_net_rx_alloc_one() - Allocate and map skb for RX

 * nfp_net_rx_alloc_one() - Allocate and map page frag for RX

 * @rx_ring:	RX ring structure of the skb

 * @dma_addr:	Pointer to storage for DMA address (output param)

 * @fl_bufsz:	size of freelist buffers

 *

 * This function will allcate a new skb, map it for DMA.

 * This function will allcate a new page frag, map it for DMA.

 *

 * Return: allocated skb or NULL on failure.

 * Return: allocated page frag or NULL on failure.

 */

static struct sk_buff *

static void *

nfp_net_rx_alloc_one(struct nfp_net_rx_ring *rx_ring, dma_addr_t *dma_addr,

		     unsigned int fl_bufsz)

{

	struct nfp_net *nn = rx_ring->r_vec->nfp_net;

	struct sk_buff *skb;

	void *frag;



	skb = netdev_alloc_skb(nn->netdev, fl_bufsz);

	if (!skb) {

		nn_warn_ratelimit(nn, "Failed to alloc receive SKB\n");

	frag = netdev_alloc_frag(fl_bufsz);

	if (!frag) {

		nn_warn_ratelimit(nn, "Failed to alloc receive page frag\n");

		return NULL;

	}



	*dma_addr = dma_map_single(&nn->pdev->dev, skb->data,

				   fl_bufsz, DMA_FROM_DEVICE);

	*dma_addr = nfp_net_dma_map_rx(nn, frag, fl_bufsz, DMA_FROM_DEVICE);

	if (dma_mapping_error(&nn->pdev->dev, *dma_addr)) {

		dev_kfree_skb_any(skb);

		skb_free_frag(frag);

		nn_warn_ratelimit(nn, "Failed to map DMA RX buffer\n");

		return NULL;

	}



	return skb;

	return frag;

}



/**

 * nfp_net_rx_give_one() - Put mapped skb on the software and hardware rings

 * @rx_ring:	RX ring structure

 * @skb:	Skb to put on rings

 * @frag:	page fragment buffer

 * @dma_addr:	DMA address of skb mapping

 */

static void nfp_net_rx_give_one(struct nfp_net_rx_ring *rx_ring,

				struct sk_buff *skb, dma_addr_t dma_addr)

				void *frag, dma_addr_t dma_addr)

{

	unsigned int wr_idx;



	wr_idx = rx_ring->wr_p % rx_ring->cnt;



	/* Stash SKB and DMA address away */

	rx_ring->rxbufs[wr_idx].skb = skb;

	rx_ring->rxbufs[wr_idx].frag = frag;

	rx_ring->rxbufs[wr_idx].dma_addr = dma_addr;



	/* Fill freelist descriptor */
@@ -1127,9 +1147,9 @@ static void nfp_net_rx_ring_reset(struct nfp_net_rx_ring *rx_ring) 
	wr_idx = rx_ring->wr_p % rx_ring->cnt;

	last_idx = rx_ring->cnt - 1;

	rx_ring->rxbufs[wr_idx].dma_addr = rx_ring->rxbufs[last_idx].dma_addr;

	rx_ring->rxbufs[wr_idx].skb = rx_ring->rxbufs[last_idx].skb;

	rx_ring->rxbufs[wr_idx].frag = rx_ring->rxbufs[last_idx].frag;

	rx_ring->rxbufs[last_idx].dma_addr = 0;

	rx_ring->rxbufs[last_idx].skb = NULL;

	rx_ring->rxbufs[last_idx].frag = NULL;



	memset(rx_ring->rxds, 0, sizeof(*rx_ring->rxds) * rx_ring->cnt);

	rx_ring->wr_p = 0;
@@ -1149,7 +1169,6 @@ static void nfp_net_rx_ring_reset(struct nfp_net_rx_ring *rx_ring) 
static void

nfp_net_rx_ring_bufs_free(struct nfp_net *nn, struct nfp_net_rx_ring *rx_ring)

{

	struct pci_dev *pdev = nn->pdev;

	unsigned int i;



	for (i = 0; i < rx_ring->cnt - 1; i++) {
@@ -1157,14 +1176,14 @@ nfp_net_rx_ring_bufs_free(struct nfp_net *nn, struct nfp_net_rx_ring *rx_ring) 
		 * fails to allocate enough buffers and calls here to free

		 * already allocated ones.

		 */

		if (!rx_ring->rxbufs[i].skb)

		if (!rx_ring->rxbufs[i].frag)

			continue;



		dma_unmap_single(&pdev->dev, rx_ring->rxbufs[i].dma_addr,

		nfp_net_dma_unmap_rx(nn, rx_ring->rxbufs[i].dma_addr,

				     rx_ring->bufsz, DMA_FROM_DEVICE);

		dev_kfree_skb_any(rx_ring->rxbufs[i].skb);

		skb_free_frag(rx_ring->rxbufs[i].frag);

		rx_ring->rxbufs[i].dma_addr = 0;

		rx_ring->rxbufs[i].skb = NULL;

		rx_ring->rxbufs[i].frag = NULL;

	}

}
@@ -1182,10 +1201,10 @@ nfp_net_rx_ring_bufs_alloc(struct nfp_net *nn, struct nfp_net_rx_ring *rx_ring) 
	rxbufs = rx_ring->rxbufs;



	for (i = 0; i < rx_ring->cnt - 1; i++) {

		rxbufs[i].skb =

		rxbufs[i].frag =

			nfp_net_rx_alloc_one(rx_ring, &rxbufs[i].dma_addr,

					     rx_ring->bufsz);

		if (!rxbufs[i].skb) {

		if (!rxbufs[i].frag) {

			nfp_net_rx_ring_bufs_free(nn, rx_ring);

			return -ENOMEM;

		}
@@ -1203,7 +1222,7 @@ static void nfp_net_rx_ring_fill_freelist(struct nfp_net_rx_ring *rx_ring) 
	unsigned int i;



	for (i = 0; i < rx_ring->cnt - 1; i++)

		nfp_net_rx_give_one(rx_ring, rx_ring->rxbufs[i].skb,

		nfp_net_rx_give_one(rx_ring, rx_ring->rxbufs[i].frag,

				    rx_ring->rxbufs[i].dma_addr);

}
@@ -1338,8 +1357,13 @@ nfp_net_rx_drop(struct nfp_net_r_vector *r_vec, struct nfp_net_rx_ring *rx_ring, 
	r_vec->rx_drops++;

	u64_stats_update_end(&r_vec->rx_sync);



	/* skb is build based on the frag, free_skb() would free the frag

	 * so to be able to reuse it we need an extra ref.

	 */

	if (skb && rxbuf && skb->head == rxbuf->frag)

		page_ref_inc(virt_to_head_page(rxbuf->frag));

	if (rxbuf)

		nfp_net_rx_give_one(rx_ring, rxbuf->skb, rxbuf->dma_addr);

		nfp_net_rx_give_one(rx_ring, rxbuf->frag, rxbuf->dma_addr);

	if (skb)

		dev_kfree_skb_any(skb);

}
@@ -1360,10 +1384,12 @@ static int nfp_net_rx(struct nfp_net_rx_ring *rx_ring, int budget) 
	struct nfp_net_r_vector *r_vec = rx_ring->r_vec;

	struct nfp_net *nn = r_vec->nfp_net;

	unsigned int data_len, meta_len;

	struct sk_buff *skb, *new_skb;

	struct nfp_net_rx_buf *rxbuf;

	struct nfp_net_rx_desc *rxd;

	dma_addr_t new_dma_addr;

	struct sk_buff *skb;

	int pkts_polled = 0;

	void *new_frag;

	int idx;



	while (pkts_polled < budget) {
@@ -1381,21 +1407,23 @@ static int nfp_net_rx(struct nfp_net_rx_ring *rx_ring, int budget) 
		rx_ring->rd_p++;

		pkts_polled++;



		skb = rx_ring->rxbufs[idx].skb;



		new_skb = nfp_net_rx_alloc_one(rx_ring, &new_dma_addr,

		rxbuf =	&rx_ring->rxbufs[idx];

		skb = build_skb(rxbuf->frag, nn->fl_bufsz);

		if (unlikely(!skb)) {

			nfp_net_rx_drop(r_vec, rx_ring, rxbuf, NULL);

			continue;

		}

		new_frag = nfp_net_rx_alloc_one(rx_ring, &new_dma_addr,

						nn->fl_bufsz);

		if (!new_skb) {

			nfp_net_rx_drop(r_vec, rx_ring, &rx_ring->rxbufs[idx],

					NULL);

		if (unlikely(!new_frag)) {

			nfp_net_rx_drop(r_vec, rx_ring, rxbuf, skb);

			continue;

		}



		dma_unmap_single(&nn->pdev->dev,

				 rx_ring->rxbufs[idx].dma_addr,

		nfp_net_dma_unmap_rx(nn, rx_ring->rxbufs[idx].dma_addr,

				     nn->fl_bufsz, DMA_FROM_DEVICE);



		nfp_net_rx_give_one(rx_ring, new_skb, new_dma_addr);

		nfp_net_rx_give_one(rx_ring, new_frag, new_dma_addr);



		/*         < meta_len >

		 *  <-- [rx_offset] -->
@@ -1413,9 +1441,10 @@ static int nfp_net_rx(struct nfp_net_rx_ring *rx_ring, int budget) 
		data_len = le16_to_cpu(rxd->rxd.data_len);



		if (nn->rx_offset == NFP_NET_CFG_RX_OFFSET_DYNAMIC)

			skb_reserve(skb, meta_len);

			skb_reserve(skb, NFP_NET_RX_BUF_HEADROOM + meta_len);

		else

			skb_reserve(skb, nn->rx_offset);

			skb_reserve(skb,

				    NFP_NET_RX_BUF_HEADROOM + nn->rx_offset);

		skb_put(skb, data_len - meta_len);



		/* Stats update */

drivers/net/ethernet/netronome/nfp/nfp_net_debugfs.c

+4 −5
Original line number Diff line number Diff line
@@ -44,8 +44,8 @@ static int nfp_net_debugfs_rx_q_read(struct seq_file *file, void *data) 
	struct nfp_net_r_vector *r_vec = file->private;

	struct nfp_net_rx_ring *rx_ring;

	struct nfp_net_rx_desc *rxd;

	struct sk_buff *skb;

	struct nfp_net *nn;

	void *frag;

	int i;



	rtnl_lock();
@@ -73,10 +73,9 @@ static int nfp_net_debugfs_rx_q_read(struct seq_file *file, void *data) 
		seq_printf(file, "%04d: 0x%08x 0x%08x", i,

			   rxd->vals[0], rxd->vals[1]);



		skb = READ_ONCE(rx_ring->rxbufs[i].skb);

		if (skb)

			seq_printf(file, " skb->head=%p skb->data=%p",

				   skb->head, skb->data);

		frag = READ_ONCE(rx_ring->rxbufs[i].frag);

		if (frag)

			seq_printf(file, " frag=%p", frag);



		if (rx_ring->rxbufs[i].dma_addr)

			seq_printf(file, " dma_addr=%pad",