Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 925f5690 authored by Emil Tantilov's avatar Emil Tantilov Committed by Jeff Kirsher
Browse files

ixgbevf: break out Rx buffer page management 



Based on commit e0142726 ("igb: Break out Rx buffer page management")

Consolidate Rx code paths to reduce duplication when we expand them in
the future.

Signed-off-by: default avatarEmil Tantilov <emil.s.tantilov@intel.com>
Tested-by: default avatarKrishneil Singh <krishneil.k.singh@intel.com>
Signed-off-by: default avatarJeff Kirsher <jeffrey.t.kirsher@intel.com>
parent 21c046e4

drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c

+114 −113
Original line number Diff line number Diff line
@@ -130,6 +130,9 @@ static void ixgbevf_service_event_complete(struct ixgbevf_adapter *adapter) 
static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter);

static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector);

static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter);

static bool ixgbevf_can_reuse_rx_page(struct ixgbevf_rx_buffer *rx_buffer);

static void ixgbevf_reuse_rx_page(struct ixgbevf_ring *rx_ring,

				  struct ixgbevf_rx_buffer *old_buff);



static void ixgbevf_remove_adapter(struct ixgbe_hw *hw)

{
@@ -527,6 +530,49 @@ static void ixgbevf_process_skb_fields(struct ixgbevf_ring *rx_ring, 
	skb->protocol = eth_type_trans(skb, rx_ring->netdev);

}



static

struct ixgbevf_rx_buffer *ixgbevf_get_rx_buffer(struct ixgbevf_ring *rx_ring,

						const unsigned int size)

{

	struct ixgbevf_rx_buffer *rx_buffer;



	rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];

	prefetchw(rx_buffer->page);



	/* we are reusing so sync this buffer for CPU use */

	dma_sync_single_range_for_cpu(rx_ring->dev,

				      rx_buffer->dma,

				      rx_buffer->page_offset,

				      size,

				      DMA_FROM_DEVICE);



	rx_buffer->pagecnt_bias--;



	return rx_buffer;

}



static void ixgbevf_put_rx_buffer(struct ixgbevf_ring *rx_ring,

				  struct ixgbevf_rx_buffer *rx_buffer)

{

	if (ixgbevf_can_reuse_rx_page(rx_buffer)) {

		/* hand second half of page back to the ring */

		ixgbevf_reuse_rx_page(rx_ring, rx_buffer);

	} else {

		/* We are not reusing the buffer so unmap it and free

		 * any references we are holding to it

		 */

		dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,

				     ixgbevf_rx_pg_size(rx_ring),

				     DMA_FROM_DEVICE,

				     IXGBEVF_RX_DMA_ATTR);

		__page_frag_cache_drain(rx_buffer->page,

					rx_buffer->pagecnt_bias);

	}



	/* clear contents of rx_buffer */

	rx_buffer->page = NULL;

}



/**

 * ixgbevf_is_non_eop - process handling of non-EOP buffers

 * @rx_ring: Rx ring being processed
@@ -740,11 +786,10 @@ static inline bool ixgbevf_page_is_reserved(struct page *page) 
	return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page);

}



static bool ixgbevf_can_reuse_rx_page(struct ixgbevf_rx_buffer *rx_buffer,

				      struct page *page,

				      const unsigned int truesize)

static bool ixgbevf_can_reuse_rx_page(struct ixgbevf_rx_buffer *rx_buffer)

{

	unsigned int pagecnt_bias = rx_buffer->pagecnt_bias--;

	unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;

	struct page *page = rx_buffer->page;



	/* avoid re-using remote pages */

	if (unlikely(ixgbevf_page_is_reserved(page)))
@@ -752,16 +797,9 @@ static bool ixgbevf_can_reuse_rx_page(struct ixgbevf_rx_buffer *rx_buffer, 


#if (PAGE_SIZE < 8192)

	/* if we are only owner of page we can reuse it */

	if (unlikely(page_ref_count(page) != pagecnt_bias))

	if (unlikely((page_ref_count(page) - pagecnt_bias) > 1))

		return false;



	/* flip page offset to other buffer */

	rx_buffer->page_offset ^= truesize;



#else

	/* move offset up to the next cache line */

	rx_buffer->page_offset += truesize;



#define IXGBEVF_LAST_OFFSET \

	(SKB_WITH_OVERHEAD(PAGE_SIZE) - IXGBEVF_RXBUFFER_2048)
@@ -774,7 +812,7 @@ static bool ixgbevf_can_reuse_rx_page(struct ixgbevf_rx_buffer *rx_buffer, 
	 * the pagecnt_bias and page count so that we fully restock the

	 * number of references the driver holds.

	 */

	if (unlikely(pagecnt_bias == 1)) {

	if (unlikely(!pagecnt_bias)) {

		page_ref_add(page, USHRT_MAX);

		rx_buffer->pagecnt_bias = USHRT_MAX;

	}
@@ -786,25 +824,16 @@ static bool ixgbevf_can_reuse_rx_page(struct ixgbevf_rx_buffer *rx_buffer, 
 * ixgbevf_add_rx_frag - Add contents of Rx buffer to sk_buff

 * @rx_ring: rx descriptor ring to transact packets on

 * @rx_buffer: buffer containing page to add

 * @rx_desc: descriptor containing length of buffer written by hardware

 * @skb: sk_buff to place the data into

 * @size: size of buffer to be added

 *

 * This function will add the data contained in rx_buffer->page to the skb.

 * This is done either through a direct copy if the data in the buffer is

 * less than the skb header size, otherwise it will just attach the page as

 * a frag to the skb.

 *

 * The function will then update the page offset if necessary and return

 * true if the buffer can be reused by the adapter.

 **/

static bool ixgbevf_add_rx_frag(struct ixgbevf_ring *rx_ring,

static void ixgbevf_add_rx_frag(struct ixgbevf_ring *rx_ring,

				struct ixgbevf_rx_buffer *rx_buffer,

				u16 size,

				union ixgbe_adv_rx_desc *rx_desc,

				struct sk_buff *skb)

				struct sk_buff *skb,

				unsigned int size)

{

	struct page *page = rx_buffer->page;

	void *va = page_address(page) + rx_buffer->page_offset;

#if (PAGE_SIZE < 8192)

	unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2;

#else
@@ -812,62 +841,29 @@ static bool ixgbevf_add_rx_frag(struct ixgbevf_ring *rx_ring, 
				SKB_DATA_ALIGN(IXGBEVF_SKB_PAD + size) :

				SKB_DATA_ALIGN(size);

#endif

	unsigned int pull_len;



	if (unlikely(skb_is_nonlinear(skb)))

		goto add_tail_frag;



	if (likely(size <= IXGBEVF_RX_HDR_SIZE)) {

		memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long)));



		/* page is not reserved, we can reuse buffer as is */

		if (likely(!ixgbevf_page_is_reserved(page)))

			return true;



		/* this page cannot be reused so discard it */

		return false;

	}



	/* we need the header to contain the greater of either ETH_HLEN or

	 * 60 bytes if the skb->len is less than 60 for skb_pad.

	 */

	pull_len = eth_get_headlen(va, IXGBEVF_RX_HDR_SIZE);



	/* align pull length to size of long to optimize memcpy performance */

	memcpy(__skb_put(skb, pull_len), va, ALIGN(pull_len, sizeof(long)));



	/* update all of the pointers */

	va += pull_len;

	size -= pull_len;



add_tail_frag:

	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,

			va - page_address(page), size, truesize);



	return ixgbevf_can_reuse_rx_page(rx_buffer, page, truesize);

	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,

			rx_buffer->page_offset, size, truesize);

#if (PAGE_SIZE < 8192)

	rx_buffer->page_offset ^= truesize;

#else

	rx_buffer->page_offset += truesize;

#endif

}



static struct sk_buff *ixgbevf_fetch_rx_buffer(struct ixgbevf_ring *rx_ring,

static

struct sk_buff *ixgbevf_construct_skb(struct ixgbevf_ring *rx_ring,

				      struct ixgbevf_rx_buffer *rx_buffer,

				      union ixgbe_adv_rx_desc *rx_desc,

					       struct sk_buff *skb)

				      unsigned int size)

{

	struct ixgbevf_rx_buffer *rx_buffer;

	struct page *page;

	u16 size = le16_to_cpu(rx_desc->wb.upper.length);



	rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];

	page = rx_buffer->page;

	prefetchw(page);



	/* we are reusing so sync this buffer for CPU use */

	dma_sync_single_range_for_cpu(rx_ring->dev,

				      rx_buffer->dma,

				      rx_buffer->page_offset,

				      size,

				      DMA_FROM_DEVICE);



	if (likely(!skb)) {

		void *va = page_address(page) + rx_buffer->page_offset;

	void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;

#if (PAGE_SIZE < 8192)

	unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2;

#else

	unsigned int truesize = SKB_DATA_ALIGN(size);

#endif

	unsigned int headlen;

	struct sk_buff *skb;



	/* prefetch first cache line of first page */

	prefetch(va);
@@ -876,38 +872,33 @@ static struct sk_buff *ixgbevf_fetch_rx_buffer(struct ixgbevf_ring *rx_ring, 
#endif



	/* allocate a skb to store the frags */

		skb = netdev_alloc_skb_ip_align(rx_ring->netdev,

						IXGBEVF_RX_HDR_SIZE);

		if (unlikely(!skb)) {

			rx_ring->rx_stats.alloc_rx_buff_failed++;

	skb = napi_alloc_skb(&rx_ring->q_vector->napi, IXGBEVF_RX_HDR_SIZE);

	if (unlikely(!skb))

		return NULL;

		}



		/* we will be copying header into skb->data in

		 * pskb_may_pull so it is in our interest to prefetch

		 * it now to avoid a possible cache miss

		 */

		prefetchw(skb->data);

	}

	/* Determine available headroom for copy */

	headlen = size;

	if (headlen > IXGBEVF_RX_HDR_SIZE)

		headlen = eth_get_headlen(va, IXGBEVF_RX_HDR_SIZE);



	/* pull page into skb */

	if (ixgbevf_add_rx_frag(rx_ring, rx_buffer, size, rx_desc, skb)) {

		/* hand second half of page back to the ring */

		ixgbevf_reuse_rx_page(rx_ring, rx_buffer);

	/* align pull length to size of long to optimize memcpy performance */

	memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long)));



	/* update all of the pointers */

	size -= headlen;

	if (size) {

		skb_add_rx_frag(skb, 0, rx_buffer->page,

				(va + headlen) - page_address(rx_buffer->page),

				size, truesize);

#if (PAGE_SIZE < 8192)

		rx_buffer->page_offset ^= truesize;

#else

		rx_buffer->page_offset += truesize;

#endif

	} else {

		/* We are not reusing the buffer so unmap it and free

		 * any references we are holding to it

		 */

		dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,

				     ixgbevf_rx_pg_size(rx_ring),

				     DMA_FROM_DEVICE, IXGBEVF_RX_DMA_ATTR);

		__page_frag_cache_drain(page, rx_buffer->pagecnt_bias);

		rx_buffer->pagecnt_bias++;

	}



	/* clear contents of buffer_info */

	rx_buffer->dma = 0;

	rx_buffer->page = NULL;



	return skb;

}
@@ -929,6 +920,8 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, 


	while (likely(total_rx_packets < budget)) {

		union ixgbe_adv_rx_desc *rx_desc;

		struct ixgbevf_rx_buffer *rx_buffer;

		unsigned int size;



		/* return some buffers to hardware, one at a time is too slow */

		if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) {
@@ -937,8 +930,8 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, 
		}



		rx_desc = IXGBEVF_RX_DESC(rx_ring, rx_ring->next_to_clean);



		if (!rx_desc->wb.upper.length)

		size = le16_to_cpu(rx_desc->wb.upper.length);

		if (!size)

			break;



		/* This memory barrier is needed to keep us from reading
@@ -947,15 +940,23 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, 
		 */

		rmb();



		rx_buffer = ixgbevf_get_rx_buffer(rx_ring, size);



		/* retrieve a buffer from the ring */

		skb = ixgbevf_fetch_rx_buffer(rx_ring, rx_desc, skb);

		if (skb)

			ixgbevf_add_rx_frag(rx_ring, rx_buffer, skb, size);

		else

			skb = ixgbevf_construct_skb(rx_ring, rx_buffer,

						    rx_desc, size);



		/* exit if we failed to retrieve a buffer */

		if (!skb) {

			rx_ring->rx_stats.alloc_rx_buff_failed++;

			rx_buffer->pagecnt_bias++;

			break;

		}



		ixgbevf_put_rx_buffer(rx_ring, rx_buffer);

		cleaned_count++;



		/* fetch next buffer in frame if non-eop */