igb: Break out Rx buffer page management (e0142726) · Commits · e / devices / android_kernel_fairphone_FP4

drivers/net/ethernet/intel/igb/igb_main.c

+121 −114

Original line number	Original line	Diff line number	Diff line
	@@ -6886,8 +6886,14 @@ static void igb_reuse_rx_page(struct igb_ring *rx_ring,
	nta++;		nta++;
	rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;		rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;

	/* transfer page from old buffer to new buffer */		/* Transfer page from old buffer to new buffer.
	new_buff = old_buff;		* Move each member individually to avoid possible store
			* forwarding stalls.
			*/
			new_buff->dma = old_buff->dma;
			new_buff->page = old_buff->page;
			new_buff->page_offset = old_buff->page_offset;
			new_buff->pagecnt_bias = old_buff->pagecnt_bias;
	}		}

	static inline bool igb_page_is_reserved(struct page *page)		static inline bool igb_page_is_reserved(struct page *page)
	@@ -6895,11 +6901,10 @@ static inline bool igb_page_is_reserved(struct page *page)
	return (page_to_nid(page) != numa_mem_id()) \|\| page_is_pfmemalloc(page);		return (page_to_nid(page) != numa_mem_id()) \|\| page_is_pfmemalloc(page);
	}		}

	static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer,		static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer)
	struct page *page,
	const unsigned int truesize)
	{		{
	unsigned int pagecnt_bias = rx_buffer->pagecnt_bias--;		unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
			struct page *page = rx_buffer->page;

	/* avoid re-using remote pages */		/* avoid re-using remote pages */
	if (unlikely(igb_page_is_reserved(page)))		if (unlikely(igb_page_is_reserved(page)))
	@@ -6907,14 +6912,9 @@ static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer,

	#if (PAGE_SIZE < 8192)		#if (PAGE_SIZE < 8192)
	/* if we are only owner of page we can reuse it */		/* if we are only owner of page we can reuse it */
	if (unlikely(page_ref_count(page) != pagecnt_bias))		if (unlikely((page_ref_count(page) - pagecnt_bias) > 1))
	return false;		return false;

	/* flip page offset to other buffer */
	rx_buffer->page_offset ^= truesize;
	#else		#else
	/* move offset up to the next cache line */
	rx_buffer->page_offset += truesize;
	#define IGB_LAST_OFFSET \		#define IGB_LAST_OFFSET \
	(SKB_WITH_OVERHEAD(PAGE_SIZE) - IGB_RXBUFFER_2048)		(SKB_WITH_OVERHEAD(PAGE_SIZE) - IGB_RXBUFFER_2048)

	@@ -6926,7 +6926,7 @@ static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer,
	* the pagecnt_bias and page count so that we fully restock the		* the pagecnt_bias and page count so that we fully restock the
	* number of references the driver holds.		* number of references the driver holds.
	*/		*/
	if (unlikely(pagecnt_bias == 1)) {		if (unlikely(!pagecnt_bias)) {
	page_ref_add(page, USHRT_MAX);		page_ref_add(page, USHRT_MAX);
	rx_buffer->pagecnt_bias = USHRT_MAX;		rx_buffer->pagecnt_bias = USHRT_MAX;
	}		}
	@@ -6938,25 +6938,16 @@ static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer,
	* igb_add_rx_frag - Add contents of Rx buffer to sk_buff		* igb_add_rx_frag - Add contents of Rx buffer to sk_buff
	* @rx_ring: rx descriptor ring to transact packets on		* @rx_ring: rx descriptor ring to transact packets on
	* @rx_buffer: buffer containing page to add		* @rx_buffer: buffer containing page to add
	* @rx_desc: descriptor containing length of buffer written by hardware
	* @skb: sk_buff to place the data into		* @skb: sk_buff to place the data into
			* @size: size of buffer to be added
	*		*
	* This function will add the data contained in rx_buffer->page to the skb.		* This function will add the data contained in rx_buffer->page to the skb.
	* This is done either through a direct copy if the data in the buffer is
	* less than the skb header size, otherwise it will just attach the page as
	* a frag to the skb.
	*
	* The function will then update the page offset if necessary and return
	* true if the buffer can be reused by the adapter.
	**/		**/
	static bool igb_add_rx_frag(struct igb_ring *rx_ring,		static void igb_add_rx_frag(struct igb_ring *rx_ring,
	struct igb_rx_buffer *rx_buffer,		struct igb_rx_buffer *rx_buffer,
	unsigned int size,		struct sk_buff *skb,
	union e1000_adv_rx_desc *rx_desc,		unsigned int size)
	struct sk_buff *skb)
	{		{
	struct page *page = rx_buffer->page;
	void *va = page_address(page) + rx_buffer->page_offset;
	#if (PAGE_SIZE < 8192)		#if (PAGE_SIZE < 8192)
	unsigned int truesize = igb_rx_pg_size(rx_ring) / 2;		unsigned int truesize = igb_rx_pg_size(rx_ring) / 2;
	#else		#else
	@@ -6964,68 +6955,28 @@ static bool igb_add_rx_frag(struct igb_ring *rx_ring,
	SKB_DATA_ALIGN(IGB_SKB_PAD + size) :		SKB_DATA_ALIGN(IGB_SKB_PAD + size) :
	SKB_DATA_ALIGN(size);		SKB_DATA_ALIGN(size);
	#endif		#endif
	unsigned int pull_len;		skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
			rx_buffer->page_offset, size, truesize);
	if (unlikely(skb_is_nonlinear(skb)))		#if (PAGE_SIZE < 8192)
	goto add_tail_frag;		rx_buffer->page_offset ^= truesize;
			#else
	if (unlikely(igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP))) {		rx_buffer->page_offset += truesize;
	igb_ptp_rx_pktstamp(rx_ring->q_vector, va, skb);		#endif
	va += IGB_TS_HDR_LEN;
	size -= IGB_TS_HDR_LEN;
	}

	if (likely(size <= IGB_RX_HDR_LEN)) {
	memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long)));

	/* page is not reserved, we can reuse buffer as-is */
	if (likely(!igb_page_is_reserved(page)))
	return true;

	/* this page cannot be reused so discard it */
	return false;
	}

	/* we need the header to contain the greater of either ETH_HLEN or
	* 60 bytes if the skb->len is less than 60 for skb_pad.
	*/
	pull_len = eth_get_headlen(va, IGB_RX_HDR_LEN);

	/* align pull length to size of long to optimize memcpy performance */
	memcpy(__skb_put(skb, pull_len), va, ALIGN(pull_len, sizeof(long)));

	/* update all of the pointers */
	va += pull_len;
	size -= pull_len;

	add_tail_frag:
	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
	va - page_address(page), size, truesize);

	return igb_can_reuse_rx_page(rx_buffer, page, truesize);
	}		}

	static struct sk_buff igb_fetch_rx_buffer(struct igb_ring rx_ring,		static struct sk_buff igb_construct_skb(struct igb_ring rx_ring,
			struct igb_rx_buffer *rx_buffer,
	union e1000_adv_rx_desc *rx_desc,		union e1000_adv_rx_desc *rx_desc,
	struct sk_buff *skb)		unsigned int size)
	{		{
	unsigned int size = le16_to_cpu(rx_desc->wb.upper.length);		void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
	struct igb_rx_buffer *rx_buffer;		#if (PAGE_SIZE < 8192)
	struct page *page;		unsigned int truesize = igb_rx_pg_size(rx_ring) / 2;
			#else
	rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];		unsigned int truesize = SKB_DATA_ALIGN(size);
	page = rx_buffer->page;		#endif
	prefetchw(page);		unsigned int headlen;
			struct sk_buff *skb;
	/* we are reusing so sync this buffer for CPU use */
	dma_sync_single_range_for_cpu(rx_ring->dev,
	rx_buffer->dma,
	rx_buffer->page_offset,
	size,
	DMA_FROM_DEVICE);

	if (likely(!skb)) {
	void *va = page_address(page) + rx_buffer->page_offset;

	/* prefetch first cache line of first page */		/* prefetch first cache line of first page */
	prefetch(va);		prefetch(va);
	@@ -7035,35 +6986,38 @@ static struct sk_buff igb_fetch_rx_buffer(struct igb_ring rx_ring,

	/* allocate a skb to store the frags */		/* allocate a skb to store the frags */
	skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGB_RX_HDR_LEN);		skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGB_RX_HDR_LEN);
	if (unlikely(!skb)) {		if (unlikely(!skb))
	rx_ring->rx_stats.alloc_failed++;
	return NULL;		return NULL;
	}

	/* we will be copying header into skb->data in		if (unlikely(igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP))) {
	* pskb_may_pull so it is in our interest to prefetch		igb_ptp_rx_pktstamp(rx_ring->q_vector, va, skb);
	* it now to avoid a possible cache miss		va += IGB_TS_HDR_LEN;
	*/		size -= IGB_TS_HDR_LEN;
	prefetchw(skb->data);
	}		}

	/* pull page into skb */		/* Determine available headroom for copy */
	if (igb_add_rx_frag(rx_ring, rx_buffer, size, rx_desc, skb)) {		headlen = size;
	/* hand second half of page back to the ring */		if (headlen > IGB_RX_HDR_LEN)
	igb_reuse_rx_page(rx_ring, rx_buffer);		headlen = eth_get_headlen(va, IGB_RX_HDR_LEN);

			/* align pull length to size of long to optimize memcpy performance */
			memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long)));

			/* update all of the pointers */
			size -= headlen;
			if (size) {
			skb_add_rx_frag(skb, 0, rx_buffer->page,
			(va + headlen) - page_address(rx_buffer->page),
			size, truesize);
			#if (PAGE_SIZE < 8192)
			rx_buffer->page_offset ^= truesize;
			#else
			rx_buffer->page_offset += truesize;
			#endif
	} else {		} else {
	/* We are not reusing the buffer so unmap it and free		rx_buffer->pagecnt_bias++;
	* any references we are holding to it
	*/
	dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
	igb_rx_pg_size(rx_ring), DMA_FROM_DEVICE,
	IGB_RX_DMA_ATTR);
	__page_frag_cache_drain(page, rx_buffer->pagecnt_bias);
	}		}

	/* clear contents of rx_buffer */
	rx_buffer->page = NULL;

	return skb;		return skb;
	}		}

	@@ -7221,6 +7175,47 @@ static void igb_process_skb_fields(struct igb_ring *rx_ring,
	skb->protocol = eth_type_trans(skb, rx_ring->netdev);		skb->protocol = eth_type_trans(skb, rx_ring->netdev);
	}		}

			static struct igb_rx_buffer igb_get_rx_buffer(struct igb_ring rx_ring,
			const unsigned int size)
			{
			struct igb_rx_buffer *rx_buffer;

			rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
			prefetchw(rx_buffer->page);

			/* we are reusing so sync this buffer for CPU use */
			dma_sync_single_range_for_cpu(rx_ring->dev,
			rx_buffer->dma,
			rx_buffer->page_offset,
			size,
			DMA_FROM_DEVICE);

			rx_buffer->pagecnt_bias--;

			return rx_buffer;
			}

			static void igb_put_rx_buffer(struct igb_ring *rx_ring,
			struct igb_rx_buffer *rx_buffer)
			{
			if (igb_can_reuse_rx_page(rx_buffer)) {
			/* hand second half of page back to the ring */
			igb_reuse_rx_page(rx_ring, rx_buffer);
			} else {
			/* We are not reusing the buffer so unmap it and free
			* any references we are holding to it
			*/
			dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
			igb_rx_pg_size(rx_ring), DMA_FROM_DEVICE,
			IGB_RX_DMA_ATTR);
			__page_frag_cache_drain(rx_buffer->page,
			rx_buffer->pagecnt_bias);
			}

			/* clear contents of rx_buffer */
			rx_buffer->page = NULL;
			}

	static int igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget)		static int igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget)
	{		{
	struct igb_ring *rx_ring = q_vector->rx.ring;		struct igb_ring *rx_ring = q_vector->rx.ring;
	@@ -7230,6 +7225,8 @@ static int igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget)

	while (likely(total_packets < budget)) {		while (likely(total_packets < budget)) {
	union e1000_adv_rx_desc *rx_desc;		union e1000_adv_rx_desc *rx_desc;
			struct igb_rx_buffer *rx_buffer;
			unsigned int size;

	/* return some buffers to hardware, one at a time is too slow */		/* return some buffers to hardware, one at a time is too slow */
	if (cleaned_count >= IGB_RX_BUFFER_WRITE) {		if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
	@@ -7238,8 +7235,8 @@ static int igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget)
	}		}

	rx_desc = IGB_RX_DESC(rx_ring, rx_ring->next_to_clean);		rx_desc = IGB_RX_DESC(rx_ring, rx_ring->next_to_clean);
			size = le16_to_cpu(rx_desc->wb.upper.length);
	if (!rx_desc->wb.upper.length)		if (!size)
	break;		break;

	/* This memory barrier is needed to keep us from reading		/* This memory barrier is needed to keep us from reading
	@@ -7248,13 +7245,23 @@ static int igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget)
	*/		*/
	dma_rmb();		dma_rmb();

			rx_buffer = igb_get_rx_buffer(rx_ring, size);

	/* retrieve a buffer from the ring */		/* retrieve a buffer from the ring */
	skb = igb_fetch_rx_buffer(rx_ring, rx_desc, skb);		if (skb)
			igb_add_rx_frag(rx_ring, rx_buffer, skb, size);
			else
			skb = igb_construct_skb(rx_ring, rx_buffer,
			rx_desc, size);

	/* exit if we failed to retrieve a buffer */		/* exit if we failed to retrieve a buffer */
	if (!skb)		if (!skb) {
			rx_ring->rx_stats.alloc_failed++;
			rx_buffer->pagecnt_bias++;
	break;		break;
			}

			igb_put_rx_buffer(rx_ring, rx_buffer);
	cleaned_count++;		cleaned_count++;

	/* fetch next buffer in frame if non-eop */		/* fetch next buffer in frame if non-eop */