Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 85c72ba1 authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'net-generic-busy-polling' 



Eric Dumazet says:

====================
net: extend busy polling support

This patch series extends busy polling range to tunnels devices,
and adds busy polling generic support to all NAPI drivers.

No need to provide ndo_busy_poll() method and extra synchronization
between ndo_busy_poll() and normal napi->poll() method.
This was proven very difficult and bug prone.

mlx5 driver is changed to support busy polling using this new method,
and a second mlx5 patch adds napi_complete_done() support and proper
SNMP accounting.

bnx2x and mlx4 drivers are converted to new infrastructure,
reducing kernel bloat and improving performance.

Latest patch, adding generic support, adds a new requirement :

 -free_netdev() and netif_napi_del() must be called from process context.

Since this might not be the case in some drivers, we might have to
either : fix the non conformant drivers (by disabling busy polling on them)
or revert this last patch.
====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents d37b4c0a 93d05d4a

drivers/net/ethernet/amd/xgbe/xgbe-drv.c

+0 −1
Original line number Diff line number Diff line
@@ -2024,7 +2024,6 @@ static int xgbe_rx_poll(struct xgbe_channel *channel, int budget) 
		skb->dev = netdev;

		skb->protocol = eth_type_trans(skb, netdev);

		skb_record_rx_queue(skb, channel->queue_index);

		skb_mark_napi_id(skb, napi);



		napi_gro_receive(napi, skb);

drivers/net/ethernet/broadcom/bcmsysport.c

+1 −1
Original line number Diff line number Diff line
@@ -1216,7 +1216,7 @@ static int bcm_sysport_init_tx_ring(struct bcm_sysport_priv *priv, 
	/* Initialize SW view of the ring */

	spin_lock_init(&ring->lock);

	ring->priv = priv;

	netif_napi_add(priv->netdev, &ring->napi, bcm_sysport_tx_poll, 64);

	netif_tx_napi_add(priv->netdev, &ring->napi, bcm_sysport_tx_poll, 64);

	ring->index = index;

	ring->size = size;

	ring->alloc_size = ring->size;

drivers/net/ethernet/broadcom/bnx2x/bnx2x.h

+0 −113
Original line number Diff line number Diff line
@@ -540,10 +540,6 @@ struct bnx2x_fastpath { 


	struct napi_struct	napi;



#ifdef CONFIG_NET_RX_BUSY_POLL

	unsigned long		busy_poll_state;

#endif



	union host_hc_status_block	status_blk;

	/* chip independent shortcuts into sb structure */

	__le16			*sb_index_values;
@@ -617,115 +613,6 @@ struct bnx2x_fastpath { 
#define bnx2x_fp_stats(bp, fp)	(&((bp)->fp_stats[(fp)->index]))

#define bnx2x_fp_qstats(bp, fp)	(&((bp)->fp_stats[(fp)->index].eth_q_stats))



#ifdef CONFIG_NET_RX_BUSY_POLL



enum bnx2x_fp_state {

	BNX2X_STATE_FP_NAPI	= BIT(0), /* NAPI handler owns the queue */



	BNX2X_STATE_FP_NAPI_REQ_BIT = 1, /* NAPI would like to own the queue */

	BNX2X_STATE_FP_NAPI_REQ = BIT(1),



	BNX2X_STATE_FP_POLL_BIT = 2,

	BNX2X_STATE_FP_POLL     = BIT(2), /* busy_poll owns the queue */



	BNX2X_STATE_FP_DISABLE_BIT = 3, /* queue is dismantled */

};



static inline void bnx2x_fp_busy_poll_init(struct bnx2x_fastpath *fp)

{

	WRITE_ONCE(fp->busy_poll_state, 0);

}



/* called from the device poll routine to get ownership of a FP */

static inline bool bnx2x_fp_lock_napi(struct bnx2x_fastpath *fp)

{

	unsigned long prev, old = READ_ONCE(fp->busy_poll_state);



	while (1) {

		switch (old) {

		case BNX2X_STATE_FP_POLL:

			/* make sure bnx2x_fp_lock_poll() wont starve us */

			set_bit(BNX2X_STATE_FP_NAPI_REQ_BIT,

				&fp->busy_poll_state);

			/* fallthrough */

		case BNX2X_STATE_FP_POLL | BNX2X_STATE_FP_NAPI_REQ:

			return false;

		default:

			break;

		}

		prev = cmpxchg(&fp->busy_poll_state, old, BNX2X_STATE_FP_NAPI);

		if (unlikely(prev != old)) {

			old = prev;

			continue;

		}

		return true;

	}

}



static inline void bnx2x_fp_unlock_napi(struct bnx2x_fastpath *fp)

{

	smp_wmb();

	fp->busy_poll_state = 0;

}



/* called from bnx2x_low_latency_poll() */

static inline bool bnx2x_fp_lock_poll(struct bnx2x_fastpath *fp)

{

	return cmpxchg(&fp->busy_poll_state, 0, BNX2X_STATE_FP_POLL) == 0;

}



static inline void bnx2x_fp_unlock_poll(struct bnx2x_fastpath *fp)

{

	smp_mb__before_atomic();

	clear_bit(BNX2X_STATE_FP_POLL_BIT, &fp->busy_poll_state);

}



/* true if a socket is polling */

static inline bool bnx2x_fp_ll_polling(struct bnx2x_fastpath *fp)

{

	return READ_ONCE(fp->busy_poll_state) & BNX2X_STATE_FP_POLL;

}



/* false if fp is currently owned */

static inline bool bnx2x_fp_ll_disable(struct bnx2x_fastpath *fp)

{

	set_bit(BNX2X_STATE_FP_DISABLE_BIT, &fp->busy_poll_state);

	return !bnx2x_fp_ll_polling(fp);



}

#else

static inline void bnx2x_fp_busy_poll_init(struct bnx2x_fastpath *fp)

{

}



static inline bool bnx2x_fp_lock_napi(struct bnx2x_fastpath *fp)

{

	return true;

}



static inline void bnx2x_fp_unlock_napi(struct bnx2x_fastpath *fp)

{

}



static inline bool bnx2x_fp_lock_poll(struct bnx2x_fastpath *fp)

{

	return false;

}



static inline void bnx2x_fp_unlock_poll(struct bnx2x_fastpath *fp)

{

}



static inline bool bnx2x_fp_ll_polling(struct bnx2x_fastpath *fp)

{

	return false;

}

static inline bool bnx2x_fp_ll_disable(struct bnx2x_fastpath *fp)

{

	return true;

}

#endif /* CONFIG_NET_RX_BUSY_POLL */



/* Use 2500 as a mini-jumbo MTU for FCoE */

#define BNX2X_FCOE_MINI_JUMBO_MTU	2500

drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c

+2 −48
Original line number Diff line number Diff line
@@ -46,7 +46,6 @@ static void bnx2x_add_all_napi_cnic(struct bnx2x *bp) 
	for_each_rx_queue_cnic(bp, i) {

		netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),

			       bnx2x_poll, NAPI_POLL_WEIGHT);

		napi_hash_add(&bnx2x_fp(bp, i, napi));

	}

}
@@ -58,7 +57,6 @@ static void bnx2x_add_all_napi(struct bnx2x *bp) 
	for_each_eth_queue(bp, i) {

		netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),

			       bnx2x_poll, NAPI_POLL_WEIGHT);

		napi_hash_add(&bnx2x_fp(bp, i, napi));

	}

}
@@ -1094,11 +1092,6 @@ static int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget) 
			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),

					       le16_to_cpu(cqe_fp->vlan_tag));



		skb_mark_napi_id(skb, &fp->napi);



		if (bnx2x_fp_ll_polling(fp))

			netif_receive_skb(skb);

		else

		napi_gro_receive(&fp->napi, skb);

next_rx:

		rx_buf->data = NULL;
@@ -1869,7 +1862,6 @@ static void bnx2x_napi_enable_cnic(struct bnx2x *bp) 
	int i;



	for_each_rx_queue_cnic(bp, i) {

		bnx2x_fp_busy_poll_init(&bp->fp[i]);

		napi_enable(&bnx2x_fp(bp, i, napi));

	}

}
@@ -1879,7 +1871,6 @@ static void bnx2x_napi_enable(struct bnx2x *bp) 
	int i;



	for_each_eth_queue(bp, i) {

		bnx2x_fp_busy_poll_init(&bp->fp[i]);

		napi_enable(&bnx2x_fp(bp, i, napi));

	}

}
@@ -1890,8 +1881,6 @@ static void bnx2x_napi_disable_cnic(struct bnx2x *bp) 


	for_each_rx_queue_cnic(bp, i) {

		napi_disable(&bnx2x_fp(bp, i, napi));

		while (!bnx2x_fp_ll_disable(&bp->fp[i]))

			usleep_range(1000, 2000);

	}

}
@@ -1901,8 +1890,6 @@ static void bnx2x_napi_disable(struct bnx2x *bp) 


	for_each_eth_queue(bp, i) {

		napi_disable(&bnx2x_fp(bp, i, napi));

		while (!bnx2x_fp_ll_disable(&bp->fp[i]))

			usleep_range(1000, 2000);

	}

}
@@ -3232,9 +3219,6 @@ static int bnx2x_poll(struct napi_struct *napi, int budget) 
			return 0;

		}

#endif

		if (!bnx2x_fp_lock_napi(fp))

			return budget;



		for_each_cos_in_tx_queue(fp, cos)

			if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))

				bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
@@ -3243,13 +3227,9 @@ static int bnx2x_poll(struct napi_struct *napi, int budget) 
			work_done += bnx2x_rx_int(fp, budget - work_done);



			/* must not complete if we consumed full budget */

			if (work_done >= budget) {

				bnx2x_fp_unlock_napi(fp);

			if (work_done >= budget)

				break;

		}

		}



		bnx2x_fp_unlock_napi(fp);



		/* Fall out from the NAPI loop if needed */

		if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
@@ -3294,32 +3274,6 @@ static int bnx2x_poll(struct napi_struct *napi, int budget) 
	return work_done;

}



#ifdef CONFIG_NET_RX_BUSY_POLL

/* must be called with local_bh_disable()d */

int bnx2x_low_latency_recv(struct napi_struct *napi)

{

	struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,

						 napi);

	struct bnx2x *bp = fp->bp;

	int found = 0;



	if ((bp->state == BNX2X_STATE_CLOSED) ||

	    (bp->state == BNX2X_STATE_ERROR) ||

	    (bp->dev->features & (NETIF_F_LRO | NETIF_F_GRO)))

		return LL_FLUSH_FAILED;



	if (!bnx2x_fp_lock_poll(fp))

		return LL_FLUSH_BUSY;



	if (bnx2x_has_rx_work(fp))

		found = bnx2x_rx_int(fp, 4);



	bnx2x_fp_unlock_poll(fp);



	return found;

}

#endif



/* we split the first BD into headers and data BDs

 * to ease the pain of our fellow microcode engineers

 * we use one mapping for both BDs

drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.h

+0 −7
Original line number Diff line number Diff line
@@ -569,13 +569,6 @@ int bnx2x_enable_msix(struct bnx2x *bp); 
 */

int bnx2x_enable_msi(struct bnx2x *bp);



/**

 * bnx2x_low_latency_recv - LL callback

 *

 * @napi:	napi structure

 */

int bnx2x_low_latency_recv(struct napi_struct *napi);



/**

 * bnx2x_alloc_mem_bp - allocate memories outsize main driver structure

 *