Merge branch 'mlx5e-xdp' (b80b8d7a) · Commits · e / devices / android_kernel_teracube_emerald

drivers/net/ethernet/mellanox/mlx5/core/en.h

+56 −13

Original line number	Diff line number	Diff line
		@@ -65,6 +65,8 @@
		#define MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE_MPW 0x3
		#define MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE_MPW 0x6

		#define MLX5_RX_HEADROOM NET_SKB_PAD

		#define MLX5_MPWRQ_LOG_STRIDE_SIZE 6 /* >= 6, HW restriction */
		#define MLX5_MPWRQ_LOG_STRIDE_SIZE_CQE_COMPRESS 8 /* >= 6, HW restriction */
		#define MLX5_MPWRQ_LOG_WQE_SZ 18
		@@ -99,6 +101,18 @@
		#define MLX5E_UPDATE_STATS_INTERVAL 200 /* msecs */
		#define MLX5E_SQ_BF_BUDGET 16

		#define MLX5E_ICOSQ_MAX_WQEBBS \
		(DIV_ROUND_UP(sizeof(struct mlx5e_umr_wqe), MLX5_SEND_WQE_BB))

		#define MLX5E_XDP_MIN_INLINE (ETH_HLEN + VLAN_HLEN)
		#define MLX5E_XDP_IHS_DS_COUNT \
		DIV_ROUND_UP(MLX5E_XDP_MIN_INLINE - 2, MLX5_SEND_WQE_DS)
		#define MLX5E_XDP_TX_DS_COUNT \
		(MLX5E_XDP_IHS_DS_COUNT + \
		(sizeof(struct mlx5e_tx_wqe) / MLX5_SEND_WQE_DS) + 1 /* SG DS */)
		#define MLX5E_XDP_TX_WQEBBS \
		DIV_ROUND_UP(MLX5E_XDP_TX_DS_COUNT, MLX5_SEND_WQEBB_NUM_DS)

		#define MLX5E_NUM_MAIN_GROUPS 9

		static inline u16 mlx5_min_rx_wqes(int wq_type, u32 wq_size)
		@@ -302,10 +316,20 @@ struct mlx5e_page_cache {
		struct mlx5e_rq {
		/* data path */
		struct mlx5_wq_ll wq;
		u32 wqe_sz;
		struct sk_buff **skb;
		struct mlx5e_mpw_info *wqe_info;

		union {
		struct mlx5e_dma_info *dma_info;
		struct {
		struct mlx5e_mpw_info *info;
		void *mtt_no_align;
		u32 mtt_offset;
		} mpwqe;
		};
		struct {
		u8 page_order;
		u32 wqe_sz; /* wqe data buffer size */
		u8 map_dir; /* dma map direction */
		} buff;
		__be32 mkey_be;

		struct device *pdev;
		@@ -321,9 +345,9 @@ struct mlx5e_rq {

		unsigned long state;
		int ix;
		u32 mpwqe_mtt_offset;

		struct mlx5e_rx_am am; /* Adaptive Moderation */
		struct bpf_prog *xdp_prog;

		/* control */
		struct mlx5_wq_ctrl wq_ctrl;
		@@ -370,11 +394,17 @@ enum {
		MLX5E_SQ_STATE_BF_ENABLE,
		};

		struct mlx5e_ico_wqe_info {
		struct mlx5e_sq_wqe_info {
		u8 opcode;
		u8 num_wqebbs;
		};

		enum mlx5e_sq_type {
		MLX5E_SQ_TXQ,
		MLX5E_SQ_ICO,
		MLX5E_SQ_XDP
		};

		struct mlx5e_sq {
		/* data path */

		@@ -392,10 +422,20 @@ struct mlx5e_sq {

		struct mlx5e_cq cq;

		/* pointers to per packet info: write@xmit, read@completion */
		/* pointers to per tx element info: write@xmit, read@completion */
		union {
		struct {
		struct sk_buff **skb;
		struct mlx5e_sq_dma *dma_fifo;
		struct mlx5e_tx_wqe_info *wqe_info;
		} txq;
		struct mlx5e_sq_wqe_info *ico_wqe;
		struct {
		struct mlx5e_sq_wqe_info *wqe_info;
		struct mlx5e_dma_info *di;
		bool doorbell;
		} xdp;
		} db;

		/* read only */
		struct mlx5_wq_cyc wq;
		@@ -417,8 +457,8 @@ struct mlx5e_sq {
		struct mlx5_uar uar;
		struct mlx5e_channel *channel;
		int tc;
		struct mlx5e_ico_wqe_info *ico_wqe_info;
		u32 rate_limit;
		u8 type;
		} ____cacheline_aligned_in_smp;

		static inline bool mlx5e_sq_has_room_for(struct mlx5e_sq *sq, u16 n)
		@@ -434,8 +474,10 @@ enum channel_flags {
		struct mlx5e_channel {
		/* data path */
		struct mlx5e_rq rq;
		struct mlx5e_sq xdp_sq;
		struct mlx5e_sq sq[MLX5E_MAX_NUM_TC];
		struct mlx5e_sq icosq; /* internal control operations */
		bool xdp;
		struct napi_struct napi;
		struct device *pdev;
		struct net_device *netdev;
		@@ -617,6 +659,7 @@ struct mlx5e_priv {
		/* priv data path fields - start */
		struct mlx5e_sq **txq_to_sq_map;
		int channeltc_to_txq_map[MLX5E_MAX_NUM_CHANNELS][MLX5E_MAX_NUM_TC];
		struct bpf_prog *xdp_prog;
		/* priv data path fields - end */

		unsigned long state;
		@@ -663,7 +706,7 @@ void mlx5e_cq_error_event(struct mlx5_core_cq *mcq, enum mlx5_event event);
		int mlx5e_napi_poll(struct napi_struct *napi, int budget);
		bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget);
		int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget);
		void mlx5e_free_tx_descs(struct mlx5e_sq *sq);
		void mlx5e_free_sq_descs(struct mlx5e_sq *sq);

		void mlx5e_page_release(struct mlx5e_rq rq, struct mlx5e_dma_info dma_info,
		bool recycle);
		@@ -764,7 +807,7 @@ static inline void mlx5e_cq_arm(struct mlx5e_cq *cq)

		static inline u32 mlx5e_get_wqe_mtt_offset(struct mlx5e_rq *rq, u16 wqe_ix)
		{
		return rq->mpwqe_mtt_offset +
		return rq->mpwqe.mtt_offset +
		wqe_ix * ALIGN(MLX5_MPWRQ_PAGES_PER_WQE, 8);
		}

drivers/net/ethernet/mellanox/mlx5/core/en_main.c

+349 −113

File changed.

Preview size limit exceeded, changes collapsed.

drivers/net/ethernet/mellanox/mlx5/core/en_rx.c

+226 −123

Original line number	Diff line number	Diff line
		@@ -179,50 +179,99 @@ void mlx5e_modify_rx_cqe_compression(struct mlx5e_priv *priv, bool val)
		mutex_unlock(&priv->state_lock);
		}

		int mlx5e_alloc_rx_wqe(struct mlx5e_rq rq, struct mlx5e_rx_wqe wqe, u16 ix)
		#define RQ_PAGE_SIZE(rq) ((1 << rq->buff.page_order) << PAGE_SHIFT)

		static inline bool mlx5e_rx_cache_put(struct mlx5e_rq *rq,
		struct mlx5e_dma_info *dma_info)
		{
		struct sk_buff *skb;
		dma_addr_t dma_addr;
		struct mlx5e_page_cache *cache = &rq->page_cache;
		u32 tail_next = (cache->tail + 1) & (MLX5E_CACHE_SIZE - 1);

		skb = napi_alloc_skb(rq->cq.napi, rq->wqe_sz);
		if (unlikely(!skb))
		return -ENOMEM;
		if (tail_next == cache->head) {
		rq->stats.cache_full++;
		return false;
		}

		cache->page_cache[cache->tail] = *dma_info;
		cache->tail = tail_next;
		return true;
		}

		static inline bool mlx5e_rx_cache_get(struct mlx5e_rq *rq,
		struct mlx5e_dma_info *dma_info)
		{
		struct mlx5e_page_cache *cache = &rq->page_cache;

		if (unlikely(cache->head == cache->tail)) {
		rq->stats.cache_empty++;
		return false;
		}

		if (page_ref_count(cache->page_cache[cache->head].page) != 1) {
		rq->stats.cache_busy++;
		return false;
		}

		dma_addr = dma_map_single(rq->pdev,
		/* hw start padding */
		skb->data,
		/* hw end padding */
		rq->wqe_sz,
		*dma_info = cache->page_cache[cache->head];
		cache->head = (cache->head + 1) & (MLX5E_CACHE_SIZE - 1);
		rq->stats.cache_reuse++;

		dma_sync_single_for_device(rq->pdev, dma_info->addr,
		RQ_PAGE_SIZE(rq),
		DMA_FROM_DEVICE);
		return true;
		}

		if (unlikely(dma_mapping_error(rq->pdev, dma_addr)))
		goto err_free_skb;
		static inline int mlx5e_page_alloc_mapped(struct mlx5e_rq *rq,
		struct mlx5e_dma_info *dma_info)
		{
		struct page *page;

		((dma_addr_t )skb->cb) = dma_addr;
		wqe->data.addr = cpu_to_be64(dma_addr);
		if (mlx5e_rx_cache_get(rq, dma_info))
		return 0;

		page = dev_alloc_pages(rq->buff.page_order);
		if (unlikely(!page))
		return -ENOMEM;

		rq->skb[ix] = skb;
		dma_info->page = page;
		dma_info->addr = dma_map_page(rq->pdev, page, 0,
		RQ_PAGE_SIZE(rq), rq->buff.map_dir);
		if (unlikely(dma_mapping_error(rq->pdev, dma_info->addr))) {
		put_page(page);
		return -ENOMEM;
		}

		return 0;
		}

		err_free_skb:
		dev_kfree_skb(skb);
		void mlx5e_page_release(struct mlx5e_rq rq, struct mlx5e_dma_info dma_info,
		bool recycle)
		{
		if (likely(recycle) && mlx5e_rx_cache_put(rq, dma_info))
		return;

		dma_unmap_page(rq->pdev, dma_info->addr, RQ_PAGE_SIZE(rq),
		rq->buff.map_dir);
		put_page(dma_info->page);
		}

		int mlx5e_alloc_rx_wqe(struct mlx5e_rq rq, struct mlx5e_rx_wqe wqe, u16 ix)
		{
		struct mlx5e_dma_info *di = &rq->dma_info[ix];

		if (unlikely(mlx5e_page_alloc_mapped(rq, di)))
		return -ENOMEM;

		wqe->data.addr = cpu_to_be64(di->addr + MLX5_RX_HEADROOM);
		return 0;
		}

		void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix)
		{
		struct sk_buff *skb = rq->skb[ix];
		struct mlx5e_dma_info *di = &rq->dma_info[ix];

		if (skb) {
		rq->skb[ix] = NULL;
		dma_unmap_single(rq->pdev,
		((dma_addr_t )skb->cb),
		rq->wqe_sz,
		DMA_FROM_DEVICE);
		dev_kfree_skb(skb);
		}
		mlx5e_page_release(rq, di, true);
		}

		static inline int mlx5e_mpwqe_strides_per_page(struct mlx5e_rq *rq)
		@@ -279,7 +328,7 @@ mlx5e_copy_skb_header_mpwqe(struct device *pdev,

		static inline void mlx5e_post_umr_wqe(struct mlx5e_rq *rq, u16 ix)
		{
		struct mlx5e_mpw_info *wi = &rq->wqe_info[ix];
		struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
		struct mlx5e_sq *sq = &rq->channel->icosq;
		struct mlx5_wq_cyc *wq = &sq->wq;
		struct mlx5e_umr_wqe *wqe;
		@@ -288,8 +337,8 @@ static inline void mlx5e_post_umr_wqe(struct mlx5e_rq *rq, u16 ix)

		/* fill sq edge with nops to avoid wqe wrap around */
		while ((pi = (sq->pc & wq->sz_m1)) > sq->edge) {
		sq->ico_wqe_info[pi].opcode = MLX5_OPCODE_NOP;
		sq->ico_wqe_info[pi].num_wqebbs = 1;
		sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_NOP;
		sq->db.ico_wqe[pi].num_wqebbs = 1;
		mlx5e_send_nop(sq, true);
		}

		@@ -299,90 +348,17 @@ static inline void mlx5e_post_umr_wqe(struct mlx5e_rq *rq, u16 ix)
		cpu_to_be32((sq->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) \|
		MLX5_OPCODE_UMR);

		sq->ico_wqe_info[pi].opcode = MLX5_OPCODE_UMR;
		sq->ico_wqe_info[pi].num_wqebbs = num_wqebbs;
		sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_UMR;
		sq->db.ico_wqe[pi].num_wqebbs = num_wqebbs;
		sq->pc += num_wqebbs;
		mlx5e_tx_notify_hw(sq, &wqe->ctrl, 0);
		}

		static inline bool mlx5e_rx_cache_put(struct mlx5e_rq *rq,
		struct mlx5e_dma_info *dma_info)
		{
		struct mlx5e_page_cache *cache = &rq->page_cache;
		u32 tail_next = (cache->tail + 1) & (MLX5E_CACHE_SIZE - 1);

		if (tail_next == cache->head) {
		rq->stats.cache_full++;
		return false;
		}

		cache->page_cache[cache->tail] = *dma_info;
		cache->tail = tail_next;
		return true;
		}

		static inline bool mlx5e_rx_cache_get(struct mlx5e_rq *rq,
		struct mlx5e_dma_info *dma_info)
		{
		struct mlx5e_page_cache *cache = &rq->page_cache;

		if (unlikely(cache->head == cache->tail)) {
		rq->stats.cache_empty++;
		return false;
		}

		if (page_ref_count(cache->page_cache[cache->head].page) != 1) {
		rq->stats.cache_busy++;
		return false;
		}

		*dma_info = cache->page_cache[cache->head];
		cache->head = (cache->head + 1) & (MLX5E_CACHE_SIZE - 1);
		rq->stats.cache_reuse++;

		dma_sync_single_for_device(rq->pdev, dma_info->addr, PAGE_SIZE,
		DMA_FROM_DEVICE);
		return true;
		}

		static inline int mlx5e_page_alloc_mapped(struct mlx5e_rq *rq,
		struct mlx5e_dma_info *dma_info)
		{
		struct page *page;

		if (mlx5e_rx_cache_get(rq, dma_info))
		return 0;

		page = dev_alloc_page();
		if (unlikely(!page))
		return -ENOMEM;

		dma_info->page = page;
		dma_info->addr = dma_map_page(rq->pdev, page, 0, PAGE_SIZE,
		DMA_FROM_DEVICE);
		if (unlikely(dma_mapping_error(rq->pdev, dma_info->addr))) {
		put_page(page);
		return -ENOMEM;
		}

		return 0;
		}

		void mlx5e_page_release(struct mlx5e_rq rq, struct mlx5e_dma_info dma_info,
		bool recycle)
		{
		if (likely(recycle) && mlx5e_rx_cache_put(rq, dma_info))
		return;

		dma_unmap_page(rq->pdev, dma_info->addr, PAGE_SIZE, DMA_FROM_DEVICE);
		put_page(dma_info->page);
		}

		static int mlx5e_alloc_rx_umr_mpwqe(struct mlx5e_rq *rq,
		struct mlx5e_rx_wqe *wqe,
		u16 ix)
		{
		struct mlx5e_mpw_info *wi = &rq->wqe_info[ix];
		struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
		u64 dma_offset = (u64)mlx5e_get_wqe_mtt_offset(rq, ix) << PAGE_SHIFT;
		int pg_strides = mlx5e_mpwqe_strides_per_page(rq);
		int err;
		@@ -436,7 +412,7 @@ void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq)
		clear_bit(MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS, &rq->state);

		if (unlikely(test_bit(MLX5E_RQ_STATE_FLUSH, &rq->state))) {
		mlx5e_free_rx_mpwqe(rq, &rq->wqe_info[wq->head]);
		mlx5e_free_rx_mpwqe(rq, &rq->mpwqe.info[wq->head]);
		return;
		}

		@@ -462,7 +438,7 @@ int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq rq, struct mlx5e_rx_wqe wqe, u16 ix)

		void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
		{
		struct mlx5e_mpw_info *wi = &rq->wqe_info[ix];
		struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];

		mlx5e_free_rx_mpwqe(rq, wi);
		}
		@@ -656,33 +632,154 @@ static inline void mlx5e_complete_rx_cqe(struct mlx5e_rq *rq,
		napi_gro_receive(rq->cq.napi, skb);
		}

		static inline void mlx5e_xmit_xdp_doorbell(struct mlx5e_sq *sq)
		{
		struct mlx5_wq_cyc *wq = &sq->wq;
		struct mlx5e_tx_wqe *wqe;
		u16 pi = (sq->pc - MLX5E_XDP_TX_WQEBBS) & wq->sz_m1; /* last pi */

		wqe = mlx5_wq_cyc_get_wqe(wq, pi);

		wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
		mlx5e_tx_notify_hw(sq, &wqe->ctrl, 0);
		}

		static inline void mlx5e_xmit_xdp_frame(struct mlx5e_rq *rq,
		struct mlx5e_dma_info *di,
		unsigned int data_offset,
		int len)
		{
		struct mlx5e_sq *sq = &rq->channel->xdp_sq;
		struct mlx5_wq_cyc *wq = &sq->wq;
		u16 pi = sq->pc & wq->sz_m1;
		struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi);
		struct mlx5e_sq_wqe_info *wi = &sq->db.xdp.wqe_info[pi];

		struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
		struct mlx5_wqe_eth_seg *eseg = &wqe->eth;
		struct mlx5_wqe_data_seg *dseg;

		dma_addr_t dma_addr = di->addr + data_offset + MLX5E_XDP_MIN_INLINE;
		unsigned int dma_len = len - MLX5E_XDP_MIN_INLINE;
		void *data = page_address(di->page) + data_offset;

		if (unlikely(!mlx5e_sq_has_room_for(sq, MLX5E_XDP_TX_WQEBBS))) {
		if (sq->db.xdp.doorbell) {
		/* SQ is full, ring doorbell */
		mlx5e_xmit_xdp_doorbell(sq);
		sq->db.xdp.doorbell = false;
		}
		rq->stats.xdp_tx_full++;
		mlx5e_page_release(rq, di, true);
		return;
		}

		dma_sync_single_for_device(sq->pdev, dma_addr, dma_len,
		PCI_DMA_TODEVICE);

		memset(wqe, 0, sizeof(*wqe));

		/* copy the inline part */
		memcpy(eseg->inline_hdr_start, data, MLX5E_XDP_MIN_INLINE);
		eseg->inline_hdr_sz = cpu_to_be16(MLX5E_XDP_MIN_INLINE);

		dseg = (struct mlx5_wqe_data_seg *)cseg + (MLX5E_XDP_TX_DS_COUNT - 1);

		/* write the dma part */
		dseg->addr = cpu_to_be64(dma_addr);
		dseg->byte_count = cpu_to_be32(dma_len);
		dseg->lkey = sq->mkey_be;

		cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) \| MLX5_OPCODE_SEND);
		cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) \| MLX5E_XDP_TX_DS_COUNT);

		sq->db.xdp.di[pi] = *di;
		wi->opcode = MLX5_OPCODE_SEND;
		wi->num_wqebbs = MLX5E_XDP_TX_WQEBBS;
		sq->pc += MLX5E_XDP_TX_WQEBBS;

		sq->db.xdp.doorbell = true;
		rq->stats.xdp_tx++;
		}

		/* returns true if packet was consumed by xdp */
		static inline bool mlx5e_xdp_handle(struct mlx5e_rq *rq,
		const struct bpf_prog *prog,
		struct mlx5e_dma_info *di,
		void *data, u16 len)
		{
		struct xdp_buff xdp;
		u32 act;

		if (!prog)
		return false;

		xdp.data = data;
		xdp.data_end = xdp.data + len;
		act = bpf_prog_run_xdp(prog, &xdp);
		switch (act) {
		case XDP_PASS:
		return false;
		case XDP_TX:
		mlx5e_xmit_xdp_frame(rq, di, MLX5_RX_HEADROOM, len);
		return true;
		default:
		bpf_warn_invalid_xdp_action(act);
		case XDP_ABORTED:
		case XDP_DROP:
		rq->stats.xdp_drop++;
		mlx5e_page_release(rq, di, true);
		return true;
		}
		}

		void mlx5e_handle_rx_cqe(struct mlx5e_rq rq, struct mlx5_cqe64 cqe)
		{
		struct bpf_prog *xdp_prog = READ_ONCE(rq->xdp_prog);
		struct mlx5e_dma_info *di;
		struct mlx5e_rx_wqe *wqe;
		struct sk_buff *skb;
		__be16 wqe_counter_be;
		struct sk_buff *skb;
		u16 wqe_counter;
		void va, data;
		u32 cqe_bcnt;

		wqe_counter_be = cqe->wqe_counter;
		wqe_counter = be16_to_cpu(wqe_counter_be);
		wqe = mlx5_wq_ll_get_wqe(&rq->wq, wqe_counter);
		skb = rq->skb[wqe_counter];
		prefetch(skb->data);
		rq->skb[wqe_counter] = NULL;

		dma_unmap_single(rq->pdev,
		((dma_addr_t )skb->cb),
		rq->wqe_sz,
		di = &rq->dma_info[wqe_counter];
		va = page_address(di->page);
		data = va + MLX5_RX_HEADROOM;

		dma_sync_single_range_for_cpu(rq->pdev,
		di->addr,
		MLX5_RX_HEADROOM,
		rq->buff.wqe_sz,
		DMA_FROM_DEVICE);
		prefetch(data);
		cqe_bcnt = be32_to_cpu(cqe->byte_cnt);

		if (unlikely((cqe->op_own >> 4) != MLX5_CQE_RESP_SEND)) {
		rq->stats.wqe_err++;
		dev_kfree_skb(skb);
		mlx5e_page_release(rq, di, true);
		goto wq_ll_pop;
		}

		cqe_bcnt = be32_to_cpu(cqe->byte_cnt);
		if (mlx5e_xdp_handle(rq, xdp_prog, di, data, cqe_bcnt))
		goto wq_ll_pop; /* page/packet was consumed by XDP */

		skb = build_skb(va, RQ_PAGE_SIZE(rq));
		if (unlikely(!skb)) {
		rq->stats.buff_alloc_err++;
		mlx5e_page_release(rq, di, true);
		goto wq_ll_pop;
		}

		/* queue up for recycling ..*/
		page_ref_inc(di->page);
		mlx5e_page_release(rq, di, true);

		skb_reserve(skb, MLX5_RX_HEADROOM);
		skb_put(skb, cqe_bcnt);

		mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
		@@ -734,7 +831,7 @@ void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq rq, struct mlx5_cqe64 cqe)
		{
		u16 cstrides = mpwrq_get_cqe_consumed_strides(cqe);
		u16 wqe_id = be16_to_cpu(cqe->wqe_id);
		struct mlx5e_mpw_info *wi = &rq->wqe_info[wqe_id];
		struct mlx5e_mpw_info *wi = &rq->mpwqe.info[wqe_id];
		struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, wqe_id);
		struct sk_buff *skb;
		u16 cqe_bcnt;
		@@ -776,6 +873,7 @@ void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq rq, struct mlx5_cqe64 cqe)
		int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget)
		{
		struct mlx5e_rq *rq = container_of(cq, struct mlx5e_rq, cq);
		struct mlx5e_sq *xdp_sq = &rq->channel->xdp_sq;
		int work_done = 0;

		if (unlikely(test_bit(MLX5E_RQ_STATE_FLUSH, &rq->state)))
		@@ -802,6 +900,11 @@ int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget)
		rq->handle_rx_cqe(rq, cqe);
		}

		if (xdp_sq->db.xdp.doorbell) {
		mlx5e_xmit_xdp_doorbell(xdp_sq);
		xdp_sq->db.xdp.doorbell = false;
		}

		mlx5_cqwq_update_db_record(&cq->wq);

		/* ensure cq space is freed before enabling more cqes */

drivers/net/ethernet/mellanox/mlx5/core/en_stats.h

+12 −0

Original line number	Diff line number	Diff line
		@@ -65,6 +65,9 @@ struct mlx5e_sw_stats {
		u64 rx_csum_none;
		u64 rx_csum_complete;
		u64 rx_csum_unnecessary_inner;
		u64 rx_xdp_drop;
		u64 rx_xdp_tx;
		u64 rx_xdp_tx_full;
		u64 tx_csum_partial;
		u64 tx_csum_partial_inner;
		u64 tx_queue_stopped;
		@@ -100,6 +103,9 @@ static const struct counter_desc sw_stats_desc[] = {
		{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_csum_none) },
		{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_csum_complete) },
		{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_csum_unnecessary_inner) },
		{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_drop) },
		{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx) },
		{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_xdp_tx_full) },
		{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_csum_partial) },
		{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_csum_partial_inner) },
		{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_queue_stopped) },
		@@ -278,6 +284,9 @@ struct mlx5e_rq_stats {
		u64 csum_none;
		u64 lro_packets;
		u64 lro_bytes;
		u64 xdp_drop;
		u64 xdp_tx;
		u64 xdp_tx_full;
		u64 wqe_err;
		u64 mpwqe_filler;
		u64 buff_alloc_err;
		@@ -295,6 +304,9 @@ static const struct counter_desc rq_stats_desc[] = {
		{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, csum_complete) },
		{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, csum_unnecessary_inner) },
		{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, csum_none) },
		{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, xdp_drop) },
		{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, xdp_tx) },
		{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, xdp_tx_full) },
		{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, lro_packets) },
		{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, lro_bytes) },
		{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, wqe_err) },

drivers/net/ethernet/mellanox/mlx5/core/en_tx.c

+50 −13

Original line number	Diff line number	Diff line
		@@ -52,7 +52,6 @@ void mlx5e_send_nop(struct mlx5e_sq *sq, bool notify_hw)
		cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) \| MLX5_OPCODE_NOP);
		cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) \| 0x01);

		sq->skb[pi] = NULL;
		sq->pc++;
		sq->stats.nop++;

		@@ -82,15 +81,17 @@ static inline void mlx5e_dma_push(struct mlx5e_sq *sq,
		u32 size,
		enum mlx5e_dma_map_type map_type)
		{
		sq->dma_fifo[sq->dma_fifo_pc & sq->dma_fifo_mask].addr = addr;
		sq->dma_fifo[sq->dma_fifo_pc & sq->dma_fifo_mask].size = size;
		sq->dma_fifo[sq->dma_fifo_pc & sq->dma_fifo_mask].type = map_type;
		u32 i = sq->dma_fifo_pc & sq->dma_fifo_mask;

		sq->db.txq.dma_fifo[i].addr = addr;
		sq->db.txq.dma_fifo[i].size = size;
		sq->db.txq.dma_fifo[i].type = map_type;
		sq->dma_fifo_pc++;
		}

		static inline struct mlx5e_sq_dma mlx5e_dma_get(struct mlx5e_sq sq, u32 i)
		{
		return &sq->dma_fifo[i & sq->dma_fifo_mask];
		return &sq->db.txq.dma_fifo[i & sq->dma_fifo_mask];
		}

		static void mlx5e_dma_unmap_wqe_err(struct mlx5e_sq *sq, u8 num_dma)
		@@ -221,7 +222,7 @@ static netdev_tx_t mlx5e_sq_xmit(struct mlx5e_sq sq, struct sk_buff skb)

		u16 pi = sq->pc & wq->sz_m1;
		struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(wq, pi);
		struct mlx5e_tx_wqe_info *wi = &sq->wqe_info[pi];
		struct mlx5e_tx_wqe_info *wi = &sq->db.txq.wqe_info[pi];

		struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
		struct mlx5_wqe_eth_seg *eseg = &wqe->eth;
		@@ -341,7 +342,7 @@ static netdev_tx_t mlx5e_sq_xmit(struct mlx5e_sq sq, struct sk_buff skb)
		cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) \| opcode);
		cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) \| ds_cnt);

		sq->skb[pi] = skb;
		sq->db.txq.skb[pi] = skb;

		wi->num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
		sq->pc += wi->num_wqebbs;
		@@ -368,8 +369,10 @@ static netdev_tx_t mlx5e_sq_xmit(struct mlx5e_sq sq, struct sk_buff skb)
		}

		/* fill sq edge with nops to avoid wqe wrap around */
		while ((sq->pc & wq->sz_m1) > sq->edge)
		while ((pi = (sq->pc & wq->sz_m1)) > sq->edge) {
		sq->db.txq.skb[pi] = NULL;
		mlx5e_send_nop(sq, false);
		}

		if (bf)
		sq->bf_budget--;
		@@ -442,8 +445,8 @@ bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget)
		last_wqe = (sqcc == wqe_counter);

		ci = sqcc & sq->wq.sz_m1;
		skb = sq->skb[ci];
		wi = &sq->wqe_info[ci];
		skb = sq->db.txq.skb[ci];
		wi = &sq->db.txq.wqe_info[ci];

		if (unlikely(!skb)) { /* nop */
		sqcc++;
		@@ -492,7 +495,7 @@ bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget)
		return (i == MLX5E_TX_CQ_POLL_BUDGET);
		}

		void mlx5e_free_tx_descs(struct mlx5e_sq *sq)
		static void mlx5e_free_txq_sq_descs(struct mlx5e_sq *sq)
		{
		struct mlx5e_tx_wqe_info *wi;
		struct sk_buff *skb;
		@@ -501,8 +504,8 @@ void mlx5e_free_tx_descs(struct mlx5e_sq *sq)

		while (sq->cc != sq->pc) {
		ci = sq->cc & sq->wq.sz_m1;
		skb = sq->skb[ci];
		wi = &sq->wqe_info[ci];
		skb = sq->db.txq.skb[ci];
		wi = &sq->db.txq.wqe_info[ci];

		if (!skb) { /* nop */
		sq->cc++;
		@@ -520,3 +523,37 @@ void mlx5e_free_tx_descs(struct mlx5e_sq *sq)
		sq->cc += wi->num_wqebbs;
		}
		}

		static void mlx5e_free_xdp_sq_descs(struct mlx5e_sq *sq)
		{
		struct mlx5e_sq_wqe_info *wi;
		struct mlx5e_dma_info *di;
		u16 ci;

		while (sq->cc != sq->pc) {
		ci = sq->cc & sq->wq.sz_m1;
		di = &sq->db.xdp.di[ci];
		wi = &sq->db.xdp.wqe_info[ci];

		if (wi->opcode == MLX5_OPCODE_NOP) {
		sq->cc++;
		continue;
		}

		sq->cc += wi->num_wqebbs;

		mlx5e_page_release(&sq->channel->rq, di, false);
		}
		}

		void mlx5e_free_sq_descs(struct mlx5e_sq *sq)
		{
		switch (sq->type) {
		case MLX5E_SQ_TXQ:
		mlx5e_free_txq_sq_descs(sq);
		break;
		case MLX5E_SQ_XDP:
		mlx5e_free_xdp_sq_descs(sq);
		break;
		}
		}