Merge tag 'mlx5-updates-2017-09-03' of git://git.kernel.org/pub/scm/linux/kernel/git/saeed/linux (18a4ded9) · Commits · e / devices / android_kernel_teracube_emerald

drivers/net/ethernet/mellanox/mlx5/core/en.h

+20 −29

Original line number	Diff line number	Diff line
		@@ -291,10 +291,11 @@ struct mlx5e_tstamp {

		enum {
		MLX5E_RQ_STATE_ENABLED,
		MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS,
		MLX5E_RQ_STATE_AM,
		};

		#define MLX5E_TEST_BIT(state, nr) (state & BIT(nr))

		struct mlx5e_cq {
		/* data path - accessed per cqe */
		struct mlx5_cqwq wq;
		@@ -342,7 +343,6 @@ enum {

		struct mlx5e_sq_wqe_info {
		u8 opcode;
		u8 num_wqebbs;
		};

		struct mlx5e_txqsq {
		@@ -418,13 +418,8 @@ struct mlx5e_xdpsq {
		struct mlx5e_icosq {
		/* data path */

		/* dirtied @completion */
		u16 cc;

		/* dirtied @xmit */
		u16 pc ____cacheline_aligned_in_smp;
		u32 dma_fifo_pc;
		u16 prev_cc;

		struct mlx5e_cq cq;

		@@ -438,7 +433,6 @@ struct mlx5e_icosq {
		void __iomem *uar_map;
		u32 sqn;
		u16 edge;
		struct device *pdev;
		__be32 mkey_be;
		unsigned long state;

		@@ -507,7 +501,7 @@ struct mlx5e_rx_am { /* Adaptive Moderation */
		*/
		#define MLX5E_CACHE_UNIT (MLX5_MPWRQ_PAGES_PER_WQE > NAPI_POLL_WEIGHT ? \
		MLX5_MPWRQ_PAGES_PER_WQE : NAPI_POLL_WEIGHT)
		#define MLX5E_CACHE_SIZE (2 * roundup_pow_of_two(MLX5E_CACHE_UNIT))
		#define MLX5E_CACHE_SIZE (4 * roundup_pow_of_two(MLX5E_CACHE_UNIT))
		struct mlx5e_page_cache {
		u32 head;
		u32 tail;
		@@ -516,7 +510,7 @@ struct mlx5e_page_cache {

		struct mlx5e_rq;
		typedef void (mlx5e_fp_handle_rx_cqe)(struct mlx5e_rq, struct mlx5_cqe64*);
		typedef int (mlx5e_fp_alloc_wqe)(struct mlx5e_rq, struct mlx5e_rx_wqe*, u16);
		typedef bool (mlx5e_fp_post_rx_wqes)(struct mlx5e_rq rq);
		typedef void (mlx5e_fp_dealloc_wqe)(struct mlx5e_rq, u16);

		struct mlx5e_rq {
		@@ -527,21 +521,26 @@ struct mlx5e_rq {
		struct {
		struct mlx5e_wqe_frag_info *frag_info;
		u32 frag_sz; /* max possible skb frag_sz */
		union {
		bool page_reuse;
		bool xdp_xmit;
		};
		} wqe;
		struct {
		struct mlx5e_mpw_info *info;
		void *mtt_no_align;
		u16 num_strides;
		u8 log_stride_sz;
		bool umr_in_progress;
		} mpwqe;
		};
		struct {
		u16 headroom;
		u8 page_order;
		u32 wqe_sz; /* wqe data buffer size */
		u8 map_dir; /* dma map direction */
		} buff;
		__be32 mkey_be;

		struct mlx5e_channel *channel;
		struct device *pdev;
		struct net_device *netdev;
		struct mlx5e_tstamp *tstamp;
		@@ -550,12 +549,11 @@ struct mlx5e_rq {
		struct mlx5e_page_cache page_cache;

		mlx5e_fp_handle_rx_cqe handle_rx_cqe;
		mlx5e_fp_alloc_wqe alloc_wqe;
		mlx5e_fp_post_rx_wqes post_wqes;
		mlx5e_fp_dealloc_wqe dealloc_wqe;

		unsigned long state;
		int ix;
		u16 rx_headroom;

		struct mlx5e_rx_am am; /* Adaptive Moderation */

		@@ -565,19 +563,13 @@ struct mlx5e_rq {

		/* control */
		struct mlx5_wq_ctrl wq_ctrl;
		__be32 mkey_be;
		u8 wq_type;
		u32 mpwqe_stride_sz;
		u32 mpwqe_num_strides;
		u32 rqn;
		struct mlx5e_channel *channel;
		struct mlx5_core_dev *mdev;
		struct mlx5_core_mkey umr_mkey;
		} ____cacheline_aligned_in_smp;

		enum channel_flags {
		MLX5E_CHANNEL_NAPI_SCHED = 1,
		};

		struct mlx5e_channel {
		/* data path */
		struct mlx5e_rq rq;
		@@ -589,7 +581,9 @@ struct mlx5e_channel {
		struct net_device *netdev;
		__be32 mkey_be;
		u8 num_tc;
		unsigned long flags;

		/* data path - accessed per napi poll */
		struct irq_desc *irq_desc;

		/* control */
		struct mlx5e_priv *priv;
		@@ -850,11 +844,9 @@ void mlx5e_page_release(struct mlx5e_rq rq, struct mlx5e_dma_info dma_info,
		void mlx5e_handle_rx_cqe(struct mlx5e_rq rq, struct mlx5_cqe64 cqe);
		void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq rq, struct mlx5_cqe64 cqe);
		bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq);
		int mlx5e_alloc_rx_wqe(struct mlx5e_rq rq, struct mlx5e_rx_wqe wqe, u16 ix);
		int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq rq, struct mlx5e_rx_wqe wqe, u16 ix);
		bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq);
		void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix);
		void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix);
		void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq);
		void mlx5e_free_rx_mpwqe(struct mlx5e_rq rq, struct mlx5e_mpw_info wi);

		void mlx5e_rx_am(struct mlx5e_rq *rq);
		@@ -933,8 +925,7 @@ void mlx5e_switch_priv_channels(struct mlx5e_priv *priv,
		void mlx5e_activate_priv_channels(struct mlx5e_priv *priv);
		void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv);

		void mlx5e_build_default_indir_rqt(struct mlx5_core_dev *mdev,
		u32 *indirection_rqt, int len,
		void mlx5e_build_default_indir_rqt(u32 *indirection_rqt, int len,
		int num_channels);
		int mlx5e_get_max_linkspeed(struct mlx5_core_dev mdev, u32 speed);

drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c

+1 −2

Original line number	Diff line number	Diff line
		@@ -663,8 +663,7 @@ int mlx5e_ethtool_set_channels(struct mlx5e_priv *priv,
		new_channels.params = priv->channels.params;
		new_channels.params.num_channels = count;
		if (!netif_is_rxfh_configured(priv->netdev))
		mlx5e_build_default_indir_rqt(priv->mdev,
		new_channels.params.indirection_rqt,
		mlx5e_build_default_indir_rqt(new_channels.params.indirection_rqt,
		MLX5E_INDIR_RQT_SIZE, count);

		if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {

drivers/net/ethernet/mellanox/mlx5/core/en_main.c

+26 −29

Original line number	Diff line number	Diff line
		@@ -208,6 +208,7 @@ static void mlx5e_update_sw_counters(struct mlx5e_priv *priv)
		s->rx_cache_full += rq_stats->cache_full;
		s->rx_cache_empty += rq_stats->cache_empty;
		s->rx_cache_busy += rq_stats->cache_busy;
		s->rx_cache_waive += rq_stats->cache_waive;

		for (j = 0; j < priv->channels.params.num_tc; j++) {
		sq_stats = &c->sq[j].stats;
		@@ -593,12 +594,12 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
		}

		rq->buff.map_dir = rq->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
		rq->rx_headroom = params->rq_headroom;
		rq->buff.headroom = params->rq_headroom;

		switch (rq->wq_type) {
		case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:

		rq->alloc_wqe = mlx5e_alloc_rx_mpwqe;
		rq->post_wqes = mlx5e_post_rx_mpwqes;
		rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe;

		rq->handle_rx_cqe = c->priv->profile->rx_handlers.handle_rx_cqe_mpwqe;
		@@ -615,11 +616,10 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
		goto err_rq_wq_destroy;
		}

		rq->mpwqe_stride_sz = BIT(params->mpwqe_log_stride_sz);
		rq->mpwqe_num_strides = BIT(params->mpwqe_log_num_strides);
		rq->mpwqe.log_stride_sz = params->mpwqe_log_stride_sz;
		rq->mpwqe.num_strides = BIT(params->mpwqe_log_num_strides);

		rq->buff.wqe_sz = rq->mpwqe_stride_sz * rq->mpwqe_num_strides;
		byte_count = rq->buff.wqe_sz;
		byte_count = rq->mpwqe.num_strides << rq->mpwqe.log_stride_sz;

		err = mlx5e_create_rq_umr_mkey(mdev, rq);
		if (err)
		@@ -638,7 +638,7 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
		err = -ENOMEM;
		goto err_rq_wq_destroy;
		}
		rq->alloc_wqe = mlx5e_alloc_rx_wqe;
		rq->post_wqes = mlx5e_post_rx_wqes;
		rq->dealloc_wqe = mlx5e_dealloc_rx_wqe;

		#ifdef CONFIG_MLX5_EN_IPSEC
		@@ -654,18 +654,17 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
		goto err_rq_wq_destroy;
		}

		rq->buff.wqe_sz = params->lro_en ?
		byte_count = params->lro_en ?
		params->lro_wqe_sz :
		MLX5E_SW2HW_MTU(c->priv, c->netdev->mtu);
		#ifdef CONFIG_MLX5_EN_IPSEC
		if (MLX5_IPSEC_DEV(mdev))
		rq->buff.wqe_sz += MLX5E_METADATA_ETHER_LEN;
		byte_count += MLX5E_METADATA_ETHER_LEN;
		#endif
		rq->wqe.page_reuse = !params->xdp_prog && !params->lro_en;
		byte_count = rq->buff.wqe_sz;

		/* calc the required page order */
		rq->wqe.frag_sz = MLX5_SKB_FRAG_SZ(rq->rx_headroom + byte_count);
		rq->wqe.frag_sz = MLX5_SKB_FRAG_SZ(rq->buff.headroom + byte_count);
		npages = DIV_ROUND_UP(rq->wqe.frag_sz, PAGE_SIZE);
		rq->buff.page_order = order_base_2(npages);

		@@ -676,6 +675,12 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
		for (i = 0; i < wq_sz; i++) {
		struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, i);

		if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
		u64 dma_offset = (u64)mlx5e_get_wqe_mtt_offset(rq, i) << PAGE_SHIFT;

		wqe->data.addr = cpu_to_be64(dma_offset);
		}

		wqe->data.byte_count = cpu_to_be32(byte_count);
		wqe->data.lkey = rq->mkey_be;
		}
		@@ -882,7 +887,8 @@ static void mlx5e_free_rx_descs(struct mlx5e_rq *rq)
		u16 wqe_ix;

		/* UMR WQE (if in progress) is always at wq->head */
		if (test_bit(MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS, &rq->state))
		if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ &&
		rq->mpwqe.umr_in_progress)
		mlx5e_free_rx_mpwqe(rq, &rq->mpwqe.info[wq->head]);

		while (!mlx5_wq_ll_is_empty(wq)) {
		@@ -925,7 +931,7 @@ static int mlx5e_open_rq(struct mlx5e_channel *c,
		goto err_destroy_rq;

		if (params->rx_am_enabled)
		set_bit(MLX5E_RQ_STATE_AM, &c->rq.state);
		c->rq.state \|= BIT(MLX5E_RQ_STATE_AM);

		return 0;

		@@ -945,7 +951,6 @@ static void mlx5e_activate_rq(struct mlx5e_rq *rq)

		set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
		sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_NOP;
		sq->db.ico_wqe[pi].num_wqebbs = 1;
		nopwqe = mlx5e_post_nop(&sq->wq, sq->sqn, &sq->pc);
		mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, &nopwqe->ctrl);
		}
		@@ -1047,7 +1052,6 @@ static int mlx5e_alloc_icosq(struct mlx5e_channel *c,
		struct mlx5_core_dev *mdev = c->mdev;
		int err;

		sq->pdev = c->pdev;
		sq->mkey_be = c->mkey_be;
		sq->channel = c;
		sq->uar_map = mdev->mlx5e_res.bfreg.map;
		@@ -1757,7 +1761,9 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
		struct net_device *netdev = priv->netdev;
		int cpu = mlx5e_get_cpu(priv, ix);
		struct mlx5e_channel *c;
		unsigned int irq;
		int err;
		int eqn;

		c = kzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu));
		if (!c)
		@@ -1774,6 +1780,9 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
		c->num_tc = params->num_tc;
		c->xdp = !!params->xdp_prog;

		mlx5_vector2eqn(priv->mdev, ix, &eqn, &irq);
		c->irq_desc = irq_to_desc(irq);

		netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64);

		err = mlx5e_open_cq(c, icocq_moder, &cparam->icosq_cq, &c->icosq.cq);
		@@ -3693,7 +3702,6 @@ static int mlx5e_xdp_set(struct net_device netdev, struct bpf_prog prog)

		set_bit(MLX5E_RQ_STATE_ENABLED, &c->rq.state);
		/* napi_schedule in case we have missed anything */
		set_bit(MLX5E_CHANNEL_NAPI_SCHED, &c->flags);
		napi_schedule(&c->napi);

		if (old_prog)
		@@ -3825,22 +3833,11 @@ u16 mlx5e_get_max_inline_cap(struct mlx5_core_dev *mdev)
		2 /sizeof(mlx5e_tx_wqe.inline_hdr_start)/;
		}

		void mlx5e_build_default_indir_rqt(struct mlx5_core_dev *mdev,
		u32 *indirection_rqt, int len,
		void mlx5e_build_default_indir_rqt(u32 *indirection_rqt, int len,
		int num_channels)
		{
		int node = mdev->priv.numa_node;
		int node_num_of_cores;
		int i;

		if (node == -1)
		node = first_online_node;

		node_num_of_cores = cpumask_weight(cpumask_of_node(node));

		if (node_num_of_cores)
		num_channels = min_t(int, num_channels, node_num_of_cores);

		for (i = 0; i < len; i++)
		indirection_rqt[i] = i % num_channels;
		}
		@@ -3979,7 +3976,7 @@ void mlx5e_build_nic_params(struct mlx5_core_dev *mdev,
		/* RSS */
		params->rss_hfunc = ETH_RSS_HASH_XOR;
		netdev_rss_key_fill(params->toeplitz_hash_key, sizeof(params->toeplitz_hash_key));
		mlx5e_build_default_indir_rqt(mdev, params->indirection_rqt,
		mlx5e_build_default_indir_rqt(params->indirection_rqt,
		MLX5E_INDIR_RQT_SIZE, max_channels);
		}

drivers/net/ethernet/mellanox/mlx5/core/en_rx.c

+137 −88

Original line number	Diff line number	Diff line
		@@ -163,7 +163,7 @@ static inline u32 mlx5e_decompress_cqes_start(struct mlx5e_rq *rq,

		static inline bool mlx5e_page_is_reserved(struct page *page)
		{
		return page_is_pfmemalloc(page) \|\| page_to_nid(page) != numa_node_id();
		return page_is_pfmemalloc(page) \|\| page_to_nid(page) != numa_mem_id();
		}

		static inline bool mlx5e_rx_cache_put(struct mlx5e_rq *rq,
		@@ -177,8 +177,10 @@ static inline bool mlx5e_rx_cache_put(struct mlx5e_rq *rq,
		return false;
		}

		if (unlikely(page_is_pfmemalloc(dma_info->page)))
		if (unlikely(mlx5e_page_is_reserved(dma_info->page))) {
		rq->stats.cache_waive++;
		return false;
		}

		cache->page_cache[cache->tail] = *dma_info;
		cache->tail = tail_next;
		@@ -252,7 +254,7 @@ static inline bool mlx5e_page_reuse(struct mlx5e_rq *rq,
		!mlx5e_page_is_reserved(wi->di.page);
		}

		int mlx5e_alloc_rx_wqe(struct mlx5e_rq rq, struct mlx5e_rx_wqe wqe, u16 ix)
		static int mlx5e_alloc_rx_wqe(struct mlx5e_rq rq, struct mlx5e_rx_wqe wqe, u16 ix)
		{
		struct mlx5e_wqe_frag_info *wi = &rq->wqe.frag_info[ix];

		@@ -263,8 +265,7 @@ int mlx5e_alloc_rx_wqe(struct mlx5e_rq rq, struct mlx5e_rx_wqe wqe, u16 ix)
		wi->offset = 0;
		}

		wqe->data.addr = cpu_to_be64(wi->di.addr + wi->offset +
		rq->rx_headroom);
		wqe->data.addr = cpu_to_be64(wi->di.addr + wi->offset + rq->buff.headroom);
		return 0;
		}

		@@ -296,7 +297,7 @@ void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix)

		static inline int mlx5e_mpwqe_strides_per_page(struct mlx5e_rq *rq)
		{
		return rq->mpwqe_num_strides >> MLX5_MPWRQ_WQE_PAGE_ORDER;
		return rq->mpwqe.num_strides >> MLX5_MPWRQ_WQE_PAGE_ORDER;
		}

		static inline void mlx5e_add_skb_frag_mpwqe(struct mlx5e_rq *rq,
		@@ -305,7 +306,7 @@ static inline void mlx5e_add_skb_frag_mpwqe(struct mlx5e_rq *rq,
		u32 page_idx, u32 frag_offset,
		u32 len)
		{
		unsigned int truesize = ALIGN(len, rq->mpwqe_stride_sz);
		unsigned int truesize = ALIGN(len, BIT(rq->mpwqe.log_stride_sz));

		dma_sync_single_for_cpu(rq->pdev,
		wi->umr.dma_info[page_idx].addr + frag_offset,
		@@ -358,7 +359,6 @@ static inline void mlx5e_post_umr_wqe(struct mlx5e_rq *rq, u16 ix)
		/* fill sq edge with nops to avoid wqe wrap around */
		while ((pi = (sq->pc & wq->sz_m1)) > sq->edge) {
		sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_NOP;
		sq->db.ico_wqe[pi].num_wqebbs = 1;
		mlx5e_post_nop(wq, sq->sqn, &sq->pc);
		}

		@@ -369,41 +369,35 @@ static inline void mlx5e_post_umr_wqe(struct mlx5e_rq *rq, u16 ix)
		MLX5_OPCODE_UMR);

		sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_UMR;
		sq->db.ico_wqe[pi].num_wqebbs = num_wqebbs;
		sq->pc += num_wqebbs;
		mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, &wqe->ctrl);
		}

		static int mlx5e_alloc_rx_umr_mpwqe(struct mlx5e_rq *rq,
		struct mlx5e_rx_wqe *wqe,
		u16 ix)
		{
		struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
		u64 dma_offset = (u64)mlx5e_get_wqe_mtt_offset(rq, ix) << PAGE_SHIFT;
		int pg_strides = mlx5e_mpwqe_strides_per_page(rq);
		struct mlx5e_dma_info *dma_info = &wi->umr.dma_info[0];
		int err;
		int i;

		for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++) {
		struct mlx5e_dma_info *dma_info = &wi->umr.dma_info[i];

		for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++, dma_info++) {
		err = mlx5e_page_alloc_mapped(rq, dma_info);
		if (unlikely(err))
		goto err_unmap;
		wi->umr.mtt[i] = cpu_to_be64(dma_info->addr \| MLX5_EN_WR);
		page_ref_add(dma_info->page, pg_strides);
		wi->skbs_frags[i] = 0;
		}

		memset(wi->skbs_frags, 0, sizeof(wi->skbs_frags) MLX5_MPWRQ_PAGES_PER_WQE);
		wi->consumed_strides = 0;
		wqe->data.addr = cpu_to_be64(dma_offset);

		return 0;

		err_unmap:
		while (--i >= 0) {
		struct mlx5e_dma_info *dma_info = &wi->umr.dma_info[i];

		dma_info--;
		page_ref_sub(dma_info->page, pg_strides);
		mlx5e_page_release(rq, dma_info, true);
		}
		@@ -414,27 +408,21 @@ static int mlx5e_alloc_rx_umr_mpwqe(struct mlx5e_rq *rq,
		void mlx5e_free_rx_mpwqe(struct mlx5e_rq rq, struct mlx5e_mpw_info wi)
		{
		int pg_strides = mlx5e_mpwqe_strides_per_page(rq);
		struct mlx5e_dma_info *dma_info = &wi->umr.dma_info[0];
		int i;

		for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++) {
		struct mlx5e_dma_info *dma_info = &wi->umr.dma_info[i];

		for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++, dma_info++) {
		page_ref_sub(dma_info->page, pg_strides - wi->skbs_frags[i]);
		mlx5e_page_release(rq, dma_info, true);
		}
		}

		void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq)
		static void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq)
		{
		struct mlx5_wq_ll *wq = &rq->wq;
		struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(wq, wq->head);

		clear_bit(MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS, &rq->state);

		if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state))) {
		mlx5e_free_rx_mpwqe(rq, &rq->mpwqe.info[wq->head]);
		return;
		}
		rq->mpwqe.umr_in_progress = false;

		mlx5_wq_ll_push(wq, be16_to_cpu(wqe->next.next_wqe_index));

		@@ -444,16 +432,18 @@ void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq)
		mlx5_wq_ll_update_db_record(wq);
		}

		int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq rq, struct mlx5e_rx_wqe wqe, u16 ix)
		static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
		{
		int err;

		err = mlx5e_alloc_rx_umr_mpwqe(rq, wqe, ix);
		if (unlikely(err))
		err = mlx5e_alloc_rx_umr_mpwqe(rq, ix);
		if (unlikely(err)) {
		rq->stats.buff_alloc_err++;
		return err;
		set_bit(MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS, &rq->state);
		}
		rq->mpwqe.umr_in_progress = true;
		mlx5e_post_umr_wqe(rq, ix);
		return -EBUSY;
		return 0;
		}

		void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
		@@ -463,94 +453,150 @@ void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
		mlx5e_free_rx_mpwqe(rq, wi);
		}

		#define RQ_CANNOT_POST(rq) \
		(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state) \|\| \
		test_bit(MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS, &rq->state))

		bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq)
		{
		struct mlx5_wq_ll *wq = &rq->wq;
		int err;

		if (unlikely(!MLX5E_TEST_BIT(rq->state, MLX5E_RQ_STATE_ENABLED)))
		return false;

		if (unlikely(RQ_CANNOT_POST(rq)))
		if (mlx5_wq_ll_is_full(wq))
		return false;

		while (!mlx5_wq_ll_is_full(wq)) {
		do {
		struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(wq, wq->head);
		int err;

		err = rq->alloc_wqe(rq, wqe, wq->head);
		if (err == -EBUSY)
		return true;
		err = mlx5e_alloc_rx_wqe(rq, wqe, wq->head);
		if (unlikely(err)) {
		rq->stats.buff_alloc_err++;
		break;
		}

		mlx5_wq_ll_push(wq, be16_to_cpu(wqe->next.next_wqe_index));
		}
		} while (!mlx5_wq_ll_is_full(wq));

		/* ensure wqes are visible to device before updating doorbell record */
		dma_wmb();

		mlx5_wq_ll_update_db_record(wq);

		return !mlx5_wq_ll_is_full(wq);
		return !!err;
		}

		static inline void mlx5e_poll_ico_single_cqe(struct mlx5e_cq *cq,
		struct mlx5e_icosq *sq,
		struct mlx5e_rq *rq,
		struct mlx5_cqe64 *cqe)
		{
		struct mlx5_wq_cyc *wq = &sq->wq;
		u16 ci = be16_to_cpu(cqe->wqe_counter) & wq->sz_m1;
		struct mlx5e_sq_wqe_info *icowi = &sq->db.ico_wqe[ci];

		mlx5_cqwq_pop(&cq->wq);

		if (unlikely((cqe->op_own >> 4) != MLX5_CQE_REQ)) {
		WARN_ONCE(true, "mlx5e: Bad OP in ICOSQ CQE: 0x%x\n",
		cqe->op_own);
		return;
		}

		if (likely(icowi->opcode == MLX5_OPCODE_UMR)) {
		mlx5e_post_rx_mpwqe(rq);
		return;
		}

		if (unlikely(icowi->opcode != MLX5_OPCODE_NOP))
		WARN_ONCE(true,
		"mlx5e: Bad OPCODE in ICOSQ WQE info: 0x%x\n",
		icowi->opcode);
		}

		static void mlx5e_poll_ico_cq(struct mlx5e_cq cq, struct mlx5e_rq rq)
		{
		struct mlx5e_icosq *sq = container_of(cq, struct mlx5e_icosq, cq);
		struct mlx5_cqe64 *cqe;

		if (unlikely(!MLX5E_TEST_BIT(sq->state, MLX5E_SQ_STATE_ENABLED)))
		return;

		cqe = mlx5_cqwq_get_cqe(&cq->wq);
		if (likely(!cqe))
		return;

		/* by design, there's only a single cqe */
		mlx5e_poll_ico_single_cqe(cq, sq, rq, cqe);

		mlx5_cqwq_update_db_record(&cq->wq);
		}

		bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq)
		{
		struct mlx5_wq_ll *wq = &rq->wq;

		if (unlikely(!MLX5E_TEST_BIT(rq->state, MLX5E_RQ_STATE_ENABLED)))
		return false;

		mlx5e_poll_ico_cq(&rq->channel->icosq.cq, rq);

		if (mlx5_wq_ll_is_full(wq))
		return false;

		if (!rq->mpwqe.umr_in_progress)
		mlx5e_alloc_rx_mpwqe(rq, wq->head);

		return true;
		}

		static void mlx5e_lro_update_hdr(struct sk_buff skb, struct mlx5_cqe64 cqe,
		u32 cqe_bcnt)
		{
		struct ethhdr eth = (struct ethhdr )(skb->data);
		struct iphdr *ipv4;
		struct ipv6hdr *ipv6;
		struct tcphdr *tcp;
		int network_depth = 0;
		__be16 proto;
		u16 tot_len;
		void *ip_p;

		u8 l4_hdr_type = get_cqe_l4_hdr_type(cqe);
		int tcp_ack = ((l4_hdr_type == CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA) \|\|
		(l4_hdr_type == CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA));
		u8 tcp_ack = (l4_hdr_type == CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA) \|\|
		(l4_hdr_type == CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA);

		skb->mac_len = ETH_HLEN;
		proto = __vlan_get_protocol(skb, eth->h_proto, &network_depth);

		ipv4 = (struct iphdr *)(skb->data + network_depth);
		ipv6 = (struct ipv6hdr *)(skb->data + network_depth);
		tot_len = cqe_bcnt - network_depth;
		ip_p = skb->data + network_depth;

		if (proto == htons(ETH_P_IP)) {
		tcp = (struct tcphdr *)(skb->data + network_depth +
		sizeof(struct iphdr));
		ipv6 = NULL;
		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
		} else {
		tcp = (struct tcphdr *)(skb->data + network_depth +
		sizeof(struct ipv6hdr));
		ipv4 = NULL;
		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
		}
		struct iphdr *ipv4 = ip_p;

		if (get_cqe_lro_tcppsh(cqe))
		tcp->psh = 1;

		if (tcp_ack) {
		tcp->ack = 1;
		tcp->ack_seq = cqe->lro_ack_seq_num;
		tcp->window = cqe->lro_tcp_win;
		}
		tcp = ip_p + sizeof(struct iphdr);
		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;

		if (ipv4) {
		ipv4->ttl = cqe->lro_min_ttl;
		ipv4->tot_len = cpu_to_be16(tot_len);
		ipv4->check = 0;
		ipv4->check = ip_fast_csum((unsigned char *)ipv4,
		ipv4->ihl);
		} else {
		struct ipv6hdr *ipv6 = ip_p;

		tcp = ip_p + sizeof(struct ipv6hdr);
		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;

		ipv6->hop_limit = cqe->lro_min_ttl;
		ipv6->payload_len = cpu_to_be16(tot_len -
		sizeof(struct ipv6hdr));
		}

		tcp->psh = get_cqe_lro_tcppsh(cqe);

		if (tcp_ack) {
		tcp->ack = 1;
		tcp->ack_seq = cqe->lro_ack_seq_num;
		tcp->window = cqe->lro_tcp_win;
		}
		}

		static inline void mlx5e_skb_set_hash(struct mlx5_cqe64 *cqe,
		@@ -776,9 +822,9 @@ struct sk_buff skb_from_cqe(struct mlx5e_rq rq, struct mlx5_cqe64 *cqe,
		struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt)
		{
		struct mlx5e_dma_info *di = &wi->di;
		u16 rx_headroom = rq->buff.headroom;
		struct sk_buff *skb;
		void va, data;
		u16 rx_headroom = rq->rx_headroom;
		bool consumed;
		u32 frag_size;

		@@ -911,7 +957,7 @@ static inline void mlx5e_mpwqe_fill_rx_skb(struct mlx5e_rq *rq,
		struct sk_buff *skb)
		{
		u16 stride_ix = mpwrq_get_cqe_stride_index(cqe);
		u32 wqe_offset = stride_ix * rq->mpwqe_stride_sz;
		u32 wqe_offset = stride_ix << rq->mpwqe.log_stride_sz;
		u32 head_offset = wqe_offset & (PAGE_SIZE - 1);
		u32 page_idx = wqe_offset >> PAGE_SHIFT;
		u32 head_page_idx = page_idx;
		@@ -979,7 +1025,7 @@ void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq rq, struct mlx5_cqe64 cqe)
		napi_gro_receive(rq->cq.napi, skb);

		mpwrq_cqe_out:
		if (likely(wi->consumed_strides < rq->mpwqe_num_strides))
		if (likely(wi->consumed_strides < rq->mpwqe.num_strides))
		return;

		mlx5e_free_rx_mpwqe(rq, wi);
		@@ -989,21 +1035,23 @@ void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq rq, struct mlx5_cqe64 cqe)
		int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget)
		{
		struct mlx5e_rq *rq = container_of(cq, struct mlx5e_rq, cq);
		struct mlx5e_xdpsq *xdpsq = &rq->xdpsq;
		struct mlx5e_xdpsq *xdpsq;
		struct mlx5_cqe64 *cqe;
		int work_done = 0;

		if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state)))
		if (unlikely(!MLX5E_TEST_BIT(rq->state, MLX5E_RQ_STATE_ENABLED)))
		return 0;

		if (cq->decmprs_left)
		work_done += mlx5e_decompress_cqes_cont(rq, cq, 0, budget);

		for (; work_done < budget; work_done++) {
		struct mlx5_cqe64 *cqe = mlx5_cqwq_get_cqe(&cq->wq);

		cqe = mlx5_cqwq_get_cqe(&cq->wq);
		if (!cqe)
		break;
		return 0;

		xdpsq = &rq->xdpsq;

		do {
		if (mlx5_get_cqe_format(cqe) == MLX5_COMPRESSED) {
		work_done +=
		mlx5e_decompress_cqes_start(rq, cq,
		@@ -1014,7 +1062,7 @@ int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget)
		mlx5_cqwq_pop(&cq->wq);

		rq->handle_rx_cqe(rq, cqe);
		}
		} while ((++work_done < budget) && (cqe = mlx5_cqwq_get_cqe(&cq->wq)));

		if (xdpsq->db.doorbell) {
		mlx5e_xmit_xdp_doorbell(xdpsq);
		@@ -1032,13 +1080,18 @@ int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget)
		bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq)
		{
		struct mlx5e_xdpsq *sq;
		struct mlx5_cqe64 *cqe;
		struct mlx5e_rq *rq;
		u16 sqcc;
		int i;

		sq = container_of(cq, struct mlx5e_xdpsq, cq);

		if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)))
		if (unlikely(!MLX5E_TEST_BIT(sq->state, MLX5E_SQ_STATE_ENABLED)))
		return false;

		cqe = mlx5_cqwq_get_cqe(&cq->wq);
		if (!cqe)
		return false;

		rq = container_of(sq, struct mlx5e_rq, xdpsq);
		@@ -1048,15 +1101,11 @@ bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq)
		*/
		sqcc = sq->cc;

		for (i = 0; i < MLX5E_TX_CQ_POLL_BUDGET; i++) {
		struct mlx5_cqe64 *cqe;
		i = 0;
		do {
		u16 wqe_counter;
		bool last_wqe;

		cqe = mlx5_cqwq_get_cqe(&cq->wq);
		if (!cqe)
		break;

		mlx5_cqwq_pop(&cq->wq);

		wqe_counter = be16_to_cpu(cqe->wqe_counter);
		@@ -1074,7 +1123,7 @@ bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq)
		/* Recycle RX page */
		mlx5e_page_release(rq, di, true);
		} while (!last_wqe);
		}
		} while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq)));

		mlx5_cqwq_update_db_record(&cq->wq);

drivers/net/ethernet/mellanox/mlx5/core/en_stats.h

+4 −0

Original line number	Diff line number	Diff line
		@@ -84,6 +84,7 @@ struct mlx5e_sw_stats {
		u64 rx_cache_full;
		u64 rx_cache_empty;
		u64 rx_cache_busy;
		u64 rx_cache_waive;

		/* Special handling counters */
		u64 link_down_events_phy;
		@@ -123,6 +124,7 @@ static const struct counter_desc sw_stats_desc[] = {
		{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_full) },
		{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_empty) },
		{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_busy) },
		{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_waive) },
		{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, link_down_events_phy) },
		};

		@@ -354,6 +356,7 @@ struct mlx5e_rq_stats {
		u64 cache_full;
		u64 cache_empty;
		u64 cache_busy;
		u64 cache_waive;
		};

		static const struct counter_desc rq_stats_desc[] = {
		@@ -377,6 +380,7 @@ static const struct counter_desc rq_stats_desc[] = {
		{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_full) },
		{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_empty) },
		{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_busy) },
		{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cache_waive) },
		};

		struct mlx5e_sq_stats {