Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit f10b7cc7 authored by Saeed Mahameed's avatar Saeed Mahameed Committed by David S. Miller
Browse files

net/mlx5e: Have a clear separation between different SQ types 



Make a clear separate between Regular SQ (TXQ) and ICO SQ creation,
destruction and union their mutual information structures.

Don't allocate redundant TXQ skb/wqe_info/dma_fifo arrays for ICO SQ.
And have a different SQ edge for ICO SQ than TXQ SQ, to be more
accurate.

In preparation for XDP TX support.

Signed-off-by: default avatarSaeed Mahameed <saeedm@mellanox.com>
Signed-off-by: default avatarTariq Toukan <tariqt@mellanox.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 86994156

drivers/net/ethernet/mellanox/mlx5/core/en.h

+18 −5
Original line number Diff line number Diff line
@@ -101,6 +101,9 @@ 
#define MLX5E_UPDATE_STATS_INTERVAL    200 /* msecs */

#define MLX5E_SQ_BF_BUDGET             16



#define MLX5E_ICOSQ_MAX_WQEBBS \

	(DIV_ROUND_UP(sizeof(struct mlx5e_umr_wqe), MLX5_SEND_WQE_BB))



#define MLX5E_NUM_MAIN_GROUPS 9



static inline u16 mlx5_min_rx_wqes(int wq_type, u32 wq_size)
@@ -386,6 +389,11 @@ struct mlx5e_ico_wqe_info { 
	u8  num_wqebbs;

};



enum mlx5e_sq_type {

	MLX5E_SQ_TXQ,

	MLX5E_SQ_ICO

};



struct mlx5e_sq {

	/* data path */
@@ -403,10 +411,15 @@ struct mlx5e_sq { 


	struct mlx5e_cq            cq;



	/* pointers to per packet info: write@xmit, read@completion */

	/* pointers to per tx element info: write@xmit, read@completion */

	union {

		struct {

			struct sk_buff           **skb;

			struct mlx5e_sq_dma       *dma_fifo;

			struct mlx5e_tx_wqe_info  *wqe_info;

		} txq;

		struct mlx5e_ico_wqe_info *ico_wqe;

	} db;



	/* read only */

	struct mlx5_wq_cyc         wq;
@@ -428,8 +441,8 @@ struct mlx5e_sq { 
	struct mlx5_uar            uar;

	struct mlx5e_channel      *channel;

	int                        tc;

	struct mlx5e_ico_wqe_info *ico_wqe_info;

	u32                        rate_limit;

	u8                         type;

} ____cacheline_aligned_in_smp;



static inline bool mlx5e_sq_has_room_for(struct mlx5e_sq *sq, u16 n)

drivers/net/ethernet/mellanox/mlx5/core/en_main.c

+79 −42
Original line number Diff line number Diff line
@@ -51,7 +51,7 @@ struct mlx5e_sq_param { 
	struct mlx5_wq_param       wq;

	u16                        max_inline;

	u8                         min_inline_mode;

	bool                       icosq;

	enum mlx5e_sq_type         type;

};



struct mlx5e_cq_param {
@@ -740,8 +740,8 @@ static int mlx5e_open_rq(struct mlx5e_channel *c, 
	if (param->am_enabled)

		set_bit(MLX5E_RQ_STATE_AM, &c->rq.state);



	sq->ico_wqe_info[pi].opcode     = MLX5_OPCODE_NOP;

	sq->ico_wqe_info[pi].num_wqebbs = 1;

	sq->db.ico_wqe[pi].opcode     = MLX5_OPCODE_NOP;

	sq->db.ico_wqe[pi].num_wqebbs = 1;

	mlx5e_send_nop(sq, true); /* trigger mlx5e_post_rx_wqes() */



	return 0;
@@ -765,26 +765,43 @@ static void mlx5e_close_rq(struct mlx5e_rq *rq) 
	mlx5e_destroy_rq(rq);

}



static void mlx5e_free_sq_db(struct mlx5e_sq *sq)

static void mlx5e_free_sq_ico_db(struct mlx5e_sq *sq)

{

	kfree(sq->wqe_info);

	kfree(sq->dma_fifo);

	kfree(sq->skb);

	kfree(sq->db.ico_wqe);

}



static int mlx5e_alloc_sq_db(struct mlx5e_sq *sq, int numa)

static int mlx5e_alloc_sq_ico_db(struct mlx5e_sq *sq, int numa)

{

	u8 wq_sz = mlx5_wq_cyc_get_size(&sq->wq);



	sq->db.ico_wqe = kzalloc_node(sizeof(*sq->db.ico_wqe) * wq_sz,

				      GFP_KERNEL, numa);

	if (!sq->db.ico_wqe)

		return -ENOMEM;



	return 0;

}



static void mlx5e_free_sq_txq_db(struct mlx5e_sq *sq)

{

	kfree(sq->db.txq.wqe_info);

	kfree(sq->db.txq.dma_fifo);

	kfree(sq->db.txq.skb);

}



static int mlx5e_alloc_sq_txq_db(struct mlx5e_sq *sq, int numa)

{

	int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);

	int df_sz = wq_sz * MLX5_SEND_WQEBB_NUM_DS;



	sq->skb = kzalloc_node(wq_sz * sizeof(*sq->skb), GFP_KERNEL, numa);

	sq->dma_fifo = kzalloc_node(df_sz * sizeof(*sq->dma_fifo), GFP_KERNEL,

				    numa);

	sq->wqe_info = kzalloc_node(wq_sz * sizeof(*sq->wqe_info), GFP_KERNEL,

				    numa);



	if (!sq->skb || !sq->dma_fifo || !sq->wqe_info) {

		mlx5e_free_sq_db(sq);

	sq->db.txq.skb = kzalloc_node(wq_sz * sizeof(*sq->db.txq.skb),

				      GFP_KERNEL, numa);

	sq->db.txq.dma_fifo = kzalloc_node(df_sz * sizeof(*sq->db.txq.dma_fifo),

					   GFP_KERNEL, numa);

	sq->db.txq.wqe_info = kzalloc_node(wq_sz * sizeof(*sq->db.txq.wqe_info),

					   GFP_KERNEL, numa);

	if (!sq->db.txq.skb || !sq->db.txq.dma_fifo || !sq->db.txq.wqe_info) {

		mlx5e_free_sq_txq_db(sq);

		return -ENOMEM;

	}
@@ -793,6 +810,30 @@ static int mlx5e_alloc_sq_db(struct mlx5e_sq *sq, int numa) 
	return 0;

}



static void mlx5e_free_sq_db(struct mlx5e_sq *sq)

{

	switch (sq->type) {

	case MLX5E_SQ_TXQ:

		mlx5e_free_sq_txq_db(sq);

		break;

	case MLX5E_SQ_ICO:

		mlx5e_free_sq_ico_db(sq);

		break;

	}

}



static int mlx5e_alloc_sq_db(struct mlx5e_sq *sq, int numa)

{

	switch (sq->type) {

	case MLX5E_SQ_TXQ:

		return mlx5e_alloc_sq_txq_db(sq, numa);

	case MLX5E_SQ_ICO:

		return mlx5e_alloc_sq_ico_db(sq, numa);

	}



	return 0;

}



static int mlx5e_create_sq(struct mlx5e_channel *c,

			   int tc,

			   struct mlx5e_sq_param *param,
@@ -803,8 +844,16 @@ static int mlx5e_create_sq(struct mlx5e_channel *c, 


	void *sqc = param->sqc;

	void *sqc_wq = MLX5_ADDR_OF(sqc, sqc, wq);

	u16 sq_max_wqebbs;

	int err;



	sq->type      = param->type;

	sq->pdev      = c->pdev;

	sq->tstamp    = &priv->tstamp;

	sq->mkey_be   = c->mkey_be;

	sq->channel   = c;

	sq->tc        = tc;



	err = mlx5_alloc_map_uar(mdev, &sq->uar, !!MLX5_CAP_GEN(mdev, bf));

	if (err)

		return err;
@@ -833,18 +882,8 @@ static int mlx5e_create_sq(struct mlx5e_channel *c, 
	if (err)

		goto err_sq_wq_destroy;



	if (param->icosq) {

		u8 wq_sz = mlx5_wq_cyc_get_size(&sq->wq);



		sq->ico_wqe_info = kzalloc_node(sizeof(*sq->ico_wqe_info) *

						wq_sz,

						GFP_KERNEL,

						cpu_to_node(c->cpu));

		if (!sq->ico_wqe_info) {

			err = -ENOMEM;

			goto err_free_sq_db;

		}

	} else {

	sq_max_wqebbs = MLX5_SEND_WQE_MAX_WQEBBS;

	if (sq->type == MLX5E_SQ_TXQ) {

		int txq_ix;



		txq_ix = c->ix + tc * priv->params.num_channels;
@@ -852,19 +891,14 @@ static int mlx5e_create_sq(struct mlx5e_channel *c, 
		priv->txq_to_sq_map[txq_ix] = sq;

	}



	sq->pdev      = c->pdev;

	sq->tstamp    = &priv->tstamp;

	sq->mkey_be   = c->mkey_be;

	sq->channel   = c;

	sq->tc        = tc;

	sq->edge      = (sq->wq.sz_m1 + 1) - MLX5_SEND_WQE_MAX_WQEBBS;

	if (sq->type == MLX5E_SQ_ICO)

		sq_max_wqebbs = MLX5E_ICOSQ_MAX_WQEBBS;



	sq->edge      = (sq->wq.sz_m1 + 1) - sq_max_wqebbs;

	sq->bf_budget = MLX5E_SQ_BF_BUDGET;



	return 0;



err_free_sq_db:

	mlx5e_free_sq_db(sq);



err_sq_wq_destroy:

	mlx5_wq_destroy(&sq->wq_ctrl);
@@ -879,7 +913,6 @@ static void mlx5e_destroy_sq(struct mlx5e_sq *sq) 
	struct mlx5e_channel *c = sq->channel;

	struct mlx5e_priv *priv = c->priv;



	kfree(sq->ico_wqe_info);

	mlx5e_free_sq_db(sq);

	mlx5_wq_destroy(&sq->wq_ctrl);

	mlx5_unmap_free_uar(priv->mdev, &sq->uar);
@@ -908,11 +941,12 @@ static int mlx5e_enable_sq(struct mlx5e_sq *sq, struct mlx5e_sq_param *param) 


	memcpy(sqc, param->sqc, sizeof(param->sqc));



	MLX5_SET(sqc,  sqc, tis_num_0, param->icosq ? 0 : priv->tisn[sq->tc]);

	MLX5_SET(sqc,  sqc, tis_num_0, param->type == MLX5E_SQ_ICO ?

				       0 : priv->tisn[sq->tc]);

	MLX5_SET(sqc,  sqc, cqn,		sq->cq.mcq.cqn);

	MLX5_SET(sqc,  sqc, min_wqe_inline_mode, sq->min_inline_mode);

	MLX5_SET(sqc,  sqc, state,		MLX5_SQC_STATE_RST);

	MLX5_SET(sqc,  sqc, tis_lst_sz,		param->icosq ? 0 : 1);

	MLX5_SET(sqc,  sqc, tis_lst_sz, param->type == MLX5E_SQ_ICO ? 0 : 1);

	MLX5_SET(sqc,  sqc, flush_in_error_en,	1);



	MLX5_SET(wq,   wq, wq_type,       MLX5_WQ_TYPE_CYCLIC);
@@ -1027,9 +1061,11 @@ static void mlx5e_close_sq(struct mlx5e_sq *sq) 
		netif_tx_disable_queue(sq->txq);



		/* last doorbell out, godspeed .. */

		if (mlx5e_sq_has_room_for(sq, 1))

		if (mlx5e_sq_has_room_for(sq, 1)) {

			sq->db.txq.skb[(sq->pc & sq->wq.sz_m1)] = NULL;

			mlx5e_send_nop(sq, true);

		}

	}



	mlx5e_disable_sq(sq);

	mlx5e_free_tx_descs(sq);
@@ -1505,6 +1541,7 @@ static void mlx5e_build_sq_param(struct mlx5e_priv *priv, 


	param->max_inline = priv->params.tx_max_inline;

	param->min_inline_mode = priv->params.tx_min_inline_mode;

	param->type = MLX5E_SQ_TXQ;

}



static void mlx5e_build_common_cq_param(struct mlx5e_priv *priv,
@@ -1578,7 +1615,7 @@ static void mlx5e_build_icosq_param(struct mlx5e_priv *priv, 
	MLX5_SET(wq, wq, log_wq_sz, log_wq_size);

	MLX5_SET(sqc, sqc, reg_umr, MLX5_CAP_ETH(priv->mdev, reg_umr_sq));



	param->icosq = true;

	param->type = MLX5E_SQ_ICO;

}



static void mlx5e_build_channel_param(struct mlx5e_priv *priv, struct mlx5e_channel_param *cparam)

drivers/net/ethernet/mellanox/mlx5/core/en_rx.c

+4 −4
Original line number Diff line number Diff line
@@ -337,8 +337,8 @@ static inline void mlx5e_post_umr_wqe(struct mlx5e_rq *rq, u16 ix) 


	/* fill sq edge with nops to avoid wqe wrap around */

	while ((pi = (sq->pc & wq->sz_m1)) > sq->edge) {

		sq->ico_wqe_info[pi].opcode = MLX5_OPCODE_NOP;

		sq->ico_wqe_info[pi].num_wqebbs = 1;

		sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_NOP;

		sq->db.ico_wqe[pi].num_wqebbs = 1;

		mlx5e_send_nop(sq, true);

	}
@@ -348,8 +348,8 @@ static inline void mlx5e_post_umr_wqe(struct mlx5e_rq *rq, u16 ix) 
		cpu_to_be32((sq->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) |

			    MLX5_OPCODE_UMR);



	sq->ico_wqe_info[pi].opcode = MLX5_OPCODE_UMR;

	sq->ico_wqe_info[pi].num_wqebbs = num_wqebbs;

	sq->db.ico_wqe[pi].opcode = MLX5_OPCODE_UMR;

	sq->db.ico_wqe[pi].num_wqebbs = num_wqebbs;

	sq->pc += num_wqebbs;

	mlx5e_tx_notify_hw(sq, &wqe->ctrl, 0);

}

drivers/net/ethernet/mellanox/mlx5/core/en_tx.c

+18 −12
Original line number Diff line number Diff line
@@ -52,7 +52,6 @@ void mlx5e_send_nop(struct mlx5e_sq *sq, bool notify_hw) 
	cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_NOP);

	cseg->qpn_ds           = cpu_to_be32((sq->sqn << 8) | 0x01);



	sq->skb[pi] = NULL;

	sq->pc++;

	sq->stats.nop++;
@@ -82,15 +81,17 @@ static inline void mlx5e_dma_push(struct mlx5e_sq *sq, 
				  u32 size,

				  enum mlx5e_dma_map_type map_type)

{

	sq->dma_fifo[sq->dma_fifo_pc & sq->dma_fifo_mask].addr = addr;

	sq->dma_fifo[sq->dma_fifo_pc & sq->dma_fifo_mask].size = size;

	sq->dma_fifo[sq->dma_fifo_pc & sq->dma_fifo_mask].type = map_type;

	u32 i = sq->dma_fifo_pc & sq->dma_fifo_mask;



	sq->db.txq.dma_fifo[i].addr = addr;

	sq->db.txq.dma_fifo[i].size = size;

	sq->db.txq.dma_fifo[i].type = map_type;

	sq->dma_fifo_pc++;

}



static inline struct mlx5e_sq_dma *mlx5e_dma_get(struct mlx5e_sq *sq, u32 i)

{

	return &sq->dma_fifo[i & sq->dma_fifo_mask];

	return &sq->db.txq.dma_fifo[i & sq->dma_fifo_mask];

}



static void mlx5e_dma_unmap_wqe_err(struct mlx5e_sq *sq, u8 num_dma)
@@ -221,7 +222,7 @@ static netdev_tx_t mlx5e_sq_xmit(struct mlx5e_sq *sq, struct sk_buff *skb) 


	u16 pi = sq->pc & wq->sz_m1;

	struct mlx5e_tx_wqe      *wqe  = mlx5_wq_cyc_get_wqe(wq, pi);

	struct mlx5e_tx_wqe_info *wi   = &sq->wqe_info[pi];

	struct mlx5e_tx_wqe_info *wi   = &sq->db.txq.wqe_info[pi];



	struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;

	struct mlx5_wqe_eth_seg  *eseg = &wqe->eth;
@@ -341,7 +342,7 @@ static netdev_tx_t mlx5e_sq_xmit(struct mlx5e_sq *sq, struct sk_buff *skb) 
	cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | opcode);

	cseg->qpn_ds           = cpu_to_be32((sq->sqn << 8) | ds_cnt);



	sq->skb[pi] = skb;

	sq->db.txq.skb[pi] = skb;



	wi->num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);

	sq->pc += wi->num_wqebbs;
@@ -368,8 +369,10 @@ static netdev_tx_t mlx5e_sq_xmit(struct mlx5e_sq *sq, struct sk_buff *skb) 
	}



	/* fill sq edge with nops to avoid wqe wrap around */

	while ((sq->pc & wq->sz_m1) > sq->edge)

	while ((pi = (sq->pc & wq->sz_m1)) > sq->edge) {

		sq->db.txq.skb[pi] = NULL;

		mlx5e_send_nop(sq, false);

	}



	if (bf)

		sq->bf_budget--;
@@ -442,8 +445,8 @@ bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget) 
			last_wqe = (sqcc == wqe_counter);



			ci = sqcc & sq->wq.sz_m1;

			skb = sq->skb[ci];

			wi = &sq->wqe_info[ci];

			skb = sq->db.txq.skb[ci];

			wi = &sq->db.txq.wqe_info[ci];



			if (unlikely(!skb)) { /* nop */

				sqcc++;
@@ -499,10 +502,13 @@ void mlx5e_free_tx_descs(struct mlx5e_sq *sq) 
	u16 ci;

	int i;



	if (sq->type != MLX5E_SQ_TXQ)

		return;



	while (sq->cc != sq->pc) {

		ci = sq->cc & sq->wq.sz_m1;

		skb = sq->skb[ci];

		wi = &sq->wqe_info[ci];

		skb = sq->db.txq.skb[ci];

		wi = &sq->db.txq.wqe_info[ci];



		if (!skb) { /* nop */

			sq->cc++;

drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c

+1 −1
Original line number Diff line number Diff line
@@ -72,7 +72,7 @@ static void mlx5e_poll_ico_cq(struct mlx5e_cq *cq) 


	do {

		u16 ci = be16_to_cpu(cqe->wqe_counter) & wq->sz_m1;

		struct mlx5e_ico_wqe_info *icowi = &sq->ico_wqe_info[ci];

		struct mlx5e_ico_wqe_info *icowi = &sq->db.ico_wqe[ci];



		mlx5_cqwq_pop(&cq->wq);

		sqcc += icowi->num_wqebbs;