Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit f02d0d6e authored by Leon Romanovsky's avatar Leon Romanovsky
Browse files

net/mlx5: Move SRQ functions to RDMA part 



There is no need to keep SRQ which is RDMA object in mlx5_core.
In this patch, we partially move the execution code, while next patches
will move table initialization/release logic too.

Reviewed-by: default avatarMark Bloch <markb@mellanox.com>
Signed-off-by: default avatarLeon Romanovsky <leonro@mellanox.com>
parent c23f88cb

drivers/infiniband/hw/mlx5/Makefile

+3 −1
Original line number Diff line number Diff line 
obj-$(CONFIG_MLX5_INFINIBAND)	+= mlx5_ib.o



mlx5_ib-y :=	main.o cq.o doorbell.o qp.o mem.o srq.o mr.o ah.o mad.o gsi.o ib_virt.o cmd.o cong.o

mlx5_ib-y :=	main.o cq.o doorbell.o qp.o mem.o srq_cmd.o \

		srq.o mr.o ah.o mad.o gsi.o ib_virt.o cmd.o \

		cong.o

mlx5_ib-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += odp.o

mlx5_ib-$(CONFIG_MLX5_ESWITCH) += ib_rep.o

mlx5_ib-$(CONFIG_INFINIBAND_USER_ACCESS) += devx.o

drivers/infiniband/hw/mlx5/cq.c

+1 −0
Original line number Diff line number Diff line
@@ -35,6 +35,7 @@ 
#include <rdma/ib_user_verbs.h>

#include <rdma/ib_cache.h>

#include "mlx5_ib.h"

#include "srq.h"



static void mlx5_ib_cq_comp(struct mlx5_core_cq *cq)

{

drivers/infiniband/hw/mlx5/srq.c

+1 −1
Original line number Diff line number Diff line
@@ -9,8 +9,8 @@ 
#include <linux/slab.h>

#include <rdma/ib_umem.h>

#include <rdma/ib_user_verbs.h>



#include "mlx5_ib.h"

#include "srq.h"



/* not supported currently */

static int srq_signature;

drivers/infiniband/hw/mlx5/srq.h

0 → 100644
+46 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */

/*

 * Copyright (c) 2013-2018, Mellanox Technologies. All rights reserved.

 */



#ifndef MLX5_IB_SRQ_H

#define MLX5_IB_SRQ_H



enum {

	MLX5_SRQ_FLAG_ERR    = (1 << 0),

	MLX5_SRQ_FLAG_WQ_SIG = (1 << 1),

	MLX5_SRQ_FLAG_RNDV   = (1 << 2),

};



struct mlx5_srq_attr {

	u32 type;

	u32 flags;

	u32 log_size;

	u32 wqe_shift;

	u32 log_page_size;

	u32 wqe_cnt;

	u32 srqn;

	u32 xrcd;

	u32 page_offset;

	u32 cqn;

	u32 pd;

	u32 lwm;

	u32 user_index;

	u64 db_record;

	__be64 *pas;

	u32 tm_log_list_size;

	u32 tm_next_tag;

	u32 tm_hw_phase_cnt;

	u32 tm_sw_phase_cnt;

	u16 uid;

};



int mlx5_core_create_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,

			 struct mlx5_srq_attr *in);

int mlx5_core_destroy_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq);

int mlx5_core_query_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,

			struct mlx5_srq_attr *out);

int mlx5_core_arm_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,

		      u16 lwm, int is_srq);

struct mlx5_core_srq *mlx5_core_get_srq(struct mlx5_core_dev *dev, u32 srqn);

#endif /* MLX5_IB_SRQ_H */

drivers/infiniband/hw/mlx5/srq_cmd.c

0 → 100644
+666 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB

/*

 * Copyright (c) 2013-2018, Mellanox Technologies inc.  All rights reserved.

 */



#include <linux/kernel.h>

#include <linux/mlx5/driver.h>

#include <linux/mlx5/cmd.h>

#include "srq.h"



static int get_pas_size(struct mlx5_srq_attr *in)

{

	u32 log_page_size = in->log_page_size + 12;

	u32 log_srq_size  = in->log_size;

	u32 log_rq_stride = in->wqe_shift;

	u32 page_offset   = in->page_offset;

	u32 po_quanta	  = 1 << (log_page_size - 6);

	u32 rq_sz	  = 1 << (log_srq_size + 4 + log_rq_stride);

	u32 page_size	  = 1 << log_page_size;

	u32 rq_sz_po      = rq_sz + (page_offset * po_quanta);

	u32 rq_num_pas    = DIV_ROUND_UP(rq_sz_po, page_size);



	return rq_num_pas * sizeof(u64);

}



static void set_wq(void *wq, struct mlx5_srq_attr *in)

{

	MLX5_SET(wq,   wq, wq_signature,  !!(in->flags

		 & MLX5_SRQ_FLAG_WQ_SIG));

	MLX5_SET(wq,   wq, log_wq_pg_sz,  in->log_page_size);

	MLX5_SET(wq,   wq, log_wq_stride, in->wqe_shift + 4);

	MLX5_SET(wq,   wq, log_wq_sz,     in->log_size);

	MLX5_SET(wq,   wq, page_offset,   in->page_offset);

	MLX5_SET(wq,   wq, lwm,		  in->lwm);

	MLX5_SET(wq,   wq, pd,		  in->pd);

	MLX5_SET64(wq, wq, dbr_addr,	  in->db_record);

}



static void set_srqc(void *srqc, struct mlx5_srq_attr *in)

{

	MLX5_SET(srqc,   srqc, wq_signature,  !!(in->flags

		 & MLX5_SRQ_FLAG_WQ_SIG));

	MLX5_SET(srqc,   srqc, log_page_size, in->log_page_size);

	MLX5_SET(srqc,   srqc, log_rq_stride, in->wqe_shift);

	MLX5_SET(srqc,   srqc, log_srq_size,  in->log_size);

	MLX5_SET(srqc,   srqc, page_offset,   in->page_offset);

	MLX5_SET(srqc,	 srqc, lwm,	      in->lwm);

	MLX5_SET(srqc,	 srqc, pd,	      in->pd);

	MLX5_SET64(srqc, srqc, dbr_addr,      in->db_record);

	MLX5_SET(srqc,	 srqc, xrcd,	      in->xrcd);

	MLX5_SET(srqc,	 srqc, cqn,	      in->cqn);

}



static void get_wq(void *wq, struct mlx5_srq_attr *in)

{

	if (MLX5_GET(wq, wq, wq_signature))

		in->flags &= MLX5_SRQ_FLAG_WQ_SIG;

	in->log_page_size = MLX5_GET(wq,   wq, log_wq_pg_sz);

	in->wqe_shift	  = MLX5_GET(wq,   wq, log_wq_stride) - 4;

	in->log_size	  = MLX5_GET(wq,   wq, log_wq_sz);

	in->page_offset   = MLX5_GET(wq,   wq, page_offset);

	in->lwm		  = MLX5_GET(wq,   wq, lwm);

	in->pd		  = MLX5_GET(wq,   wq, pd);

	in->db_record	  = MLX5_GET64(wq, wq, dbr_addr);

}



static void get_srqc(void *srqc, struct mlx5_srq_attr *in)

{

	if (MLX5_GET(srqc, srqc, wq_signature))

		in->flags &= MLX5_SRQ_FLAG_WQ_SIG;

	in->log_page_size = MLX5_GET(srqc,   srqc, log_page_size);

	in->wqe_shift	  = MLX5_GET(srqc,   srqc, log_rq_stride);

	in->log_size	  = MLX5_GET(srqc,   srqc, log_srq_size);

	in->page_offset   = MLX5_GET(srqc,   srqc, page_offset);

	in->lwm		  = MLX5_GET(srqc,   srqc, lwm);

	in->pd		  = MLX5_GET(srqc,   srqc, pd);

	in->db_record	  = MLX5_GET64(srqc, srqc, dbr_addr);

}



struct mlx5_core_srq *mlx5_core_get_srq(struct mlx5_core_dev *dev, u32 srqn)

{

	struct mlx5_srq_table *table = &dev->priv.srq_table;

	struct mlx5_core_srq *srq;



	spin_lock(&table->lock);



	srq = radix_tree_lookup(&table->tree, srqn);

	if (srq)

		atomic_inc(&srq->refcount);



	spin_unlock(&table->lock);



	return srq;

}



static int create_srq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,

			  struct mlx5_srq_attr *in)

{

	u32 create_out[MLX5_ST_SZ_DW(create_srq_out)] = {0};

	void *create_in;

	void *srqc;

	void *pas;

	int pas_size;

	int inlen;

	int err;



	pas_size  = get_pas_size(in);

	inlen	  = MLX5_ST_SZ_BYTES(create_srq_in) + pas_size;

	create_in = kvzalloc(inlen, GFP_KERNEL);

	if (!create_in)

		return -ENOMEM;



	MLX5_SET(create_srq_in, create_in, uid, in->uid);

	srqc = MLX5_ADDR_OF(create_srq_in, create_in, srq_context_entry);

	pas = MLX5_ADDR_OF(create_srq_in, create_in, pas);



	set_srqc(srqc, in);

	memcpy(pas, in->pas, pas_size);



	MLX5_SET(create_srq_in, create_in, opcode,

		 MLX5_CMD_OP_CREATE_SRQ);



	err = mlx5_cmd_exec(dev, create_in, inlen, create_out,

			    sizeof(create_out));

	kvfree(create_in);

	if (!err) {

		srq->srqn = MLX5_GET(create_srq_out, create_out, srqn);

		srq->uid = in->uid;

	}



	return err;

}



static int destroy_srq_cmd(struct mlx5_core_dev *dev,

			   struct mlx5_core_srq *srq)

{

	u32 srq_in[MLX5_ST_SZ_DW(destroy_srq_in)] = {0};

	u32 srq_out[MLX5_ST_SZ_DW(destroy_srq_out)] = {0};



	MLX5_SET(destroy_srq_in, srq_in, opcode,

		 MLX5_CMD_OP_DESTROY_SRQ);

	MLX5_SET(destroy_srq_in, srq_in, srqn, srq->srqn);

	MLX5_SET(destroy_srq_in, srq_in, uid, srq->uid);



	return mlx5_cmd_exec(dev, srq_in, sizeof(srq_in),

			     srq_out, sizeof(srq_out));

}



static int arm_srq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,

		       u16 lwm, int is_srq)

{

	u32 srq_in[MLX5_ST_SZ_DW(arm_rq_in)] = {0};

	u32 srq_out[MLX5_ST_SZ_DW(arm_rq_out)] = {0};



	MLX5_SET(arm_rq_in, srq_in, opcode, MLX5_CMD_OP_ARM_RQ);

	MLX5_SET(arm_rq_in, srq_in, op_mod, MLX5_ARM_RQ_IN_OP_MOD_SRQ);

	MLX5_SET(arm_rq_in, srq_in, srq_number, srq->srqn);

	MLX5_SET(arm_rq_in, srq_in, lwm,      lwm);

	MLX5_SET(arm_rq_in, srq_in, uid, srq->uid);



	return  mlx5_cmd_exec(dev, srq_in, sizeof(srq_in),

			      srq_out, sizeof(srq_out));

}



static int query_srq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,

			 struct mlx5_srq_attr *out)

{

	u32 srq_in[MLX5_ST_SZ_DW(query_srq_in)] = {0};

	u32 *srq_out;

	void *srqc;

	int err;



	srq_out = kvzalloc(MLX5_ST_SZ_BYTES(query_srq_out), GFP_KERNEL);

	if (!srq_out)

		return -ENOMEM;



	MLX5_SET(query_srq_in, srq_in, opcode,

		 MLX5_CMD_OP_QUERY_SRQ);

	MLX5_SET(query_srq_in, srq_in, srqn, srq->srqn);

	err =  mlx5_cmd_exec(dev, srq_in, sizeof(srq_in),

			     srq_out, MLX5_ST_SZ_BYTES(query_srq_out));

	if (err)

		goto out;



	srqc = MLX5_ADDR_OF(query_srq_out, srq_out, srq_context_entry);

	get_srqc(srqc, out);

	if (MLX5_GET(srqc, srqc, state) != MLX5_SRQC_STATE_GOOD)

		out->flags |= MLX5_SRQ_FLAG_ERR;

out:

	kvfree(srq_out);

	return err;

}



static int create_xrc_srq_cmd(struct mlx5_core_dev *dev,

			      struct mlx5_core_srq *srq,

			      struct mlx5_srq_attr *in)

{

	u32 create_out[MLX5_ST_SZ_DW(create_xrc_srq_out)];

	void *create_in;

	void *xrc_srqc;

	void *pas;

	int pas_size;

	int inlen;

	int err;



	pas_size  = get_pas_size(in);

	inlen	  = MLX5_ST_SZ_BYTES(create_xrc_srq_in) + pas_size;

	create_in = kvzalloc(inlen, GFP_KERNEL);

	if (!create_in)

		return -ENOMEM;



	MLX5_SET(create_xrc_srq_in, create_in, uid, in->uid);

	xrc_srqc = MLX5_ADDR_OF(create_xrc_srq_in, create_in,

				xrc_srq_context_entry);

	pas	 = MLX5_ADDR_OF(create_xrc_srq_in, create_in, pas);



	set_srqc(xrc_srqc, in);

	MLX5_SET(xrc_srqc, xrc_srqc, user_index, in->user_index);

	memcpy(pas, in->pas, pas_size);

	MLX5_SET(create_xrc_srq_in, create_in, opcode,

		 MLX5_CMD_OP_CREATE_XRC_SRQ);



	memset(create_out, 0, sizeof(create_out));

	err = mlx5_cmd_exec(dev, create_in, inlen, create_out,

			    sizeof(create_out));

	if (err)

		goto out;



	srq->srqn = MLX5_GET(create_xrc_srq_out, create_out, xrc_srqn);

	srq->uid = in->uid;

out:

	kvfree(create_in);

	return err;

}



static int destroy_xrc_srq_cmd(struct mlx5_core_dev *dev,

			       struct mlx5_core_srq *srq)

{

	u32 xrcsrq_in[MLX5_ST_SZ_DW(destroy_xrc_srq_in)]   = {0};

	u32 xrcsrq_out[MLX5_ST_SZ_DW(destroy_xrc_srq_out)] = {0};



	MLX5_SET(destroy_xrc_srq_in, xrcsrq_in, opcode,

		 MLX5_CMD_OP_DESTROY_XRC_SRQ);

	MLX5_SET(destroy_xrc_srq_in, xrcsrq_in, xrc_srqn, srq->srqn);

	MLX5_SET(destroy_xrc_srq_in, xrcsrq_in, uid, srq->uid);



	return mlx5_cmd_exec(dev, xrcsrq_in, sizeof(xrcsrq_in),

			     xrcsrq_out, sizeof(xrcsrq_out));

}



static int arm_xrc_srq_cmd(struct mlx5_core_dev *dev,

			   struct mlx5_core_srq *srq, u16 lwm)

{

	u32 xrcsrq_in[MLX5_ST_SZ_DW(arm_xrc_srq_in)]   = {0};

	u32 xrcsrq_out[MLX5_ST_SZ_DW(arm_xrc_srq_out)] = {0};



	MLX5_SET(arm_xrc_srq_in, xrcsrq_in, opcode,   MLX5_CMD_OP_ARM_XRC_SRQ);

	MLX5_SET(arm_xrc_srq_in, xrcsrq_in, op_mod,   MLX5_ARM_XRC_SRQ_IN_OP_MOD_XRC_SRQ);

	MLX5_SET(arm_xrc_srq_in, xrcsrq_in, xrc_srqn, srq->srqn);

	MLX5_SET(arm_xrc_srq_in, xrcsrq_in, lwm,      lwm);

	MLX5_SET(arm_xrc_srq_in, xrcsrq_in, uid, srq->uid);



	return  mlx5_cmd_exec(dev, xrcsrq_in, sizeof(xrcsrq_in),

			      xrcsrq_out, sizeof(xrcsrq_out));

}



static int query_xrc_srq_cmd(struct mlx5_core_dev *dev,

			     struct mlx5_core_srq *srq,

			     struct mlx5_srq_attr *out)

{

	u32 xrcsrq_in[MLX5_ST_SZ_DW(query_xrc_srq_in)];

	u32 *xrcsrq_out;

	void *xrc_srqc;

	int err;



	xrcsrq_out = kvzalloc(MLX5_ST_SZ_BYTES(query_xrc_srq_out), GFP_KERNEL);

	if (!xrcsrq_out)

		return -ENOMEM;

	memset(xrcsrq_in, 0, sizeof(xrcsrq_in));



	MLX5_SET(query_xrc_srq_in, xrcsrq_in, opcode,

		 MLX5_CMD_OP_QUERY_XRC_SRQ);

	MLX5_SET(query_xrc_srq_in, xrcsrq_in, xrc_srqn, srq->srqn);



	err =  mlx5_cmd_exec(dev, xrcsrq_in, sizeof(xrcsrq_in), xrcsrq_out,

			     MLX5_ST_SZ_BYTES(query_xrc_srq_out));

	if (err)

		goto out;



	xrc_srqc = MLX5_ADDR_OF(query_xrc_srq_out, xrcsrq_out,

				xrc_srq_context_entry);

	get_srqc(xrc_srqc, out);

	if (MLX5_GET(xrc_srqc, xrc_srqc, state) != MLX5_XRC_SRQC_STATE_GOOD)

		out->flags |= MLX5_SRQ_FLAG_ERR;



out:

	kvfree(xrcsrq_out);

	return err;

}



static int mlx5_core_create_rmp(struct mlx5_core_dev *dev, u32 *in, int inlen,

				u32 *rmpn)

{

	u32 out[MLX5_ST_SZ_DW(create_rmp_out)] = { 0 };

	int err;



	MLX5_SET(create_rmp_in, in, opcode, MLX5_CMD_OP_CREATE_RMP);

	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));

	if (!err)

		*rmpn = MLX5_GET(create_rmp_out, out, rmpn);



	return err;

}



static int mlx5_core_modify_rmp(struct mlx5_core_dev *dev, u32 *in, int inlen)

{

	u32 out[MLX5_ST_SZ_DW(modify_rmp_out)] = {0};



	MLX5_SET(modify_rmp_in, in, opcode, MLX5_CMD_OP_MODIFY_RMP);

	return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));

}



static int mlx5_core_query_rmp(struct mlx5_core_dev *dev, u32 rmpn, u32 *out)

{

	u32 in[MLX5_ST_SZ_DW(query_rmp_in)] = {0};

	int outlen = MLX5_ST_SZ_BYTES(query_rmp_out);



	MLX5_SET(query_rmp_in, in, opcode, MLX5_CMD_OP_QUERY_RMP);

	MLX5_SET(query_rmp_in, in, rmpn,   rmpn);

	return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);

}



static int create_rmp_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,

			  struct mlx5_srq_attr *in)

{

	void *create_in;

	void *rmpc;

	void *wq;

	int pas_size;

	int inlen;

	int err;



	pas_size = get_pas_size(in);

	inlen = MLX5_ST_SZ_BYTES(create_rmp_in) + pas_size;

	create_in = kvzalloc(inlen, GFP_KERNEL);

	if (!create_in)

		return -ENOMEM;



	rmpc = MLX5_ADDR_OF(create_rmp_in, create_in, ctx);

	wq = MLX5_ADDR_OF(rmpc, rmpc, wq);



	MLX5_SET(rmpc, rmpc, state, MLX5_RMPC_STATE_RDY);

	MLX5_SET(create_rmp_in, create_in, uid, in->uid);

	set_wq(wq, in);

	memcpy(MLX5_ADDR_OF(rmpc, rmpc, wq.pas), in->pas, pas_size);



	err = mlx5_core_create_rmp(dev, create_in, inlen, &srq->srqn);

	if (!err)

		srq->uid = in->uid;



	kvfree(create_in);

	return err;

}



static int destroy_rmp_cmd(struct mlx5_core_dev *dev,

			   struct mlx5_core_srq *srq)

{

	u32 in[MLX5_ST_SZ_DW(destroy_rmp_in)]   = {};

	u32 out[MLX5_ST_SZ_DW(destroy_rmp_out)] = {};



	MLX5_SET(destroy_rmp_in, in, opcode, MLX5_CMD_OP_DESTROY_RMP);

	MLX5_SET(destroy_rmp_in, in, rmpn, srq->srqn);

	MLX5_SET(destroy_rmp_in, in, uid, srq->uid);

	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));

}



static int arm_rmp_cmd(struct mlx5_core_dev *dev,

		       struct mlx5_core_srq *srq,

		       u16 lwm)

{

	void *in;

	void *rmpc;

	void *wq;

	void *bitmask;

	int err;



	in = kvzalloc(MLX5_ST_SZ_BYTES(modify_rmp_in), GFP_KERNEL);

	if (!in)

		return -ENOMEM;



	rmpc =	  MLX5_ADDR_OF(modify_rmp_in,   in,   ctx);

	bitmask = MLX5_ADDR_OF(modify_rmp_in,   in,   bitmask);

	wq   =	  MLX5_ADDR_OF(rmpc,	        rmpc, wq);



	MLX5_SET(modify_rmp_in, in,	 rmp_state, MLX5_RMPC_STATE_RDY);

	MLX5_SET(modify_rmp_in, in,	 rmpn,      srq->srqn);

	MLX5_SET(modify_rmp_in, in, uid, srq->uid);

	MLX5_SET(wq,		wq,	 lwm,	    lwm);

	MLX5_SET(rmp_bitmask,	bitmask, lwm,	    1);

	MLX5_SET(rmpc, rmpc, state, MLX5_RMPC_STATE_RDY);



	err = mlx5_core_modify_rmp(dev, in, MLX5_ST_SZ_BYTES(modify_rmp_in));



	kvfree(in);

	return err;

}



static int query_rmp_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,

			 struct mlx5_srq_attr *out)

{

	u32 *rmp_out;

	void *rmpc;

	int err;



	rmp_out =  kvzalloc(MLX5_ST_SZ_BYTES(query_rmp_out), GFP_KERNEL);

	if (!rmp_out)

		return -ENOMEM;



	err = mlx5_core_query_rmp(dev, srq->srqn, rmp_out);

	if (err)

		goto out;



	rmpc = MLX5_ADDR_OF(query_rmp_out, rmp_out, rmp_context);

	get_wq(MLX5_ADDR_OF(rmpc, rmpc, wq), out);

	if (MLX5_GET(rmpc, rmpc, state) != MLX5_RMPC_STATE_RDY)

		out->flags |= MLX5_SRQ_FLAG_ERR;



out:

	kvfree(rmp_out);

	return err;

}



static int create_xrq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,

			  struct mlx5_srq_attr *in)

{

	u32 create_out[MLX5_ST_SZ_DW(create_xrq_out)] = {0};

	void *create_in;

	void *xrqc;

	void *wq;

	int pas_size;

	int inlen;

	int err;



	pas_size = get_pas_size(in);

	inlen = MLX5_ST_SZ_BYTES(create_xrq_in) + pas_size;

	create_in = kvzalloc(inlen, GFP_KERNEL);

	if (!create_in)

		return -ENOMEM;



	xrqc = MLX5_ADDR_OF(create_xrq_in, create_in, xrq_context);

	wq = MLX5_ADDR_OF(xrqc, xrqc, wq);



	set_wq(wq, in);

	memcpy(MLX5_ADDR_OF(xrqc, xrqc, wq.pas), in->pas, pas_size);



	if (in->type == IB_SRQT_TM) {

		MLX5_SET(xrqc, xrqc, topology, MLX5_XRQC_TOPOLOGY_TAG_MATCHING);

		if (in->flags & MLX5_SRQ_FLAG_RNDV)

			MLX5_SET(xrqc, xrqc, offload, MLX5_XRQC_OFFLOAD_RNDV);

		MLX5_SET(xrqc, xrqc,

			 tag_matching_topology_context.log_matching_list_sz,

			 in->tm_log_list_size);

	}

	MLX5_SET(xrqc, xrqc, user_index, in->user_index);

	MLX5_SET(xrqc, xrqc, cqn, in->cqn);

	MLX5_SET(create_xrq_in, create_in, opcode, MLX5_CMD_OP_CREATE_XRQ);

	MLX5_SET(create_xrq_in, create_in, uid, in->uid);

	err = mlx5_cmd_exec(dev, create_in, inlen, create_out,

			    sizeof(create_out));

	kvfree(create_in);

	if (!err) {

		srq->srqn = MLX5_GET(create_xrq_out, create_out, xrqn);

		srq->uid = in->uid;

	}



	return err;

}



static int destroy_xrq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq)

{

	u32 in[MLX5_ST_SZ_DW(destroy_xrq_in)] = {0};

	u32 out[MLX5_ST_SZ_DW(destroy_xrq_out)] = {0};



	MLX5_SET(destroy_xrq_in, in, opcode, MLX5_CMD_OP_DESTROY_XRQ);

	MLX5_SET(destroy_xrq_in, in, xrqn,   srq->srqn);

	MLX5_SET(destroy_xrq_in, in, uid, srq->uid);



	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));

}



static int arm_xrq_cmd(struct mlx5_core_dev *dev,

		       struct mlx5_core_srq *srq,

		       u16 lwm)

{

	u32 out[MLX5_ST_SZ_DW(arm_rq_out)] = {0};

	u32 in[MLX5_ST_SZ_DW(arm_rq_in)] = {0};



	MLX5_SET(arm_rq_in, in, opcode,     MLX5_CMD_OP_ARM_RQ);

	MLX5_SET(arm_rq_in, in, op_mod,     MLX5_ARM_RQ_IN_OP_MOD_XRQ);

	MLX5_SET(arm_rq_in, in, srq_number, srq->srqn);

	MLX5_SET(arm_rq_in, in, lwm,	    lwm);

	MLX5_SET(arm_rq_in, in, uid, srq->uid);



	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));

}



static int query_xrq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,

			 struct mlx5_srq_attr *out)

{

	u32 in[MLX5_ST_SZ_DW(query_xrq_in)] = {0};

	u32 *xrq_out;

	int outlen = MLX5_ST_SZ_BYTES(query_xrq_out);

	void *xrqc;

	int err;



	xrq_out = kvzalloc(outlen, GFP_KERNEL);

	if (!xrq_out)

		return -ENOMEM;



	MLX5_SET(query_xrq_in, in, opcode, MLX5_CMD_OP_QUERY_XRQ);

	MLX5_SET(query_xrq_in, in, xrqn, srq->srqn);



	err = mlx5_cmd_exec(dev, in, sizeof(in), xrq_out, outlen);

	if (err)

		goto out;



	xrqc = MLX5_ADDR_OF(query_xrq_out, xrq_out, xrq_context);

	get_wq(MLX5_ADDR_OF(xrqc, xrqc, wq), out);

	if (MLX5_GET(xrqc, xrqc, state) != MLX5_XRQC_STATE_GOOD)

		out->flags |= MLX5_SRQ_FLAG_ERR;

	out->tm_next_tag =

		MLX5_GET(xrqc, xrqc,

			 tag_matching_topology_context.append_next_index);

	out->tm_hw_phase_cnt =

		MLX5_GET(xrqc, xrqc,

			 tag_matching_topology_context.hw_phase_cnt);

	out->tm_sw_phase_cnt =

		MLX5_GET(xrqc, xrqc,

			 tag_matching_topology_context.sw_phase_cnt);



out:

	kvfree(xrq_out);

	return err;

}



static int create_srq_split(struct mlx5_core_dev *dev,

			    struct mlx5_core_srq *srq,

			    struct mlx5_srq_attr *in)

{

	if (!dev->issi)

		return create_srq_cmd(dev, srq, in);

	switch (srq->common.res) {

	case MLX5_RES_XSRQ:

		return create_xrc_srq_cmd(dev, srq, in);

	case MLX5_RES_XRQ:

		return create_xrq_cmd(dev, srq, in);

	default:

		return create_rmp_cmd(dev, srq, in);

	}

}



static int destroy_srq_split(struct mlx5_core_dev *dev,

			     struct mlx5_core_srq *srq)

{

	if (!dev->issi)

		return destroy_srq_cmd(dev, srq);

	switch (srq->common.res) {

	case MLX5_RES_XSRQ:

		return destroy_xrc_srq_cmd(dev, srq);

	case MLX5_RES_XRQ:

		return destroy_xrq_cmd(dev, srq);

	default:

		return destroy_rmp_cmd(dev, srq);

	}

}



int mlx5_core_create_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,

			 struct mlx5_srq_attr *in)

{

	int err;

	struct mlx5_srq_table *table = &dev->priv.srq_table;



	switch (in->type) {

	case IB_SRQT_XRC:

		srq->common.res = MLX5_RES_XSRQ;

		break;

	case IB_SRQT_TM:

		srq->common.res = MLX5_RES_XRQ;

		break;

	default:

		srq->common.res = MLX5_RES_SRQ;

	}



	err = create_srq_split(dev, srq, in);

	if (err)

		return err;



	atomic_set(&srq->refcount, 1);

	init_completion(&srq->free);



	spin_lock_irq(&table->lock);

	err = radix_tree_insert(&table->tree, srq->srqn, srq);

	spin_unlock_irq(&table->lock);

	if (err)

		goto err_destroy_srq_split;



	return 0;



err_destroy_srq_split:

	destroy_srq_split(dev, srq);



	return err;

}



int mlx5_core_destroy_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq)

{

	struct mlx5_srq_table *table = &dev->priv.srq_table;

	struct mlx5_core_srq *tmp;

	int err;



	spin_lock_irq(&table->lock);

	tmp = radix_tree_delete(&table->tree, srq->srqn);

	spin_unlock_irq(&table->lock);

	if (!tmp || tmp != srq)

		return -EINVAL;



	err = destroy_srq_split(dev, srq);

	if (err)

		return err;



	if (atomic_dec_and_test(&srq->refcount))

		complete(&srq->free);

	wait_for_completion(&srq->free);



	return 0;

}



int mlx5_core_query_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,

			struct mlx5_srq_attr *out)

{

	if (!dev->issi)

		return query_srq_cmd(dev, srq, out);

	switch (srq->common.res) {

	case MLX5_RES_XSRQ:

		return query_xrc_srq_cmd(dev, srq, out);

	case MLX5_RES_XRQ:

		return query_xrq_cmd(dev, srq, out);

	default:

		return query_rmp_cmd(dev, srq, out);

	}

}



int mlx5_core_arm_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,

		      u16 lwm, int is_srq)

{

	if (!dev->issi)

		return arm_srq_cmd(dev, srq, lwm, is_srq);

	switch (srq->common.res) {

	case MLX5_RES_XSRQ:

		return arm_xrc_srq_cmd(dev, srq, lwm);

	case MLX5_RES_XRQ:

		return arm_xrq_cmd(dev, srq, lwm);

	default:

		return arm_rmp_cmd(dev, srq, lwm);

	}

}