qedr: Add support for PD,PKEY and CQ verbs (a7efd777) · Commits · e / devices / android_kernel_oneplus_sm7250

drivers/infiniband/hw/qedr/main.c

+48 −1

Original line number	Diff line number	Diff line
		@@ -87,7 +87,14 @@ static int qedr_register_device(struct qedr_dev *dev)

		dev->ibdev.uverbs_cmd_mask = QEDR_UVERBS(GET_CONTEXT) \|
		QEDR_UVERBS(QUERY_DEVICE) \|
		QEDR_UVERBS(QUERY_PORT);
		QEDR_UVERBS(QUERY_PORT) \|
		QEDR_UVERBS(ALLOC_PD) \|
		QEDR_UVERBS(DEALLOC_PD) \|
		QEDR_UVERBS(CREATE_COMP_CHANNEL) \|
		QEDR_UVERBS(CREATE_CQ) \|
		QEDR_UVERBS(RESIZE_CQ) \|
		QEDR_UVERBS(DESTROY_CQ) \|
		QEDR_UVERBS(REQ_NOTIFY_CQ);

		dev->ibdev.phys_port_cnt = 1;
		dev->ibdev.num_comp_vectors = dev->num_cnq;
		@@ -105,6 +112,16 @@ static int qedr_register_device(struct qedr_dev *dev)
		dev->ibdev.dealloc_ucontext = qedr_dealloc_ucontext;
		dev->ibdev.mmap = qedr_mmap;

		dev->ibdev.alloc_pd = qedr_alloc_pd;
		dev->ibdev.dealloc_pd = qedr_dealloc_pd;

		dev->ibdev.create_cq = qedr_create_cq;
		dev->ibdev.destroy_cq = qedr_destroy_cq;
		dev->ibdev.resize_cq = qedr_resize_cq;
		dev->ibdev.req_notify_cq = qedr_arm_cq;

		dev->ibdev.query_pkey = qedr_query_pkey;

		dev->ibdev.dma_device = &dev->pdev->dev;

		dev->ibdev.get_link_layer = qedr_link_layer;
		@@ -322,6 +339,8 @@ static irqreturn_t qedr_irq_handler(int irq, void *handle)
		{
		u16 hw_comp_cons, sw_comp_cons;
		struct qedr_cnq *cnq = handle;
		struct regpair *cq_handle;
		struct qedr_cq *cq;

		qed_sb_ack(cnq->sb, IGU_INT_DISABLE, 0);

		@@ -334,8 +353,36 @@ static irqreturn_t qedr_irq_handler(int irq, void *handle)
		rmb();

		while (sw_comp_cons != hw_comp_cons) {
		cq_handle = (struct regpair *)qed_chain_consume(&cnq->pbl);
		cq = (struct qedr_cq *)(uintptr_t)HILO_U64(cq_handle->hi,
		cq_handle->lo);

		if (cq == NULL) {
		DP_ERR(cnq->dev,
		"Received NULL CQ cq_handle->hi=%d cq_handle->lo=%d sw_comp_cons=%d hw_comp_cons=%d\n",
		cq_handle->hi, cq_handle->lo, sw_comp_cons,
		hw_comp_cons);

		break;
		}

		if (cq->sig != QEDR_CQ_MAGIC_NUMBER) {
		DP_ERR(cnq->dev,
		"Problem with cq signature, cq_handle->hi=%d ch_handle->lo=%d cq=%p\n",
		cq_handle->hi, cq_handle->lo, cq);
		break;
		}

		cq->arm_flags = 0;

		if (cq->ibcq.comp_handler)
		(*cq->ibcq.comp_handler)
		(&cq->ibcq, cq->ibcq.cq_context);

		sw_comp_cons = qed_chain_get_cons_idx(&cnq->pbl);

		cnq->n_comp++;

		}

		qed_ops->rdma_cnq_prod_update(cnq->dev->rdma_ctx, cnq->index,

drivers/infiniband/hw/qedr/qedr.h

+78 −0

Original line number	Diff line number	Diff line
		@@ -50,6 +50,10 @@

		#define QEDR_MSG_INIT "INIT"
		#define QEDR_MSG_MISC "MISC"
		#define QEDR_MSG_CQ " CQ"
		#define QEDR_MSG_MR " MR"

		#define QEDR_CQ_MAGIC_NUMBER (0x11223344)

		struct qedr_dev;

		@@ -181,6 +185,12 @@ struct qedr_dev {
		#define QEDR_ROCE_PKEY_TABLE_LEN 1
		#define QEDR_ROCE_PKEY_DEFAULT 0xffff

		struct qedr_pbl {
		struct list_head list_entry;
		void *va;
		dma_addr_t pa;
		};

		struct qedr_ucontext {
		struct ib_ucontext ibucontext;
		struct qedr_dev *dev;
		@@ -196,6 +206,64 @@ struct qedr_ucontext {
		struct mutex mm_list_lock;
		};

		union db_prod64 {
		struct rdma_pwm_val32_data data;
		u64 raw;
		};

		enum qedr_cq_type {
		QEDR_CQ_TYPE_GSI,
		QEDR_CQ_TYPE_KERNEL,
		QEDR_CQ_TYPE_USER,
		};

		struct qedr_pbl_info {
		u32 num_pbls;
		u32 num_pbes;
		u32 pbl_size;
		u32 pbe_size;
		bool two_layered;
		};

		struct qedr_userq {
		struct ib_umem *umem;
		struct qedr_pbl_info pbl_info;
		struct qedr_pbl *pbl_tbl;
		u64 buf_addr;
		size_t buf_len;
		};

		struct qedr_cq {
		struct ib_cq ibcq;

		enum qedr_cq_type cq_type;
		u32 sig;

		u16 icid;

		/* Lock to protect multiplem CQ's */
		spinlock_t cq_lock;
		u8 arm_flags;
		struct qed_chain pbl;

		void __iomem *db_addr;
		union db_prod64 db;

		u8 pbl_toggle;
		union rdma_cqe *latest_cqe;
		union rdma_cqe *toggle_cqe;

		u32 cq_cons;

		struct qedr_userq q;
		};

		struct qedr_pd {
		struct ib_pd ibpd;
		u32 pd_id;
		struct qedr_ucontext *uctx;
		};

		struct qedr_mm {
		struct {
		u64 phy_addr;
		@@ -215,4 +283,14 @@ static inline struct qedr_dev get_qedr_dev(struct ib_device ibdev)
		return container_of(ibdev, struct qedr_dev, ibdev);
		}

		static inline struct qedr_pd get_qedr_pd(struct ib_pd ibpd)
		{
		return container_of(ibpd, struct qedr_pd, ibpd);
		}

		static inline struct qedr_cq get_qedr_cq(struct ib_cq ibcq)
		{
		return container_of(ibcq, struct qedr_cq, ibcq);
		}

		#endif

drivers/infiniband/hw/qedr/qedr_hsi_rdma.h

+79 −0

Original line number	Diff line number	Diff line
		@@ -47,6 +47,19 @@ struct rdma_cqe_responder {
		__le32 imm_data_hi;
		__le16 rq_cons;
		u8 flags;
		#define RDMA_CQE_RESPONDER_TOGGLE_BIT_MASK 0x1
		#define RDMA_CQE_RESPONDER_TOGGLE_BIT_SHIFT 0
		#define RDMA_CQE_RESPONDER_TYPE_MASK 0x3
		#define RDMA_CQE_RESPONDER_TYPE_SHIFT 1
		#define RDMA_CQE_RESPONDER_INV_FLG_MASK 0x1
		#define RDMA_CQE_RESPONDER_INV_FLG_SHIFT 3
		#define RDMA_CQE_RESPONDER_IMM_FLG_MASK 0x1
		#define RDMA_CQE_RESPONDER_IMM_FLG_SHIFT 4
		#define RDMA_CQE_RESPONDER_RDMA_FLG_MASK 0x1
		#define RDMA_CQE_RESPONDER_RDMA_FLG_SHIFT 5
		#define RDMA_CQE_RESPONDER_RESERVED2_MASK 0x3
		#define RDMA_CQE_RESPONDER_RESERVED2_SHIFT 6
		u8 status;
		};

		struct rdma_cqe_requester {
		@@ -58,6 +71,12 @@ struct rdma_cqe_requester {
		__le32 reserved3;
		__le16 reserved4;
		u8 flags;
		#define RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK 0x1
		#define RDMA_CQE_REQUESTER_TOGGLE_BIT_SHIFT 0
		#define RDMA_CQE_REQUESTER_TYPE_MASK 0x3
		#define RDMA_CQE_REQUESTER_TYPE_SHIFT 1
		#define RDMA_CQE_REQUESTER_RESERVED5_MASK 0x1F
		#define RDMA_CQE_REQUESTER_RESERVED5_SHIFT 3
		u8 status;
		};

		@@ -66,6 +85,12 @@ struct rdma_cqe_common {
		struct regpair qp_handle;
		__le16 reserved1[7];
		u8 flags;
		#define RDMA_CQE_COMMON_TOGGLE_BIT_MASK 0x1
		#define RDMA_CQE_COMMON_TOGGLE_BIT_SHIFT 0
		#define RDMA_CQE_COMMON_TYPE_MASK 0x3
		#define RDMA_CQE_COMMON_TYPE_SHIFT 1
		#define RDMA_CQE_COMMON_RESERVED2_MASK 0x1F
		#define RDMA_CQE_COMMON_RESERVED2_SHIFT 3
		u8 status;
		};

		@@ -76,6 +101,45 @@ union rdma_cqe {
		struct rdma_cqe_common cmn;
		};

		/* * CQE requester status enumeration */
		enum rdma_cqe_requester_status_enum {
		RDMA_CQE_REQ_STS_OK,
		RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR,
		RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR,
		RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR,
		RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR,
		RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR,
		RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR,
		RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR,
		RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR,
		RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR,
		RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR,
		RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR,
		MAX_RDMA_CQE_REQUESTER_STATUS_ENUM
		};

		/* CQE responder status enumeration */
		enum rdma_cqe_responder_status_enum {
		RDMA_CQE_RESP_STS_OK,
		RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR,
		RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR,
		RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR,
		RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR,
		RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR,
		RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR,
		RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR,
		MAX_RDMA_CQE_RESPONDER_STATUS_ENUM
		};

		/* CQE type enumeration */
		enum rdma_cqe_type {
		RDMA_CQE_TYPE_REQUESTER,
		RDMA_CQE_TYPE_RESPONDER_RQ,
		RDMA_CQE_TYPE_RESPONDER_SRQ,
		RDMA_CQE_TYPE_INVALID,
		MAX_RDMA_CQE_TYPE
		};

		struct rdma_sq_sge {
		__le32 length;
		struct regpair addr;
		@@ -93,4 +157,19 @@ struct rdma_srq_sge {
		__le32 length;
		__le32 l_key;
		};

		/* Rdma doorbell data for CQ */
		struct rdma_pwm_val32_data {
		__le16 icid;
		u8 agg_flags;
		u8 params;
		#define RDMA_PWM_VAL32_DATA_AGG_CMD_MASK 0x3
		#define RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT 0
		#define RDMA_PWM_VAL32_DATA_BYPASS_EN_MASK 0x1
		#define RDMA_PWM_VAL32_DATA_BYPASS_EN_SHIFT 2
		#define RDMA_PWM_VAL32_DATA_RESERVED_MASK 0x1F
		#define RDMA_PWM_VAL32_DATA_RESERVED_SHIFT 3
		__le32 value;
		};

		#endif /* __QED_HSI_RDMA__ */

drivers/infiniband/hw/qedr/verbs.c

+539 −0

Original line number	Diff line number	Diff line
		@@ -49,6 +49,17 @@
		#include "verbs.h"
		#include <rdma/qedr-abi.h>

		#define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)

		int qedr_query_pkey(struct ib_device ibdev, u8 port, u16 index, u16 pkey)
		{
		if (index > QEDR_ROCE_PKEY_TABLE_LEN)
		return -EINVAL;

		*pkey = QEDR_ROCE_PKEY_DEFAULT;
		return 0;
		}

		int qedr_query_gid(struct ib_device *ibdev, u8 port, int index,
		union ib_gid *sgid)
		{
		@@ -454,3 +465,531 @@ int qedr_mmap(struct ib_ucontext context, struct vm_area_struct vma)
		DP_DEBUG(dev, QEDR_MSG_INIT, "qedr_mmap return code: %d\n", rc);
		return rc;
		}

		struct ib_pd qedr_alloc_pd(struct ib_device ibdev,
		struct ib_ucontext context, struct ib_udata udata)
		{
		struct qedr_dev *dev = get_qedr_dev(ibdev);
		struct qedr_ucontext *uctx = NULL;
		struct qedr_alloc_pd_uresp uresp;
		struct qedr_pd *pd;
		u16 pd_id;
		int rc;

		DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
		(udata && context) ? "User Lib" : "Kernel");

		if (!dev->rdma_ctx) {
		DP_ERR(dev, "invlaid RDMA context\n");
		return ERR_PTR(-EINVAL);
		}

		pd = kzalloc(sizeof(*pd), GFP_KERNEL);
		if (!pd)
		return ERR_PTR(-ENOMEM);

		dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);

		uresp.pd_id = pd_id;
		pd->pd_id = pd_id;

		if (udata && context) {
		rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
		if (rc)
		DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
		uctx = get_qedr_ucontext(context);
		uctx->pd = pd;
		pd->uctx = uctx;
		}

		return &pd->ibpd;
		}

		int qedr_dealloc_pd(struct ib_pd *ibpd)
		{
		struct qedr_dev *dev = get_qedr_dev(ibpd->device);
		struct qedr_pd *pd = get_qedr_pd(ibpd);

		if (!pd)
		pr_err("Invalid PD received in dealloc_pd\n");

		DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
		dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);

		kfree(pd);

		return 0;
		}

		static void qedr_free_pbl(struct qedr_dev *dev,
		struct qedr_pbl_info pbl_info, struct qedr_pbl pbl)
		{
		struct pci_dev *pdev = dev->pdev;
		int i;

		for (i = 0; i < pbl_info->num_pbls; i++) {
		if (!pbl[i].va)
		continue;
		dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
		pbl[i].va, pbl[i].pa);
		}

		kfree(pbl);
		}

		#define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
		#define MAX_FW_PBL_PAGE_SIZE (64 * 1024)

		#define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
		#define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
		#define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)

		static struct qedr_pbl qedr_alloc_pbl_tbl(struct qedr_dev dev,
		struct qedr_pbl_info *pbl_info,
		gfp_t flags)
		{
		struct pci_dev *pdev = dev->pdev;
		struct qedr_pbl *pbl_table;
		dma_addr_t *pbl_main_tbl;
		dma_addr_t pa;
		void *va;
		int i;

		pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
		if (!pbl_table)
		return ERR_PTR(-ENOMEM);

		for (i = 0; i < pbl_info->num_pbls; i++) {
		va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size,
		&pa, flags);
		if (!va)
		goto err;

		memset(va, 0, pbl_info->pbl_size);
		pbl_table[i].va = va;
		pbl_table[i].pa = pa;
		}

		/* Two-Layer PBLs, if we have more than one pbl we need to initialize
		* the first one with physical pointers to all of the rest
		*/
		pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
		for (i = 0; i < pbl_info->num_pbls - 1; i++)
		pbl_main_tbl[i] = pbl_table[i + 1].pa;

		return pbl_table;

		err:
		for (i--; i >= 0; i--)
		dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
		pbl_table[i].va, pbl_table[i].pa);

		qedr_free_pbl(dev, pbl_info, pbl_table);

		return ERR_PTR(-ENOMEM);
		}

		static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
		struct qedr_pbl_info *pbl_info,
		u32 num_pbes, int two_layer_capable)
		{
		u32 pbl_capacity;
		u32 pbl_size;
		u32 num_pbls;

		if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
		if (num_pbes > MAX_PBES_TWO_LAYER) {
		DP_ERR(dev, "prepare pbl table: too many pages %d\n",
		num_pbes);
		return -EINVAL;
		}

		/* calculate required pbl page size */
		pbl_size = MIN_FW_PBL_PAGE_SIZE;
		pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
		NUM_PBES_ON_PAGE(pbl_size);

		while (pbl_capacity < num_pbes) {
		pbl_size *= 2;
		pbl_capacity = pbl_size / sizeof(u64);
		pbl_capacity = pbl_capacity * pbl_capacity;
		}

		num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
		num_pbls++; /* One for the layer0 ( points to the pbls) */
		pbl_info->two_layered = true;
		} else {
		/* One layered PBL */
		num_pbls = 1;
		pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
		roundup_pow_of_two((num_pbes * sizeof(u64))));
		pbl_info->two_layered = false;
		}

		pbl_info->num_pbls = num_pbls;
		pbl_info->pbl_size = pbl_size;
		pbl_info->num_pbes = num_pbes;

		DP_DEBUG(dev, QEDR_MSG_MR,
		"prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
		pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);

		return 0;
		}

		static void qedr_populate_pbls(struct qedr_dev dev, struct ib_umem umem,
		struct qedr_pbl *pbl,
		struct qedr_pbl_info *pbl_info)
		{
		int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
		struct qedr_pbl *pbl_tbl;
		struct scatterlist *sg;
		struct regpair *pbe;
		int entry;
		u32 addr;

		if (!pbl_info->num_pbes)
		return;

		/* If we have a two layered pbl, the first pbl points to the rest
		* of the pbls and the first entry lays on the second pbl in the table
		*/
		if (pbl_info->two_layered)
		pbl_tbl = &pbl[1];
		else
		pbl_tbl = pbl;

		pbe = (struct regpair *)pbl_tbl->va;
		if (!pbe) {
		DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
		return;
		}

		pbe_cnt = 0;

		shift = ilog2(umem->page_size);

		for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
		pages = sg_dma_len(sg) >> shift;
		for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
		/* store the page address in pbe */
		pbe->lo = cpu_to_le32(sg_dma_address(sg) +
		umem->page_size * pg_cnt);
		addr = upper_32_bits(sg_dma_address(sg) +
		umem->page_size * pg_cnt);
		pbe->hi = cpu_to_le32(addr);
		pbe_cnt++;
		total_num_pbes++;
		pbe++;

		if (total_num_pbes == pbl_info->num_pbes)
		return;

		/* If the given pbl is full storing the pbes,
		* move to next pbl.
		*/
		if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
		pbl_tbl++;
		pbe = (struct regpair *)pbl_tbl->va;
		pbe_cnt = 0;
		}
		}
		}
		}

		static int qedr_copy_cq_uresp(struct qedr_dev *dev,
		struct qedr_cq cq, struct ib_udata udata)
		{
		struct qedr_create_cq_uresp uresp;
		int rc;

		memset(&uresp, 0, sizeof(uresp));

		uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
		uresp.icid = cq->icid;

		rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
		if (rc)
		DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);

		return rc;
		}

		static void consume_cqe(struct qedr_cq *cq)
		{
		if (cq->latest_cqe == cq->toggle_cqe)
		cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;

		cq->latest_cqe = qed_chain_consume(&cq->pbl);
		}

		static inline int qedr_align_cq_entries(int entries)
		{
		u64 size, aligned_size;

		/* We allocate an extra entry that we don't report to the FW. */
		size = (entries + 1) * QEDR_CQE_SIZE;
		aligned_size = ALIGN(size, PAGE_SIZE);

		return aligned_size / QEDR_CQE_SIZE;
		}

		static inline int qedr_init_user_queue(struct ib_ucontext *ib_ctx,
		struct qedr_dev *dev,
		struct qedr_userq *q,
		u64 buf_addr, size_t buf_len,
		int access, int dmasync)
		{
		int page_cnt;
		int rc;

		q->buf_addr = buf_addr;
		q->buf_len = buf_len;
		q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync);
		if (IS_ERR(q->umem)) {
		DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
		PTR_ERR(q->umem));
		return PTR_ERR(q->umem);
		}

		page_cnt = ib_umem_page_count(q->umem);
		rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, page_cnt, 0);
		if (rc)
		goto err0;

		q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
		if (IS_ERR_OR_NULL(q->pbl_tbl))
		goto err0;

		qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info);

		return 0;

		err0:
		ib_umem_release(q->umem);

		return rc;
		}

		static inline void qedr_init_cq_params(struct qedr_cq *cq,
		struct qedr_ucontext *ctx,
		struct qedr_dev *dev, int vector,
		int chain_entries, int page_cnt,
		u64 pbl_ptr,
		struct qed_rdma_create_cq_in_params
		*params)
		{
		memset(params, 0, sizeof(*params));
		params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
		params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
		params->cnq_id = vector;
		params->cq_size = chain_entries - 1;
		params->dpi = (ctx) ? ctx->dpi : dev->dpi;
		params->pbl_num_pages = page_cnt;
		params->pbl_ptr = pbl_ptr;
		params->pbl_two_level = 0;
		}

		static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
		{
		/* Flush data before signalling doorbell */
		wmb();
		cq->db.data.agg_flags = flags;
		cq->db.data.value = cpu_to_le32(cons);
		writeq(cq->db.raw, cq->db_addr);

		/* Make sure write would stick */
		mmiowb();
		}

		int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
		{
		struct qedr_cq *cq = get_qedr_cq(ibcq);
		unsigned long sflags;

		if (cq->cq_type == QEDR_CQ_TYPE_GSI)
		return 0;

		spin_lock_irqsave(&cq->cq_lock, sflags);

		cq->arm_flags = 0;

		if (flags & IB_CQ_SOLICITED)
		cq->arm_flags \|= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;

		if (flags & IB_CQ_NEXT_COMP)
		cq->arm_flags \|= DQ_UCM_ROCE_CQ_ARM_CF_CMD;

		doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);

		spin_unlock_irqrestore(&cq->cq_lock, sflags);

		return 0;
		}

		struct ib_cq qedr_create_cq(struct ib_device ibdev,
		const struct ib_cq_init_attr *attr,
		struct ib_ucontext ib_ctx, struct ib_udata udata)
		{
		struct qedr_ucontext *ctx = get_qedr_ucontext(ib_ctx);
		struct qed_rdma_destroy_cq_out_params destroy_oparams;
		struct qed_rdma_destroy_cq_in_params destroy_iparams;
		struct qedr_dev *dev = get_qedr_dev(ibdev);
		struct qed_rdma_create_cq_in_params params;
		struct qedr_create_cq_ureq ureq;
		int vector = attr->comp_vector;
		int entries = attr->cqe;
		struct qedr_cq *cq;
		int chain_entries;
		int page_cnt;
		u64 pbl_ptr;
		u16 icid;
		int rc;

		DP_DEBUG(dev, QEDR_MSG_INIT,
		"create_cq: called from %s. entries=%d, vector=%d\n",
		udata ? "User Lib" : "Kernel", entries, vector);

		if (entries > QEDR_MAX_CQES) {
		DP_ERR(dev,
		"create cq: the number of entries %d is too high. Must be equal or below %d.\n",
		entries, QEDR_MAX_CQES);
		return ERR_PTR(-EINVAL);
		}

		chain_entries = qedr_align_cq_entries(entries);
		chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);

		cq = kzalloc(sizeof(*cq), GFP_KERNEL);
		if (!cq)
		return ERR_PTR(-ENOMEM);

		if (udata) {
		memset(&ureq, 0, sizeof(ureq));
		if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
		DP_ERR(dev,
		"create cq: problem copying data from user space\n");
		goto err0;
		}

		if (!ureq.len) {
		DP_ERR(dev,
		"create cq: cannot create a cq with 0 entries\n");
		goto err0;
		}

		cq->cq_type = QEDR_CQ_TYPE_USER;

		rc = qedr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr,
		ureq.len, IB_ACCESS_LOCAL_WRITE, 1);
		if (rc)
		goto err0;

		pbl_ptr = cq->q.pbl_tbl->pa;
		page_cnt = cq->q.pbl_info.num_pbes;
		} else {
		cq->cq_type = QEDR_CQ_TYPE_KERNEL;

		rc = dev->ops->common->chain_alloc(dev->cdev,
		QED_CHAIN_USE_TO_CONSUME,
		QED_CHAIN_MODE_PBL,
		QED_CHAIN_CNT_TYPE_U32,
		chain_entries,
		sizeof(union rdma_cqe),
		&cq->pbl);
		if (rc)
		goto err1;

		page_cnt = qed_chain_get_page_cnt(&cq->pbl);
		pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
		}

		qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
		pbl_ptr, &params);

		rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
		if (rc)
		goto err2;

		cq->icid = icid;
		cq->sig = QEDR_CQ_MAGIC_NUMBER;
		spin_lock_init(&cq->cq_lock);

		if (ib_ctx) {
		rc = qedr_copy_cq_uresp(dev, cq, udata);
		if (rc)
		goto err3;
		} else {
		/* Generate doorbell address. */
		cq->db_addr = dev->db_addr +
		DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
		cq->db.data.icid = cq->icid;
		cq->db.data.params = DB_AGG_CMD_SET <<
		RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;

		/* point to the very last element, passing it we will toggle */
		cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
		cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
		cq->latest_cqe = NULL;
		consume_cqe(cq);
		cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
		}

		DP_DEBUG(dev, QEDR_MSG_CQ,
		"create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
		cq->icid, cq, params.cq_size);

		return &cq->ibcq;

		err3:
		destroy_iparams.icid = cq->icid;
		dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
		&destroy_oparams);
		err2:
		if (udata)
		qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
		else
		dev->ops->common->chain_free(dev->cdev, &cq->pbl);
		err1:
		if (udata)
		ib_umem_release(cq->q.umem);
		err0:
		kfree(cq);
		return ERR_PTR(-EINVAL);
		}

		int qedr_resize_cq(struct ib_cq ibcq, int new_cnt, struct ib_udata udata)
		{
		struct qedr_dev *dev = get_qedr_dev(ibcq->device);
		struct qedr_cq *cq = get_qedr_cq(ibcq);

		DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);

		return 0;
		}

		int qedr_destroy_cq(struct ib_cq *ibcq)
		{
		struct qedr_dev *dev = get_qedr_dev(ibcq->device);
		struct qed_rdma_destroy_cq_out_params oparams;
		struct qed_rdma_destroy_cq_in_params iparams;
		struct qedr_cq *cq = get_qedr_cq(ibcq);

		DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq: cq_id %d", cq->icid);

		/* GSIs CQs are handled by driver, so they don't exist in the FW */
		if (cq->cq_type != QEDR_CQ_TYPE_GSI) {
		iparams.icid = cq->icid;
		dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
		dev->ops->common->chain_free(dev->cdev, &cq->pbl);
		}

		if (ibcq->uobject && ibcq->uobject->context) {
		qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
		ib_umem_release(cq->q.umem);
		}

		kfree(cq);

		return 0;
		}

drivers/infiniband/hw/qedr/verbs.h

+14 −0

Original line number	Diff line number	Diff line
		@@ -40,6 +40,8 @@ int qedr_modify_port(struct ib_device *, u8 port, int mask,

		int qedr_query_gid(struct ib_device , u8 port, int index, union ib_gid gid);

		int qedr_query_pkey(struct ib_device , u8 port, u16 index, u16 pkey);

		struct ib_ucontext qedr_alloc_ucontext(struct ib_device , struct ib_udata *);
		int qedr_dealloc_ucontext(struct ib_ucontext *);

		@@ -49,4 +51,16 @@ int qedr_del_gid(struct ib_device *device, u8 port_num,
		int qedr_add_gid(struct ib_device *device, u8 port_num,
		unsigned int index, const union ib_gid *gid,
		const struct ib_gid_attr attr, void *context);
		struct ib_pd qedr_alloc_pd(struct ib_device ,
		struct ib_ucontext , struct ib_udata );
		int qedr_dealloc_pd(struct ib_pd *pd);

		struct ib_cq qedr_create_cq(struct ib_device ibdev,
		const struct ib_cq_init_attr *attr,
		struct ib_ucontext *ib_ctx,
		struct ib_udata *udata);
		int qedr_resize_cq(struct ib_cq , int cqe, struct ib_udata );
		int qedr_destroy_cq(struct ib_cq *);
		int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);

		#endif