Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit bbf8eed1 authored by Vladimir Sokolovsky's avatar Vladimir Sokolovsky Committed by Roland Dreier
Browse files

IB/mlx4: Add support for resizing CQs 



Signed-off-by: default avatarVladimir Sokolovsky <vlad@mellanox.co.il>
Signed-off-by: default avatarRoland Dreier <rolandd@cisco.com>
parent 3fdcb97f

drivers/infiniband/hw/mlx4/cq.c

+259 −33
Original line number Diff line number Diff line
@@ -93,6 +93,74 @@ int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period) 
	return mlx4_cq_modify(dev->dev, &mcq->mcq, cq_count, cq_period);

}



static int mlx4_ib_alloc_cq_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq_buf *buf, int nent)

{

	int err;



	err = mlx4_buf_alloc(dev->dev, nent * sizeof(struct mlx4_cqe),

			     PAGE_SIZE * 2, &buf->buf);



	if (err)

		goto out;



	err = mlx4_mtt_init(dev->dev, buf->buf.npages, buf->buf.page_shift,

				    &buf->mtt);

	if (err)

		goto err_buf;



	err = mlx4_buf_write_mtt(dev->dev, &buf->mtt, &buf->buf);

	if (err)

		goto err_mtt;



	return 0;



err_mtt:

	mlx4_mtt_cleanup(dev->dev, &buf->mtt);



err_buf:

	mlx4_buf_free(dev->dev, nent * sizeof(struct mlx4_cqe),

			      &buf->buf);



out:

	return err;

}



static void mlx4_ib_free_cq_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq_buf *buf, int cqe)

{

	mlx4_buf_free(dev->dev, (cqe + 1) * sizeof(struct mlx4_cqe), &buf->buf);

}



static int mlx4_ib_get_cq_umem(struct mlx4_ib_dev *dev, struct ib_ucontext *context,

			       struct mlx4_ib_cq_buf *buf, struct ib_umem **umem,

			       u64 buf_addr, int cqe)

{

	int err;



	*umem = ib_umem_get(context, buf_addr, cqe * sizeof (struct mlx4_cqe),

			    IB_ACCESS_LOCAL_WRITE);

	if (IS_ERR(*umem))

		return PTR_ERR(*umem);



	err = mlx4_mtt_init(dev->dev, ib_umem_page_count(*umem),

			    ilog2((*umem)->page_size), &buf->mtt);

	if (err)

		goto err_buf;



	err = mlx4_ib_umem_write_mtt(dev, &buf->mtt, *umem);

	if (err)

		goto err_mtt;



	return 0;



err_mtt:

	mlx4_mtt_cleanup(dev->dev, &buf->mtt);



err_buf:

	ib_umem_release(*umem);



	return err;

}



struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, int entries, int vector,

				struct ib_ucontext *context,

				struct ib_udata *udata)
@@ -100,7 +168,6 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, int entries, int vector 
	struct mlx4_ib_dev *dev = to_mdev(ibdev);

	struct mlx4_ib_cq *cq;

	struct mlx4_uar *uar;

	int buf_size;

	int err;



	if (entries < 1 || entries > dev->dev->caps.max_cqes)
@@ -112,8 +179,10 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, int entries, int vector 


	entries      = roundup_pow_of_two(entries + 1);

	cq->ibcq.cqe = entries - 1;

	buf_size     = entries * sizeof (struct mlx4_cqe);

	mutex_init(&cq->resize_mutex);

	spin_lock_init(&cq->lock);

	cq->resize_buf = NULL;

	cq->resize_umem = NULL;



	if (context) {

		struct mlx4_ib_create_cq ucmd;
@@ -123,21 +192,10 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, int entries, int vector 
			goto err_cq;

		}



		cq->umem = ib_umem_get(context, ucmd.buf_addr, buf_size,

				       IB_ACCESS_LOCAL_WRITE);

		if (IS_ERR(cq->umem)) {

			err = PTR_ERR(cq->umem);

			goto err_cq;

		}



		err = mlx4_mtt_init(dev->dev, ib_umem_page_count(cq->umem),

				    ilog2(cq->umem->page_size), &cq->buf.mtt);

		err = mlx4_ib_get_cq_umem(dev, context, &cq->buf, &cq->umem,

					  ucmd.buf_addr, entries);

		if (err)

			goto err_buf;



		err = mlx4_ib_umem_write_mtt(dev, &cq->buf.mtt, cq->umem);

		if (err)

			goto err_mtt;

			goto err_cq;



		err = mlx4_ib_db_map_user(to_mucontext(context), ucmd.db_addr,

					  &cq->db);
@@ -155,19 +213,9 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, int entries, int vector 
		*cq->mcq.set_ci_db = 0;

		*cq->mcq.arm_db    = 0;



		if (mlx4_buf_alloc(dev->dev, buf_size, PAGE_SIZE * 2, &cq->buf.buf)) {

			err = -ENOMEM;

			goto err_db;

		}



		err = mlx4_mtt_init(dev->dev, cq->buf.buf.npages, cq->buf.buf.page_shift,

				    &cq->buf.mtt);

		if (err)

			goto err_buf;



		err = mlx4_buf_write_mtt(dev->dev, &cq->buf.mtt, &cq->buf.buf);

		err = mlx4_ib_alloc_cq_buf(dev, &cq->buf, entries);

		if (err)

			goto err_mtt;

			goto err_db;



		uar = &dev->priv_uar;

	}
@@ -195,12 +243,10 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, int entries, int vector 
err_mtt:

	mlx4_mtt_cleanup(dev->dev, &cq->buf.mtt);



err_buf:

	if (context)

		ib_umem_release(cq->umem);

	else

		mlx4_buf_free(dev->dev, entries * sizeof (struct mlx4_cqe),

			      &cq->buf.buf);

		mlx4_ib_free_cq_buf(dev, &cq->buf, entries);



err_db:

	if (!context)
@@ -212,6 +258,170 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, int entries, int vector 
	return ERR_PTR(err);

}



static int mlx4_alloc_resize_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq,

				  int entries)

{

	int err;



	if (cq->resize_buf)

		return -EBUSY;



	cq->resize_buf = kmalloc(sizeof *cq->resize_buf, GFP_ATOMIC);

	if (!cq->resize_buf)

		return -ENOMEM;



	err = mlx4_ib_alloc_cq_buf(dev, &cq->resize_buf->buf, entries);

	if (err) {

		kfree(cq->resize_buf);

		cq->resize_buf = NULL;

		return err;

	}



	cq->resize_buf->cqe = entries - 1;



	return 0;

}



static int mlx4_alloc_resize_umem(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq,

				   int entries, struct ib_udata *udata)

{

	struct mlx4_ib_resize_cq ucmd;

	int err;



	if (cq->resize_umem)

		return -EBUSY;



	if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd))

		return -EFAULT;



	cq->resize_buf = kmalloc(sizeof *cq->resize_buf, GFP_ATOMIC);

	if (!cq->resize_buf)

		return -ENOMEM;



	err = mlx4_ib_get_cq_umem(dev, cq->umem->context, &cq->resize_buf->buf,

				  &cq->resize_umem, ucmd.buf_addr, entries);

	if (err) {

		kfree(cq->resize_buf);

		cq->resize_buf = NULL;

		return err;

	}



	cq->resize_buf->cqe = entries - 1;



	return 0;

}



static int mlx4_ib_get_outstanding_cqes(struct mlx4_ib_cq *cq)

{

	u32 i;



	i = cq->mcq.cons_index;

	while (get_sw_cqe(cq, i & cq->ibcq.cqe))

		++i;



	return i - cq->mcq.cons_index;

}



static void mlx4_ib_cq_resize_copy_cqes(struct mlx4_ib_cq *cq)

{

	struct mlx4_cqe *cqe;

	int i;



	i = cq->mcq.cons_index;

	cqe = get_cqe(cq, i & cq->ibcq.cqe);

	while ((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) != MLX4_CQE_OPCODE_RESIZE) {

		memcpy(get_cqe_from_buf(&cq->resize_buf->buf,

					(i + 1) & cq->resize_buf->cqe),

			get_cqe(cq, i & cq->ibcq.cqe), sizeof(struct mlx4_cqe));

		cqe = get_cqe(cq, ++i & cq->ibcq.cqe);

	}

	++cq->mcq.cons_index;

}



int mlx4_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata)

{

	struct mlx4_ib_dev *dev = to_mdev(ibcq->device);

	struct mlx4_ib_cq *cq = to_mcq(ibcq);

	int outst_cqe;

	int err;



	mutex_lock(&cq->resize_mutex);



	if (entries < 1 || entries > dev->dev->caps.max_cqes) {

		err = -EINVAL;

		goto out;

	}



	entries = roundup_pow_of_two(entries + 1);

	if (entries == ibcq->cqe + 1) {

		err = 0;

		goto out;

	}



	if (ibcq->uobject) {

		err = mlx4_alloc_resize_umem(dev, cq, entries, udata);

		if (err)

			goto out;

	} else {

		/* Can't be smaller then the number of outstanding CQEs */

		outst_cqe = mlx4_ib_get_outstanding_cqes(cq);

		if (entries < outst_cqe + 1) {

			err = 0;

			goto out;

		}



		err = mlx4_alloc_resize_buf(dev, cq, entries);

		if (err)

			goto out;

	}



	err = mlx4_cq_resize(dev->dev, &cq->mcq, entries, &cq->resize_buf->buf.mtt);

	if (err)

		goto err_buf;



	if (ibcq->uobject) {

		cq->buf      = cq->resize_buf->buf;

		cq->ibcq.cqe = cq->resize_buf->cqe;

		ib_umem_release(cq->umem);

		cq->umem     = cq->resize_umem;



		kfree(cq->resize_buf);

		cq->resize_buf = NULL;

		cq->resize_umem = NULL;

	} else {

		spin_lock_irq(&cq->lock);

		if (cq->resize_buf) {

			mlx4_ib_cq_resize_copy_cqes(cq);

			mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe);

			cq->buf      = cq->resize_buf->buf;

			cq->ibcq.cqe = cq->resize_buf->cqe;



			kfree(cq->resize_buf);

			cq->resize_buf = NULL;

		}

		spin_unlock_irq(&cq->lock);

	}



	goto out;



err_buf:

	if (!ibcq->uobject)

		mlx4_ib_free_cq_buf(dev, &cq->resize_buf->buf,

				    cq->resize_buf->cqe);



	kfree(cq->resize_buf);

	cq->resize_buf = NULL;



	if (cq->resize_umem) {

		ib_umem_release(cq->resize_umem);

		cq->resize_umem = NULL;

	}



out:

	mutex_unlock(&cq->resize_mutex);

	return err;

}



int mlx4_ib_destroy_cq(struct ib_cq *cq)

{

	struct mlx4_ib_dev *dev = to_mdev(cq->device);
@@ -224,8 +434,7 @@ int mlx4_ib_destroy_cq(struct ib_cq *cq) 
		mlx4_ib_db_unmap_user(to_mucontext(cq->uobject->context), &mcq->db);

		ib_umem_release(mcq->umem);

	} else {

		mlx4_buf_free(dev->dev, (cq->cqe + 1) * sizeof (struct mlx4_cqe),

			      &mcq->buf.buf);

		mlx4_ib_free_cq_buf(dev, &mcq->buf, cq->cqe + 1);

		mlx4_ib_db_free(dev, &mcq->db);

	}
@@ -332,6 +541,7 @@ static int mlx4_ib_poll_one(struct mlx4_ib_cq *cq, 
	u32 g_mlpath_rqpn;

	u16 wqe_ctr;



repoll:

	cqe = next_cqe_sw(cq);

	if (!cqe)

		return -EAGAIN;
@@ -354,6 +564,22 @@ static int mlx4_ib_poll_one(struct mlx4_ib_cq *cq, 
		return -EINVAL;

	}



	/* Resize CQ in progress */

	if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == MLX4_CQE_OPCODE_RESIZE)) {

		if (cq->resize_buf) {

			struct mlx4_ib_dev *dev = to_mdev(cq->ibcq.device);



			mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe);

			cq->buf      = cq->resize_buf->buf;

			cq->ibcq.cqe = cq->resize_buf->cqe;



			kfree(cq->resize_buf);

			cq->resize_buf = NULL;

		}



		goto repoll;

	}



	if (!*cur_qp ||

	    (be32_to_cpu(cqe->my_qpn) & 0xffffff) != (*cur_qp)->mqp.qpn) {

		/*

drivers/infiniband/hw/mlx4/main.c

+2 −0
Original line number Diff line number Diff line
@@ -571,6 +571,7 @@ static void *mlx4_ib_add(struct mlx4_dev *dev) 
		(1ull << IB_USER_VERBS_CMD_DEREG_MR)		|

		(1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL)	|

		(1ull << IB_USER_VERBS_CMD_CREATE_CQ)		|

		(1ull << IB_USER_VERBS_CMD_RESIZE_CQ)		|

		(1ull << IB_USER_VERBS_CMD_DESTROY_CQ)		|

		(1ull << IB_USER_VERBS_CMD_CREATE_QP)		|

		(1ull << IB_USER_VERBS_CMD_MODIFY_QP)		|
@@ -610,6 +611,7 @@ static void *mlx4_ib_add(struct mlx4_dev *dev) 
	ibdev->ib_dev.post_recv		= mlx4_ib_post_recv;

	ibdev->ib_dev.create_cq		= mlx4_ib_create_cq;

	ibdev->ib_dev.modify_cq		= mlx4_ib_modify_cq;

	ibdev->ib_dev.resize_cq		= mlx4_ib_resize_cq;

	ibdev->ib_dev.destroy_cq	= mlx4_ib_destroy_cq;

	ibdev->ib_dev.poll_cq		= mlx4_ib_poll_cq;

	ibdev->ib_dev.req_notify_cq	= mlx4_ib_arm_cq;

drivers/infiniband/hw/mlx4/mlx4_ib.h

+9 −0
Original line number Diff line number Diff line
@@ -78,13 +78,21 @@ struct mlx4_ib_cq_buf { 
	struct mlx4_mtt		mtt;

};



struct mlx4_ib_cq_resize {

	struct mlx4_ib_cq_buf	buf;

	int			cqe;

};



struct mlx4_ib_cq {

	struct ib_cq		ibcq;

	struct mlx4_cq		mcq;

	struct mlx4_ib_cq_buf	buf;

	struct mlx4_ib_cq_resize *resize_buf;

	struct mlx4_ib_db	db;

	spinlock_t		lock;

	struct mutex		resize_mutex;

	struct ib_umem	       *umem;

	struct ib_umem	       *resize_umem;

};



struct mlx4_ib_mr {
@@ -255,6 +263,7 @@ struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 
int mlx4_ib_dereg_mr(struct ib_mr *mr);



int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period);

int mlx4_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata);

struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, int entries, int vector,

				struct ib_ucontext *context,

				struct ib_udata *udata);

drivers/net/mlx4/cq.c

+28 −0
Original line number Diff line number Diff line
@@ -159,6 +159,34 @@ int mlx4_cq_modify(struct mlx4_dev *dev, struct mlx4_cq *cq, 
}

EXPORT_SYMBOL_GPL(mlx4_cq_modify);



int mlx4_cq_resize(struct mlx4_dev *dev, struct mlx4_cq *cq,

		   int entries, struct mlx4_mtt *mtt)

{

	struct mlx4_cmd_mailbox *mailbox;

	struct mlx4_cq_context *cq_context;

	u64 mtt_addr;

	int err;



	mailbox = mlx4_alloc_cmd_mailbox(dev);

	if (IS_ERR(mailbox))

		return PTR_ERR(mailbox);



	cq_context = mailbox->buf;

	memset(cq_context, 0, sizeof *cq_context);



	cq_context->logsize_usrpage = cpu_to_be32(ilog2(entries) << 24);

	cq_context->log_page_size   = mtt->page_shift - 12;

	mtt_addr = mlx4_mtt_addr(dev, mtt);

	cq_context->mtt_base_addr_h = mtt_addr >> 32;

	cq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff);



	err = mlx4_MODIFY_CQ(dev, mailbox, cq->cqn, 1);



	mlx4_free_cmd_mailbox(dev, mailbox);

	return err;

}

EXPORT_SYMBOL_GPL(mlx4_cq_resize);



int mlx4_cq_alloc(struct mlx4_dev *dev, int nent, struct mlx4_mtt *mtt,

		  struct mlx4_uar *uar, u64 db_rec, struct mlx4_cq *cq)

{

include/linux/mlx4/cq.h

+2 −0
Original line number Diff line number Diff line
@@ -132,5 +132,7 @@ enum { 


int mlx4_cq_modify(struct mlx4_dev *dev, struct mlx4_cq *cq,

		   u16 count, u16 period);

int mlx4_cq_resize(struct mlx4_dev *dev, struct mlx4_cq *cq,

		   int entries, struct mlx4_mtt *mtt);



#endif /* MLX4_CQ_H */