Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 1b2cd0fc authored by Sagi Grimberg's avatar Sagi Grimberg Committed by Doug Ledford
Browse files

IB/mlx4: Support the new memory registration API 



Support the new memory registration API by allocating a
private page list array in mlx4_ib_mr and populate it when
mlx4_ib_map_mr_sg is invoked. Also, support IB_WR_REG_MR
by setting the exact WQE as IB_WR_FAST_REG_MR, just take the
needed information from different places:
- page_size, iova, length, access flags (ib_mr)
- page array (mlx4_ib_mr)
- key (ib_reg_wr)

The IB_WR_FAST_REG_MR handlers will be removed later when
all the ULPs will be converted.

Signed-off-by: default avatarSagi Grimberg <sagig@mellanox.com>
Tested-by: default avatarChristoph Hellwig <hch@lst.de>
Signed-off-by: default avatarDoug Ledford <dledford@redhat.com>
parent 8a187ee5

drivers/infiniband/hw/mlx4/cq.c

+1 −0
Original line number Diff line number Diff line
@@ -819,6 +819,7 @@ static int mlx4_ib_poll_one(struct mlx4_ib_cq *cq, 
			break;

		case MLX4_OPCODE_FMR:

			wc->opcode    = IB_WC_FAST_REG_MR;

			/* TODO: wc->opcode    = IB_WC_REG_MR; */

			break;

		case MLX4_OPCODE_LOCAL_INVAL:

			wc->opcode    = IB_WC_LOCAL_INV;

drivers/infiniband/hw/mlx4/main.c

+1 −0
Original line number Diff line number Diff line
@@ -2266,6 +2266,7 @@ static void *mlx4_ib_add(struct mlx4_dev *dev) 
	ibdev->ib_dev.rereg_user_mr	= mlx4_ib_rereg_user_mr;

	ibdev->ib_dev.dereg_mr		= mlx4_ib_dereg_mr;

	ibdev->ib_dev.alloc_mr		= mlx4_ib_alloc_mr;

	ibdev->ib_dev.map_mr_sg		= mlx4_ib_map_mr_sg;

	ibdev->ib_dev.alloc_fast_reg_page_list = mlx4_ib_alloc_fast_reg_page_list;

	ibdev->ib_dev.free_fast_reg_page_list  = mlx4_ib_free_fast_reg_page_list;

	ibdev->ib_dev.attach_mcast	= mlx4_ib_mcg_attach;

drivers/infiniband/hw/mlx4/mlx4_ib.h

+10 −0
Original line number Diff line number Diff line
@@ -129,10 +129,17 @@ struct mlx4_ib_cq { 
	struct list_head		recv_qp_list;

};



#define MLX4_MR_PAGES_ALIGN 0x40



struct mlx4_ib_mr {

	struct ib_mr		ibmr;

	__be64			*pages;

	dma_addr_t		page_map;

	u32			npages;

	u32			max_pages;

	struct mlx4_mr		mmr;

	struct ib_umem	       *umem;

	void			*pages_alloc;

};



struct mlx4_ib_mw {
@@ -714,6 +721,9 @@ int mlx4_ib_dealloc_mw(struct ib_mw *mw); 
struct ib_mr *mlx4_ib_alloc_mr(struct ib_pd *pd,

			       enum ib_mr_type mr_type,

			       u32 max_num_sg);

int mlx4_ib_map_mr_sg(struct ib_mr *ibmr,

		      struct scatterlist *sg,

		      int sg_nents);

struct ib_fast_reg_page_list *mlx4_ib_alloc_fast_reg_page_list(struct ib_device *ibdev,

							       int page_list_len);

void mlx4_ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list);

drivers/infiniband/hw/mlx4/mr.c

+95 −6
Original line number Diff line number Diff line
@@ -59,7 +59,7 @@ struct ib_mr *mlx4_ib_get_dma_mr(struct ib_pd *pd, int acc) 
	struct mlx4_ib_mr *mr;

	int err;



	mr = kmalloc(sizeof *mr, GFP_KERNEL);

	mr = kzalloc(sizeof(*mr), GFP_KERNEL);

	if (!mr)

		return ERR_PTR(-ENOMEM);
@@ -140,7 +140,7 @@ struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 
	int err;

	int n;



	mr = kmalloc(sizeof *mr, GFP_KERNEL);

	mr = kzalloc(sizeof(*mr), GFP_KERNEL);

	if (!mr)

		return ERR_PTR(-ENOMEM);
@@ -271,11 +271,59 @@ int mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags, 
	return err;

}



static int

mlx4_alloc_priv_pages(struct ib_device *device,

		      struct mlx4_ib_mr *mr,

		      int max_pages)

{

	int size = max_pages * sizeof(u64);

	int add_size;

	int ret;



	add_size = max_t(int, MLX4_MR_PAGES_ALIGN - ARCH_KMALLOC_MINALIGN, 0);



	mr->pages_alloc = kzalloc(size + add_size, GFP_KERNEL);

	if (!mr->pages_alloc)

		return -ENOMEM;



	mr->pages = PTR_ALIGN(mr->pages_alloc, MLX4_MR_PAGES_ALIGN);



	mr->page_map = dma_map_single(device->dma_device, mr->pages,

				      size, DMA_TO_DEVICE);



	if (dma_mapping_error(device->dma_device, mr->page_map)) {

		ret = -ENOMEM;

		goto err;

	}



	return 0;

err:

	kfree(mr->pages_alloc);



	return ret;

}



static void

mlx4_free_priv_pages(struct mlx4_ib_mr *mr)

{

	if (mr->pages) {

		struct ib_device *device = mr->ibmr.device;

		int size = mr->max_pages * sizeof(u64);



		dma_unmap_single(device->dma_device, mr->page_map,

				 size, DMA_TO_DEVICE);

		kfree(mr->pages_alloc);

		mr->pages = NULL;

	}

}



int mlx4_ib_dereg_mr(struct ib_mr *ibmr)

{

	struct mlx4_ib_mr *mr = to_mmr(ibmr);

	int ret;



	mlx4_free_priv_pages(mr);



	ret = mlx4_mr_free(to_mdev(ibmr->device)->dev, &mr->mmr);

	if (ret)

		return ret;
@@ -362,7 +410,7 @@ struct ib_mr *mlx4_ib_alloc_mr(struct ib_pd *pd, 
	    max_num_sg > MLX4_MAX_FAST_REG_PAGES)

		return ERR_PTR(-EINVAL);



	mr = kmalloc(sizeof *mr, GFP_KERNEL);

	mr = kzalloc(sizeof(*mr), GFP_KERNEL);

	if (!mr)

		return ERR_PTR(-ENOMEM);
@@ -371,18 +419,25 @@ struct ib_mr *mlx4_ib_alloc_mr(struct ib_pd *pd, 
	if (err)

		goto err_free;



	err = mlx4_alloc_priv_pages(pd->device, mr, max_num_sg);

	if (err)

		goto err_free_mr;



	mr->max_pages = max_num_sg;



	err = mlx4_mr_enable(dev->dev, &mr->mmr);

	if (err)

		goto err_mr;

		goto err_free_pl;



	mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key;

	mr->umem = NULL;



	return &mr->ibmr;



err_mr:

err_free_pl:

	mlx4_free_priv_pages(mr);

err_free_mr:

	(void) mlx4_mr_free(dev->dev, &mr->mmr);



err_free:

	kfree(mr);

	return ERR_PTR(err);
@@ -528,3 +583,37 @@ int mlx4_ib_fmr_dealloc(struct ib_fmr *ibfmr) 


	return err;

}



static int mlx4_set_page(struct ib_mr *ibmr, u64 addr)

{

	struct mlx4_ib_mr *mr = to_mmr(ibmr);



	if (unlikely(mr->npages == mr->max_pages))

		return -ENOMEM;



	mr->pages[mr->npages++] = cpu_to_be64(addr | MLX4_MTT_FLAG_PRESENT);



	return 0;

}



int mlx4_ib_map_mr_sg(struct ib_mr *ibmr,

		      struct scatterlist *sg,

		      int sg_nents)

{

	struct mlx4_ib_mr *mr = to_mmr(ibmr);

	int rc;



	mr->npages = 0;



	ib_dma_sync_single_for_cpu(ibmr->device, mr->page_map,

				   sizeof(u64) * mr->max_pages,

				   DMA_TO_DEVICE);



	rc = ib_sg_to_pages(ibmr, sg, sg_nents, mlx4_set_page);



	ib_dma_sync_single_for_device(ibmr->device, mr->page_map,

				      sizeof(u64) * mr->max_pages,

				      DMA_TO_DEVICE);



	return rc;

}

drivers/infiniband/hw/mlx4/qp.c

+25 −0
Original line number Diff line number Diff line
@@ -112,6 +112,7 @@ static const __be32 mlx4_ib_opcode[] = { 
	[IB_WR_SEND_WITH_INV]			= cpu_to_be32(MLX4_OPCODE_SEND_INVAL),

	[IB_WR_LOCAL_INV]			= cpu_to_be32(MLX4_OPCODE_LOCAL_INVAL),

	[IB_WR_FAST_REG_MR]			= cpu_to_be32(MLX4_OPCODE_FMR),

	[IB_WR_REG_MR]				= cpu_to_be32(MLX4_OPCODE_FMR),

	[IB_WR_MASKED_ATOMIC_CMP_AND_SWP]	= cpu_to_be32(MLX4_OPCODE_MASKED_ATOMIC_CS),

	[IB_WR_MASKED_ATOMIC_FETCH_AND_ADD]	= cpu_to_be32(MLX4_OPCODE_MASKED_ATOMIC_FA),

	[IB_WR_BIND_MW]				= cpu_to_be32(MLX4_OPCODE_BIND_MW),
@@ -2505,6 +2506,22 @@ static __be32 convert_access(int acc) 
		cpu_to_be32(MLX4_WQE_FMR_PERM_LOCAL_READ);

}



static void set_reg_seg(struct mlx4_wqe_fmr_seg *fseg,

			struct ib_reg_wr *wr)

{

	struct mlx4_ib_mr *mr = to_mmr(wr->mr);



	fseg->flags		= convert_access(wr->access);

	fseg->mem_key		= cpu_to_be32(wr->key);

	fseg->buf_list		= cpu_to_be64(mr->page_map);

	fseg->start_addr	= cpu_to_be64(mr->ibmr.iova);

	fseg->reg_len		= cpu_to_be64(mr->ibmr.length);

	fseg->offset		= 0; /* XXX -- is this just for ZBVA? */

	fseg->page_size		= cpu_to_be32(ilog2(mr->ibmr.page_size));

	fseg->reserved[0]	= 0;

	fseg->reserved[1]	= 0;

}



static void set_fmr_seg(struct mlx4_wqe_fmr_seg *fseg,

		struct ib_fast_reg_wr *wr)

{
@@ -2866,6 +2883,14 @@ int mlx4_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 
				size += sizeof (struct mlx4_wqe_fmr_seg) / 16;

				break;



			case IB_WR_REG_MR:

				ctrl->srcrb_flags |=

					cpu_to_be32(MLX4_WQE_CTRL_STRONG_ORDER);

				set_reg_seg(wqe, reg_wr(wr));

				wqe  += sizeof(struct mlx4_wqe_fmr_seg);

				size += sizeof(struct mlx4_wqe_fmr_seg) / 16;

				break;



			case IB_WR_BIND_MW:

				ctrl->srcrb_flags |=

					cpu_to_be32(MLX4_WQE_CTRL_STRONG_ORDER);