Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit bb6c96d7 authored by Chuck Lever's avatar Chuck Lever Committed by Doug Ledford
Browse files

xprtrdma: Replace global lkey with lkey local to PD 



The core API has changed so that devices that do not have a global
DMA lkey automatically create an mr, per-PD, and make that lkey
available. The global DMA lkey interface is going away in favor of
the per-PD DMA lkey.

The per-PD DMA lkey is always available. Convert xprtrdma to use the
device's per-PD DMA lkey for regbufs, no matter which memory
registration scheme is in use.

Signed-off-by: default avatarChuck Lever <chuck.lever@oracle.com>
Signed-off-by: default avatarSagi Grimberg <sagig@mellanox.com>
Cc: linux-nfs <linux-nfs@vger.kernel.org>
Acked-by: default avatarAnna Schumaker <Anna.Schumaker@Netapp.com>
Signed-off-by: default avatarDoug Ledford <dledford@redhat.com>
parent 310b7cec

net/sunrpc/xprtrdma/fmr_ops.c

+0 −19
Original line number Diff line number Diff line
@@ -39,25 +39,6 @@ static int 
fmr_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep,

	    struct rpcrdma_create_data_internal *cdata)

{

	struct ib_device_attr *devattr = &ia->ri_devattr;

	struct ib_mr *mr;



	/* Obtain an lkey to use for the regbufs, which are

	 * protected from remote access.

	 */

	if (devattr->device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY) {

		ia->ri_dma_lkey = ia->ri_device->local_dma_lkey;

	} else {

		mr = ib_get_dma_mr(ia->ri_pd, IB_ACCESS_LOCAL_WRITE);

		if (IS_ERR(mr)) {

			pr_err("%s: ib_get_dma_mr for failed with %lX\n",

			       __func__, PTR_ERR(mr));

			return -ENOMEM;

		}

		ia->ri_dma_lkey = ia->ri_dma_mr->lkey;

		ia->ri_dma_mr = mr;

	}



	return 0;

}

net/sunrpc/xprtrdma/frwr_ops.c

+0 −5
Original line number Diff line number Diff line
@@ -189,11 +189,6 @@ frwr_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep, 
	struct ib_device_attr *devattr = &ia->ri_devattr;

	int depth, delta;



	/* Obtain an lkey to use for the regbufs, which are

	 * protected from remote access.

	 */

	ia->ri_dma_lkey = ia->ri_device->local_dma_lkey;



	ia->ri_max_frmr_depth =

			min_t(unsigned int, RPCRDMA_MAX_DATA_SEGS,

			      devattr->max_fast_reg_page_list_len);

net/sunrpc/xprtrdma/physical_ops.c

+1 −9
Original line number Diff line number Diff line
@@ -23,7 +23,6 @@ static int 
physical_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep,

		 struct rpcrdma_create_data_internal *cdata)

{

	struct ib_device_attr *devattr = &ia->ri_devattr;

	struct ib_mr *mr;



	/* Obtain an rkey to use for RPC data payloads.
@@ -37,15 +36,8 @@ physical_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep, 
		       __func__, PTR_ERR(mr));

		return -ENOMEM;

	}

	ia->ri_dma_mr = mr;



	/* Obtain an lkey to use for regbufs.

	 */

	if (devattr->device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY)

		ia->ri_dma_lkey = ia->ri_device->local_dma_lkey;

	else

		ia->ri_dma_lkey = ia->ri_dma_mr->lkey;



	ia->ri_dma_mr = mr;

	return 0;

}

net/sunrpc/xprtrdma/verbs.c

+1 −1
Original line number Diff line number Diff line
@@ -1252,7 +1252,7 @@ rpcrdma_alloc_regbuf(struct rpcrdma_ia *ia, size_t size, gfp_t flags) 
		goto out_free;



	iov->length = size;

	iov->lkey = ia->ri_dma_lkey;

	iov->lkey = ia->ri_pd->local_dma_lkey;

	rb->rg_size = size;

	rb->rg_owner = NULL;

	return rb;

net/sunrpc/xprtrdma/xprt_rdma.h

+0 −1
Original line number Diff line number Diff line
@@ -65,7 +65,6 @@ struct rpcrdma_ia { 
	struct rdma_cm_id 	*ri_id;

	struct ib_pd		*ri_pd;

	struct ib_mr		*ri_dma_mr;

	u32			ri_dma_lkey;

	struct completion	ri_done;

	int			ri_async_rc;

	unsigned int		ri_max_frmr_depth;