Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 96ceddea authored by Chuck Lever's avatar Chuck Lever Committed by Anna Schumaker
Browse files

xprtrdma: Remove usage of "mw" 



Clean up: struct rpcrdma_mw was named after Memory Windows, but
xprtrdma no longer supports a Memory Window registration mode.
Rename rpcrdma_mw and its fields to reduce confusion and make
the code more sensible to read.

Renaming "mw" was suggested by Tom Talpey, the author of the
original xprtrdma implementation. It's a good idea, but I haven't
done this until now because it's a huge diffstat for no benefit
other than code readability.

However, I'm about to introduce static trace points that expose
a few of xprtrdma's internal data structures. They should make sense
in the trace report, and it's reasonable to treat trace points as a
kernel API contract which might be difficult to change later.

While I'm churning things up, two additional changes:
- rename variables unhelpfully called "r" to "mr", to improve code
  clarity, and
- rename the MR-related helper functions using the form
  "rpcrdma_mr_<verb>", to be consistent with other areas of the
  code.

Signed-off-by: default avatarChuck Lever <chuck.lever@oracle.com>
Signed-off-by: default avatarAnna Schumaker <Anna.Schumaker@Netapp.com>
parent ce5b3717

net/sunrpc/xprtrdma/fmr_ops.c

+74 −74
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0

/*

 * Copyright (c) 2015 Oracle.  All rights reserved.

 * Copyright (c) 2015, 2017 Oracle.  All rights reserved.

 * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.

 */
@@ -47,7 +47,7 @@ fmr_is_supported(struct rpcrdma_ia *ia) 
}



static int

fmr_op_init_mr(struct rpcrdma_ia *ia, struct rpcrdma_mw *mw)

fmr_op_init_mr(struct rpcrdma_ia *ia, struct rpcrdma_mr *mr)

{

	static struct ib_fmr_attr fmr_attr = {

		.max_pages	= RPCRDMA_MAX_FMR_SGES,
@@ -55,106 +55,106 @@ fmr_op_init_mr(struct rpcrdma_ia *ia, struct rpcrdma_mw *mw) 
		.page_shift	= PAGE_SHIFT

	};



	mw->fmr.fm_physaddrs = kcalloc(RPCRDMA_MAX_FMR_SGES,

	mr->fmr.fm_physaddrs = kcalloc(RPCRDMA_MAX_FMR_SGES,

				       sizeof(u64), GFP_KERNEL);

	if (!mw->fmr.fm_physaddrs)

	if (!mr->fmr.fm_physaddrs)

		goto out_free;



	mw->mw_sg = kcalloc(RPCRDMA_MAX_FMR_SGES,

			    sizeof(*mw->mw_sg), GFP_KERNEL);

	if (!mw->mw_sg)

	mr->mr_sg = kcalloc(RPCRDMA_MAX_FMR_SGES,

			    sizeof(*mr->mr_sg), GFP_KERNEL);

	if (!mr->mr_sg)

		goto out_free;



	sg_init_table(mw->mw_sg, RPCRDMA_MAX_FMR_SGES);

	sg_init_table(mr->mr_sg, RPCRDMA_MAX_FMR_SGES);



	mw->fmr.fm_mr = ib_alloc_fmr(ia->ri_pd, RPCRDMA_FMR_ACCESS_FLAGS,

	mr->fmr.fm_mr = ib_alloc_fmr(ia->ri_pd, RPCRDMA_FMR_ACCESS_FLAGS,

				     &fmr_attr);

	if (IS_ERR(mw->fmr.fm_mr))

	if (IS_ERR(mr->fmr.fm_mr))

		goto out_fmr_err;



	return 0;



out_fmr_err:

	dprintk("RPC:       %s: ib_alloc_fmr returned %ld\n", __func__,

		PTR_ERR(mw->fmr.fm_mr));

		PTR_ERR(mr->fmr.fm_mr));



out_free:

	kfree(mw->mw_sg);

	kfree(mw->fmr.fm_physaddrs);

	kfree(mr->mr_sg);

	kfree(mr->fmr.fm_physaddrs);

	return -ENOMEM;

}



static int

__fmr_unmap(struct rpcrdma_mw *mw)

__fmr_unmap(struct rpcrdma_mr *mr)

{

	LIST_HEAD(l);

	int rc;



	list_add(&mw->fmr.fm_mr->list, &l);

	list_add(&mr->fmr.fm_mr->list, &l);

	rc = ib_unmap_fmr(&l);

	list_del(&mw->fmr.fm_mr->list);

	list_del(&mr->fmr.fm_mr->list);

	return rc;

}



static void

fmr_op_release_mr(struct rpcrdma_mw *r)

fmr_op_release_mr(struct rpcrdma_mr *mr)

{

	LIST_HEAD(unmap_list);

	int rc;



	/* Ensure MW is not on any rl_registered list */

	if (!list_empty(&r->mw_list))

		list_del(&r->mw_list);

	if (!list_empty(&mr->mr_list))

		list_del(&mr->mr_list);



	kfree(r->fmr.fm_physaddrs);

	kfree(r->mw_sg);

	kfree(mr->fmr.fm_physaddrs);

	kfree(mr->mr_sg);



	/* In case this one was left mapped, try to unmap it

	 * to prevent dealloc_fmr from failing with EBUSY

	 */

	rc = __fmr_unmap(r);

	rc = __fmr_unmap(mr);

	if (rc)

		pr_err("rpcrdma: final ib_unmap_fmr for %p failed %i\n",

		       r, rc);

		       mr, rc);



	rc = ib_dealloc_fmr(r->fmr.fm_mr);

	rc = ib_dealloc_fmr(mr->fmr.fm_mr);

	if (rc)

		pr_err("rpcrdma: final ib_dealloc_fmr for %p returned %i\n",

		       r, rc);

		       mr, rc);



	kfree(r);

	kfree(mr);

}



/* Reset of a single FMR.

 */

static void

fmr_op_recover_mr(struct rpcrdma_mw *mw)

fmr_op_recover_mr(struct rpcrdma_mr *mr)

{

	struct rpcrdma_xprt *r_xprt = mw->mw_xprt;

	struct rpcrdma_xprt *r_xprt = mr->mr_xprt;

	int rc;



	/* ORDER: invalidate first */

	rc = __fmr_unmap(mw);

	rc = __fmr_unmap(mr);



	/* ORDER: then DMA unmap */

	ib_dma_unmap_sg(r_xprt->rx_ia.ri_device,

			mw->mw_sg, mw->mw_nents, mw->mw_dir);

			mr->mr_sg, mr->mr_nents, mr->mr_dir);

	if (rc)

		goto out_release;



	rpcrdma_put_mw(r_xprt, mw);

	rpcrdma_mr_put(mr);

	r_xprt->rx_stats.mrs_recovered++;

	return;



out_release:

	pr_err("rpcrdma: FMR reset failed (%d), %p released\n", rc, mw);

	pr_err("rpcrdma: FMR reset failed (%d), %p released\n", rc, mr);

	r_xprt->rx_stats.mrs_orphaned++;



	spin_lock(&r_xprt->rx_buf.rb_mwlock);

	list_del(&mw->mw_all);

	spin_unlock(&r_xprt->rx_buf.rb_mwlock);

	spin_lock(&r_xprt->rx_buf.rb_mrlock);

	list_del(&mr->mr_all);

	spin_unlock(&r_xprt->rx_buf.rb_mrlock);



	fmr_op_release_mr(mw);

	fmr_op_release_mr(mr);

}



static int
@@ -180,15 +180,15 @@ fmr_op_maxpages(struct rpcrdma_xprt *r_xprt) 
 */

static struct rpcrdma_mr_seg *

fmr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg,

	   int nsegs, bool writing, struct rpcrdma_mw **out)

	   int nsegs, bool writing, struct rpcrdma_mr **out)

{

	struct rpcrdma_mr_seg *seg1 = seg;

	int len, pageoff, i, rc;

	struct rpcrdma_mw *mw;

	struct rpcrdma_mr *mr;

	u64 *dma_pages;



	mw = rpcrdma_get_mw(r_xprt);

	if (!mw)

	mr = rpcrdma_mr_get(r_xprt);

	if (!mr)

		return ERR_PTR(-ENOBUFS);



	pageoff = offset_in_page(seg1->mr_offset);
@@ -199,12 +199,12 @@ fmr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg, 
		nsegs = RPCRDMA_MAX_FMR_SGES;

	for (i = 0; i < nsegs;) {

		if (seg->mr_page)

			sg_set_page(&mw->mw_sg[i],

			sg_set_page(&mr->mr_sg[i],

				    seg->mr_page,

				    seg->mr_len,

				    offset_in_page(seg->mr_offset));

		else

			sg_set_buf(&mw->mw_sg[i], seg->mr_offset,

			sg_set_buf(&mr->mr_sg[i], seg->mr_offset,

				   seg->mr_len);

		len += seg->mr_len;

		++seg;
@@ -214,40 +214,40 @@ fmr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg, 
		    offset_in_page((seg-1)->mr_offset + (seg-1)->mr_len))

			break;

	}

	mw->mw_dir = rpcrdma_data_dir(writing);

	mr->mr_dir = rpcrdma_data_dir(writing);



	mw->mw_nents = ib_dma_map_sg(r_xprt->rx_ia.ri_device,

				     mw->mw_sg, i, mw->mw_dir);

	if (!mw->mw_nents)

	mr->mr_nents = ib_dma_map_sg(r_xprt->rx_ia.ri_device,

				     mr->mr_sg, i, mr->mr_dir);

	if (!mr->mr_nents)

		goto out_dmamap_err;



	for (i = 0, dma_pages = mw->fmr.fm_physaddrs; i < mw->mw_nents; i++)

		dma_pages[i] = sg_dma_address(&mw->mw_sg[i]);

	rc = ib_map_phys_fmr(mw->fmr.fm_mr, dma_pages, mw->mw_nents,

	for (i = 0, dma_pages = mr->fmr.fm_physaddrs; i < mr->mr_nents; i++)

		dma_pages[i] = sg_dma_address(&mr->mr_sg[i]);

	rc = ib_map_phys_fmr(mr->fmr.fm_mr, dma_pages, mr->mr_nents,

			     dma_pages[0]);

	if (rc)

		goto out_maperr;



	mw->mw_handle = mw->fmr.fm_mr->rkey;

	mw->mw_length = len;

	mw->mw_offset = dma_pages[0] + pageoff;

	mr->mr_handle = mr->fmr.fm_mr->rkey;

	mr->mr_length = len;

	mr->mr_offset = dma_pages[0] + pageoff;



	*out = mw;

	*out = mr;

	return seg;



out_dmamap_err:

	pr_err("rpcrdma: failed to DMA map sg %p sg_nents %d\n",

	       mw->mw_sg, i);

	rpcrdma_put_mw(r_xprt, mw);

	       mr->mr_sg, i);

	rpcrdma_mr_put(mr);

	return ERR_PTR(-EIO);



out_maperr:

	pr_err("rpcrdma: ib_map_phys_fmr %u@0x%llx+%i (%d) status %i\n",

	       len, (unsigned long long)dma_pages[0],

	       pageoff, mw->mw_nents, rc);

	       pageoff, mr->mr_nents, rc);

	ib_dma_unmap_sg(r_xprt->rx_ia.ri_device,

			mw->mw_sg, mw->mw_nents, mw->mw_dir);

	rpcrdma_put_mw(r_xprt, mw);

			mr->mr_sg, mr->mr_nents, mr->mr_dir);

	rpcrdma_mr_put(mr);

	return ERR_PTR(-EIO);

}
@@ -256,13 +256,13 @@ fmr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg, 
 * Sleeps until it is safe for the host CPU to access the

 * previously mapped memory regions.

 *

 * Caller ensures that @mws is not empty before the call. This

 * Caller ensures that @mrs is not empty before the call. This

 * function empties the list.

 */

static void

fmr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct list_head *mws)

fmr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct list_head *mrs)

{

	struct rpcrdma_mw *mw;

	struct rpcrdma_mr *mr;

	LIST_HEAD(unmap_list);

	int rc;
@@ -271,10 +271,10 @@ fmr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct list_head *mws) 
	 * ib_unmap_fmr() is slow, so use a single call instead

	 * of one call per mapped FMR.

	 */

	list_for_each_entry(mw, mws, mw_list) {

	list_for_each_entry(mr, mrs, mr_list) {

		dprintk("RPC:       %s: unmapping fmr %p\n",

			__func__, &mw->fmr);

		list_add_tail(&mw->fmr.fm_mr->list, &unmap_list);

			__func__, &mr->fmr);

		list_add_tail(&mr->fmr.fm_mr->list, &unmap_list);

	}

	r_xprt->rx_stats.local_inv_needed++;

	rc = ib_unmap_fmr(&unmap_list);
@@ -284,14 +284,14 @@ fmr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct list_head *mws) 
	/* ORDER: Now DMA unmap all of the req's MRs, and return

	 * them to the free MW list.

	 */

	while (!list_empty(mws)) {

		mw = rpcrdma_pop_mw(mws);

	while (!list_empty(mrs)) {

		mr = rpcrdma_mr_pop(mrs);

		dprintk("RPC:       %s: DMA unmapping fmr %p\n",

			__func__, &mw->fmr);

		list_del(&mw->fmr.fm_mr->list);

			__func__, &mr->fmr);

		list_del(&mr->fmr.fm_mr->list);

		ib_dma_unmap_sg(r_xprt->rx_ia.ri_device,

				mw->mw_sg, mw->mw_nents, mw->mw_dir);

		rpcrdma_put_mw(r_xprt, mw);

				mr->mr_sg, mr->mr_nents, mr->mr_dir);

		rpcrdma_mr_put(mr);

	}



	return;
@@ -299,10 +299,10 @@ fmr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct list_head *mws) 
out_reset:

	pr_err("rpcrdma: ib_unmap_fmr failed (%i)\n", rc);



	while (!list_empty(mws)) {

		mw = rpcrdma_pop_mw(mws);

		list_del(&mw->fmr.fm_mr->list);

		fmr_op_recover_mr(mw);

	while (!list_empty(mrs)) {

		mr = rpcrdma_mr_pop(mrs);

		list_del(&mr->fmr.fm_mr->list);

		fmr_op_recover_mr(mr);

	}

}

net/sunrpc/xprtrdma/frwr_ops.c

+89 −88
Original line number Diff line number Diff line
@@ -17,7 +17,7 @@ 
 * A Memory Region is prepared for RDMA READ or WRITE using a FAST_REG

 * Work Request (frwr_op_map). When the RDMA operation is finished, this

 * Memory Region is invalidated using a LOCAL_INV Work Request

 * (frwr_op_unmap).

 * (frwr_op_unmap_sync).

 *

 * Typically these Work Requests are not signaled, and neither are RDMA

 * SEND Work Requests (with the exception of signaling occasionally to
@@ -26,7 +26,7 @@ 
 *

 * As an optimization, frwr_op_unmap marks MRs INVALID before the

 * LOCAL_INV WR is posted. If posting succeeds, the MR is placed on

 * rb_mws immediately so that no work (like managing a linked list

 * rb_mrs immediately so that no work (like managing a linked list

 * under a spinlock) is needed in the completion upcall.

 *

 * But this means that frwr_op_map() can occasionally encounter an MR
@@ -60,7 +60,7 @@ 
 * When frwr_op_map encounters FLUSHED and VALID MRs, they are recovered

 * with ib_dereg_mr and then are re-initialized. Because MR recovery

 * allocates fresh resources, it is deferred to a workqueue, and the

 * recovered MRs are placed back on the rb_mws list when recovery is

 * recovered MRs are placed back on the rb_mrs list when recovery is

 * complete. frwr_op_map allocates another MR for the current RPC while

 * the broken MR is reset.

 *
@@ -96,21 +96,21 @@ frwr_is_supported(struct rpcrdma_ia *ia) 
}



static int

frwr_op_init_mr(struct rpcrdma_ia *ia, struct rpcrdma_mw *r)

frwr_op_init_mr(struct rpcrdma_ia *ia, struct rpcrdma_mr *mr)

{

	unsigned int depth = ia->ri_max_frwr_depth;

	struct rpcrdma_frwr *frwr = &r->frwr;

	struct rpcrdma_frwr *frwr = &mr->frwr;

	int rc;



	frwr->fr_mr = ib_alloc_mr(ia->ri_pd, ia->ri_mrtype, depth);

	if (IS_ERR(frwr->fr_mr))

		goto out_mr_err;



	r->mw_sg = kcalloc(depth, sizeof(*r->mw_sg), GFP_KERNEL);

	if (!r->mw_sg)

	mr->mr_sg = kcalloc(depth, sizeof(*mr->mr_sg), GFP_KERNEL);

	if (!mr->mr_sg)

		goto out_list_err;



	sg_init_table(r->mw_sg, depth);

	sg_init_table(mr->mr_sg, depth);

	init_completion(&frwr->fr_linv_done);

	return 0;
@@ -129,32 +129,32 @@ frwr_op_init_mr(struct rpcrdma_ia *ia, struct rpcrdma_mw *r) 
}



static void

frwr_op_release_mr(struct rpcrdma_mw *r)

frwr_op_release_mr(struct rpcrdma_mr *mr)

{

	int rc;



	/* Ensure MW is not on any rl_registered list */

	if (!list_empty(&r->mw_list))

		list_del(&r->mw_list);

	/* Ensure MR is not on any rl_registered list */

	if (!list_empty(&mr->mr_list))

		list_del(&mr->mr_list);



	rc = ib_dereg_mr(r->frwr.fr_mr);

	rc = ib_dereg_mr(mr->frwr.fr_mr);

	if (rc)

		pr_err("rpcrdma: final ib_dereg_mr for %p returned %i\n",

		       r, rc);

	kfree(r->mw_sg);

	kfree(r);

		       mr, rc);

	kfree(mr->mr_sg);

	kfree(mr);

}



static int

__frwr_reset_mr(struct rpcrdma_ia *ia, struct rpcrdma_mw *r)

__frwr_mr_reset(struct rpcrdma_ia *ia, struct rpcrdma_mr *mr)

{

	struct rpcrdma_frwr *frwr = &r->frwr;

	struct rpcrdma_frwr *frwr = &mr->frwr;

	int rc;



	rc = ib_dereg_mr(frwr->fr_mr);

	if (rc) {

		pr_warn("rpcrdma: ib_dereg_mr status %d, frwr %p orphaned\n",

			rc, r);

			rc, mr);

		return rc;

	}
@@ -162,7 +162,7 @@ __frwr_reset_mr(struct rpcrdma_ia *ia, struct rpcrdma_mw *r) 
				  ia->ri_max_frwr_depth);

	if (IS_ERR(frwr->fr_mr)) {

		pr_warn("rpcrdma: ib_alloc_mr status %ld, frwr %p orphaned\n",

			PTR_ERR(frwr->fr_mr), r);

			PTR_ERR(frwr->fr_mr), mr);

		return PTR_ERR(frwr->fr_mr);

	}
@@ -174,33 +174,33 @@ __frwr_reset_mr(struct rpcrdma_ia *ia, struct rpcrdma_mw *r) 
/* Reset of a single FRWR. Generate a fresh rkey by replacing the MR.

 */

static void

frwr_op_recover_mr(struct rpcrdma_mw *mw)

frwr_op_recover_mr(struct rpcrdma_mr *mr)

{

	enum rpcrdma_frwr_state state = mw->frwr.fr_state;

	struct rpcrdma_xprt *r_xprt = mw->mw_xprt;

	enum rpcrdma_frwr_state state = mr->frwr.fr_state;

	struct rpcrdma_xprt *r_xprt = mr->mr_xprt;

	struct rpcrdma_ia *ia = &r_xprt->rx_ia;

	int rc;



	rc = __frwr_reset_mr(ia, mw);

	rc = __frwr_mr_reset(ia, mr);

	if (state != FRWR_FLUSHED_LI)

		ib_dma_unmap_sg(ia->ri_device,

				mw->mw_sg, mw->mw_nents, mw->mw_dir);

				mr->mr_sg, mr->mr_nents, mr->mr_dir);

	if (rc)

		goto out_release;



	rpcrdma_put_mw(r_xprt, mw);

	rpcrdma_mr_put(mr);

	r_xprt->rx_stats.mrs_recovered++;

	return;



out_release:

	pr_err("rpcrdma: FRWR reset failed %d, %p release\n", rc, mw);

	pr_err("rpcrdma: FRWR reset failed %d, %p release\n", rc, mr);

	r_xprt->rx_stats.mrs_orphaned++;



	spin_lock(&r_xprt->rx_buf.rb_mwlock);

	list_del(&mw->mw_all);

	spin_unlock(&r_xprt->rx_buf.rb_mwlock);

	spin_lock(&r_xprt->rx_buf.rb_mrlock);

	list_del(&mr->mr_all);

	spin_unlock(&r_xprt->rx_buf.rb_mrlock);



	frwr_op_release_mr(mw);

	frwr_op_release_mr(mr);

}



static int
@@ -347,40 +347,39 @@ frwr_wc_localinv_wake(struct ib_cq *cq, struct ib_wc *wc) 
 */

static struct rpcrdma_mr_seg *

frwr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg,

	    int nsegs, bool writing, struct rpcrdma_mw **out)

	    int nsegs, bool writing, struct rpcrdma_mr **out)

{

	struct rpcrdma_ia *ia = &r_xprt->rx_ia;

	bool holes_ok = ia->ri_mrtype == IB_MR_TYPE_SG_GAPS;

	struct rpcrdma_frwr *frwr;

	struct rpcrdma_mw *mw;

	struct ib_mr *mr;

	struct rpcrdma_mr *mr;

	struct ib_mr *ibmr;

	struct ib_reg_wr *reg_wr;

	struct ib_send_wr *bad_wr;

	int rc, i, n;

	u8 key;



	mw = NULL;

	mr = NULL;

	do {

		if (mw)

			rpcrdma_defer_mr_recovery(mw);

		mw = rpcrdma_get_mw(r_xprt);

		if (!mw)

		if (mr)

			rpcrdma_mr_defer_recovery(mr);

		mr = rpcrdma_mr_get(r_xprt);

		if (!mr)

			return ERR_PTR(-ENOBUFS);

	} while (mw->frwr.fr_state != FRWR_IS_INVALID);

	frwr = &mw->frwr;

	} while (mr->frwr.fr_state != FRWR_IS_INVALID);

	frwr = &mr->frwr;

	frwr->fr_state = FRWR_IS_VALID;

	mr = frwr->fr_mr;



	if (nsegs > ia->ri_max_frwr_depth)

		nsegs = ia->ri_max_frwr_depth;

	for (i = 0; i < nsegs;) {

		if (seg->mr_page)

			sg_set_page(&mw->mw_sg[i],

			sg_set_page(&mr->mr_sg[i],

				    seg->mr_page,

				    seg->mr_len,

				    offset_in_page(seg->mr_offset));

		else

			sg_set_buf(&mw->mw_sg[i], seg->mr_offset,

			sg_set_buf(&mr->mr_sg[i], seg->mr_offset,

				   seg->mr_len);



		++seg;
@@ -391,21 +390,22 @@ frwr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg, 
		    offset_in_page((seg-1)->mr_offset + (seg-1)->mr_len))

			break;

	}

	mw->mw_dir = rpcrdma_data_dir(writing);

	mr->mr_dir = rpcrdma_data_dir(writing);



	mw->mw_nents = ib_dma_map_sg(ia->ri_device, mw->mw_sg, i, mw->mw_dir);

	if (!mw->mw_nents)

	mr->mr_nents = ib_dma_map_sg(ia->ri_device, mr->mr_sg, i, mr->mr_dir);

	if (!mr->mr_nents)

		goto out_dmamap_err;



	n = ib_map_mr_sg(mr, mw->mw_sg, mw->mw_nents, NULL, PAGE_SIZE);

	if (unlikely(n != mw->mw_nents))

	ibmr = frwr->fr_mr;

	n = ib_map_mr_sg(ibmr, mr->mr_sg, mr->mr_nents, NULL, PAGE_SIZE);

	if (unlikely(n != mr->mr_nents))

		goto out_mapmr_err;



	dprintk("RPC:       %s: Using frwr %p to map %u segments (%llu bytes)\n",

		__func__, frwr, mw->mw_nents, mr->length);

		__func__, frwr, mr->mr_nents, ibmr->length);



	key = (u8)(mr->rkey & 0x000000FF);

	ib_update_fast_reg_key(mr, ++key);

	key = (u8)(ibmr->rkey & 0x000000FF);

	ib_update_fast_reg_key(ibmr, ++key);



	reg_wr = &frwr->fr_regwr;

	reg_wr->wr.next = NULL;
@@ -414,8 +414,8 @@ frwr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg, 
	reg_wr->wr.wr_cqe = &frwr->fr_cqe;

	reg_wr->wr.num_sge = 0;

	reg_wr->wr.send_flags = 0;

	reg_wr->mr = mr;

	reg_wr->key = mr->rkey;

	reg_wr->mr = ibmr;

	reg_wr->key = ibmr->rkey;

	reg_wr->access = writing ?

			 IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE :

			 IB_ACCESS_REMOTE_READ;
@@ -424,48 +424,48 @@ frwr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg, 
	if (rc)

		goto out_senderr;



	mw->mw_handle = mr->rkey;

	mw->mw_length = mr->length;

	mw->mw_offset = mr->iova;

	mr->mr_handle = ibmr->rkey;

	mr->mr_length = ibmr->length;

	mr->mr_offset = ibmr->iova;



	*out = mw;

	*out = mr;

	return seg;



out_dmamap_err:

	pr_err("rpcrdma: failed to DMA map sg %p sg_nents %d\n",

	       mw->mw_sg, i);

	       mr->mr_sg, i);

	frwr->fr_state = FRWR_IS_INVALID;

	rpcrdma_put_mw(r_xprt, mw);

	rpcrdma_mr_put(mr);

	return ERR_PTR(-EIO);



out_mapmr_err:

	pr_err("rpcrdma: failed to map mr %p (%d/%d)\n",

	       frwr->fr_mr, n, mw->mw_nents);

	rpcrdma_defer_mr_recovery(mw);

	       frwr->fr_mr, n, mr->mr_nents);

	rpcrdma_mr_defer_recovery(mr);

	return ERR_PTR(-EIO);



out_senderr:

	pr_err("rpcrdma: FRWR registration ib_post_send returned %i\n", rc);

	rpcrdma_defer_mr_recovery(mw);

	rpcrdma_mr_defer_recovery(mr);

	return ERR_PTR(-ENOTCONN);

}



/* Handle a remotely invalidated mw on the @mws list

/* Handle a remotely invalidated mr on the @mrs list

 */

static void

frwr_op_reminv(struct rpcrdma_rep *rep, struct list_head *mws)

frwr_op_reminv(struct rpcrdma_rep *rep, struct list_head *mrs)

{

	struct rpcrdma_mw *mw;

	struct rpcrdma_mr *mr;



	list_for_each_entry(mw, mws, mw_list)

		if (mw->mw_handle == rep->rr_inv_rkey) {

			struct rpcrdma_xprt *r_xprt = mw->mw_xprt;

	list_for_each_entry(mr, mrs, mr_list)

		if (mr->mr_handle == rep->rr_inv_rkey) {

			struct rpcrdma_xprt *r_xprt = mr->mr_xprt;



			list_del(&mw->mw_list);

			mw->frwr.fr_state = FRWR_IS_INVALID;

			list_del(&mr->mr_list);

			mr->frwr.fr_state = FRWR_IS_INVALID;

			ib_dma_unmap_sg(r_xprt->rx_ia.ri_device,

					mw->mw_sg, mw->mw_nents, mw->mw_dir);

			rpcrdma_put_mw(r_xprt, mw);

					mr->mr_sg, mr->mr_nents, mr->mr_dir);

			rpcrdma_mr_put(mr);

			break;	/* only one invalidated MR per RPC */

		}

}
@@ -475,16 +475,16 @@ frwr_op_reminv(struct rpcrdma_rep *rep, struct list_head *mws) 
 * Sleeps until it is safe for the host CPU to access the

 * previously mapped memory regions.

 *

 * Caller ensures that @mws is not empty before the call. This

 * Caller ensures that @mrs is not empty before the call. This

 * function empties the list.

 */

static void

frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct list_head *mws)

frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct list_head *mrs)

{

	struct ib_send_wr *first, **prev, *last, *bad_wr;

	struct rpcrdma_ia *ia = &r_xprt->rx_ia;

	struct rpcrdma_frwr *frwr;

	struct rpcrdma_mw *mw;

	struct rpcrdma_mr *mr;

	int count, rc;



	/* ORDER: Invalidate all of the MRs first
@@ -495,10 +495,11 @@ frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct list_head *mws) 
	frwr = NULL;

	count = 0;

	prev = &first;

	list_for_each_entry(mw, mws, mw_list) {

		mw->frwr.fr_state = FRWR_IS_INVALID;

	list_for_each_entry(mr, mrs, mr_list) {

		mr->frwr.fr_state = FRWR_IS_INVALID;



		frwr = &mr->frwr;



		frwr = &mw->frwr;

		dprintk("RPC:       %s: invalidating frwr %p\n",

			__func__, frwr);
@@ -507,7 +508,7 @@ frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct list_head *mws) 
		memset(last, 0, sizeof(*last));

		last->wr_cqe = &frwr->fr_cqe;

		last->opcode = IB_WR_LOCAL_INV;

		last->ex.invalidate_rkey = mw->mw_handle;

		last->ex.invalidate_rkey = mr->mr_handle;

		count++;



		*prev = last;
@@ -537,16 +538,16 @@ frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct list_head *mws) 
		goto reset_mrs;



	/* ORDER: Now DMA unmap all of the MRs, and return

	 * them to the free MW list.

	 * them to the free MR list.

	 */

unmap:

	while (!list_empty(mws)) {

		mw = rpcrdma_pop_mw(mws);

	while (!list_empty(mrs)) {

		mr = rpcrdma_mr_pop(mrs);

		dprintk("RPC:       %s: DMA unmapping frwr %p\n",

			__func__, &mw->frwr);

			__func__, &mr->frwr);

		ib_dma_unmap_sg(ia->ri_device,

				mw->mw_sg, mw->mw_nents, mw->mw_dir);

		rpcrdma_put_mw(r_xprt, mw);

				mr->mr_sg, mr->mr_nents, mr->mr_dir);

		rpcrdma_mr_put(mr);

	}

	return;
@@ -559,9 +560,9 @@ frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct list_head *mws) 
	while (bad_wr) {

		frwr = container_of(bad_wr, struct rpcrdma_frwr,

				    fr_invwr);

		mw = container_of(frwr, struct rpcrdma_mw, frwr);

		mr = container_of(frwr, struct rpcrdma_mr, frwr);



		__frwr_reset_mr(ia, mw);

		__frwr_mr_reset(ia, mr);



		bad_wr = bad_wr->next;

	}

net/sunrpc/xprtrdma/rpc_rdma.c

+32 −32
Original line number Diff line number Diff line
@@ -292,15 +292,15 @@ encode_item_not_present(struct xdr_stream *xdr) 
}



static void

xdr_encode_rdma_segment(__be32 *iptr, struct rpcrdma_mw *mw)

xdr_encode_rdma_segment(__be32 *iptr, struct rpcrdma_mr *mr)

{

	*iptr++ = cpu_to_be32(mw->mw_handle);

	*iptr++ = cpu_to_be32(mw->mw_length);

	xdr_encode_hyper(iptr, mw->mw_offset);

	*iptr++ = cpu_to_be32(mr->mr_handle);

	*iptr++ = cpu_to_be32(mr->mr_length);

	xdr_encode_hyper(iptr, mr->mr_offset);

}



static int

encode_rdma_segment(struct xdr_stream *xdr, struct rpcrdma_mw *mw)

encode_rdma_segment(struct xdr_stream *xdr, struct rpcrdma_mr *mr)

{

	__be32 *p;
@@ -308,12 +308,12 @@ encode_rdma_segment(struct xdr_stream *xdr, struct rpcrdma_mw *mw) 
	if (unlikely(!p))

		return -EMSGSIZE;



	xdr_encode_rdma_segment(p, mw);

	xdr_encode_rdma_segment(p, mr);

	return 0;

}



static int

encode_read_segment(struct xdr_stream *xdr, struct rpcrdma_mw *mw,

encode_read_segment(struct xdr_stream *xdr, struct rpcrdma_mr *mr,

		    u32 position)

{

	__be32 *p;
@@ -324,7 +324,7 @@ encode_read_segment(struct xdr_stream *xdr, struct rpcrdma_mw *mw, 


	*p++ = xdr_one;			/* Item present */

	*p++ = cpu_to_be32(position);

	xdr_encode_rdma_segment(p, mw);

	xdr_encode_rdma_segment(p, mr);

	return 0;

}
@@ -348,7 +348,7 @@ rpcrdma_encode_read_list(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req, 
{

	struct xdr_stream *xdr = &req->rl_stream;

	struct rpcrdma_mr_seg *seg;

	struct rpcrdma_mw *mw;

	struct rpcrdma_mr *mr;

	unsigned int pos;

	int nsegs;
@@ -363,21 +363,21 @@ rpcrdma_encode_read_list(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req, 


	do {

		seg = r_xprt->rx_ia.ri_ops->ro_map(r_xprt, seg, nsegs,

						   false, &mw);

						   false, &mr);

		if (IS_ERR(seg))

			return PTR_ERR(seg);

		rpcrdma_push_mw(mw, &req->rl_registered);

		rpcrdma_mr_push(mr, &req->rl_registered);



		if (encode_read_segment(xdr, mw, pos) < 0)

		if (encode_read_segment(xdr, mr, pos) < 0)

			return -EMSGSIZE;



		dprintk("RPC: %5u %s: pos %u %u@0x%016llx:0x%08x (%s)\n",

			rqst->rq_task->tk_pid, __func__, pos,

			mw->mw_length, (unsigned long long)mw->mw_offset,

			mw->mw_handle, mw->mw_nents < nsegs ? "more" : "last");

			mr->mr_length, (unsigned long long)mr->mr_offset,

			mr->mr_handle, mr->mr_nents < nsegs ? "more" : "last");



		r_xprt->rx_stats.read_chunk_count++;

		nsegs -= mw->mw_nents;

		nsegs -= mr->mr_nents;

	} while (nsegs);



	return 0;
@@ -404,7 +404,7 @@ rpcrdma_encode_write_list(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req, 
{

	struct xdr_stream *xdr = &req->rl_stream;

	struct rpcrdma_mr_seg *seg;

	struct rpcrdma_mw *mw;

	struct rpcrdma_mr *mr;

	int nsegs, nchunks;

	__be32 *segcount;
@@ -425,23 +425,23 @@ rpcrdma_encode_write_list(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req, 
	nchunks = 0;

	do {

		seg = r_xprt->rx_ia.ri_ops->ro_map(r_xprt, seg, nsegs,

						   true, &mw);

						   true, &mr);

		if (IS_ERR(seg))

			return PTR_ERR(seg);

		rpcrdma_push_mw(mw, &req->rl_registered);

		rpcrdma_mr_push(mr, &req->rl_registered);



		if (encode_rdma_segment(xdr, mw) < 0)

		if (encode_rdma_segment(xdr, mr) < 0)

			return -EMSGSIZE;



		dprintk("RPC: %5u %s: %u@0x016%llx:0x%08x (%s)\n",

			rqst->rq_task->tk_pid, __func__,

			mw->mw_length, (unsigned long long)mw->mw_offset,

			mw->mw_handle, mw->mw_nents < nsegs ? "more" : "last");

			mr->mr_length, (unsigned long long)mr->mr_offset,

			mr->mr_handle, mr->mr_nents < nsegs ? "more" : "last");



		r_xprt->rx_stats.write_chunk_count++;

		r_xprt->rx_stats.total_rdma_request += seg->mr_len;

		nchunks++;

		nsegs -= mw->mw_nents;

		nsegs -= mr->mr_nents;

	} while (nsegs);



	/* Update count of segments in this Write chunk */
@@ -468,7 +468,7 @@ rpcrdma_encode_reply_chunk(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req, 
{

	struct xdr_stream *xdr = &req->rl_stream;

	struct rpcrdma_mr_seg *seg;

	struct rpcrdma_mw *mw;

	struct rpcrdma_mr *mr;

	int nsegs, nchunks;

	__be32 *segcount;
@@ -487,23 +487,23 @@ rpcrdma_encode_reply_chunk(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req, 
	nchunks = 0;

	do {

		seg = r_xprt->rx_ia.ri_ops->ro_map(r_xprt, seg, nsegs,

						   true, &mw);

						   true, &mr);

		if (IS_ERR(seg))

			return PTR_ERR(seg);

		rpcrdma_push_mw(mw, &req->rl_registered);

		rpcrdma_mr_push(mr, &req->rl_registered);



		if (encode_rdma_segment(xdr, mw) < 0)

		if (encode_rdma_segment(xdr, mr) < 0)

			return -EMSGSIZE;



		dprintk("RPC: %5u %s: %u@0x%016llx:0x%08x (%s)\n",

			rqst->rq_task->tk_pid, __func__,

			mw->mw_length, (unsigned long long)mw->mw_offset,

			mw->mw_handle, mw->mw_nents < nsegs ? "more" : "last");

			mr->mr_length, (unsigned long long)mr->mr_offset,

			mr->mr_handle, mr->mr_nents < nsegs ? "more" : "last");



		r_xprt->rx_stats.reply_chunk_count++;

		r_xprt->rx_stats.total_rdma_request += seg->mr_len;

		nchunks++;

		nsegs -= mw->mw_nents;

		nsegs -= mr->mr_nents;

	} while (nsegs);



	/* Update count of segments in the Reply chunk */
@@ -821,10 +821,10 @@ rpcrdma_marshal_req(struct rpcrdma_xprt *r_xprt, struct rpc_rqst *rqst) 
	 * so these registrations are invalid and unusable.

	 */

	while (unlikely(!list_empty(&req->rl_registered))) {

		struct rpcrdma_mw *mw;

		struct rpcrdma_mr *mr;



		mw = rpcrdma_pop_mw(&req->rl_registered);

		rpcrdma_defer_mr_recovery(mw);

		mr = rpcrdma_mr_pop(&req->rl_registered);

		rpcrdma_mr_defer_recovery(mr);

	}



	/* This implementation supports the following combinations

net/sunrpc/xprtrdma/verbs.c

+66 −53

File changed.

Preview size limit exceeded, changes collapsed.

net/sunrpc/xprtrdma/xprt_rdma.h

+31 −31

File changed.

Preview size limit exceeded, changes collapsed.