Merge tag 'nfs-rdma-for-4.20-1' of git://git.linux-nfs.org/projects/anna/linux-nfs (93bdcf9f) · Commits · e / devices / android_kernel_fairphone_FP5

include/trace/events/rpcrdma.h

+9 −9

Original line number	Original line	Diff line number	Diff line
	@@ -263,7 +263,7 @@ DECLARE_EVENT_CLASS(xprtrdma_mr,
	);		);

	#define DEFINE_MR_EVENT(name) \		#define DEFINE_MR_EVENT(name) \
	DEFINE_EVENT(xprtrdma_mr, name, \		DEFINE_EVENT(xprtrdma_mr, xprtrdma_mr_##name, \
	TP_PROTO( \		TP_PROTO( \
	const struct rpcrdma_mr *mr \		const struct rpcrdma_mr *mr \
	), \		), \
	@@ -306,7 +306,7 @@ DECLARE_EVENT_CLASS(xprtrdma_cb_event,
	** Connection events		** Connection events
	**/		**/

	TRACE_EVENT(xprtrdma_conn_upcall,		TRACE_EVENT(xprtrdma_cm_event,
	TP_PROTO(		TP_PROTO(
	const struct rpcrdma_xprt *r_xprt,		const struct rpcrdma_xprt *r_xprt,
	struct rdma_cm_event *event		struct rdma_cm_event *event
	@@ -377,7 +377,7 @@ DEFINE_RXPRT_EVENT(xprtrdma_reinsert);
	DEFINE_RXPRT_EVENT(xprtrdma_reconnect);		DEFINE_RXPRT_EVENT(xprtrdma_reconnect);
	DEFINE_RXPRT_EVENT(xprtrdma_inject_dsc);		DEFINE_RXPRT_EVENT(xprtrdma_inject_dsc);

	TRACE_EVENT(xprtrdma_qp_error,		TRACE_EVENT(xprtrdma_qp_event,
	TP_PROTO(		TP_PROTO(
	const struct rpcrdma_xprt *r_xprt,		const struct rpcrdma_xprt *r_xprt,
	const struct ib_event *event		const struct ib_event *event
	@@ -509,7 +509,7 @@ TRACE_EVENT(xprtrdma_post_send,
	TP_STRUCT__entry(		TP_STRUCT__entry(
	__field(const void *, req)		__field(const void *, req)
	__field(int, num_sge)		__field(int, num_sge)
	__field(bool, signaled)		__field(int, signaled)
	__field(int, status)		__field(int, status)
	),		),

	@@ -651,11 +651,11 @@ DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_fastreg);
	DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_li);		DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_li);
	DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_li_wake);		DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_li_wake);

	DEFINE_MR_EVENT(xprtrdma_localinv);		DEFINE_MR_EVENT(localinv);
	DEFINE_MR_EVENT(xprtrdma_dma_map);		DEFINE_MR_EVENT(map);
	DEFINE_MR_EVENT(xprtrdma_dma_unmap);		DEFINE_MR_EVENT(unmap);
	DEFINE_MR_EVENT(xprtrdma_remoteinv);		DEFINE_MR_EVENT(remoteinv);
	DEFINE_MR_EVENT(xprtrdma_recover_mr);		DEFINE_MR_EVENT(recycle);

	/**		/**
	** Reply events		** Reply events

net/sunrpc/xprt.c

+4 −10

Original line number	Original line	Diff line number	Diff line
	@@ -834,17 +834,11 @@ void xprt_connect(struct rpc_task *task)

	static void xprt_connect_status(struct rpc_task *task)		static void xprt_connect_status(struct rpc_task *task)
	{		{
	struct rpc_xprt *xprt = task->tk_rqstp->rq_xprt;		switch (task->tk_status) {
			case 0:
	if (task->tk_status == 0) {
	xprt->stat.connect_count++;
	xprt->stat.connect_time += (long)jiffies - xprt->stat.connect_start;
	dprintk("RPC: %5u xprt_connect_status: connection established\n",		dprintk("RPC: %5u xprt_connect_status: connection established\n",
	task->tk_pid);		task->tk_pid);
	return;		break;
	}

	switch (task->tk_status) {
	case -ECONNREFUSED:		case -ECONNREFUSED:
	case -ECONNRESET:		case -ECONNRESET:
	case -ECONNABORTED:		case -ECONNABORTED:
	@@ -861,7 +855,7 @@ static void xprt_connect_status(struct rpc_task *task)
	default:		default:
	dprintk("RPC: %5u xprt_connect_status: error %d connecting to "		dprintk("RPC: %5u xprt_connect_status: error %d connecting to "
	"server %s\n", task->tk_pid, -task->tk_status,		"server %s\n", task->tk_pid, -task->tk_status,
	xprt->servername);		task->tk_rqstp->rq_xprt->servername);
	task->tk_status = -EIO;		task->tk_status = -EIO;
	}		}
	}		}

net/sunrpc/xprtrdma/backchannel.c

+8 −8

Original line number	Original line	Diff line number	Diff line
	@@ -53,9 +53,9 @@ static int rpcrdma_bc_setup_reqs(struct rpcrdma_xprt *r_xprt,
	rqst->rq_xprt = xprt;		rqst->rq_xprt = xprt;
	INIT_LIST_HEAD(&rqst->rq_bc_list);		INIT_LIST_HEAD(&rqst->rq_bc_list);
	__set_bit(RPC_BC_PA_IN_USE, &rqst->rq_bc_pa_state);		__set_bit(RPC_BC_PA_IN_USE, &rqst->rq_bc_pa_state);
	spin_lock_bh(&xprt->bc_pa_lock);		spin_lock(&xprt->bc_pa_lock);
	list_add(&rqst->rq_bc_pa_list, &xprt->bc_pa_list);		list_add(&rqst->rq_bc_pa_list, &xprt->bc_pa_list);
	spin_unlock_bh(&xprt->bc_pa_lock);		spin_unlock(&xprt->bc_pa_lock);

	size = r_xprt->rx_data.inline_rsize;		size = r_xprt->rx_data.inline_rsize;
	rb = rpcrdma_alloc_regbuf(size, DMA_TO_DEVICE, GFP_KERNEL);		rb = rpcrdma_alloc_regbuf(size, DMA_TO_DEVICE, GFP_KERNEL);
	@@ -230,16 +230,16 @@ void xprt_rdma_bc_destroy(struct rpc_xprt *xprt, unsigned int reqs)
	struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);		struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);
	struct rpc_rqst rqst, tmp;		struct rpc_rqst rqst, tmp;

	spin_lock_bh(&xprt->bc_pa_lock);		spin_lock(&xprt->bc_pa_lock);
	list_for_each_entry_safe(rqst, tmp, &xprt->bc_pa_list, rq_bc_pa_list) {		list_for_each_entry_safe(rqst, tmp, &xprt->bc_pa_list, rq_bc_pa_list) {
	list_del(&rqst->rq_bc_pa_list);		list_del(&rqst->rq_bc_pa_list);
	spin_unlock_bh(&xprt->bc_pa_lock);		spin_unlock(&xprt->bc_pa_lock);

	rpcrdma_bc_free_rqst(r_xprt, rqst);		rpcrdma_bc_free_rqst(r_xprt, rqst);

	spin_lock_bh(&xprt->bc_pa_lock);		spin_lock(&xprt->bc_pa_lock);
	}		}
	spin_unlock_bh(&xprt->bc_pa_lock);		spin_unlock(&xprt->bc_pa_lock);
	}		}

	/**		/**
	@@ -257,9 +257,9 @@ void xprt_rdma_bc_free_rqst(struct rpc_rqst *rqst)
	rpcrdma_recv_buffer_put(req->rl_reply);		rpcrdma_recv_buffer_put(req->rl_reply);
	req->rl_reply = NULL;		req->rl_reply = NULL;

	spin_lock_bh(&xprt->bc_pa_lock);		spin_lock(&xprt->bc_pa_lock);
	list_add_tail(&rqst->rq_bc_pa_list, &xprt->bc_pa_list);		list_add_tail(&rqst->rq_bc_pa_list, &xprt->bc_pa_list);
	spin_unlock_bh(&xprt->bc_pa_lock);		spin_unlock(&xprt->bc_pa_lock);
	}		}

	/**		/**

net/sunrpc/xprtrdma/fmr_ops.c

+60 −71

Original line number	Original line	Diff line number	Diff line
	@@ -49,46 +49,7 @@ fmr_is_supported(struct rpcrdma_ia *ia)
	return true;		return true;
	}		}

	static int		static void
	fmr_op_init_mr(struct rpcrdma_ia ia, struct rpcrdma_mr mr)
	{
	static struct ib_fmr_attr fmr_attr = {
	.max_pages = RPCRDMA_MAX_FMR_SGES,
	.max_maps = 1,
	.page_shift = PAGE_SHIFT
	};

	mr->fmr.fm_physaddrs = kcalloc(RPCRDMA_MAX_FMR_SGES,
	sizeof(u64), GFP_KERNEL);
	if (!mr->fmr.fm_physaddrs)
	goto out_free;

	mr->mr_sg = kcalloc(RPCRDMA_MAX_FMR_SGES,
	sizeof(*mr->mr_sg), GFP_KERNEL);
	if (!mr->mr_sg)
	goto out_free;

	sg_init_table(mr->mr_sg, RPCRDMA_MAX_FMR_SGES);

	mr->fmr.fm_mr = ib_alloc_fmr(ia->ri_pd, RPCRDMA_FMR_ACCESS_FLAGS,
	&fmr_attr);
	if (IS_ERR(mr->fmr.fm_mr))
	goto out_fmr_err;

	INIT_LIST_HEAD(&mr->mr_list);
	return 0;

	out_fmr_err:
	dprintk("RPC: %s: ib_alloc_fmr returned %ld\n", __func__,
	PTR_ERR(mr->fmr.fm_mr));

	out_free:
	kfree(mr->mr_sg);
	kfree(mr->fmr.fm_physaddrs);
	return -ENOMEM;
	}

	static int
	__fmr_unmap(struct rpcrdma_mr *mr)		__fmr_unmap(struct rpcrdma_mr *mr)
	{		{
	LIST_HEAD(l);		LIST_HEAD(l);
	@@ -97,13 +58,16 @@ __fmr_unmap(struct rpcrdma_mr *mr)
	list_add(&mr->fmr.fm_mr->list, &l);		list_add(&mr->fmr.fm_mr->list, &l);
	rc = ib_unmap_fmr(&l);		rc = ib_unmap_fmr(&l);
	list_del(&mr->fmr.fm_mr->list);		list_del(&mr->fmr.fm_mr->list);
	return rc;		if (rc)
			pr_err("rpcrdma: final ib_unmap_fmr for %p failed %i\n",
			mr, rc);
	}		}

			/* Release an MR.
			*/
	static void		static void
	fmr_op_release_mr(struct rpcrdma_mr *mr)		fmr_op_release_mr(struct rpcrdma_mr *mr)
	{		{
	LIST_HEAD(unmap_list);
	int rc;		int rc;

	kfree(mr->fmr.fm_physaddrs);		kfree(mr->fmr.fm_physaddrs);
	@@ -112,10 +76,7 @@ fmr_op_release_mr(struct rpcrdma_mr *mr)
	/* In case this one was left mapped, try to unmap it		/* In case this one was left mapped, try to unmap it
	* to prevent dealloc_fmr from failing with EBUSY		* to prevent dealloc_fmr from failing with EBUSY
	*/		*/
	rc = __fmr_unmap(mr);		__fmr_unmap(mr);
	if (rc)
	pr_err("rpcrdma: final ib_unmap_fmr for %p failed %i\n",
	mr, rc);

	rc = ib_dealloc_fmr(mr->fmr.fm_mr);		rc = ib_dealloc_fmr(mr->fmr.fm_mr);
	if (rc)		if (rc)
	@@ -125,40 +86,68 @@ fmr_op_release_mr(struct rpcrdma_mr *mr)
	kfree(mr);		kfree(mr);
	}		}

	/* Reset of a single FMR.		/* MRs are dynamically allocated, so simply clean up and release the MR.
			* A replacement MR will subsequently be allocated on demand.
	*/		*/
	static void		static void
	fmr_op_recover_mr(struct rpcrdma_mr *mr)		fmr_mr_recycle_worker(struct work_struct *work)
	{		{
			struct rpcrdma_mr *mr = container_of(work, struct rpcrdma_mr, mr_recycle);
	struct rpcrdma_xprt *r_xprt = mr->mr_xprt;		struct rpcrdma_xprt *r_xprt = mr->mr_xprt;
	int rc;

	/* ORDER: invalidate first */		trace_xprtrdma_mr_recycle(mr);
	rc = __fmr_unmap(mr);
	if (rc)
	goto out_release;

	/* ORDER: then DMA unmap */
	rpcrdma_mr_unmap_and_put(mr);

	r_xprt->rx_stats.mrs_recovered++;		trace_xprtrdma_mr_unmap(mr);
	return;

	out_release:
	pr_err("rpcrdma: FMR reset failed (%d), %p released\n", rc, mr);
	r_xprt->rx_stats.mrs_orphaned++;

	trace_xprtrdma_dma_unmap(mr);
	ib_dma_unmap_sg(r_xprt->rx_ia.ri_device,		ib_dma_unmap_sg(r_xprt->rx_ia.ri_device,
	mr->mr_sg, mr->mr_nents, mr->mr_dir);		mr->mr_sg, mr->mr_nents, mr->mr_dir);

	spin_lock(&r_xprt->rx_buf.rb_mrlock);		spin_lock(&r_xprt->rx_buf.rb_mrlock);
	list_del(&mr->mr_all);		list_del(&mr->mr_all);
			r_xprt->rx_stats.mrs_recycled++;
	spin_unlock(&r_xprt->rx_buf.rb_mrlock);		spin_unlock(&r_xprt->rx_buf.rb_mrlock);

	fmr_op_release_mr(mr);		fmr_op_release_mr(mr);
	}		}

			static int
			fmr_op_init_mr(struct rpcrdma_ia ia, struct rpcrdma_mr mr)
			{
			static struct ib_fmr_attr fmr_attr = {
			.max_pages = RPCRDMA_MAX_FMR_SGES,
			.max_maps = 1,
			.page_shift = PAGE_SHIFT
			};

			mr->fmr.fm_physaddrs = kcalloc(RPCRDMA_MAX_FMR_SGES,
			sizeof(u64), GFP_KERNEL);
			if (!mr->fmr.fm_physaddrs)
			goto out_free;

			mr->mr_sg = kcalloc(RPCRDMA_MAX_FMR_SGES,
			sizeof(*mr->mr_sg), GFP_KERNEL);
			if (!mr->mr_sg)
			goto out_free;

			sg_init_table(mr->mr_sg, RPCRDMA_MAX_FMR_SGES);

			mr->fmr.fm_mr = ib_alloc_fmr(ia->ri_pd, RPCRDMA_FMR_ACCESS_FLAGS,
			&fmr_attr);
			if (IS_ERR(mr->fmr.fm_mr))
			goto out_fmr_err;

			INIT_LIST_HEAD(&mr->mr_list);
			INIT_WORK(&mr->mr_recycle, fmr_mr_recycle_worker);
			return 0;

			out_fmr_err:
			dprintk("RPC: %s: ib_alloc_fmr returned %ld\n", __func__,
			PTR_ERR(mr->fmr.fm_mr));

			out_free:
			kfree(mr->mr_sg);
			kfree(mr->fmr.fm_physaddrs);
			return -ENOMEM;
			}

	/* On success, sets:		/* On success, sets:
	* ep->rep_attr.cap.max_send_wr		* ep->rep_attr.cap.max_send_wr
	* ep->rep_attr.cap.max_recv_wr		* ep->rep_attr.cap.max_recv_wr
	@@ -187,6 +176,7 @@ fmr_op_open(struct rpcrdma_ia ia, struct rpcrdma_ep ep,

	ia->ri_max_segs = max_t(unsigned int, 1, RPCRDMA_MAX_DATA_SEGS /		ia->ri_max_segs = max_t(unsigned int, 1, RPCRDMA_MAX_DATA_SEGS /
	RPCRDMA_MAX_FMR_SGES);		RPCRDMA_MAX_FMR_SGES);
			ia->ri_max_segs += 2; /* segments for head and tail buffers */
	return 0;		return 0;
	}		}

	@@ -244,7 +234,7 @@ fmr_op_map(struct rpcrdma_xprt r_xprt, struct rpcrdma_mr_seg seg,
	mr->mr_sg, i, mr->mr_dir);		mr->mr_sg, i, mr->mr_dir);
	if (!mr->mr_nents)		if (!mr->mr_nents)
	goto out_dmamap_err;		goto out_dmamap_err;
	trace_xprtrdma_dma_map(mr);		trace_xprtrdma_mr_map(mr);

	for (i = 0, dma_pages = mr->fmr.fm_physaddrs; i < mr->mr_nents; i++)		for (i = 0, dma_pages = mr->fmr.fm_physaddrs; i < mr->mr_nents; i++)
	dma_pages[i] = sg_dma_address(&mr->mr_sg[i]);		dma_pages[i] = sg_dma_address(&mr->mr_sg[i]);
	@@ -305,13 +295,13 @@ fmr_op_unmap_sync(struct rpcrdma_xprt r_xprt, struct list_head mrs)
	list_for_each_entry(mr, mrs, mr_list) {		list_for_each_entry(mr, mrs, mr_list) {
	dprintk("RPC: %s: unmapping fmr %p\n",		dprintk("RPC: %s: unmapping fmr %p\n",
	__func__, &mr->fmr);		__func__, &mr->fmr);
	trace_xprtrdma_localinv(mr);		trace_xprtrdma_mr_localinv(mr);
	list_add_tail(&mr->fmr.fm_mr->list, &unmap_list);		list_add_tail(&mr->fmr.fm_mr->list, &unmap_list);
	}		}
	r_xprt->rx_stats.local_inv_needed++;		r_xprt->rx_stats.local_inv_needed++;
	rc = ib_unmap_fmr(&unmap_list);		rc = ib_unmap_fmr(&unmap_list);
	if (rc)		if (rc)
	goto out_reset;		goto out_release;

	/* ORDER: Now DMA unmap all of the req's MRs, and return		/* ORDER: Now DMA unmap all of the req's MRs, and return
	* them to the free MW list.		* them to the free MW list.
	@@ -324,13 +314,13 @@ fmr_op_unmap_sync(struct rpcrdma_xprt r_xprt, struct list_head mrs)

	return;		return;

	out_reset:		out_release:
	pr_err("rpcrdma: ib_unmap_fmr failed (%i)\n", rc);		pr_err("rpcrdma: ib_unmap_fmr failed (%i)\n", rc);

	while (!list_empty(mrs)) {		while (!list_empty(mrs)) {
	mr = rpcrdma_mr_pop(mrs);		mr = rpcrdma_mr_pop(mrs);
	list_del(&mr->fmr.fm_mr->list);		list_del(&mr->fmr.fm_mr->list);
	fmr_op_recover_mr(mr);		rpcrdma_mr_recycle(mr);
	}		}
	}		}

	@@ -338,7 +328,6 @@ const struct rpcrdma_memreg_ops rpcrdma_fmr_memreg_ops = {
	.ro_map = fmr_op_map,		.ro_map = fmr_op_map,
	.ro_send = fmr_op_send,		.ro_send = fmr_op_send,
	.ro_unmap_sync = fmr_op_unmap_sync,		.ro_unmap_sync = fmr_op_unmap_sync,
	.ro_recover_mr = fmr_op_recover_mr,
	.ro_open = fmr_op_open,		.ro_open = fmr_op_open,
	.ro_maxpages = fmr_op_maxpages,		.ro_maxpages = fmr_op_maxpages,
	.ro_init_mr = fmr_op_init_mr,		.ro_init_mr = fmr_op_init_mr,

net/sunrpc/xprtrdma/frwr_ops.c

+51 −86

Original line number	Original line	Diff line number	Diff line
	@@ -97,6 +97,44 @@ frwr_is_supported(struct rpcrdma_ia *ia)
	return false;		return false;
	}		}

			static void
			frwr_op_release_mr(struct rpcrdma_mr *mr)
			{
			int rc;

			rc = ib_dereg_mr(mr->frwr.fr_mr);
			if (rc)
			pr_err("rpcrdma: final ib_dereg_mr for %p returned %i\n",
			mr, rc);
			kfree(mr->mr_sg);
			kfree(mr);
			}

			/* MRs are dynamically allocated, so simply clean up and release the MR.
			* A replacement MR will subsequently be allocated on demand.
			*/
			static void
			frwr_mr_recycle_worker(struct work_struct *work)
			{
			struct rpcrdma_mr *mr = container_of(work, struct rpcrdma_mr, mr_recycle);
			enum rpcrdma_frwr_state state = mr->frwr.fr_state;
			struct rpcrdma_xprt *r_xprt = mr->mr_xprt;

			trace_xprtrdma_mr_recycle(mr);

			if (state != FRWR_FLUSHED_LI) {
			trace_xprtrdma_mr_unmap(mr);
			ib_dma_unmap_sg(r_xprt->rx_ia.ri_device,
			mr->mr_sg, mr->mr_nents, mr->mr_dir);
			}

			spin_lock(&r_xprt->rx_buf.rb_mrlock);
			list_del(&mr->mr_all);
			r_xprt->rx_stats.mrs_recycled++;
			spin_unlock(&r_xprt->rx_buf.rb_mrlock);
			frwr_op_release_mr(mr);
			}

	static int		static int
	frwr_op_init_mr(struct rpcrdma_ia ia, struct rpcrdma_mr mr)		frwr_op_init_mr(struct rpcrdma_ia ia, struct rpcrdma_mr mr)
	{		{
	@@ -113,6 +151,7 @@ frwr_op_init_mr(struct rpcrdma_ia ia, struct rpcrdma_mr mr)
	goto out_list_err;		goto out_list_err;

	INIT_LIST_HEAD(&mr->mr_list);		INIT_LIST_HEAD(&mr->mr_list);
			INIT_WORK(&mr->mr_recycle, frwr_mr_recycle_worker);
	sg_init_table(mr->mr_sg, depth);		sg_init_table(mr->mr_sg, depth);
	init_completion(&frwr->fr_linv_done);		init_completion(&frwr->fr_linv_done);
	return 0;		return 0;
	@@ -131,79 +170,6 @@ frwr_op_init_mr(struct rpcrdma_ia ia, struct rpcrdma_mr mr)
	return rc;		return rc;
	}		}

	static void
	frwr_op_release_mr(struct rpcrdma_mr *mr)
	{
	int rc;

	rc = ib_dereg_mr(mr->frwr.fr_mr);
	if (rc)
	pr_err("rpcrdma: final ib_dereg_mr for %p returned %i\n",
	mr, rc);
	kfree(mr->mr_sg);
	kfree(mr);
	}

	static int
	__frwr_mr_reset(struct rpcrdma_ia ia, struct rpcrdma_mr mr)
	{
	struct rpcrdma_frwr *frwr = &mr->frwr;
	int rc;

	rc = ib_dereg_mr(frwr->fr_mr);
	if (rc) {
	pr_warn("rpcrdma: ib_dereg_mr status %d, frwr %p orphaned\n",
	rc, mr);
	return rc;
	}

	frwr->fr_mr = ib_alloc_mr(ia->ri_pd, ia->ri_mrtype,
	ia->ri_max_frwr_depth);
	if (IS_ERR(frwr->fr_mr)) {
	pr_warn("rpcrdma: ib_alloc_mr status %ld, frwr %p orphaned\n",
	PTR_ERR(frwr->fr_mr), mr);
	return PTR_ERR(frwr->fr_mr);
	}

	dprintk("RPC: %s: recovered FRWR %p\n", __func__, frwr);
	frwr->fr_state = FRWR_IS_INVALID;
	return 0;
	}

	/* Reset of a single FRWR. Generate a fresh rkey by replacing the MR.
	*/
	static void
	frwr_op_recover_mr(struct rpcrdma_mr *mr)
	{
	enum rpcrdma_frwr_state state = mr->frwr.fr_state;
	struct rpcrdma_xprt *r_xprt = mr->mr_xprt;
	struct rpcrdma_ia *ia = &r_xprt->rx_ia;
	int rc;

	rc = __frwr_mr_reset(ia, mr);
	if (state != FRWR_FLUSHED_LI) {
	trace_xprtrdma_dma_unmap(mr);
	ib_dma_unmap_sg(ia->ri_device,
	mr->mr_sg, mr->mr_nents, mr->mr_dir);
	}
	if (rc)
	goto out_release;

	rpcrdma_mr_put(mr);
	r_xprt->rx_stats.mrs_recovered++;
	return;

	out_release:
	pr_err("rpcrdma: FRWR reset failed %d, %p released\n", rc, mr);
	r_xprt->rx_stats.mrs_orphaned++;

	spin_lock(&r_xprt->rx_buf.rb_mrlock);
	list_del(&mr->mr_all);
	spin_unlock(&r_xprt->rx_buf.rb_mrlock);

	frwr_op_release_mr(mr);
	}

	/* On success, sets:		/* On success, sets:
	* ep->rep_attr.cap.max_send_wr		* ep->rep_attr.cap.max_send_wr
	* ep->rep_attr.cap.max_recv_wr		* ep->rep_attr.cap.max_recv_wr
	@@ -276,6 +242,7 @@ frwr_op_open(struct rpcrdma_ia ia, struct rpcrdma_ep ep,

	ia->ri_max_segs = max_t(unsigned int, 1, RPCRDMA_MAX_DATA_SEGS /		ia->ri_max_segs = max_t(unsigned int, 1, RPCRDMA_MAX_DATA_SEGS /
	ia->ri_max_frwr_depth);		ia->ri_max_frwr_depth);
			ia->ri_max_segs += 2; /* segments for head and tail buffers */
	return 0;		return 0;
	}		}

	@@ -384,7 +351,7 @@ frwr_op_map(struct rpcrdma_xprt r_xprt, struct rpcrdma_mr_seg seg,
	mr = NULL;		mr = NULL;
	do {		do {
	if (mr)		if (mr)
	rpcrdma_mr_defer_recovery(mr);		rpcrdma_mr_recycle(mr);
	mr = rpcrdma_mr_get(r_xprt);		mr = rpcrdma_mr_get(r_xprt);
	if (!mr)		if (!mr)
	return ERR_PTR(-EAGAIN);		return ERR_PTR(-EAGAIN);
	@@ -417,7 +384,7 @@ frwr_op_map(struct rpcrdma_xprt r_xprt, struct rpcrdma_mr_seg seg,
	mr->mr_nents = ib_dma_map_sg(ia->ri_device, mr->mr_sg, i, mr->mr_dir);		mr->mr_nents = ib_dma_map_sg(ia->ri_device, mr->mr_sg, i, mr->mr_dir);
	if (!mr->mr_nents)		if (!mr->mr_nents)
	goto out_dmamap_err;		goto out_dmamap_err;
	trace_xprtrdma_dma_map(mr);		trace_xprtrdma_mr_map(mr);

	ibmr = frwr->fr_mr;		ibmr = frwr->fr_mr;
	n = ib_map_mr_sg(ibmr, mr->mr_sg, mr->mr_nents, NULL, PAGE_SIZE);		n = ib_map_mr_sg(ibmr, mr->mr_sg, mr->mr_nents, NULL, PAGE_SIZE);
	@@ -451,7 +418,7 @@ frwr_op_map(struct rpcrdma_xprt r_xprt, struct rpcrdma_mr_seg seg,
	out_mapmr_err:		out_mapmr_err:
	pr_err("rpcrdma: failed to map mr %p (%d/%d)\n",		pr_err("rpcrdma: failed to map mr %p (%d/%d)\n",
	frwr->fr_mr, n, mr->mr_nents);		frwr->fr_mr, n, mr->mr_nents);
	rpcrdma_mr_defer_recovery(mr);		rpcrdma_mr_recycle(mr);
	return ERR_PTR(-EIO);		return ERR_PTR(-EIO);
	}		}

	@@ -499,7 +466,7 @@ frwr_op_reminv(struct rpcrdma_rep rep, struct list_head mrs)
	list_for_each_entry(mr, mrs, mr_list)		list_for_each_entry(mr, mrs, mr_list)
	if (mr->mr_handle == rep->rr_inv_rkey) {		if (mr->mr_handle == rep->rr_inv_rkey) {
	list_del_init(&mr->mr_list);		list_del_init(&mr->mr_list);
	trace_xprtrdma_remoteinv(mr);		trace_xprtrdma_mr_remoteinv(mr);
	mr->frwr.fr_state = FRWR_IS_INVALID;		mr->frwr.fr_state = FRWR_IS_INVALID;
	rpcrdma_mr_unmap_and_put(mr);		rpcrdma_mr_unmap_and_put(mr);
	break; /* only one invalidated MR per RPC */		break; /* only one invalidated MR per RPC */
	@@ -536,7 +503,7 @@ frwr_op_unmap_sync(struct rpcrdma_xprt r_xprt, struct list_head mrs)
	mr->frwr.fr_state = FRWR_IS_INVALID;		mr->frwr.fr_state = FRWR_IS_INVALID;

	frwr = &mr->frwr;		frwr = &mr->frwr;
	trace_xprtrdma_localinv(mr);		trace_xprtrdma_mr_localinv(mr);

	frwr->fr_cqe.done = frwr_wc_localinv;		frwr->fr_cqe.done = frwr_wc_localinv;
	last = &frwr->fr_invwr;		last = &frwr->fr_invwr;
	@@ -570,7 +537,7 @@ frwr_op_unmap_sync(struct rpcrdma_xprt r_xprt, struct list_head mrs)
	if (bad_wr != first)		if (bad_wr != first)
	wait_for_completion(&frwr->fr_linv_done);		wait_for_completion(&frwr->fr_linv_done);
	if (rc)		if (rc)
	goto reset_mrs;		goto out_release;

	/* ORDER: Now DMA unmap all of the MRs, and return		/* ORDER: Now DMA unmap all of the MRs, and return
	* them to the free MR list.		* them to the free MR list.
	@@ -582,22 +549,21 @@ frwr_op_unmap_sync(struct rpcrdma_xprt r_xprt, struct list_head mrs)
	}		}
	return;		return;

	reset_mrs:		out_release:
	pr_err("rpcrdma: FRWR invalidate ib_post_send returned %i\n", rc);		pr_err("rpcrdma: FRWR invalidate ib_post_send returned %i\n", rc);

	/* Find and reset the MRs in the LOCAL_INV WRs that did not		/* Unmap and release the MRs in the LOCAL_INV WRs that did not
	* get posted.		* get posted.
	*/		*/
	while (bad_wr) {		while (bad_wr) {
	frwr = container_of(bad_wr, struct rpcrdma_frwr,		frwr = container_of(bad_wr, struct rpcrdma_frwr,
	fr_invwr);		fr_invwr);
	mr = container_of(frwr, struct rpcrdma_mr, frwr);		mr = container_of(frwr, struct rpcrdma_mr, frwr);

	__frwr_mr_reset(ia, mr);

	bad_wr = bad_wr->next;		bad_wr = bad_wr->next;

			list_del(&mr->mr_list);
			frwr_op_release_mr(mr);
	}		}
	goto unmap;
	}		}

	const struct rpcrdma_memreg_ops rpcrdma_frwr_memreg_ops = {		const struct rpcrdma_memreg_ops rpcrdma_frwr_memreg_ops = {
	@@ -605,7 +571,6 @@ const struct rpcrdma_memreg_ops rpcrdma_frwr_memreg_ops = {
	.ro_send = frwr_op_send,		.ro_send = frwr_op_send,
	.ro_reminv = frwr_op_reminv,		.ro_reminv = frwr_op_reminv,
	.ro_unmap_sync = frwr_op_unmap_sync,		.ro_unmap_sync = frwr_op_unmap_sync,
	.ro_recover_mr = frwr_op_recover_mr,
	.ro_open = frwr_op_open,		.ro_open = frwr_op_open,
	.ro_maxpages = frwr_op_maxpages,		.ro_maxpages = frwr_op_maxpages,
	.ro_init_mr = frwr_op_init_mr,		.ro_init_mr = frwr_op_init_mr,