Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit f2877623 authored by Chuck Lever's avatar Chuck Lever Committed by Anna Schumaker
Browse files

xprtrdma: Chain Send to FastReg WRs 



With FRWR, the client transport can perform memory registration and
post a Send with just a single ib_post_send.

This reduces contention between the send_request path and the Send
Completion handlers, and reduces the overhead of registering a chunk
that has multiple segments.

Signed-off-by: default avatarChuck Lever <chuck.lever@oracle.com>
Signed-off-by: default avatarAnna Schumaker <Anna.Schumaker@Netapp.com>
parent fb14ae88

net/sunrpc/xprtrdma/fmr_ops.c

+11 −0
Original line number Original line Diff line number Diff line
@@ -251,6 +251,16 @@ fmr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg, 
	return ERR_PTR(-EIO);

	return ERR_PTR(-EIO);

}

}





/* Post Send WR containing the RPC Call message.

 */

static int

fmr_op_send(struct rpcrdma_ia *ia, struct rpcrdma_req *req)

{

	struct ib_send_wr *bad_wr;



	return ib_post_send(ia->ri_id->qp, &req->rl_sendctx->sc_wr, &bad_wr);

}



/* Invalidate all memory regions that were registered for "req".

/* Invalidate all memory regions that were registered for "req".

 *

 *

 * Sleeps until it is safe for the host CPU to access the

 * Sleeps until it is safe for the host CPU to access the
@@ -305,6 +315,7 @@ fmr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct list_head *mrs) 




const struct rpcrdma_memreg_ops rpcrdma_fmr_memreg_ops = {

const struct rpcrdma_memreg_ops rpcrdma_fmr_memreg_ops = {

	.ro_map				= fmr_op_map,

	.ro_map				= fmr_op_map,

	.ro_send			= fmr_op_send,

	.ro_unmap_sync			= fmr_op_unmap_sync,

	.ro_unmap_sync			= fmr_op_unmap_sync,

	.ro_recover_mr			= fmr_op_recover_mr,

	.ro_recover_mr			= fmr_op_recover_mr,

	.ro_open			= fmr_op_open,

	.ro_open			= fmr_op_open,

net/sunrpc/xprtrdma/frwr_ops.c

+35 −16
Original line number Original line Diff line number Diff line
@@ -357,8 +357,7 @@ frwr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg, 
	struct rpcrdma_mr *mr;

	struct rpcrdma_mr *mr;

	struct ib_mr *ibmr;

	struct ib_mr *ibmr;

	struct ib_reg_wr *reg_wr;

	struct ib_reg_wr *reg_wr;

	struct ib_send_wr *bad_wr;

	int i, n;

	int rc, i, n;

	u8 key;

	u8 key;





	mr = NULL;

	mr = NULL;
@@ -407,22 +406,12 @@ frwr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg, 
	ib_update_fast_reg_key(ibmr, ++key);

	ib_update_fast_reg_key(ibmr, ++key);





	reg_wr = &frwr->fr_regwr;

	reg_wr = &frwr->fr_regwr;

	reg_wr->wr.next = NULL;

	reg_wr->wr.opcode = IB_WR_REG_MR;

	frwr->fr_cqe.done = frwr_wc_fastreg;

	reg_wr->wr.wr_cqe = &frwr->fr_cqe;

	reg_wr->wr.num_sge = 0;

	reg_wr->wr.send_flags = 0;

	reg_wr->mr = ibmr;

	reg_wr->mr = ibmr;

	reg_wr->key = ibmr->rkey;

	reg_wr->key = ibmr->rkey;

	reg_wr->access = writing ?

	reg_wr->access = writing ?

			 IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE :

			 IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE :

			 IB_ACCESS_REMOTE_READ;

			 IB_ACCESS_REMOTE_READ;





	rc = ib_post_send(ia->ri_id->qp, &reg_wr->wr, &bad_wr);

	if (rc)

		goto out_senderr;



	mr->mr_handle = ibmr->rkey;

	mr->mr_handle = ibmr->rkey;

	mr->mr_length = ibmr->length;

	mr->mr_length = ibmr->length;

	mr->mr_offset = ibmr->iova;

	mr->mr_offset = ibmr->iova;
@@ -442,11 +431,40 @@ frwr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg, 
	       frwr->fr_mr, n, mr->mr_nents);

	       frwr->fr_mr, n, mr->mr_nents);

	rpcrdma_mr_defer_recovery(mr);

	rpcrdma_mr_defer_recovery(mr);

	return ERR_PTR(-EIO);

	return ERR_PTR(-EIO);

}





out_senderr:

/* Post Send WR containing the RPC Call message.

	pr_err("rpcrdma: FRWR registration ib_post_send returned %i\n", rc);

 *

	rpcrdma_mr_defer_recovery(mr);

 * For FRMR, chain any FastReg WRs to the Send WR. Only a

	return ERR_PTR(-ENOTCONN);

 * single ib_post_send call is needed to register memory

 * and then post the Send WR.

 */

static int

frwr_op_send(struct rpcrdma_ia *ia, struct rpcrdma_req *req)

{

	struct ib_send_wr *post_wr, *bad_wr;

	struct rpcrdma_mr *mr;



	post_wr = &req->rl_sendctx->sc_wr;

	list_for_each_entry(mr, &req->rl_registered, mr_list) {

		struct rpcrdma_frwr *frwr;



		frwr = &mr->frwr;



		frwr->fr_cqe.done = frwr_wc_fastreg;

		frwr->fr_regwr.wr.next = post_wr;

		frwr->fr_regwr.wr.wr_cqe = &frwr->fr_cqe;

		frwr->fr_regwr.wr.num_sge = 0;

		frwr->fr_regwr.wr.opcode = IB_WR_REG_MR;

		frwr->fr_regwr.wr.send_flags = 0;



		post_wr = &frwr->fr_regwr.wr;

	}



	/* If ib_post_send fails, the next ->send_request for

	 * @req will queue these MWs for recovery.

	 */

	return ib_post_send(ia->ri_id->qp, post_wr, &bad_wr);

}

}





/* Handle a remotely invalidated mr on the @mrs list

/* Handle a remotely invalidated mr on the @mrs list
@@ -561,6 +579,7 @@ frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct list_head *mrs) 




const struct rpcrdma_memreg_ops rpcrdma_frwr_memreg_ops = {

const struct rpcrdma_memreg_ops rpcrdma_frwr_memreg_ops = {

	.ro_map				= frwr_op_map,

	.ro_map				= frwr_op_map,

	.ro_send			= frwr_op_send,

	.ro_reminv			= frwr_op_reminv,

	.ro_reminv			= frwr_op_reminv,

	.ro_unmap_sync			= frwr_op_unmap_sync,

	.ro_unmap_sync			= frwr_op_unmap_sync,

	.ro_recover_mr			= frwr_op_recover_mr,

	.ro_recover_mr			= frwr_op_recover_mr,

net/sunrpc/xprtrdma/verbs.c

+1 −2
Original line number Original line Diff line number Diff line
@@ -1535,7 +1535,6 @@ rpcrdma_ep_post(struct rpcrdma_ia *ia, 
		struct rpcrdma_req *req)

		struct rpcrdma_req *req)

{

{

	struct ib_send_wr *send_wr = &req->rl_sendctx->sc_wr;

	struct ib_send_wr *send_wr = &req->rl_sendctx->sc_wr;

	struct ib_send_wr *send_wr_fail;

	int rc;

	int rc;





	if (req->rl_reply) {

	if (req->rl_reply) {
@@ -1554,7 +1553,7 @@ rpcrdma_ep_post(struct rpcrdma_ia *ia, 
		--ep->rep_send_count;

		--ep->rep_send_count;

	}

	}





	rc = ib_post_send(ia->ri_id->qp, send_wr, &send_wr_fail);

	rc = ia->ri_ops->ro_send(ia, req);

	trace_xprtrdma_post_send(req, rc);

	trace_xprtrdma_post_send(req, rc);

	if (rc)

	if (rc)

		return -ENOTCONN;

		return -ENOTCONN;

net/sunrpc/xprtrdma/xprt_rdma.h

+2 −0
Original line number Original line Diff line number Diff line
@@ -472,6 +472,8 @@ struct rpcrdma_memreg_ops { 
			(*ro_map)(struct rpcrdma_xprt *,

			(*ro_map)(struct rpcrdma_xprt *,

				  struct rpcrdma_mr_seg *, int, bool,

				  struct rpcrdma_mr_seg *, int, bool,

				  struct rpcrdma_mr **);

				  struct rpcrdma_mr **);

	int		(*ro_send)(struct rpcrdma_ia *ia,

				   struct rpcrdma_req *req);

	void		(*ro_reminv)(struct rpcrdma_rep *rep,

	void		(*ro_reminv)(struct rpcrdma_rep *rep,

				     struct list_head *mrs);

				     struct list_head *mrs);

	void		(*ro_unmap_sync)(struct rpcrdma_xprt *,

	void		(*ro_unmap_sync)(struct rpcrdma_xprt *,