Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 26fb2254 authored by Chuck Lever's avatar Chuck Lever Committed by J. Bruce Fields
Browse files

svcrdma: Estimate Send Queue depth properly 



The rdma_rw API adjusts max_send_wr upwards during the
rdma_create_qp() call. If the ULP actually wants to take advantage
of these extra resources, it must increase the size of its send
completion queue (created before rdma_create_qp is called) and
increase its send queue accounting limit.

Use the new rdma_rw_mr_factor API to figure out the correct value
to use for the Send Queue and Send Completion Queue depths.

And, ensure that the chosen Send Queue depth for a newly created
transport does not overrun the QP WR limit of the underlying device.

Lastly, there's no longer a need to carry the Send Queue depth in
struct svcxprt_rdma, since the value is used only in the
svc_rdma_accept() path.

Signed-off-by: default avatarChuck Lever <chuck.lever@oracle.com>
Signed-off-by: default avatarJ. Bruce Fields <bfields@redhat.com>
parent 00628182

net/sunrpc/xprtrdma/svc_rdma_transport.c

+13 −4
Original line number Diff line number Diff line
@@ -51,6 +51,7 @@ 
#include <linux/workqueue.h>

#include <rdma/ib_verbs.h>

#include <rdma/rdma_cm.h>

#include <rdma/rw.h>

#include <linux/sunrpc/svc_rdma.h>

#include <linux/export.h>

#include "xprt_rdma.h"
@@ -713,7 +714,7 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) 
	struct ib_qp_init_attr qp_attr;

	struct ib_device *dev;

	struct sockaddr *sap;

	unsigned int i;

	unsigned int i, ctxts;

	int ret = 0;



	listen_rdma = container_of(xprt, struct svcxprt_rdma, sc_xprt);
@@ -754,7 +755,14 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) 
		newxprt->sc_max_bc_requests = 2;

	}

	newxprt->sc_fc_credits = cpu_to_be32(newxprt->sc_max_requests);

	newxprt->sc_sq_depth = newxprt->sc_rq_depth;

	ctxts = rdma_rw_mr_factor(dev, newxprt->sc_port_num, RPCSVC_MAXPAGES);

	ctxts *= newxprt->sc_max_requests;

	newxprt->sc_sq_depth = newxprt->sc_rq_depth + ctxts;

	if (newxprt->sc_sq_depth > dev->attrs.max_qp_wr) {

		pr_warn("svcrdma: reducing send depth to %d\n",

			dev->attrs.max_qp_wr);

		newxprt->sc_sq_depth = dev->attrs.max_qp_wr;

	}

	atomic_set(&newxprt->sc_sq_avail, newxprt->sc_sq_depth);



	if (!svc_rdma_prealloc_ctxts(newxprt))
@@ -789,8 +797,8 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) 
	qp_attr.event_handler = qp_event_handler;

	qp_attr.qp_context = &newxprt->sc_xprt;

	qp_attr.port_num = newxprt->sc_port_num;

	qp_attr.cap.max_rdma_ctxs = newxprt->sc_max_requests;

	qp_attr.cap.max_send_wr = newxprt->sc_sq_depth;

	qp_attr.cap.max_rdma_ctxs = ctxts;

	qp_attr.cap.max_send_wr = newxprt->sc_sq_depth - ctxts;

	qp_attr.cap.max_recv_wr = newxprt->sc_rq_depth;

	qp_attr.cap.max_send_sge = newxprt->sc_max_sge;

	qp_attr.cap.max_recv_sge = newxprt->sc_max_sge;
@@ -858,6 +866,7 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) 
	dprintk("    remote address  : %pIS:%u\n", sap, rpc_get_port(sap));

	dprintk("    max_sge         : %d\n", newxprt->sc_max_sge);

	dprintk("    sq_depth        : %d\n", newxprt->sc_sq_depth);

	dprintk("    rdma_rw_ctxs    : %d\n", ctxts);

	dprintk("    max_requests    : %d\n", newxprt->sc_max_requests);

	dprintk("    ord             : %d\n", newxprt->sc_ord);