Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 8cec3dba authored by Chuck Lever's avatar Chuck Lever Committed by Anna Schumaker
Browse files

xprtrdma: rpcrdma_regbuf alignment 



Allocate the struct rpcrdma_regbuf separately from the I/O buffer
to better guarantee the alignment of the I/O buffer and eliminate
the wasted space between the rpcrdma_regbuf metadata and the buffer
itself.

Signed-off-by: default avatarChuck Lever <chuck.lever@oracle.com>
Signed-off-by: default avatarAnna Schumaker <Anna.Schumaker@Netapp.com>
parent 23146500

net/sunrpc/xprtrdma/backchannel.c

+3 −3
Original line number Original line Diff line number Diff line
@@ -45,10 +45,10 @@ static int rpcrdma_bc_setup_reqs(struct rpcrdma_xprt *r_xprt, 




		size = r_xprt->rx_data.inline_rsize;

		size = r_xprt->rx_data.inline_rsize;

		rb = rpcrdma_alloc_regbuf(size, DMA_TO_DEVICE, GFP_KERNEL);

		rb = rpcrdma_alloc_regbuf(size, DMA_TO_DEVICE, GFP_KERNEL);

		if (IS_ERR(rb))

		if (!rb)

			goto out_fail;

			goto out_fail;

		req->rl_sendbuf = rb;

		req->rl_sendbuf = rb;

		xdr_buf_init(&rqst->rq_snd_buf, rb->rg_base,

		xdr_buf_init(&rqst->rq_snd_buf, rdmab_data(rb),

			     min_t(size_t, size, PAGE_SIZE));

			     min_t(size_t, size, PAGE_SIZE));

	}

	}

	return 0;

	return 0;
@@ -123,7 +123,7 @@ static int rpcrdma_bc_marshal_reply(struct rpc_rqst *rqst) 




	rpcrdma_set_xdrlen(&req->rl_hdrbuf, 0);

	rpcrdma_set_xdrlen(&req->rl_hdrbuf, 0);

	xdr_init_encode(&req->rl_stream, &req->rl_hdrbuf,

	xdr_init_encode(&req->rl_stream, &req->rl_hdrbuf,

			req->rl_rdmabuf->rg_base, rqst);

			rdmab_data(req->rl_rdmabuf), rqst);





	p = xdr_reserve_space(&req->rl_stream, 28);

	p = xdr_reserve_space(&req->rl_stream, 28);

	if (unlikely(!p))

	if (unlikely(!p))

net/sunrpc/xprtrdma/rpc_rdma.c

+2 −2
Original line number Original line Diff line number Diff line
@@ -747,8 +747,8 @@ rpcrdma_marshal_req(struct rpcrdma_xprt *r_xprt, struct rpc_rqst *rqst) 
	int ret;

	int ret;





	rpcrdma_set_xdrlen(&req->rl_hdrbuf, 0);

	rpcrdma_set_xdrlen(&req->rl_hdrbuf, 0);

	xdr_init_encode(xdr, &req->rl_hdrbuf,

	xdr_init_encode(xdr, &req->rl_hdrbuf, rdmab_data(req->rl_rdmabuf),

			req->rl_rdmabuf->rg_base, rqst);

			rqst);





	/* Fixed header fields */

	/* Fixed header fields */

	ret = -EMSGSIZE;

	ret = -EMSGSIZE;

net/sunrpc/xprtrdma/transport.c

+4 −4
Original line number Original line Diff line number Diff line
@@ -595,7 +595,7 @@ rpcrdma_get_sendbuf(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req, 
		return true;

		return true;





	rb = rpcrdma_alloc_regbuf(size, DMA_TO_DEVICE, flags);

	rb = rpcrdma_alloc_regbuf(size, DMA_TO_DEVICE, flags);

	if (IS_ERR(rb))

	if (!rb)

		return false;

		return false;





	rpcrdma_free_regbuf(req->rl_sendbuf);

	rpcrdma_free_regbuf(req->rl_sendbuf);
@@ -625,7 +625,7 @@ rpcrdma_get_recvbuf(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req, 
		return true;

		return true;





	rb = rpcrdma_alloc_regbuf(size, DMA_NONE, flags);

	rb = rpcrdma_alloc_regbuf(size, DMA_NONE, flags);

	if (IS_ERR(rb))

	if (!rb)

		return false;

		return false;





	rpcrdma_free_regbuf(req->rl_recvbuf);

	rpcrdma_free_regbuf(req->rl_recvbuf);
@@ -660,8 +660,8 @@ xprt_rdma_allocate(struct rpc_task *task) 
	if (!rpcrdma_get_recvbuf(r_xprt, req, rqst->rq_rcvsize, flags))

	if (!rpcrdma_get_recvbuf(r_xprt, req, rqst->rq_rcvsize, flags))

		goto out_fail;

		goto out_fail;





	rqst->rq_buffer = req->rl_sendbuf->rg_base;

	rqst->rq_buffer = rdmab_data(req->rl_sendbuf);

	rqst->rq_rbuffer = req->rl_recvbuf->rg_base;

	rqst->rq_rbuffer = rdmab_data(req->rl_recvbuf);

	trace_xprtrdma_op_allocate(task, req);

	trace_xprtrdma_op_allocate(task, req);

	return 0;

	return 0;

net/sunrpc/xprtrdma/verbs.c

+16 −11
Original line number Original line Diff line number Diff line
@@ -1013,12 +1013,12 @@ struct rpcrdma_req *rpcrdma_req_create(struct rpcrdma_xprt *r_xprt, gfp_t flags) 
		return NULL;

		return NULL;





	rb = rpcrdma_alloc_regbuf(RPCRDMA_HDRBUF_SIZE, DMA_TO_DEVICE, flags);

	rb = rpcrdma_alloc_regbuf(RPCRDMA_HDRBUF_SIZE, DMA_TO_DEVICE, flags);

	if (IS_ERR(rb)) {

	if (!rb) {

		kfree(req);

		kfree(req);

		return NULL;

		return NULL;

	}

	}

	req->rl_rdmabuf = rb;

	req->rl_rdmabuf = rb;

	xdr_buf_init(&req->rl_hdrbuf, rb->rg_base, rdmab_length(rb));

	xdr_buf_init(&req->rl_hdrbuf, rdmab_data(rb), rdmab_length(rb));

	req->rl_buffer = buffer;

	req->rl_buffer = buffer;

	INIT_LIST_HEAD(&req->rl_registered);

	INIT_LIST_HEAD(&req->rl_registered);
@@ -1040,9 +1040,9 @@ static bool rpcrdma_rep_create(struct rpcrdma_xprt *r_xprt, bool temp) 




	rep->rr_rdmabuf = rpcrdma_alloc_regbuf(cdata->inline_rsize,

	rep->rr_rdmabuf = rpcrdma_alloc_regbuf(cdata->inline_rsize,

					       DMA_FROM_DEVICE, GFP_KERNEL);

					       DMA_FROM_DEVICE, GFP_KERNEL);

	if (IS_ERR(rep->rr_rdmabuf))

	if (!rep->rr_rdmabuf)

		goto out_free;

		goto out_free;

	xdr_buf_init(&rep->rr_hdrbuf, rep->rr_rdmabuf->rg_base,

	xdr_buf_init(&rep->rr_hdrbuf, rdmab_data(rep->rr_rdmabuf),

		     rdmab_length(rep->rr_rdmabuf));

		     rdmab_length(rep->rr_rdmabuf));





	rep->rr_cqe.done = rpcrdma_wc_receive;

	rep->rr_cqe.done = rpcrdma_wc_receive;
@@ -1354,8 +1354,7 @@ rpcrdma_recv_buffer_put(struct rpcrdma_rep *rep) 
 * @direction: direction of data movement

 * @direction: direction of data movement

 * @flags: GFP flags

 * @flags: GFP flags

 *

 *

 * Returns an ERR_PTR, or a pointer to a regbuf, a buffer that

 * Returns a pointer to a rpcrdma_regbuf object, or NULL.

 * can be persistently DMA-mapped for I/O.

 *

 *

 * xprtrdma uses a regbuf for posting an outgoing RDMA SEND, or for

 * xprtrdma uses a regbuf for posting an outgoing RDMA SEND, or for

 * receiving the payload of RDMA RECV operations. During Long Calls

 * receiving the payload of RDMA RECV operations. During Long Calls
@@ -1367,14 +1366,18 @@ rpcrdma_alloc_regbuf(size_t size, enum dma_data_direction direction, 
{

{

	struct rpcrdma_regbuf *rb;

	struct rpcrdma_regbuf *rb;





	rb = kmalloc(sizeof(*rb) + size, flags);

	rb = kmalloc(sizeof(*rb), flags);

	if (rb == NULL)

	if (!rb)

		return ERR_PTR(-ENOMEM);

		return NULL;

	rb->rg_data = kmalloc(size, flags);

	if (!rb->rg_data) {

		kfree(rb);

		return NULL;

	}





	rb->rg_device = NULL;

	rb->rg_device = NULL;

	rb->rg_direction = direction;

	rb->rg_direction = direction;

	rb->rg_iov.length = size;

	rb->rg_iov.length = size;



	return rb;

	return rb;

}

}
@@ -1392,7 +1395,7 @@ __rpcrdma_dma_map_regbuf(struct rpcrdma_ia *ia, struct rpcrdma_regbuf *rb) 
		return false;

		return false;





	rb->rg_iov.addr = ib_dma_map_single(device,

	rb->rg_iov.addr = ib_dma_map_single(device,

					    (void *)rb->rg_base,

					    rdmab_data(rb),

					    rdmab_length(rb),

					    rdmab_length(rb),

					    rb->rg_direction);

					    rb->rg_direction);

	if (ib_dma_mapping_error(device, rdmab_addr(rb))) {

	if (ib_dma_mapping_error(device, rdmab_addr(rb))) {
@@ -1427,6 +1430,8 @@ void 
rpcrdma_free_regbuf(struct rpcrdma_regbuf *rb)

rpcrdma_free_regbuf(struct rpcrdma_regbuf *rb)

{

{

	rpcrdma_dma_unmap_regbuf(rb);

	rpcrdma_dma_unmap_regbuf(rb);

	if (rb)

		kfree(rb->rg_data);

	kfree(rb);

	kfree(rb);

}

}

net/sunrpc/xprtrdma/xprt_rdma.h

+10 −9
Original line number Original line Diff line number Diff line
@@ -121,33 +121,34 @@ struct rpcrdma_regbuf { 
	struct ib_sge		rg_iov;

	struct ib_sge		rg_iov;

	struct ib_device	*rg_device;

	struct ib_device	*rg_device;

	enum dma_data_direction	rg_direction;

	enum dma_data_direction	rg_direction;

	__be32			rg_base[0] __attribute__ ((aligned(256)));

	void			*rg_data;

};

};





static inline u64

static inline u64 rdmab_addr(struct rpcrdma_regbuf *rb)

rdmab_addr(struct rpcrdma_regbuf *rb)

{

{

	return rb->rg_iov.addr;

	return rb->rg_iov.addr;

}

}





static inline u32

static inline u32 rdmab_length(struct rpcrdma_regbuf *rb)

rdmab_length(struct rpcrdma_regbuf *rb)

{

{

	return rb->rg_iov.length;

	return rb->rg_iov.length;

}

}





static inline u32

static inline u32 rdmab_lkey(struct rpcrdma_regbuf *rb)

rdmab_lkey(struct rpcrdma_regbuf *rb)

{

{

	return rb->rg_iov.lkey;

	return rb->rg_iov.lkey;

}

}





static inline struct ib_device *

static inline struct ib_device *rdmab_device(struct rpcrdma_regbuf *rb)

rdmab_device(struct rpcrdma_regbuf *rb)

{

{

	return rb->rg_device;

	return rb->rg_device;

}

}





static inline void *rdmab_data(const struct rpcrdma_regbuf *rb)

{

	return rb->rg_data;

}



#define RPCRDMA_DEF_GFP		(GFP_NOIO | __GFP_NOWARN)

#define RPCRDMA_DEF_GFP		(GFP_NOIO | __GFP_NOWARN)





/* To ensure a transport can always make forward progress,

/* To ensure a transport can always make forward progress,