RPC/RDMA: refactor the inline memory registration code. (8d4ba034) · Commits · e / devices / android_kernel_teracube_2e

net/sunrpc/xprtrdma/verbs.c

+207 −158

Original line number	Original line	Diff line number	Diff line
	@@ -863,6 +863,7 @@ rpcrdma_buffer_create(struct rpcrdma_buffer buf, struct rpcrdma_ep ep,
	char *p;		char *p;
	size_t len;		size_t len;
	int i, rc;		int i, rc;
			struct rpcrdma_mw *r;

	buf->rb_max_requests = cdata->max_requests;		buf->rb_max_requests = cdata->max_requests;
	spin_lock_init(&buf->rb_lock);		spin_lock_init(&buf->rb_lock);
	@@ -873,7 +874,7 @@ rpcrdma_buffer_create(struct rpcrdma_buffer buf, struct rpcrdma_ep ep,
	* 2. arrays of struct rpcrdma_req to fill in pointers		* 2. arrays of struct rpcrdma_req to fill in pointers
	* 3. array of struct rpcrdma_rep for replies		* 3. array of struct rpcrdma_rep for replies
	* 4. padding, if any		* 4. padding, if any
	* 5. mw's, if any		* 5. mw's or fmr's, if any
	* Send/recv buffers in req/rep need to be registered		* Send/recv buffers in req/rep need to be registered
	*/		*/

	@@ -927,15 +928,13 @@ rpcrdma_buffer_create(struct rpcrdma_buffer buf, struct rpcrdma_ep ep,
	* and also reduce unbind-to-bind collision.		* and also reduce unbind-to-bind collision.
	*/		*/
	INIT_LIST_HEAD(&buf->rb_mws);		INIT_LIST_HEAD(&buf->rb_mws);
			r = (struct rpcrdma_mw *)p;
	switch (ia->ri_memreg_strategy) {		switch (ia->ri_memreg_strategy) {
	case RPCRDMA_MTHCAFMR:		case RPCRDMA_MTHCAFMR:
	{
	struct rpcrdma_mw r = (struct rpcrdma_mw )p;
	struct ib_fmr_attr fa = {
	RPCRDMA_MAX_DATA_SEGS, 1, PAGE_SHIFT
	};
	/* TBD we are perhaps overallocating here */		/* TBD we are perhaps overallocating here */
	for (i = (buf->rb_max_requests+1) * RPCRDMA_MAX_SEGS; i; i--) {		for (i = (buf->rb_max_requests+1) * RPCRDMA_MAX_SEGS; i; i--) {
			static struct ib_fmr_attr fa =
			{ RPCRDMA_MAX_DATA_SEGS, 1, PAGE_SHIFT };
	r->r.fmr = ib_alloc_fmr(ia->ri_pd,		r->r.fmr = ib_alloc_fmr(ia->ri_pd,
	IB_ACCESS_REMOTE_WRITE \| IB_ACCESS_REMOTE_READ,		IB_ACCESS_REMOTE_WRITE \| IB_ACCESS_REMOTE_READ,
	&fa);		&fa);
	@@ -948,12 +947,9 @@ rpcrdma_buffer_create(struct rpcrdma_buffer buf, struct rpcrdma_ep ep,
	list_add(&r->mw_list, &buf->rb_mws);		list_add(&r->mw_list, &buf->rb_mws);
	++r;		++r;
	}		}
	}
	break;		break;
	case RPCRDMA_MEMWINDOWS_ASYNC:		case RPCRDMA_MEMWINDOWS_ASYNC:
	case RPCRDMA_MEMWINDOWS:		case RPCRDMA_MEMWINDOWS:
	{
	struct rpcrdma_mw r = (struct rpcrdma_mw )p;
	/* Allocate one extra request's worth, for full cycling */		/* Allocate one extra request's worth, for full cycling */
	for (i = (buf->rb_max_requests+1) * RPCRDMA_MAX_SEGS; i; i--) {		for (i = (buf->rb_max_requests+1) * RPCRDMA_MAX_SEGS; i; i--) {
	r->r.mw = ib_alloc_mw(ia->ri_pd);		r->r.mw = ib_alloc_mw(ia->ri_pd);
	@@ -966,7 +962,6 @@ rpcrdma_buffer_create(struct rpcrdma_buffer buf, struct rpcrdma_ep ep,
	list_add(&r->mw_list, &buf->rb_mws);		list_add(&r->mw_list, &buf->rb_mws);
	++r;		++r;
	}		}
	}
	break;		break;
	default:		default:
	break;		break;
	@@ -1046,6 +1041,7 @@ rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf)
	{		{
	int rc, i;		int rc, i;
	struct rpcrdma_ia *ia = rdmab_to_ia(buf);		struct rpcrdma_ia *ia = rdmab_to_ia(buf);
			struct rpcrdma_mw *r;

	/* clean up in reverse order from create		/* clean up in reverse order from create
	* 1. recv mr memory (mr free, then kfree)		* 1. recv mr memory (mr free, then kfree)
	@@ -1065,7 +1061,6 @@ rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf)
	}		}
	if (buf->rb_send_bufs && buf->rb_send_bufs[i]) {		if (buf->rb_send_bufs && buf->rb_send_bufs[i]) {
	while (!list_empty(&buf->rb_mws)) {		while (!list_empty(&buf->rb_mws)) {
	struct rpcrdma_mw *r;
	r = list_entry(buf->rb_mws.next,		r = list_entry(buf->rb_mws.next,
	struct rpcrdma_mw, mw_list);		struct rpcrdma_mw, mw_list);
	list_del(&r->mw_list);		list_del(&r->mw_list);
	@@ -1115,6 +1110,8 @@ rpcrdma_buffer_get(struct rpcrdma_buffer *buffers)
	{		{
	struct rpcrdma_req *req;		struct rpcrdma_req *req;
	unsigned long flags;		unsigned long flags;
			int i;
			struct rpcrdma_mw *r;

	spin_lock_irqsave(&buffers->rb_lock, flags);		spin_lock_irqsave(&buffers->rb_lock, flags);
	if (buffers->rb_send_index == buffers->rb_max_requests) {		if (buffers->rb_send_index == buffers->rb_max_requests) {
	@@ -1135,9 +1132,8 @@ rpcrdma_buffer_get(struct rpcrdma_buffer *buffers)
	}		}
	buffers->rb_send_bufs[buffers->rb_send_index++] = NULL;		buffers->rb_send_bufs[buffers->rb_send_index++] = NULL;
	if (!list_empty(&buffers->rb_mws)) {		if (!list_empty(&buffers->rb_mws)) {
	int i = RPCRDMA_MAX_SEGS - 1;		i = RPCRDMA_MAX_SEGS - 1;
	do {		do {
	struct rpcrdma_mw *r;
	r = list_entry(buffers->rb_mws.next,		r = list_entry(buffers->rb_mws.next,
	struct rpcrdma_mw, mw_list);		struct rpcrdma_mw, mw_list);
	list_del(&r->mw_list);		list_del(&r->mw_list);
	@@ -1329,74 +1325,79 @@ rpcrdma_unmap_one(struct rpcrdma_ia ia, struct rpcrdma_mr_seg seg)
	seg->mr_dma, seg->mr_dmalen, seg->mr_dir);		seg->mr_dma, seg->mr_dmalen, seg->mr_dir);
	}		}

	int		static int
	rpcrdma_register_external(struct rpcrdma_mr_seg *seg,		rpcrdma_register_fmr_external(struct rpcrdma_mr_seg *seg,
	int nsegs, int writing, struct rpcrdma_xprt *r_xprt)		int nsegs, int writing, struct rpcrdma_ia ia)
	{		{
	struct rpcrdma_ia *ia = &r_xprt->rx_ia;
	int mem_priv = (writing ? IB_ACCESS_REMOTE_WRITE :
	IB_ACCESS_REMOTE_READ);
	struct rpcrdma_mr_seg *seg1 = seg;		struct rpcrdma_mr_seg *seg1 = seg;
	int i;
	int rc = 0;

	switch (ia->ri_memreg_strategy) {

	#if RPCRDMA_PERSISTENT_REGISTRATION
	case RPCRDMA_ALLPHYSICAL:
	rpcrdma_map_one(ia, seg, writing);
	seg->mr_rkey = ia->ri_bind_mem->rkey;
	seg->mr_base = seg->mr_dma;
	seg->mr_nsegs = 1;
	nsegs = 1;
	break;
	#endif

	/* Registration using fast memory registration */
	case RPCRDMA_MTHCAFMR:
	{
	u64 physaddrs[RPCRDMA_MAX_DATA_SEGS];		u64 physaddrs[RPCRDMA_MAX_DATA_SEGS];
	int len, pageoff = offset_in_page(seg->mr_offset);		int len, pageoff, i, rc;

			pageoff = offset_in_page(seg1->mr_offset);
	seg1->mr_offset -= pageoff; /* start of page */		seg1->mr_offset -= pageoff; /* start of page */
	seg1->mr_len += pageoff;		seg1->mr_len += pageoff;
	len = -pageoff;		len = -pageoff;
	if (nsegs > RPCRDMA_MAX_DATA_SEGS)		if (*nsegs > RPCRDMA_MAX_DATA_SEGS)
	nsegs = RPCRDMA_MAX_DATA_SEGS;		*nsegs = RPCRDMA_MAX_DATA_SEGS;
	for (i = 0; i < nsegs;) {		for (i = 0; i < *nsegs;) {
	rpcrdma_map_one(ia, seg, writing);		rpcrdma_map_one(ia, seg, writing);
	physaddrs[i] = seg->mr_dma;		physaddrs[i] = seg->mr_dma;
	len += seg->mr_len;		len += seg->mr_len;
	++seg;		++seg;
	++i;		++i;
	/* Check for holes */		/* Check for holes */
	if ((i < nsegs && offset_in_page(seg->mr_offset)) \|\|		if ((i < *nsegs && offset_in_page(seg->mr_offset)) \|\|
	offset_in_page((seg-1)->mr_offset + (seg-1)->mr_len))		offset_in_page((seg-1)->mr_offset + (seg-1)->mr_len))
	break;		break;
	}		}
	nsegs = i;
	rc = ib_map_phys_fmr(seg1->mr_chunk.rl_mw->r.fmr,		rc = ib_map_phys_fmr(seg1->mr_chunk.rl_mw->r.fmr,
	physaddrs, nsegs, seg1->mr_dma);		physaddrs, i, seg1->mr_dma);
	if (rc) {		if (rc) {
	dprintk("RPC: %s: failed ib_map_phys_fmr "		dprintk("RPC: %s: failed ib_map_phys_fmr "
	"%u@0x%llx+%i (%d)... status %i\n", __func__,		"%u@0x%llx+%i (%d)... status %i\n", __func__,
	len, (unsigned long long)seg1->mr_dma,		len, (unsigned long long)seg1->mr_dma,
	pageoff, nsegs, rc);		pageoff, i, rc);
	while (nsegs--)		while (i--)
	rpcrdma_unmap_one(ia, --seg);		rpcrdma_unmap_one(ia, --seg);
	} else {		} else {
	seg1->mr_rkey = seg1->mr_chunk.rl_mw->r.fmr->rkey;		seg1->mr_rkey = seg1->mr_chunk.rl_mw->r.fmr->rkey;
	seg1->mr_base = seg1->mr_dma + pageoff;		seg1->mr_base = seg1->mr_dma + pageoff;
	seg1->mr_nsegs = nsegs;		seg1->mr_nsegs = i;
	seg1->mr_len = len;		seg1->mr_len = len;
	}		}
			*nsegs = i;
			return rc;
	}		}
	break;

	/* Registration using memory windows */		static int
	case RPCRDMA_MEMWINDOWS_ASYNC:		rpcrdma_deregister_fmr_external(struct rpcrdma_mr_seg *seg,
	case RPCRDMA_MEMWINDOWS:		struct rpcrdma_ia *ia)
			{
			struct rpcrdma_mr_seg *seg1 = seg;
			LIST_HEAD(l);
			int rc;

			list_add(&seg1->mr_chunk.rl_mw->r.fmr->list, &l);
			rc = ib_unmap_fmr(&l);
			while (seg1->mr_nsegs--)
			rpcrdma_unmap_one(ia, seg++);
			if (rc)
			dprintk("RPC: %s: failed ib_unmap_fmr,"
			" status %i\n", __func__, rc);
			return rc;
			}

			static int
			rpcrdma_register_memwin_external(struct rpcrdma_mr_seg *seg,
			int nsegs, int writing, struct rpcrdma_ia ia,
			struct rpcrdma_xprt *r_xprt)
	{		{
			int mem_priv = (writing ? IB_ACCESS_REMOTE_WRITE :
			IB_ACCESS_REMOTE_READ);
	struct ib_mw_bind param;		struct ib_mw_bind param;
			int rc;

			*nsegs = 1;
	rpcrdma_map_one(ia, seg, writing);		rpcrdma_map_one(ia, seg, writing);
	param.mr = ia->ri_bind_mem;		param.mr = ia->ri_bind_mem;
	param.wr_id = 0ULL; /* no send cookie */		param.wr_id = 0ULL; /* no send cookie */
	@@ -1406,8 +1407,7 @@ rpcrdma_register_external(struct rpcrdma_mr_seg *seg,
	param.mw_access_flags = mem_priv;		param.mw_access_flags = mem_priv;

	DECR_CQCOUNT(&r_xprt->rx_ep);		DECR_CQCOUNT(&r_xprt->rx_ep);
	rc = ib_bind_mw(ia->ri_id->qp,		rc = ib_bind_mw(ia->ri_id->qp, seg->mr_chunk.rl_mw->r.mw, &param);
	seg->mr_chunk.rl_mw->r.mw, &param);
	if (rc) {		if (rc) {
	dprintk("RPC: %s: failed ib_bind_mw "		dprintk("RPC: %s: failed ib_bind_mw "
	"%u@0x%llx status %i\n",		"%u@0x%llx status %i\n",
	@@ -1418,19 +1418,56 @@ rpcrdma_register_external(struct rpcrdma_mr_seg *seg,
	seg->mr_rkey = seg->mr_chunk.rl_mw->r.mw->rkey;		seg->mr_rkey = seg->mr_chunk.rl_mw->r.mw->rkey;
	seg->mr_base = param.addr;		seg->mr_base = param.addr;
	seg->mr_nsegs = 1;		seg->mr_nsegs = 1;
	nsegs = 1;
	}		}
			return rc;
	}		}
	break;

	/* Default registration each time */		static int
	default:		rpcrdma_deregister_memwin_external(struct rpcrdma_mr_seg *seg,
			struct rpcrdma_ia *ia,
			struct rpcrdma_xprt r_xprt, void *r)
			{
			struct ib_mw_bind param;
			LIST_HEAD(l);
			int rc;

			BUG_ON(seg->mr_nsegs != 1);
			param.mr = ia->ri_bind_mem;
			param.addr = 0ULL; /* unbind */
			param.length = 0;
			param.mw_access_flags = 0;
			if (*r) {
			param.wr_id = (u64) (unsigned long) *r;
			param.send_flags = IB_SEND_SIGNALED;
			INIT_CQCOUNT(&r_xprt->rx_ep);
			} else {
			param.wr_id = 0ULL;
			param.send_flags = 0;
			DECR_CQCOUNT(&r_xprt->rx_ep);
			}
			rc = ib_bind_mw(ia->ri_id->qp, seg->mr_chunk.rl_mw->r.mw, &param);
			rpcrdma_unmap_one(ia, seg);
			if (rc)
			dprintk("RPC: %s: failed ib_(un)bind_mw,"
			" status %i\n", __func__, rc);
			else
			r = NULL; / will upcall on completion */
			return rc;
			}

			static int
			rpcrdma_register_default_external(struct rpcrdma_mr_seg *seg,
			int nsegs, int writing, struct rpcrdma_ia ia)
	{		{
			int mem_priv = (writing ? IB_ACCESS_REMOTE_WRITE :
			IB_ACCESS_REMOTE_READ);
			struct rpcrdma_mr_seg *seg1 = seg;
	struct ib_phys_buf ipb[RPCRDMA_MAX_DATA_SEGS];		struct ib_phys_buf ipb[RPCRDMA_MAX_DATA_SEGS];
	int len = 0;		int len, i, rc = 0;
	if (nsegs > RPCRDMA_MAX_DATA_SEGS)
	nsegs = RPCRDMA_MAX_DATA_SEGS;		if (*nsegs > RPCRDMA_MAX_DATA_SEGS)
	for (i = 0; i < nsegs;) {		*nsegs = RPCRDMA_MAX_DATA_SEGS;
			for (len = 0, i = 0; i < *nsegs;) {
	rpcrdma_map_one(ia, seg, writing);		rpcrdma_map_one(ia, seg, writing);
	ipb[i].addr = seg->mr_dma;		ipb[i].addr = seg->mr_dma;
	ipb[i].size = seg->mr_len;		ipb[i].size = seg->mr_len;
	@@ -1438,28 +1475,80 @@ rpcrdma_register_external(struct rpcrdma_mr_seg *seg,
	++seg;		++seg;
	++i;		++i;
	/* Check for holes */		/* Check for holes */
	if ((i < nsegs && offset_in_page(seg->mr_offset)) \|\|		if ((i < *nsegs && offset_in_page(seg->mr_offset)) \|\|
	offset_in_page((seg-1)->mr_offset+(seg-1)->mr_len))		offset_in_page((seg-1)->mr_offset+(seg-1)->mr_len))
	break;		break;
	}		}
	nsegs = i;
	seg1->mr_base = seg1->mr_dma;		seg1->mr_base = seg1->mr_dma;
	seg1->mr_chunk.rl_mr = ib_reg_phys_mr(ia->ri_pd,		seg1->mr_chunk.rl_mr = ib_reg_phys_mr(ia->ri_pd,
	ipb, nsegs, mem_priv, &seg1->mr_base);		ipb, i, mem_priv, &seg1->mr_base);
	if (IS_ERR(seg1->mr_chunk.rl_mr)) {		if (IS_ERR(seg1->mr_chunk.rl_mr)) {
	rc = PTR_ERR(seg1->mr_chunk.rl_mr);		rc = PTR_ERR(seg1->mr_chunk.rl_mr);
	dprintk("RPC: %s: failed ib_reg_phys_mr "		dprintk("RPC: %s: failed ib_reg_phys_mr "
	"%u@0x%llx (%d)... status %i\n",		"%u@0x%llx (%d)... status %i\n",
	__func__, len,		__func__, len,
	(unsigned long long)seg1->mr_dma, nsegs, rc);		(unsigned long long)seg1->mr_dma, i, rc);
	while (nsegs--)		while (i--)
	rpcrdma_unmap_one(ia, --seg);		rpcrdma_unmap_one(ia, --seg);
	} else {		} else {
	seg1->mr_rkey = seg1->mr_chunk.rl_mr->rkey;		seg1->mr_rkey = seg1->mr_chunk.rl_mr->rkey;
	seg1->mr_nsegs = nsegs;		seg1->mr_nsegs = i;
	seg1->mr_len = len;		seg1->mr_len = len;
	}		}
			*nsegs = i;
			return rc;
	}		}

			static int
			rpcrdma_deregister_default_external(struct rpcrdma_mr_seg *seg,
			struct rpcrdma_ia *ia)
			{
			struct rpcrdma_mr_seg *seg1 = seg;
			int rc;

			rc = ib_dereg_mr(seg1->mr_chunk.rl_mr);
			seg1->mr_chunk.rl_mr = NULL;
			while (seg1->mr_nsegs--)
			rpcrdma_unmap_one(ia, seg++);
			if (rc)
			dprintk("RPC: %s: failed ib_dereg_mr,"
			" status %i\n", __func__, rc);
			return rc;
			}

			int
			rpcrdma_register_external(struct rpcrdma_mr_seg *seg,
			int nsegs, int writing, struct rpcrdma_xprt *r_xprt)
			{
			struct rpcrdma_ia *ia = &r_xprt->rx_ia;
			int rc = 0;

			switch (ia->ri_memreg_strategy) {

			#if RPCRDMA_PERSISTENT_REGISTRATION
			case RPCRDMA_ALLPHYSICAL:
			rpcrdma_map_one(ia, seg, writing);
			seg->mr_rkey = ia->ri_bind_mem->rkey;
			seg->mr_base = seg->mr_dma;
			seg->mr_nsegs = 1;
			nsegs = 1;
			break;
			#endif

			/* Registration using fmr memory registration */
			case RPCRDMA_MTHCAFMR:
			rc = rpcrdma_register_fmr_external(seg, &nsegs, writing, ia);
			break;

			/* Registration using memory windows */
			case RPCRDMA_MEMWINDOWS_ASYNC:
			case RPCRDMA_MEMWINDOWS:
			rc = rpcrdma_register_memwin_external(seg, &nsegs, writing, ia, r_xprt);
			break;

			/* Default registration each time */
			default:
			rc = rpcrdma_register_default_external(seg, &nsegs, writing, ia);
	break;		break;
	}		}
	if (rc)		if (rc)
	@@ -1473,7 +1562,6 @@ rpcrdma_deregister_external(struct rpcrdma_mr_seg *seg,
	struct rpcrdma_xprt r_xprt, void r)		struct rpcrdma_xprt r_xprt, void r)
	{		{
	struct rpcrdma_ia *ia = &r_xprt->rx_ia;		struct rpcrdma_ia *ia = &r_xprt->rx_ia;
	struct rpcrdma_mr_seg *seg1 = seg;
	int nsegs = seg->mr_nsegs, rc;		int nsegs = seg->mr_nsegs, rc;

	switch (ia->ri_memreg_strategy) {		switch (ia->ri_memreg_strategy) {
	@@ -1487,55 +1575,16 @@ rpcrdma_deregister_external(struct rpcrdma_mr_seg *seg,
	#endif		#endif

	case RPCRDMA_MTHCAFMR:		case RPCRDMA_MTHCAFMR:
	{		rc = rpcrdma_deregister_fmr_external(seg, ia);
	LIST_HEAD(l);
	list_add(&seg->mr_chunk.rl_mw->r.fmr->list, &l);
	rc = ib_unmap_fmr(&l);
	while (seg1->mr_nsegs--)
	rpcrdma_unmap_one(ia, seg++);
	}
	if (rc)
	dprintk("RPC: %s: failed ib_unmap_fmr,"
	" status %i\n", __func__, rc);
	break;		break;

	case RPCRDMA_MEMWINDOWS_ASYNC:		case RPCRDMA_MEMWINDOWS_ASYNC:
	case RPCRDMA_MEMWINDOWS:		case RPCRDMA_MEMWINDOWS:
	{		rc = rpcrdma_deregister_memwin_external(seg, ia, r_xprt, &r);
	struct ib_mw_bind param;
	BUG_ON(nsegs != 1);
	param.mr = ia->ri_bind_mem;
	param.addr = 0ULL; /* unbind */
	param.length = 0;
	param.mw_access_flags = 0;
	if (r) {
	param.wr_id = (u64) (unsigned long) r;
	param.send_flags = IB_SEND_SIGNALED;
	INIT_CQCOUNT(&r_xprt->rx_ep);
	} else {
	param.wr_id = 0ULL;
	param.send_flags = 0;
	DECR_CQCOUNT(&r_xprt->rx_ep);
	}
	rc = ib_bind_mw(ia->ri_id->qp,
	seg->mr_chunk.rl_mw->r.mw, &param);
	rpcrdma_unmap_one(ia, seg);
	}
	if (rc)
	dprintk("RPC: %s: failed ib_(un)bind_mw,"
	" status %i\n", __func__, rc);
	else
	r = NULL; /* will upcall on completion */
	break;		break;

	default:		default:
	rc = ib_dereg_mr(seg1->mr_chunk.rl_mr);		rc = rpcrdma_deregister_default_external(seg, ia);
	seg1->mr_chunk.rl_mr = NULL;
	while (seg1->mr_nsegs--)
	rpcrdma_unmap_one(ia, seg++);
	if (rc)
	dprintk("RPC: %s: failed ib_dereg_mr,"
	" status %i\n", __func__, rc);
	break;		break;
	}		}
	if (r) {		if (r) {