Merge tag 'nfs-rdma-3.16' of git://git.linux-nfs.org/projects/anna/nfs-rdma into linux-next

#define RPC_MAX_SLOT_TABLE_LIMIT	(65536U)

#define RPC_MAX_SLOT_TABLE	RPC_MAX_SLOT_TABLE_LIMIT

#define RPC_CWNDSHIFT		(8U)

#define RPC_CWNDSCALE		(1U << RPC_CWNDSHIFT)

#define RPC_INITCWND		RPC_CWNDSCALE

#define RPC_MAXCWND(xprt)	((xprt)->max_reqs << RPC_CWNDSHIFT)

#define RPCXPRT_CONGESTED(xprt) ((xprt)->cong >= (xprt)->cwnd)

/*

 * This describes a timeout strategy

 */

static DEFINE_SPINLOCK(xprt_list_lock);

static LIST_HEAD(xprt_list);

/*

 * The transport code maintains an estimate on the maximum number of out-

 * standing RPC requests, using a smoothed version of the congestion

 * avoidance implemented in 44BSD. This is basically the Van Jacobson

 * congestion algorithm: If a retransmit occurs, the congestion window is

 * halved; otherwise, it is incremented by 1/cwnd when

 *

 *	-	a reply is received and

 *	-	a full number of requests are outstanding and

 *	-	the congestion window hasn't been updated recently.

 */

#define RPC_CWNDSHIFT		(8U)

#define RPC_CWNDSCALE		(1U << RPC_CWNDSHIFT)

#define RPC_INITCWND		RPC_CWNDSCALE

#define RPC_MAXCWND(xprt)	((xprt)->max_reqs << RPC_CWNDSHIFT)

#define RPCXPRT_CONGESTED(xprt) ((xprt)->cong >= (xprt)->cwnd)

/**

 * xprt_register_transport - register a transport implementation

 * @transport: transport to register

 * @task: recently completed RPC request used to adjust window

 * @result: result code of completed RPC request

 *

 * We use a time-smoothed congestion estimator to avoid heavy oscillation.

 * The transport code maintains an estimate on the maximum number of out-

 * standing RPC requests, using a smoothed version of the congestion

 * avoidance implemented in 44BSD. This is basically the Van Jacobson

 * congestion algorithm: If a retransmit occurs, the congestion window is

 * halved; otherwise, it is incremented by 1/cwnd when

 *

 *	-	a reply is received and

 *	-	a full number of requests are outstanding and

 *	-	the congestion window hasn't been updated recently.

 */

void xprt_adjust_cwnd(struct rpc_xprt *xprt, struct rpc_task *task, int result)

{

 * elements. Segments are then coalesced when registered, if possible

 * within the selected memreg mode.

 *

 * Note, this routine is never called if the connection's memory

 * registration strategy is 0 (bounce buffers).

 * Returns positive number of segments converted, or a negative errno.

 */

static int

	page_base = xdrbuf->page_base & ~PAGE_MASK;

	p = 0;

	while (len && n < nsegs) {

		if (!ppages[p]) {

			/* alloc the pagelist for receiving buffer */

			ppages[p] = alloc_page(GFP_ATOMIC);

			if (!ppages[p])

				return -ENOMEM;

		}

		seg[n].mr_page = ppages[p];

		seg[n].mr_offset = (void *)(unsigned long) page_base;

		seg[n].mr_len = min_t(u32, PAGE_SIZE - page_base, len);

		BUG_ON(seg[n].mr_len > PAGE_SIZE);

		if (seg[n].mr_len > PAGE_SIZE)

			return -EIO;

		len -= seg[n].mr_len;

		++n;

		++p;

	/* Message overflows the seg array */

	if (len && n == nsegs)

		return 0;

		return -EIO;

	if (xdrbuf->tail[0].iov_len) {

		/* the rpcrdma protocol allows us to omit any trailing

			return n;

		if (n == nsegs)

			/* Tail remains, but we're out of segments */

			return 0;

			return -EIO;

		seg[n].mr_page = NULL;

		seg[n].mr_offset = xdrbuf->tail[0].iov_base;

		seg[n].mr_len = xdrbuf->tail[0].iov_len;

 *  Reply chunk (a counted array):

 *   N elements:

 *    1 - N - HLOO - HLOO - ... - HLOO

 *

 * Returns positive RPC/RDMA header size, or negative errno.

 */

static unsigned int

static ssize_t

rpcrdma_create_chunks(struct rpc_rqst *rqst, struct xdr_buf *target,

		struct rpcrdma_msg *headerp, enum rpcrdma_chunktype type)

{

	struct rpcrdma_req *req = rpcr_to_rdmar(rqst);

	struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(rqst->rq_xprt);

	int nsegs, nchunks = 0;

	int n, nsegs, nchunks = 0;

	unsigned int pos;

	struct rpcrdma_mr_seg *seg = req->rl_segments;

	struct rpcrdma_read_chunk *cur_rchunk = NULL;

		pos = target->head[0].iov_len;

	nsegs = rpcrdma_convert_iovs(target, pos, type, seg, RPCRDMA_MAX_SEGS);

	if (nsegs == 0)

		return 0;

	if (nsegs < 0)

		return nsegs;

	do {

		/* bind/register the memory, then build chunk from result. */

		int n = rpcrdma_register_external(seg, nsegs,

		n = rpcrdma_register_external(seg, nsegs,

						cur_wchunk != NULL, r_xprt);

		if (n <= 0)

			goto out;

	/* success. all failures return above */

	req->rl_nchunks = nchunks;

	BUG_ON(nchunks == 0);

	BUG_ON((r_xprt->rx_ia.ri_memreg_strategy == RPCRDMA_FRMR)

	       && (nchunks > 3));

	/*

	 * finish off header. If write, marshal discrim and nchunks.

	 */

out:

	for (pos = 0; nchunks--;)

		pos += rpcrdma_deregister_external(

				&req->rl_segments[pos], r_xprt, NULL);

	return 0;

				&req->rl_segments[pos], r_xprt);

	return n;

}

/*

 *  [1] -- the RPC header/data, marshaled by RPC and the NFS protocol.

 *  [2] -- optional padding.

 *  [3] -- if padded, header only in [1] and data here.

 *

 * Returns zero on success, otherwise a negative errno.

 */

int

	struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);

	struct rpcrdma_req *req = rpcr_to_rdmar(rqst);

	char *base;

	size_t hdrlen, rpclen, padlen;

	size_t rpclen, padlen;

	ssize_t hdrlen;

	enum rpcrdma_chunktype rtype, wtype;

	struct rpcrdma_msg *headerp;

	/* The following simplification is not true forever */

	if (rtype != rpcrdma_noch && wtype == rpcrdma_replych)

		wtype = rpcrdma_noch;

	BUG_ON(rtype != rpcrdma_noch && wtype != rpcrdma_noch);

	if (r_xprt->rx_ia.ri_memreg_strategy == RPCRDMA_BOUNCEBUFFERS &&

	    (rtype != rpcrdma_noch || wtype != rpcrdma_noch)) {

		/* forced to "pure inline"? */

		dprintk("RPC:       %s: too much data (%d/%d) for inline\n",

			__func__, rqst->rq_rcv_buf.len, rqst->rq_snd_buf.len);

		return -1;

	if (rtype != rpcrdma_noch && wtype != rpcrdma_noch) {

		dprintk("RPC:       %s: cannot marshal multiple chunk lists\n",

			__func__);

		return -EIO;

	}

	hdrlen = 28; /*sizeof *headerp;*/

			headerp->rm_body.rm_padded.rm_pempty[1] = xdr_zero;

			headerp->rm_body.rm_padded.rm_pempty[2] = xdr_zero;

			hdrlen += 2 * sizeof(u32); /* extra words in padhdr */

			BUG_ON(wtype != rpcrdma_noch);

			if (wtype != rpcrdma_noch) {

				dprintk("RPC:       %s: invalid chunk list\n",

					__func__);

				return -EIO;

			}

		} else {

			headerp->rm_body.rm_nochunks.rm_empty[0] = xdr_zero;

			headerp->rm_body.rm_nochunks.rm_empty[1] = xdr_zero;

			 * on receive. Therefore, we request a reply chunk

			 * for non-writes wherever feasible and efficient.

			 */

			if (wtype == rpcrdma_noch &&

			    r_xprt->rx_ia.ri_memreg_strategy > RPCRDMA_REGISTER)

			if (wtype == rpcrdma_noch)

				wtype = rpcrdma_replych;

		}

	}

		hdrlen = rpcrdma_create_chunks(rqst,

					&rqst->rq_rcv_buf, headerp, wtype);

	}

	if (hdrlen == 0)

		return -1;

	if (hdrlen < 0)

		return hdrlen;

	dprintk("RPC:       %s: %s: hdrlen %zd rpclen %zd padlen %zd"

		" headerp 0x%p base 0x%p lkey 0x%x\n",

	rqst->rq_private_buf = rqst->rq_rcv_buf;

}

/*

 * This function is called when an async event is posted to

 * the connection which changes the connection state. All it

 * does at this point is mark the connection up/down, the rpc

 * timers do the rest.

 */

void

rpcrdma_conn_func(struct rpcrdma_ep *ep)

rpcrdma_connect_worker(struct work_struct *work)

{

	struct rpcrdma_ep *ep =

		container_of(work, struct rpcrdma_ep, rep_connect_worker.work);

	struct rpc_xprt *xprt = ep->rep_xprt;

	spin_lock_bh(&xprt->transport_lock);

}

/*

 * This function is called when memory window unbind which we are waiting

 * for completes. Just use rr_func (zeroed by upcall) to signal completion.

 * This function is called when an async event is posted to

 * the connection which changes the connection state. All it

 * does at this point is mark the connection up/down, the rpc

 * timers do the rest.

 */

static void

rpcrdma_unbind_func(struct rpcrdma_rep *rep)

void

rpcrdma_conn_func(struct rpcrdma_ep *ep)

{

	wake_up(&rep->rr_unbind);

	schedule_delayed_work(&ep->rep_connect_worker, 0);

}

/*

	struct rpc_xprt *xprt = rep->rr_xprt;

	struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);

	__be32 *iptr;

	int i, rdmalen, status;

	int rdmalen, status;

	unsigned long cwnd;

	/* Check status. If bad, signal disconnect and return rep to pool */

	if (rep->rr_len == ~0U) {

	/* from here on, the reply is no longer an orphan */

	req->rl_reply = rep;

	xprt->reestablish_timeout = 0;

	/* check for expected message types */

	/* The order of some of these tests is important. */

		break;

	}

	/* If using mw bind, start the deregister process now. */

	/* (Note: if mr_free(), cannot perform it here, in tasklet context) */

	if (req->rl_nchunks) switch (r_xprt->rx_ia.ri_memreg_strategy) {

	case RPCRDMA_MEMWINDOWS:

		for (i = 0; req->rl_nchunks-- > 1;)

			i += rpcrdma_deregister_external(

				&req->rl_segments[i], r_xprt, NULL);

		/* Optionally wait (not here) for unbinds to complete */

		rep->rr_func = rpcrdma_unbind_func;

		(void) rpcrdma_deregister_external(&req->rl_segments[i],

						   r_xprt, rep);

		break;

	case RPCRDMA_MEMWINDOWS_ASYNC:

		for (i = 0; req->rl_nchunks--;)

			i += rpcrdma_deregister_external(&req->rl_segments[i],

							 r_xprt, NULL);

		break;

	default:

		break;

	}

	cwnd = xprt->cwnd;

	xprt->cwnd = atomic_read(&r_xprt->rx_buf.rb_credits) << RPC_CWNDSHIFT;

	if (xprt->cwnd > cwnd)

		xprt_release_rqst_cong(rqst->rq_task);

	dprintk("RPC:       %s: xprt_complete_rqst(0x%p, 0x%p, %d)\n",

			__func__, xprt, rqst, status);

#endif

#define RPCRDMA_BIND_TO		(60U * HZ)

#define RPCRDMA_INIT_REEST_TO	(5U * HZ)

#define RPCRDMA_MAX_REEST_TO	(30U * HZ)

#define RPCRDMA_IDLE_DISC_TO	(5U * 60 * HZ)

static struct rpc_xprt_ops xprt_rdma_procs;	/* forward reference */

static void

xprt_rdma_destroy(struct rpc_xprt *xprt)

{

	struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);

	int rc;

	dprintk("RPC:       %s: called\n", __func__);

	xprt_clear_connected(xprt);

	rpcrdma_buffer_destroy(&r_xprt->rx_buf);

	rc = rpcrdma_ep_destroy(&r_xprt->rx_ep, &r_xprt->rx_ia);

	if (rc)

		dprintk("RPC:       %s: rpcrdma_ep_destroy returned %i\n",

			__func__, rc);

	rpcrdma_ep_destroy(&r_xprt->rx_ep, &r_xprt->rx_ia);

	rpcrdma_ia_close(&r_xprt->rx_ia);

	xprt_rdma_free_addresses(xprt);

	/* 60 second timeout, no retries */

	xprt->timeout = &xprt_rdma_default_timeout;

	xprt->bind_timeout = (60U * HZ);

	xprt->reestablish_timeout = (5U * HZ);

	xprt->idle_timeout = (5U * 60 * HZ);

	xprt->bind_timeout = RPCRDMA_BIND_TO;

	xprt->reestablish_timeout = RPCRDMA_INIT_REEST_TO;

	xprt->idle_timeout = RPCRDMA_IDLE_DISC_TO;

	xprt->resvport = 0;		/* privileged port not needed */

	xprt->tsh_size = 0;		/* RPC-RDMA handles framing */

	xprt_rdma_free_addresses(xprt);

	rc = -EINVAL;

out3:

	(void) rpcrdma_ep_destroy(new_ep, &new_xprt->rx_ia);

	rpcrdma_ep_destroy(new_ep, &new_xprt->rx_ia);

out2:

	rpcrdma_ia_close(&new_xprt->rx_ia);

out1:

		schedule_delayed_work(&r_xprt->rdma_connect,

			xprt->reestablish_timeout);

		xprt->reestablish_timeout <<= 1;

		if (xprt->reestablish_timeout > (30 * HZ))

			xprt->reestablish_timeout = (30 * HZ);

		else if (xprt->reestablish_timeout < (5 * HZ))

			xprt->reestablish_timeout = (5 * HZ);

		if (xprt->reestablish_timeout > RPCRDMA_MAX_REEST_TO)

			xprt->reestablish_timeout = RPCRDMA_MAX_REEST_TO;

		else if (xprt->reestablish_timeout < RPCRDMA_INIT_REEST_TO)

			xprt->reestablish_timeout = RPCRDMA_INIT_REEST_TO;

	} else {

		schedule_delayed_work(&r_xprt->rdma_connect, 0);

		if (!RPC_IS_ASYNC(task))

	}

}

static int

xprt_rdma_reserve_xprt(struct rpc_xprt *xprt, struct rpc_task *task)

{

	struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);

	int credits = atomic_read(&r_xprt->rx_buf.rb_credits);

	/* == RPC_CWNDSCALE @ init, but *after* setup */

	if (r_xprt->rx_buf.rb_cwndscale == 0UL) {

		r_xprt->rx_buf.rb_cwndscale = xprt->cwnd;

		dprintk("RPC:       %s: cwndscale %lu\n", __func__,

			r_xprt->rx_buf.rb_cwndscale);

		BUG_ON(r_xprt->rx_buf.rb_cwndscale <= 0);

	}

	xprt->cwnd = credits * r_xprt->rx_buf.rb_cwndscale;

	return xprt_reserve_xprt_cong(xprt, task);

}

/*

 * The RDMA allocate/free functions need the task structure as a place

 * to hide the struct rpcrdma_req, which is necessary for the actual send/recv

	struct rpcrdma_req *req, *nreq;

	req = rpcrdma_buffer_get(&rpcx_to_rdmax(xprt)->rx_buf);

	BUG_ON(NULL == req);

	if (req == NULL)

		return NULL;

	if (size > req->rl_size) {

		dprintk("RPC:       %s: size %zd too large for buffer[%zd]: "

		 * If the allocation or registration fails, the RPC framework

		 * will (doggedly) retry.

		 */

		if (rpcx_to_rdmax(xprt)->rx_ia.ri_memreg_strategy ==

				RPCRDMA_BOUNCEBUFFERS) {

			/* forced to "pure inline" */

			dprintk("RPC:       %s: too much data (%zd) for inline "

					"(r/w max %d/%d)\n", __func__, size,

					rpcx_to_rdmad(xprt).inline_rsize,

					rpcx_to_rdmad(xprt).inline_wsize);

			size = req->rl_size;

			rpc_exit(task, -EIO);		/* fail the operation */

			rpcx_to_rdmax(xprt)->rx_stats.failed_marshal_count++;

			goto out;

		}

		if (task->tk_flags & RPC_TASK_SWAPPER)

			nreq = kmalloc(sizeof *req + size, GFP_ATOMIC);

		else

		req = nreq;

	}

	dprintk("RPC:       %s: size %zd, request 0x%p\n", __func__, size, req);

out:

	req->rl_connect_cookie = 0;	/* our reserved value */

	return req->rl_xdr_buf;

		__func__, rep, (rep && rep->rr_func) ? " (with waiter)" : "");

	/*

	 * Finish the deregistration. When using mw bind, this was

	 * begun in rpcrdma_reply_handler(). In all other modes, we

	 * do it here, in thread context. The process is considered

	 * Finish the deregistration.  The process is considered

	 * complete when the rr_func vector becomes NULL - this

	 * was put in place during rpcrdma_reply_handler() - the wait

	 * call below will not block if the dereg is "done". If

	for (i = 0; req->rl_nchunks;) {

		--req->rl_nchunks;

		i += rpcrdma_deregister_external(

			&req->rl_segments[i], r_xprt, NULL);

	}

	if (rep && wait_event_interruptible(rep->rr_unbind, !rep->rr_func)) {

		rep->rr_func = NULL;	/* abandon the callback */

		req->rl_reply = NULL;

			&req->rl_segments[i], r_xprt);

	}

	if (req->rl_iov.length == 0) {	/* see allocate above */

	struct rpc_xprt *xprt = rqst->rq_xprt;

	struct rpcrdma_req *req = rpcr_to_rdmar(rqst);

	struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);

	int rc;

	/* marshal the send itself */

	if (req->rl_niovs == 0 && rpcrdma_marshal_req(rqst) != 0) {

		r_xprt->rx_stats.failed_marshal_count++;

		dprintk("RPC:       %s: rpcrdma_marshal_req failed\n",

			__func__);

		return -EIO;

	if (req->rl_niovs == 0) {

		rc = rpcrdma_marshal_req(rqst);

		if (rc < 0)

			goto failed_marshal;

	}

	if (req->rl_reply == NULL) 		/* e.g. reconnection */

	rqst->rq_bytes_sent = 0;

	return 0;

failed_marshal:

	r_xprt->rx_stats.failed_marshal_count++;

	dprintk("RPC:       %s: rpcrdma_marshal_req failed, status %i\n",

		__func__, rc);

	if (rc == -EIO)

		return -EIO;

drop_connection:

	xprt_disconnect_done(xprt);

	return -ENOTCONN;	/* implies disconnect */

 */

static struct rpc_xprt_ops xprt_rdma_procs = {

	.reserve_xprt		= xprt_rdma_reserve_xprt,

	.reserve_xprt		= xprt_reserve_xprt_cong,

	.release_xprt		= xprt_release_xprt_cong, /* sunrpc/xprt.c */

	.alloc_slot		= xprt_alloc_slot,

	.release_request	= xprt_release_rqst_cong,       /* ditto */