Merge tag 'nfs-rdma-4.12-1' of git://git.linux-nfs.org/projects/anna/nfs-rdma

EXPORT_SYMBOL_GPL(xdr_init_decode);

/**

 * xdr_init_decode - Initialize an xdr_stream for decoding data.

 * xdr_init_decode_pages - Initialize an xdr_stream for decoding into pages

 * @xdr: pointer to xdr_stream struct

 * @buf: pointer to XDR buffer from which to decode data

 * @pages: list of pages to decode into

	xprt_wake_pending_tasks(xprt, -EAGAIN);

	spin_unlock_bh(&xprt->transport_lock);

}

EXPORT_SYMBOL_GPL(xprt_force_disconnect);

/**

 * xprt_conditional_disconnect - force a transport to disconnect

	}

	sge->length = len;

	ib_dma_sync_single_for_device(ia->ri_device, sge->addr,

	ib_dma_sync_single_for_device(rdmab_device(rb), sge->addr,

				      sge->length, DMA_TO_DEVICE);

	req->rl_send_wr.num_sge++;

	return true;

	sge[sge_no].addr = rdmab_addr(rb);

	sge[sge_no].length = xdr->head[0].iov_len;

	sge[sge_no].lkey = rdmab_lkey(rb);

	ib_dma_sync_single_for_device(device, sge[sge_no].addr,

	ib_dma_sync_single_for_device(rdmab_device(rb), sge[sge_no].addr,

				      sge[sge_no].length, DMA_TO_DEVICE);

	/* If there is a Read chunk, the page list is being handled

	return 0;

out_err:

	if (PTR_ERR(iptr) != -ENOBUFS) {

		pr_err("rpcrdma: rpcrdma_marshal_req failed, status %ld\n",

		       PTR_ERR(iptr));

		r_xprt->rx_stats.failed_marshal_count++;

	}

	return PTR_ERR(iptr);

}

unsigned int xprt_rdma_max_inline_read = RPCRDMA_DEF_INLINE;

static unsigned int xprt_rdma_max_inline_write = RPCRDMA_DEF_INLINE;

static unsigned int xprt_rdma_inline_write_padding;

static unsigned int xprt_rdma_memreg_strategy = RPCRDMA_FRMR;

		int xprt_rdma_pad_optimize = 0;

unsigned int xprt_rdma_memreg_strategy		= RPCRDMA_FRMR;

int xprt_rdma_pad_optimize;

#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)

	new_xprt = rpcx_to_rdmax(xprt);

	rc = rpcrdma_ia_open(new_xprt, sap, xprt_rdma_memreg_strategy);

	rc = rpcrdma_ia_open(new_xprt, sap);

	if (rc)

		goto out1;

	return ERR_PTR(rc);

}

/*

 * Close a connection, during shutdown or timeout/reconnect

/**

 * xprt_rdma_close - Close down RDMA connection

 * @xprt: generic transport to be closed

 *

 * Called during transport shutdown reconnect, or device

 * removal. Caller holds the transport's write lock.

 */

static void

xprt_rdma_close(struct rpc_xprt *xprt)

{

	struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);

	struct rpcrdma_ep *ep = &r_xprt->rx_ep;

	struct rpcrdma_ia *ia = &r_xprt->rx_ia;

	dprintk("RPC:       %s: closing xprt %p\n", __func__, xprt);

	dprintk("RPC:       %s: closing\n", __func__);

	if (r_xprt->rx_ep.rep_connected > 0)

	if (test_and_clear_bit(RPCRDMA_IAF_REMOVING, &ia->ri_flags)) {

		xprt_clear_connected(xprt);

		rpcrdma_ia_remove(ia);

		return;

	}

	if (ep->rep_connected == -ENODEV)

		return;

	if (ep->rep_connected > 0)

		xprt->reestablish_timeout = 0;

	xprt_disconnect_done(xprt);

	rpcrdma_ep_disconnect(&r_xprt->rx_ep, &r_xprt->rx_ia);

	rpcrdma_ep_disconnect(ep, ia);

}

static void

	dprintk("RPC:       %s: %u\n", __func__, port);

}

/**

 * xprt_rdma_timer - invoked when an RPC times out

 * @xprt: controlling RPC transport

 * @task: RPC task that timed out

 *

 * Invoked when the transport is still connected, but an RPC

 * retransmit timeout occurs.

 *

 * Since RDMA connections don't have a keep-alive, forcibly

 * disconnect and retry to connect. This drives full

 * detection of the network path, and retransmissions of

 * all pending RPCs.

 */

static void

xprt_rdma_timer(struct rpc_xprt *xprt, struct rpc_task *task)

{

	dprintk("RPC: %5u %s: xprt = %p\n", task->tk_pid, __func__, xprt);

	xprt_force_disconnect(xprt);

}

static void

xprt_rdma_connect(struct rpc_xprt *xprt, struct rpc_task *task)

{

 * xprt_rdma_send_request - marshal and send an RPC request

 * @task: RPC task with an RPC message in rq_snd_buf

 *

 * Caller holds the transport's write lock.

 *

 * Return values:

 *        0:	The request has been sent

 * ENOTCONN:	Caller needs to invoke connect logic then call again

	struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);

	int rc = 0;

	if (!xprt_connected(xprt))

		goto drop_connection;

	/* On retransmit, remove any previously registered chunks */

	if (unlikely(!list_empty(&req->rl_registered)))

		r_xprt->rx_ia.ri_ops->ro_unmap_safe(r_xprt, req, false);

	.alloc_slot		= xprt_alloc_slot,

	.release_request	= xprt_release_rqst_cong,       /* ditto */

	.set_retrans_timeout	= xprt_set_retrans_timeout_def, /* ditto */

	.timer			= xprt_rdma_timer,

	.rpcbind		= rpcb_getport_async,	/* sunrpc/rpcb_clnt.c */

	.set_port		= xprt_rdma_set_port,

	.connect		= xprt_rdma_connect,

#include <linux/sunrpc/addr.h>

#include <linux/sunrpc/svc_rdma.h>

#include <asm/bitops.h>

#include <linux/module.h> /* try_module_get()/module_put() */

#include <rdma/ib_cm.h>

#include "xprt_rdma.h"

/*

 * internal functions

 */

static void rpcrdma_create_mrs(struct rpcrdma_xprt *r_xprt);

static void rpcrdma_destroy_mrs(struct rpcrdma_buffer *buf);

static void rpcrdma_dma_unmap_regbuf(struct rpcrdma_regbuf *rb);

static struct workqueue_struct *rpcrdma_receive_wq;

static struct workqueue_struct *rpcrdma_receive_wq __read_mostly;

int

rpcrdma_alloc_wq(void)

	rep->rr_wc_flags = wc->wc_flags;

	rep->rr_inv_rkey = wc->ex.invalidate_rkey;

	ib_dma_sync_single_for_cpu(rep->rr_device,

	ib_dma_sync_single_for_cpu(rdmab_device(rep->rr_rdmabuf),

				   rdmab_addr(rep->rr_rdmabuf),

				   rep->rr_len, DMA_FROM_DEVICE);

			__func__, ep);

		complete(&ia->ri_done);

		break;

	case RDMA_CM_EVENT_DEVICE_REMOVAL:

#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)

		pr_info("rpcrdma: removing device for %pIS:%u\n",

			sap, rpc_get_port(sap));

#endif

		set_bit(RPCRDMA_IAF_REMOVING, &ia->ri_flags);

		ep->rep_connected = -ENODEV;

		xprt_force_disconnect(&xprt->rx_xprt);

		wait_for_completion(&ia->ri_remove_done);

		ia->ri_id = NULL;

		ia->ri_pd = NULL;

		ia->ri_device = NULL;

		/* Return 1 to ensure the core destroys the id. */

		return 1;

	case RDMA_CM_EVENT_ESTABLISHED:

		connstate = 1;

		ib_query_qp(ia->ri_id->qp, attr,

		goto connected;

	case RDMA_CM_EVENT_DISCONNECTED:

		connstate = -ECONNABORTED;

		goto connected;

	case RDMA_CM_EVENT_DEVICE_REMOVAL:

		connstate = -ENODEV;

connected:

		dprintk("RPC:       %s: %sconnected\n",

					__func__, connstate > 0 ? "" : "dis");

	return 0;

}

static void rpcrdma_destroy_id(struct rdma_cm_id *id)

{

	if (id) {

		module_put(id->device->owner);

		rdma_destroy_id(id);

	}

}

static struct rdma_cm_id *

rpcrdma_create_id(struct rpcrdma_xprt *xprt,

			struct rpcrdma_ia *ia, struct sockaddr *addr)

	int rc;

	init_completion(&ia->ri_done);

	init_completion(&ia->ri_remove_done);

	id = rdma_create_id(&init_net, rpcrdma_conn_upcall, xprt, RDMA_PS_TCP,

			    IB_QPT_RC);

		goto out;

	}

	/* FIXME:

	 * Until xprtrdma supports DEVICE_REMOVAL, the provider must

	 * be pinned while there are active NFS/RDMA mounts to prevent

	 * hangs and crashes at umount time.

	 */

	if (!ia->ri_async_rc && !try_module_get(id->device->owner)) {

		dprintk("RPC:       %s: Failed to get device module\n",

			__func__);

		ia->ri_async_rc = -ENODEV;

	}

	rc = ia->ri_async_rc;

	if (rc)

		goto out;

	if (rc) {

		dprintk("RPC:       %s: rdma_resolve_route() failed %i\n",

			__func__, rc);

		goto put;

		goto out;

	}

	rc = wait_for_completion_interruptible_timeout(&ia->ri_done, wtimeout);

	if (rc < 0) {

		dprintk("RPC:       %s: wait() exited: %i\n",

			__func__, rc);

		goto put;

		goto out;

	}

	rc = ia->ri_async_rc;

	if (rc)

		goto put;

		goto out;

	return id;

put:

	module_put(id->device->owner);

out:

	rdma_destroy_id(id);

	return ERR_PTR(rc);

 * Exported functions.

 */

/*

 * Open and initialize an Interface Adapter.

 *  o initializes fields of struct rpcrdma_ia, including

 *    interface and provider attributes and protection zone.

/**

 * rpcrdma_ia_open - Open and initialize an Interface Adapter.

 * @xprt: controlling transport

 * @addr: IP address of remote peer

 *

 * Returns 0 on success, negative errno if an appropriate

 * Interface Adapter could not be found and opened.

 */

int

rpcrdma_ia_open(struct rpcrdma_xprt *xprt, struct sockaddr *addr, int memreg)

rpcrdma_ia_open(struct rpcrdma_xprt *xprt, struct sockaddr *addr)

{

	struct rpcrdma_ia *ia = &xprt->rx_ia;

	int rc;

	ia->ri_id = rpcrdma_create_id(xprt, ia, addr);

	if (IS_ERR(ia->ri_id)) {

		rc = PTR_ERR(ia->ri_id);

		goto out1;

		goto out_err;

	}

	ia->ri_device = ia->ri_id->device;

	if (IS_ERR(ia->ri_pd)) {

		rc = PTR_ERR(ia->ri_pd);

		pr_err("rpcrdma: ib_alloc_pd() returned %d\n", rc);

		goto out2;

		goto out_err;

	}

	switch (memreg) {

	switch (xprt_rdma_memreg_strategy) {

	case RPCRDMA_FRMR:

		if (frwr_is_supported(ia)) {

			ia->ri_ops = &rpcrdma_frwr_memreg_ops;

		}

		/*FALLTHROUGH*/

	default:

		pr_err("rpcrdma: Unsupported memory registration mode: %d\n",

		       memreg);

		pr_err("rpcrdma: Device %s does not support memreg mode %d\n",

		       ia->ri_device->name, xprt_rdma_memreg_strategy);

		rc = -EINVAL;

		goto out3;

		goto out_err;

	}

	return 0;

out3:

	ib_dealloc_pd(ia->ri_pd);

	ia->ri_pd = NULL;

out2:

	rpcrdma_destroy_id(ia->ri_id);

	ia->ri_id = NULL;

out1:

out_err:

	rpcrdma_ia_close(ia);

	return rc;

}

/*

 * Clean up/close an IA.

 *   o if event handles and PD have been initialized, free them.

 *   o close the IA

/**

 * rpcrdma_ia_remove - Handle device driver unload

 * @ia: interface adapter being removed

 *

 * Divest transport H/W resources associated with this adapter,

 * but allow it to be restored later.

 */

void

rpcrdma_ia_remove(struct rpcrdma_ia *ia)

{

	struct rpcrdma_xprt *r_xprt = container_of(ia, struct rpcrdma_xprt,

						   rx_ia);

	struct rpcrdma_ep *ep = &r_xprt->rx_ep;

	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;

	struct rpcrdma_req *req;

	struct rpcrdma_rep *rep;

	cancel_delayed_work_sync(&buf->rb_refresh_worker);

	/* This is similar to rpcrdma_ep_destroy, but:

	 * - Don't cancel the connect worker.

	 * - Don't call rpcrdma_ep_disconnect, which waits

	 *   for another conn upcall, which will deadlock.

	 * - rdma_disconnect is unneeded, the underlying

	 *   connection is already gone.

	 */

	if (ia->ri_id->qp) {

		ib_drain_qp(ia->ri_id->qp);

		rdma_destroy_qp(ia->ri_id);

		ia->ri_id->qp = NULL;

	}

	ib_free_cq(ep->rep_attr.recv_cq);

	ib_free_cq(ep->rep_attr.send_cq);

	/* The ULP is responsible for ensuring all DMA

	 * mappings and MRs are gone.

	 */

	list_for_each_entry(rep, &buf->rb_recv_bufs, rr_list)

		rpcrdma_dma_unmap_regbuf(rep->rr_rdmabuf);

	list_for_each_entry(req, &buf->rb_allreqs, rl_all) {

		rpcrdma_dma_unmap_regbuf(req->rl_rdmabuf);

		rpcrdma_dma_unmap_regbuf(req->rl_sendbuf);

		rpcrdma_dma_unmap_regbuf(req->rl_recvbuf);

	}

	rpcrdma_destroy_mrs(buf);

	/* Allow waiters to continue */

	complete(&ia->ri_remove_done);

}

/**

 * rpcrdma_ia_close - Clean up/close an IA.

 * @ia: interface adapter to close

 *

 */

void

rpcrdma_ia_close(struct rpcrdma_ia *ia)

	if (ia->ri_id != NULL && !IS_ERR(ia->ri_id)) {

		if (ia->ri_id->qp)

			rdma_destroy_qp(ia->ri_id);

		rpcrdma_destroy_id(ia->ri_id);

		ia->ri_id = NULL;

		rdma_destroy_id(ia->ri_id);

	}

	ia->ri_id = NULL;

	ia->ri_device = NULL;

	/* If the pd is still busy, xprtrdma missed freeing a resource */

	if (ia->ri_pd && !IS_ERR(ia->ri_pd))

		ib_dealloc_pd(ia->ri_pd);

	ia->ri_pd = NULL;

}

/*

	ib_free_cq(ep->rep_attr.send_cq);

}

/*

 * Connect unconnected endpoint.

/* Re-establish a connection after a device removal event.

 * Unlike a normal reconnection, a fresh PD and a new set

 * of MRs and buffers is needed.

 */

int

rpcrdma_ep_connect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)

static int

rpcrdma_ep_recreate_xprt(struct rpcrdma_xprt *r_xprt,

			 struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)

{

	struct rpcrdma_xprt *r_xprt = container_of(ia, struct rpcrdma_xprt,

						   rx_ia);

	struct sockaddr *sap = (struct sockaddr *)&r_xprt->rx_data.addr;

	int rc, err;

	pr_info("%s: r_xprt = %p\n", __func__, r_xprt);

	rc = -EHOSTUNREACH;

	if (rpcrdma_ia_open(r_xprt, sap))

		goto out1;

	rc = -ENOMEM;

	err = rpcrdma_ep_create(ep, ia, &r_xprt->rx_data);

	if (err) {

		pr_err("rpcrdma: rpcrdma_ep_create returned %d\n", err);

		goto out2;

	}

	rc = -ENETUNREACH;

	err = rdma_create_qp(ia->ri_id, ia->ri_pd, &ep->rep_attr);

	if (err) {

		pr_err("rpcrdma: rdma_create_qp returned %d\n", err);

		goto out3;

	}

	rpcrdma_create_mrs(r_xprt);

	return 0;

out3:

	rpcrdma_ep_destroy(ep, ia);

out2:

	rpcrdma_ia_close(ia);

out1:

	return rc;

}

static int

rpcrdma_ep_reconnect(struct rpcrdma_xprt *r_xprt, struct rpcrdma_ep *ep,

		     struct rpcrdma_ia *ia)

{

	struct sockaddr *sap = (struct sockaddr *)&r_xprt->rx_data.addr;

	struct rdma_cm_id *id, *old;

	struct sockaddr *sap;

	unsigned int extras;

	int rc = 0;

	int err, rc;

	if (ep->rep_connected != 0) {

retry:

	dprintk("RPC:       %s: reconnecting...\n", __func__);

	rpcrdma_ep_disconnect(ep, ia);

		sap = (struct sockaddr *)&r_xprt->rx_data.addr;

		id = rpcrdma_create_id(r_xprt, ia, sap);

		if (IS_ERR(id)) {

	rc = -EHOSTUNREACH;

	id = rpcrdma_create_id(r_xprt, ia, sap);

	if (IS_ERR(id))

		goto out;

		}

		/* TEMP TEMP TEMP - fail if new device:

		 * Deregister/remarshal *all* requests!

		 * Close and recreate adapter, pd, etc!

		 * Re-determine all attributes still sane!

		 * More stuff I haven't thought of!

		 * Rrrgh!

	/* As long as the new ID points to the same device as the

	 * old ID, we can reuse the transport's existing PD and all

	 * previously allocated MRs. Also, the same device means

	 * the transport's previous DMA mappings are still valid.

	 *

	 * This is a sanity check only. There should be no way these

	 * point to two different devices here.

	 */

		if (ia->ri_device != id->device) {

			printk("RPC:       %s: can't reconnect on "

				"different device!\n", __func__);

			rpcrdma_destroy_id(id);

	old = id;

	rc = -ENETUNREACH;

			goto out;

	if (ia->ri_device != id->device) {

		pr_err("rpcrdma: can't reconnect on different device!\n");

		goto out_destroy;

	}

		/* END TEMP */

		rc = rdma_create_qp(id, ia->ri_pd, &ep->rep_attr);

		if (rc) {

			dprintk("RPC:       %s: rdma_create_qp failed %i\n",

				__func__, rc);

			rpcrdma_destroy_id(id);

			rc = -ENETUNREACH;

			goto out;

	err = rdma_create_qp(id, ia->ri_pd, &ep->rep_attr);

	if (err) {

		dprintk("RPC:       %s: rdma_create_qp returned %d\n",

			__func__, err);

		goto out_destroy;

	}

	/* Atomically replace the transport's ID and QP. */

	rc = 0;

	old = ia->ri_id;

	ia->ri_id = id;

	rdma_destroy_qp(old);

		rpcrdma_destroy_id(old);

	} else {

out_destroy:

	rdma_destroy_id(old);

out:

	return rc;

}

/*

 * Connect unconnected endpoint.

 */

int

rpcrdma_ep_connect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)

{

	struct rpcrdma_xprt *r_xprt = container_of(ia, struct rpcrdma_xprt,

						   rx_ia);

	unsigned int extras;

	int rc;

retry:

	switch (ep->rep_connected) {

	case 0:

		dprintk("RPC:       %s: connecting...\n", __func__);

		rc = rdma_create_qp(ia->ri_id, ia->ri_pd, &ep->rep_attr);

		if (rc) {

			dprintk("RPC:       %s: rdma_create_qp failed %i\n",

				__func__, rc);

			/* do not update ep->rep_connected */

			return -ENETUNREACH;

			rc = -ENETUNREACH;

			goto out_noupdate;

		}

		break;

	case -ENODEV:

		rc = rpcrdma_ep_recreate_xprt(r_xprt, ep, ia);

		if (rc)

			goto out_noupdate;

		break;

	default:

		rc = rpcrdma_ep_reconnect(r_xprt, ep, ia);

		if (rc)

			goto out;

	}

	ep->rep_connected = 0;

out:

	if (rc)

		ep->rep_connected = rc;

out_noupdate:

	return rc;

}

rpcrdma_create_rep(struct rpcrdma_xprt *r_xprt)

{

	struct rpcrdma_create_data_internal *cdata = &r_xprt->rx_data;

	struct rpcrdma_ia *ia = &r_xprt->rx_ia;

	struct rpcrdma_rep *rep;

	int rc;

		goto out_free;

	}

	rep->rr_device = ia->ri_device;

	rep->rr_cqe.done = rpcrdma_wc_receive;

	rep->rr_rxprt = r_xprt;

	INIT_WORK(&rep->rr_work, rpcrdma_reply_handler);

rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf)

{

	cancel_delayed_work_sync(&buf->rb_recovery_worker);

	cancel_delayed_work_sync(&buf->rb_refresh_worker);

	while (!list_empty(&buf->rb_recv_bufs)) {

		struct rpcrdma_rep *rep;

out_nomws:

	dprintk("RPC:       %s: no MWs available\n", __func__);

	if (r_xprt->rx_ep.rep_connected != -ENODEV)

		schedule_delayed_work(&buf->rb_refresh_worker, 0);

	/* Allow the reply handler and refresh worker to run */

bool

__rpcrdma_dma_map_regbuf(struct rpcrdma_ia *ia, struct rpcrdma_regbuf *rb)

{

	struct ib_device *device = ia->ri_device;