RDS: Common RDMA transport code

/*

 * Copyright (c) 2009 Oracle.  All rights reserved.

 *

 * This software is available to you under a choice of one of two

 * licenses.  You may choose to be licensed under the terms of the GNU

 * General Public License (GPL) Version 2, available from the file

 * COPYING in the main directory of this source tree, or the

 * OpenIB.org BSD license below:

 *

 *     Redistribution and use in source and binary forms, with or

 *     without modification, are permitted provided that the following

 *     conditions are met:

 *

 *      - Redistributions of source code must retain the above

 *        copyright notice, this list of conditions and the following

 *        disclaimer.

 *

 *      - Redistributions in binary form must reproduce the above

 *        copyright notice, this list of conditions and the following

 *        disclaimer in the documentation and/or other materials

 *        provided with the distribution.

 *

 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,

 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF

 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND

 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS

 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN

 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN

 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

 * SOFTWARE.

 *

 */

#include <rdma/rdma_cm.h>

#include "rdma_transport.h"

static struct rdma_cm_id *rds_iw_listen_id;

int rds_rdma_cm_event_handler(struct rdma_cm_id *cm_id,

			      struct rdma_cm_event *event)

{

	/* this can be null in the listening path */

	struct rds_connection *conn = cm_id->context;

	struct rds_transport *trans;

	int ret = 0;

	rdsdebug("conn %p id %p handling event %u\n", conn, cm_id,

		 event->event);

	if (cm_id->device->node_type == RDMA_NODE_RNIC)

		trans = &rds_iw_transport;

	else

		trans = &rds_ib_transport;

	/* Prevent shutdown from tearing down the connection

	 * while we're executing. */

	if (conn) {

		mutex_lock(&conn->c_cm_lock);

		/* If the connection is being shut down, bail out

		 * right away. We return 0 so cm_id doesn't get

		 * destroyed prematurely */

		if (rds_conn_state(conn) == RDS_CONN_DISCONNECTING) {

			/* Reject incoming connections while we're tearing

			 * down an existing one. */

			if (event->event == RDMA_CM_EVENT_CONNECT_REQUEST)

				ret = 1;

			goto out;

		}

	}

	switch (event->event) {

	case RDMA_CM_EVENT_CONNECT_REQUEST:

		ret = trans->cm_handle_connect(cm_id, event);

		break;

	case RDMA_CM_EVENT_ADDR_RESOLVED:

		/* XXX do we need to clean up if this fails? */

		ret = rdma_resolve_route(cm_id,

					 RDS_RDMA_RESOLVE_TIMEOUT_MS);

		break;

	case RDMA_CM_EVENT_ROUTE_RESOLVED:

		/* XXX worry about racing with listen acceptance */

		ret = trans->cm_initiate_connect(cm_id);

		break;

	case RDMA_CM_EVENT_ESTABLISHED:

		trans->cm_connect_complete(conn, event);

		break;

	case RDMA_CM_EVENT_ADDR_ERROR:

	case RDMA_CM_EVENT_ROUTE_ERROR:

	case RDMA_CM_EVENT_CONNECT_ERROR:

	case RDMA_CM_EVENT_UNREACHABLE:

	case RDMA_CM_EVENT_REJECTED:

	case RDMA_CM_EVENT_DEVICE_REMOVAL:

	case RDMA_CM_EVENT_ADDR_CHANGE:

		if (conn)

			rds_conn_drop(conn);

		break;

	case RDMA_CM_EVENT_DISCONNECTED:

		printk(KERN_WARNING "RDS/IW: DISCONNECT event - dropping connection "

			"%pI4->%pI4\n", &conn->c_laddr,

			 &conn->c_faddr);

		rds_conn_drop(conn);

		break;

	default:

		/* things like device disconnect? */

		printk(KERN_ERR "unknown event %u\n", event->event);

		BUG();

		break;

	}

out:

	if (conn)

		mutex_unlock(&conn->c_cm_lock);

	rdsdebug("id %p event %u handling ret %d\n", cm_id, event->event, ret);

	return ret;

}

static int __init rds_rdma_listen_init(void)

{

	struct sockaddr_in sin;

	struct rdma_cm_id *cm_id;

	int ret;

	cm_id = rdma_create_id(rds_rdma_cm_event_handler, NULL, RDMA_PS_TCP);

	if (IS_ERR(cm_id)) {

		ret = PTR_ERR(cm_id);

		printk(KERN_ERR "RDS/IW: failed to setup listener, "

		       "rdma_create_id() returned %d\n", ret);

		goto out;

	}

	sin.sin_family = PF_INET,

	sin.sin_addr.s_addr = (__force u32)htonl(INADDR_ANY);

	sin.sin_port = (__force u16)htons(RDS_PORT);

	/*

	 * XXX I bet this binds the cm_id to a device.  If we want to support

	 * fail-over we'll have to take this into consideration.

	 */

	ret = rdma_bind_addr(cm_id, (struct sockaddr *)&sin);

	if (ret) {

		printk(KERN_ERR "RDS/IW: failed to setup listener, "

		       "rdma_bind_addr() returned %d\n", ret);

		goto out;

	}

	ret = rdma_listen(cm_id, 128);

	if (ret) {

		printk(KERN_ERR "RDS/IW: failed to setup listener, "

		       "rdma_listen() returned %d\n", ret);

		goto out;

	}

	rdsdebug("cm %p listening on port %u\n", cm_id, RDS_PORT);

	rds_iw_listen_id = cm_id;

	cm_id = NULL;

out:

	if (cm_id)

		rdma_destroy_id(cm_id);

	return ret;

}

static void rds_rdma_listen_stop(void)

{

	if (rds_iw_listen_id) {

		rdsdebug("cm %p\n", rds_iw_listen_id);

		rdma_destroy_id(rds_iw_listen_id);

		rds_iw_listen_id = NULL;

	}

}

int __init rds_rdma_init(void)

{

	int ret;

	ret = rds_rdma_listen_init();

	if (ret)

		goto out;

	ret = rds_iw_init();

	if (ret)

		goto err_iw_init;

	ret = rds_ib_init();

	if (ret)

		goto err_ib_init;

	goto out;

err_ib_init:

	rds_iw_exit();

err_iw_init:

	rds_rdma_listen_stop();

out:

	return ret;

}

void rds_rdma_exit(void)

{

	/* stop listening first to ensure no new connections are attempted */

	rds_rdma_listen_stop();

	rds_ib_exit();

	rds_iw_exit();

}

#ifndef _RDMA_TRANSPORT_H

#define _RDMA_TRANSPORT_H

#include <rdma/ib_verbs.h>

#include <rdma/rdma_cm.h>

#include "rds.h"

#define RDS_RDMA_RESOLVE_TIMEOUT_MS     5000

int rds_rdma_conn_connect(struct rds_connection *conn);

int rds_rdma_cm_event_handler(struct rdma_cm_id *cm_id,

			      struct rdma_cm_event *event);

/* from rdma_transport.c */

int rds_rdma_init(void);

void rds_rdma_exit(void);

/* from ib.c */

extern struct rds_transport rds_ib_transport;

int rds_ib_init(void);

void rds_ib_exit(void);

/* from iw.c */

extern struct rds_transport rds_iw_transport;

int rds_iw_init(void);

void rds_iw_exit(void);

#endif