Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 52bedf37 authored by Karsten Graul's avatar Karsten Graul Committed by David S. Miller
Browse files

net/smc: process add/delete link messages 



Add initial support for the LLC messages ADD LINK and DELETE LINK.
Introduce a link state field. Extend the initial LLC handshake with
ADD LINK processing.

Signed-off-by: default avatarKarsten Graul <kgraul@linux.vnet.ibm.com>
Signed-off-by: default avatarUrsula Braun <ubraun@linux.vnet.ibm.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 75d320d6

net/smc/af_smc.c

+42 −0
Original line number Diff line number Diff line
@@ -313,6 +313,27 @@ static int smc_clnt_conf_first_link(struct smc_sock *smc) 
	if (rc < 0)

		return SMC_CLC_DECL_TCL;



	/* receive ADD LINK request from server over RoCE fabric */

	rest = wait_for_completion_interruptible_timeout(&link->llc_add,

							 SMC_LLC_WAIT_TIME);

	if (rest <= 0) {

		struct smc_clc_msg_decline dclc;



		rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),

				      SMC_CLC_DECLINE);

		return rc;

	}



	/* send add link reject message, only one link supported for now */

	rc = smc_llc_send_add_link(link,

				   link->smcibdev->mac[link->ibport - 1],

				   &link->smcibdev->gid[link->ibport - 1],

				   SMC_LLC_RESP);

	if (rc < 0)

		return SMC_CLC_DECL_TCL;



	link->state = SMC_LNK_ACTIVE;



	return 0;

}
@@ -714,6 +735,27 @@ static int smc_serv_conf_first_link(struct smc_sock *smc) 
	if (link->llc_confirm_resp_rc)

		return SMC_CLC_DECL_RMBE_EC;



	/* send ADD LINK request to client over the RoCE fabric */

	rc = smc_llc_send_add_link(link,

				   link->smcibdev->mac[link->ibport - 1],

				   &link->smcibdev->gid[link->ibport - 1],

				   SMC_LLC_REQ);

	if (rc < 0)

		return SMC_CLC_DECL_TCL;



	/* receive ADD LINK response from client over the RoCE fabric */

	rest = wait_for_completion_interruptible_timeout(&link->llc_add_resp,

							 SMC_LLC_WAIT_TIME);

	if (rest <= 0) {

		struct smc_clc_msg_decline dclc;



		rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),

				      SMC_CLC_DECLINE);

		return rc;

	}



	link->state = SMC_LNK_ACTIVE;



	return 0;

}

net/smc/smc_core.c

+3 −0
Original line number Diff line number Diff line
@@ -176,6 +176,7 @@ static int smc_lgr_create(struct smc_sock *smc, 


	lnk = &lgr->lnk[SMC_SINGLE_LINK];

	/* initialize link */

	lnk->state = SMC_LNK_ACTIVATING;

	lnk->smcibdev = smcibdev;

	lnk->ibport = ibport;

	lnk->path_mtu = smcibdev->pattr[ibport - 1].active_mtu;
@@ -197,6 +198,8 @@ static int smc_lgr_create(struct smc_sock *smc, 
		goto destroy_qp;

	init_completion(&lnk->llc_confirm);

	init_completion(&lnk->llc_confirm_resp);

	init_completion(&lnk->llc_add);

	init_completion(&lnk->llc_add_resp);



	smc->conn.lgr = lgr;

	rwlock_init(&lgr->conns_lock);

net/smc/smc_core.h

+10 −0
Original line number Diff line number Diff line
@@ -32,6 +32,12 @@ enum smc_lgr_role { /* possible roles of a link group */ 
	SMC_SERV	/* server */

};



enum smc_link_state {			/* possible states of a link */

	SMC_LNK_INACTIVE,	/* link is inactive */

	SMC_LNK_ACTIVATING,	/* link is being activated */

	SMC_LNK_ACTIVE		/* link is active */

};



#define SMC_WR_BUF_SIZE		48	/* size of work request buffer */



struct smc_wr_buf {
@@ -87,10 +93,14 @@ struct smc_link { 
	u8			peer_mac[ETH_ALEN];	/* = gid[8:10||13:15] */

	u8			peer_gid[sizeof(union ib_gid)];	/* gid of peer*/

	u8			link_id;	/* unique # within link group */



	enum smc_link_state	state;		/* state of link */

	struct completion	llc_confirm;	/* wait for rx of conf link */

	struct completion	llc_confirm_resp; /* wait 4 rx of cnf lnk rsp */

	int			llc_confirm_rc; /* rc from confirm link msg */

	int			llc_confirm_resp_rc; /* rc from conf_resp msg */

	struct completion	llc_add;	/* wait for rx of add link */

	struct completion	llc_add_resp;	/* wait for rx of add link rsp*/

};



/* For now we just allow one parallel link per link group. The SMC protocol

net/smc/smc_llc.c

+161 −7
Original line number Diff line number Diff line
@@ -4,9 +4,6 @@ 
 *

 *  Link Layer Control (LLC)

 *

 *  For now, we only support the necessary "confirm link" functionality

 *  which happens for the first RoCE link after successful CLC handshake.

 *

 *  Copyright IBM Corp. 2016

 *

 *  Author(s):  Klaus Wacker <Klaus.Wacker@de.ibm.com>
@@ -26,7 +23,13 @@ 
struct smc_llc_hdr {

	struct smc_wr_rx_hdr common;

	u8 length;	/* 44 */

	u8 reserved;

#if defined(__BIG_ENDIAN_BITFIELD)

	u8 reserved:4,

	   add_link_rej_rsn:4;

#elif defined(__LITTLE_ENDIAN_BITFIELD)

	u8 add_link_rej_rsn:4,

	   reserved:4;

#endif

	u8 flags;

};
@@ -43,6 +46,33 @@ struct smc_llc_msg_confirm_link { /* type 0x01 */ 
	u8 reserved[9];

};



#define SMC_LLC_FLAG_ADD_LNK_REJ	0x40

#define SMC_LLC_REJ_RSN_NO_ALT_PATH	1



#define SMC_LLC_ADD_LNK_MAX_LINKS	2



struct smc_llc_msg_add_link {		/* type 0x02 */

	struct smc_llc_hdr hd;

	u8 sender_mac[ETH_ALEN];

	u8 reserved2[2];

	u8 sender_gid[SMC_GID_SIZE];

	u8 sender_qp_num[3];

	u8 link_num;

	u8 flags2;	/* QP mtu */

	u8 initial_psn[3];

	u8 reserved[8];

};



#define SMC_LLC_FLAG_DEL_LINK_ALL	0x40

#define SMC_LLC_FLAG_DEL_LINK_ORDERLY	0x20



struct smc_llc_msg_del_link {		/* type 0x04 */

	struct smc_llc_hdr hd;

	u8 link_num;

	__be32 reason;

	u8 reserved[35];

} __packed;			/* format defined in RFC7609 */



struct smc_llc_msg_test_link {		/* type 0x07 */

	struct smc_llc_hdr hd;

	u8 user_data[16];
@@ -88,6 +118,8 @@ struct smc_llc_msg_delete_rkey { /* type 0x09 */ 


union smc_llc_msg {

	struct smc_llc_msg_confirm_link confirm_link;

	struct smc_llc_msg_add_link add_link;

	struct smc_llc_msg_del_link delete_link;



	struct smc_llc_msg_confirm_rkey confirm_rkey;

	struct smc_llc_msg_confirm_rkey_cont confirm_rkey_cont;
@@ -176,7 +208,64 @@ int smc_llc_send_confirm_link(struct smc_link *link, u8 mac[], 
	hton24(confllc->sender_qp_num, link->roce_qp->qp_num);

	/* confllc->link_num = SMC_SINGLE_LINK; already done by memset above */

	memcpy(confllc->link_uid, lgr->id, SMC_LGR_ID_SIZE);

	confllc->max_links = SMC_LINKS_PER_LGR_MAX;

	confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS; /* enforce peer resp. */

	/* send llc message */

	rc = smc_wr_tx_send(link, pend);

	return rc;

}



/* send ADD LINK request or response */

int smc_llc_send_add_link(struct smc_link *link, u8 mac[],

			  union ib_gid *gid,

			  enum smc_llc_reqresp reqresp)

{

	struct smc_llc_msg_add_link *addllc;

	struct smc_wr_tx_pend_priv *pend;

	struct smc_wr_buf *wr_buf;

	int rc;



	rc = smc_llc_add_pending_send(link, &wr_buf, &pend);

	if (rc)

		return rc;

	addllc = (struct smc_llc_msg_add_link *)wr_buf;

	memset(addllc, 0, sizeof(*addllc));

	addllc->hd.common.type = SMC_LLC_ADD_LINK;

	addllc->hd.length = sizeof(struct smc_llc_msg_add_link);

	if (reqresp == SMC_LLC_RESP) {

		addllc->hd.flags |= SMC_LLC_FLAG_RESP;

		/* always reject more links for now */

		addllc->hd.flags |= SMC_LLC_FLAG_ADD_LNK_REJ;

		addllc->hd.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH;

	}

	memcpy(addllc->sender_mac, mac, ETH_ALEN);

	memcpy(addllc->sender_gid, gid, SMC_GID_SIZE);

	/* send llc message */

	rc = smc_wr_tx_send(link, pend);

	return rc;

}



/* send DELETE LINK request or response */

int smc_llc_send_delete_link(struct smc_link *link,

			     enum smc_llc_reqresp reqresp)

{

	struct smc_llc_msg_del_link *delllc;

	struct smc_wr_tx_pend_priv *pend;

	struct smc_wr_buf *wr_buf;

	int rc;



	rc = smc_llc_add_pending_send(link, &wr_buf, &pend);

	if (rc)

		return rc;

	delllc = (struct smc_llc_msg_del_link *)wr_buf;

	memset(delllc, 0, sizeof(*delllc));

	delllc->hd.common.type = SMC_LLC_DELETE_LINK;

	delllc->hd.length = sizeof(struct smc_llc_msg_add_link);

	if (reqresp == SMC_LLC_RESP)

		delllc->hd.flags |= SMC_LLC_FLAG_RESP;

	/* DEL_LINK_ALL because only 1 link supported */

	delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL;

	delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;

	delllc->link_num = link->link_id;

	/* send llc message */

	rc = smc_wr_tx_send(link, pend);

	return rc;
@@ -239,12 +328,14 @@ static void smc_llc_rx_confirm_link(struct smc_link *link, 
		conf_rc = ENOTSUPP;



	if (llc->hd.flags & SMC_LLC_FLAG_RESP) {

		if (lgr->role == SMC_SERV) {

		if (lgr->role == SMC_SERV &&

		    link->state == SMC_LNK_ACTIVATING) {

			link->llc_confirm_resp_rc = conf_rc;

			complete(&link->llc_confirm_resp);

		}

	} else {

		if (lgr->role == SMC_CLNT) {

		if (lgr->role == SMC_CLNT &&

		    link->state == SMC_LNK_ACTIVATING) {

			link->llc_confirm_rc = conf_rc;

			link->link_id = llc->link_num;

			complete(&link->llc_confirm);
@@ -252,6 +343,55 @@ static void smc_llc_rx_confirm_link(struct smc_link *link, 
	}

}



static void smc_llc_rx_add_link(struct smc_link *link,

				struct smc_llc_msg_add_link *llc)

{

	struct smc_link_group *lgr = container_of(link, struct smc_link_group,

						  lnk[SMC_SINGLE_LINK]);



	if (llc->hd.flags & SMC_LLC_FLAG_RESP) {

		if (link->state == SMC_LNK_ACTIVATING)

			complete(&link->llc_add_resp);

	} else {

		if (link->state == SMC_LNK_ACTIVATING) {

			complete(&link->llc_add);

			return;

		}



		if (lgr->role == SMC_SERV) {

			smc_llc_send_add_link(link,

					link->smcibdev->mac[link->ibport - 1],

					&link->smcibdev->gid[link->ibport - 1],

					SMC_LLC_REQ);



		} else {

			smc_llc_send_add_link(link,

					link->smcibdev->mac[link->ibport - 1],

					&link->smcibdev->gid[link->ibport - 1],

					SMC_LLC_RESP);

		}

	}

}



static void smc_llc_rx_delete_link(struct smc_link *link,

				   struct smc_llc_msg_del_link *llc)

{

	struct smc_link_group *lgr = container_of(link, struct smc_link_group,

						  lnk[SMC_SINGLE_LINK]);



	if (llc->hd.flags & SMC_LLC_FLAG_RESP) {

		if (lgr->role == SMC_SERV)

			smc_lgr_terminate(lgr);

	} else {

		if (lgr->role == SMC_SERV) {

			smc_llc_send_delete_link(link, SMC_LLC_REQ);

		} else {

			smc_llc_send_delete_link(link, SMC_LLC_RESP);

			smc_lgr_terminate(lgr);

		}

	}

}



static void smc_llc_rx_test_link(struct smc_link *link,

				 struct smc_llc_msg_test_link *llc)

{
@@ -343,6 +483,12 @@ static void smc_llc_rx_handler(struct ib_wc *wc, void *buf) 
	case SMC_LLC_CONFIRM_LINK:

		smc_llc_rx_confirm_link(link, &llc->confirm_link);

		break;

	case SMC_LLC_ADD_LINK:

		smc_llc_rx_add_link(link, &llc->add_link);

		break;

	case SMC_LLC_DELETE_LINK:

		smc_llc_rx_delete_link(link, &llc->delete_link);

		break;

	case SMC_LLC_CONFIRM_RKEY:

		smc_llc_rx_confirm_rkey(link, &llc->confirm_rkey);

		break;
@@ -366,6 +512,14 @@ static struct smc_wr_rx_handler smc_llc_rx_handlers[] = { 
		.handler	= smc_llc_rx_handler,

		.type		= SMC_LLC_TEST_LINK

	},

	{

		.handler	= smc_llc_rx_handler,

		.type		= SMC_LLC_ADD_LINK

	},

	{

		.handler	= smc_llc_rx_handler,

		.type		= SMC_LLC_DELETE_LINK

	},

	{

		.handler	= smc_llc_rx_handler,

		.type		= SMC_LLC_CONFIRM_RKEY

net/smc/smc_llc.h

+7 −0
Original line number Diff line number Diff line
@@ -18,6 +18,7 @@ 
#define SMC_LLC_FLAG_RESP		0x80



#define SMC_LLC_WAIT_FIRST_TIME		(5 * HZ)

#define SMC_LLC_WAIT_TIME		(2 * HZ)



enum smc_llc_reqresp {

	SMC_LLC_REQ,
@@ -26,6 +27,8 @@ enum smc_llc_reqresp { 


enum smc_llc_msg_type {

	SMC_LLC_CONFIRM_LINK		= 0x01,

	SMC_LLC_ADD_LINK		= 0x02,

	SMC_LLC_DELETE_LINK		= 0x04,

	SMC_LLC_CONFIRM_RKEY		= 0x06,

	SMC_LLC_TEST_LINK		= 0x07,

	SMC_LLC_CONFIRM_RKEY_CONT	= 0x08,
@@ -35,6 +38,10 @@ enum smc_llc_msg_type { 
/* transmit */

int smc_llc_send_confirm_link(struct smc_link *lnk, u8 mac[], union ib_gid *gid,

			      enum smc_llc_reqresp reqresp);

int smc_llc_send_add_link(struct smc_link *link, u8 mac[], union ib_gid *gid,

			  enum smc_llc_reqresp reqresp);

int smc_llc_send_delete_link(struct smc_link *link,

			     enum smc_llc_reqresp reqresp);

int smc_llc_send_test_link(struct smc_link *lnk, u8 user_data[16],

			   enum smc_llc_reqresp reqresp);

int smc_llc_init(void) __init;