net/smc: eliminate cursor read and write calls (bac6de7b) · Commits · e / devices / android_kernel_fairphone_FP5

net/smc/af_smc.c

+2 −6

Original line number	Original line	Diff line number	Diff line
	@@ -1755,12 +1755,8 @@ static int smc_ioctl(struct socket *sock, unsigned int cmd,
	smc->sk.sk_state == SMC_CLOSED) {		smc->sk.sk_state == SMC_CLOSED) {
	answ = 0;		answ = 0;
	} else {		} else {
	smc_curs_write(&cons,		smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
	smc_curs_read(&conn->local_tx_ctrl.cons, conn),		smc_curs_copy(&urg, &conn->urg_curs, conn);
	conn);
	smc_curs_write(&urg,
	smc_curs_read(&conn->urg_curs, conn),
	conn);
	answ = smc_curs_diff(conn->rmb_desc->len,		answ = smc_curs_diff(conn->rmb_desc->len,
	&cons, &urg) == 1;		&cons, &urg) == 1;
	}		}

net/smc/smc_cdc.c

+12 −21

Original line number	Original line	Diff line number	Diff line
	@@ -34,14 +34,15 @@ static void smc_cdc_tx_handler(struct smc_wr_tx_pend_priv *pnd_snd,
	enum ib_wc_status wc_status)		enum ib_wc_status wc_status)
	{		{
	struct smc_cdc_tx_pend cdcpend = (struct smc_cdc_tx_pend )pnd_snd;		struct smc_cdc_tx_pend cdcpend = (struct smc_cdc_tx_pend )pnd_snd;
			struct smc_connection *conn = cdcpend->conn;
	struct smc_sock *smc;		struct smc_sock *smc;
	int diff;		int diff;

	if (!cdcpend->conn)		if (!conn)
	/* already dismissed */		/* already dismissed */
	return;		return;

	smc = container_of(cdcpend->conn, struct smc_sock, conn);		smc = container_of(conn, struct smc_sock, conn);
	bh_lock_sock(&smc->sk);		bh_lock_sock(&smc->sk);
	if (!wc_status) {		if (!wc_status) {
	diff = smc_curs_diff(cdcpend->conn->sndbuf_desc->len,		diff = smc_curs_diff(cdcpend->conn->sndbuf_desc->len,
	@@ -52,9 +53,7 @@ static void smc_cdc_tx_handler(struct smc_wr_tx_pend_priv *pnd_snd,
	atomic_add(diff, &cdcpend->conn->sndbuf_space);		atomic_add(diff, &cdcpend->conn->sndbuf_space);
	/* guarantee 0 <= sndbuf_space <= sndbuf_desc->len */		/* guarantee 0 <= sndbuf_space <= sndbuf_desc->len */
	smp_mb__after_atomic();		smp_mb__after_atomic();
	smc_curs_write(&cdcpend->conn->tx_curs_fin,		smc_curs_copy(&conn->tx_curs_fin, &cdcpend->cursor, conn);
	smc_curs_read(&cdcpend->cursor, cdcpend->conn),
	cdcpend->conn);
	}		}
	smc_tx_sndbuf_nonfull(smc);		smc_tx_sndbuf_nonfull(smc);
	bh_unlock_sock(&smc->sk);		bh_unlock_sock(&smc->sk);
	@@ -110,9 +109,8 @@ int smc_cdc_msg_send(struct smc_connection *conn,
	&conn->local_tx_ctrl, conn);		&conn->local_tx_ctrl, conn);
	rc = smc_wr_tx_send(link, (struct smc_wr_tx_pend_priv *)pend);		rc = smc_wr_tx_send(link, (struct smc_wr_tx_pend_priv *)pend);
	if (!rc)		if (!rc)
	smc_curs_write(&conn->rx_curs_confirmed,		smc_curs_copy(&conn->rx_curs_confirmed,
	smc_curs_read(&conn->local_tx_ctrl.cons, conn),		&conn->local_tx_ctrl.cons, conn);
	conn);

	return rc;		return rc;
	}		}
	@@ -194,8 +192,8 @@ int smcd_cdc_msg_send(struct smc_connection *conn)
	rc = smcd_tx_ism_write(conn, &cdc, sizeof(cdc), 0, 1);		rc = smcd_tx_ism_write(conn, &cdc, sizeof(cdc), 0, 1);
	if (rc)		if (rc)
	return rc;		return rc;
	smc_curs_write(&conn->rx_curs_confirmed,		smc_curs_copy(&conn->rx_curs_confirmed, &conn->local_tx_ctrl.cons,
	smc_curs_read(&conn->local_tx_ctrl.cons, conn), conn);		conn);
	/* Calculate transmitted data and increment free send buffer space */		/* Calculate transmitted data and increment free send buffer space */
	diff = smc_curs_diff(conn->sndbuf_desc->len, &conn->tx_curs_fin,		diff = smc_curs_diff(conn->sndbuf_desc->len, &conn->tx_curs_fin,
	&conn->tx_curs_sent);		&conn->tx_curs_sent);
	@@ -204,8 +202,7 @@ int smcd_cdc_msg_send(struct smc_connection *conn)
	atomic_add(diff, &conn->sndbuf_space);		atomic_add(diff, &conn->sndbuf_space);
	/* guarantee 0 <= sndbuf_space <= sndbuf_desc->len */		/* guarantee 0 <= sndbuf_space <= sndbuf_desc->len */
	smp_mb__after_atomic();		smp_mb__after_atomic();
	smc_curs_write(&conn->tx_curs_fin,		smc_curs_copy(&conn->tx_curs_fin, &conn->tx_curs_sent, conn);
	smc_curs_read(&conn->tx_curs_sent, conn), conn);

	smc_tx_sndbuf_nonfull(smc);		smc_tx_sndbuf_nonfull(smc);
	return rc;		return rc;
	@@ -225,9 +222,7 @@ static void smc_cdc_handle_urg_data_arrival(struct smc_sock *smc,
	char *base;		char *base;

	/* new data included urgent business */		/* new data included urgent business */
	smc_curs_write(&conn->urg_curs,		smc_curs_copy(&conn->urg_curs, &conn->local_rx_ctrl.prod, conn);
	smc_curs_read(&conn->local_rx_ctrl.prod, conn),
	conn);
	conn->urg_state = SMC_URG_VALID;		conn->urg_state = SMC_URG_VALID;
	if (!sock_flag(&smc->sk, SOCK_URGINLINE))		if (!sock_flag(&smc->sk, SOCK_URGINLINE))
	/* we'll skip the urgent byte, so don't account for it */		/* we'll skip the urgent byte, so don't account for it */
	@@ -247,12 +242,8 @@ static void smc_cdc_msg_recv_action(struct smc_sock *smc,
	struct smc_connection *conn = &smc->conn;		struct smc_connection *conn = &smc->conn;
	int diff_cons, diff_prod;		int diff_cons, diff_prod;

	smc_curs_write(&prod_old,		smc_curs_copy(&prod_old, &conn->local_rx_ctrl.prod, conn);
	smc_curs_read(&conn->local_rx_ctrl.prod, conn),		smc_curs_copy(&cons_old, &conn->local_rx_ctrl.cons, conn);
	conn);
	smc_curs_write(&cons_old,
	smc_curs_read(&conn->local_rx_ctrl.cons, conn),
	conn);
	smc_cdc_msg_to_host(&conn->local_rx_ctrl, cdc, conn);		smc_cdc_msg_to_host(&conn->local_rx_ctrl, cdc, conn);

	diff_cons = smc_curs_diff(conn->peer_rmbe_size, &cons_old,		diff_cons = smc_curs_diff(conn->peer_rmbe_size, &cons_old,

net/smc/smc_cdc.h

+15 −28

Original line number	Original line	Diff line number	Diff line
	@@ -104,47 +104,34 @@ static inline u64 smc_curs_read(union smc_host_cursor *curs,
	#endif		#endif
	}		}

	static inline u64 smc_curs_read_net(union smc_cdc_cursor *curs,		/* Copy cursor src into tgt */
	struct smc_connection *conn)		static inline void smc_curs_copy(union smc_host_cursor *tgt,
	{		union smc_host_cursor *src,
	#ifndef KERNEL_HAS_ATOMIC64
	unsigned long flags;
	u64 ret;

	spin_lock_irqsave(&conn->acurs_lock, flags);
	ret = curs->acurs;
	spin_unlock_irqrestore(&conn->acurs_lock, flags);
	return ret;
	#else
	return atomic64_read(&curs->acurs);
	#endif
	}

	static inline void smc_curs_write(union smc_host_cursor *curs, u64 val,
	struct smc_connection *conn)		struct smc_connection *conn)
	{		{
	#ifndef KERNEL_HAS_ATOMIC64		#ifndef KERNEL_HAS_ATOMIC64
	unsigned long flags;		unsigned long flags;

	spin_lock_irqsave(&conn->acurs_lock, flags);		spin_lock_irqsave(&conn->acurs_lock, flags);
	curs->acurs = val;		tgt->acurs = src->acurs;
	spin_unlock_irqrestore(&conn->acurs_lock, flags);		spin_unlock_irqrestore(&conn->acurs_lock, flags);
	#else		#else
	atomic64_set(&curs->acurs, val);		atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
	#endif		#endif
	}		}

	static inline void smc_curs_write_net(union smc_cdc_cursor *curs, u64 val,		static inline void smc_curs_copy_net(union smc_cdc_cursor *tgt,
			union smc_cdc_cursor *src,
	struct smc_connection *conn)		struct smc_connection *conn)
	{		{
	#ifndef KERNEL_HAS_ATOMIC64		#ifndef KERNEL_HAS_ATOMIC64
	unsigned long flags;		unsigned long flags;

	spin_lock_irqsave(&conn->acurs_lock, flags);		spin_lock_irqsave(&conn->acurs_lock, flags);
	curs->acurs = val;		tgt->acurs = src->acurs;
	spin_unlock_irqrestore(&conn->acurs_lock, flags);		spin_unlock_irqrestore(&conn->acurs_lock, flags);
	#else		#else
	atomic64_set(&curs->acurs, val);		atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
	#endif		#endif
	}		}

	@@ -179,7 +166,7 @@ static inline void smc_host_cursor_to_cdc(union smc_cdc_cursor *peer,
	{		{
	union smc_host_cursor temp;		union smc_host_cursor temp;

	smc_curs_write(&temp, smc_curs_read(local, conn), conn);		smc_curs_copy(&temp, local, conn);
	peer->count = htonl(temp.count);		peer->count = htonl(temp.count);
	peer->wrap = htons(temp.wrap);		peer->wrap = htons(temp.wrap);
	/* peer->reserved = htons(0); must be ensured by caller */		/* peer->reserved = htons(0); must be ensured by caller */
	@@ -206,8 +193,8 @@ static inline void smc_cdc_cursor_to_host(union smc_host_cursor *local,
	union smc_host_cursor temp, old;		union smc_host_cursor temp, old;
	union smc_cdc_cursor net;		union smc_cdc_cursor net;

	smc_curs_write(&old, smc_curs_read(local, conn), conn);		smc_curs_copy(&old, local, conn);
	smc_curs_write_net(&net, smc_curs_read_net(peer, conn), conn);		smc_curs_copy_net(&net, peer, conn);
	temp.count = ntohl(net.count);		temp.count = ntohl(net.count);
	temp.wrap = ntohs(net.wrap);		temp.wrap = ntohs(net.wrap);
	if ((old.wrap > temp.wrap) && temp.wrap)		if ((old.wrap > temp.wrap) && temp.wrap)
	@@ -215,7 +202,7 @@ static inline void smc_cdc_cursor_to_host(union smc_host_cursor *local,
	if ((old.wrap == temp.wrap) &&		if ((old.wrap == temp.wrap) &&
	(old.count > temp.count))		(old.count > temp.count))
	return;		return;
	smc_curs_write(local, smc_curs_read(&temp, conn), conn);		smc_curs_copy(local, &temp, conn);
	}		}

	static inline void smcr_cdc_msg_to_host(struct smc_host_cdc_msg *local,		static inline void smcr_cdc_msg_to_host(struct smc_host_cdc_msg *local,

net/smc/smc_rx.c

+4 −11

Original line number	Original line	Diff line number	Diff line
	@@ -82,8 +82,7 @@ static int smc_rx_update_consumer(struct smc_sock *smc,
	}		}
	}		}

	smc_curs_write(&conn->local_tx_ctrl.cons, smc_curs_read(&cons, conn),		smc_curs_copy(&conn->local_tx_ctrl.cons, &cons, conn);
	conn);

	/* send consumer cursor update if required */		/* send consumer cursor update if required */
	/* similar to advertising new TCP rcv_wnd if required */		/* similar to advertising new TCP rcv_wnd if required */
	@@ -97,8 +96,7 @@ static void smc_rx_update_cons(struct smc_sock *smc, size_t len)
	struct smc_connection *conn = &smc->conn;		struct smc_connection *conn = &smc->conn;
	union smc_host_cursor cons;		union smc_host_cursor cons;

	smc_curs_write(&cons, smc_curs_read(&conn->local_tx_ctrl.cons, conn),		smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
	conn);
	smc_rx_update_consumer(smc, cons, len);		smc_rx_update_consumer(smc, cons, len);
	}		}

	@@ -245,10 +243,7 @@ static int smc_rx_recv_urg(struct smc_sock smc, struct msghdr msg, int len,
	if (!(flags & MSG_TRUNC))		if (!(flags & MSG_TRUNC))
	rc = memcpy_to_msg(msg, &conn->urg_rx_byte, 1);		rc = memcpy_to_msg(msg, &conn->urg_rx_byte, 1);
	len = 1;		len = 1;
	smc_curs_write(&cons,		smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
	smc_curs_read(&conn->local_tx_ctrl.cons,
	conn),
	conn);
	if (smc_curs_diff(conn->rmb_desc->len, &cons,		if (smc_curs_diff(conn->rmb_desc->len, &cons,
	&conn->urg_curs) > 1)		&conn->urg_curs) > 1)
	conn->urg_rx_skip_pend = true;		conn->urg_rx_skip_pend = true;
	@@ -370,9 +365,7 @@ int smc_rx_recvmsg(struct smc_sock smc, struct msghdr msg,
	continue;		continue;
	}		}

	smc_curs_write(&cons,		smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
	smc_curs_read(&conn->local_tx_ctrl.cons, conn),
	conn);
	/* subsequent splice() calls pick up where previous left */		/* subsequent splice() calls pick up where previous left */
	if (splbytes)		if (splbytes)
	smc_curs_add(conn->rmb_desc->len, &cons, splbytes);		smc_curs_add(conn->rmb_desc->len, &cons, splbytes);

net/smc/smc_tx.c

+13 −33

Original line number	Original line	Diff line number	Diff line
	@@ -181,9 +181,7 @@ int smc_tx_sendmsg(struct smc_sock smc, struct msghdr msg, size_t len)
	copylen = min_t(size_t, send_remaining, writespace);		copylen = min_t(size_t, send_remaining, writespace);
	/* determine start of sndbuf */		/* determine start of sndbuf */
	sndbuf_base = conn->sndbuf_desc->cpu_addr;		sndbuf_base = conn->sndbuf_desc->cpu_addr;
	smc_curs_write(&prep,		smc_curs_copy(&prep, &conn->tx_curs_prep, conn);
	smc_curs_read(&conn->tx_curs_prep, conn),
	conn);
	tx_cnt_prep = prep.count;		tx_cnt_prep = prep.count;
	/* determine chunks where to write into sndbuf */		/* determine chunks where to write into sndbuf */
	/* either unwrapped case, or 1st chunk of wrapped case */		/* either unwrapped case, or 1st chunk of wrapped case */
	@@ -214,9 +212,7 @@ int smc_tx_sendmsg(struct smc_sock smc, struct msghdr msg, size_t len)
	smc_sndbuf_sync_sg_for_device(conn);		smc_sndbuf_sync_sg_for_device(conn);
	/* update cursors */		/* update cursors */
	smc_curs_add(conn->sndbuf_desc->len, &prep, copylen);		smc_curs_add(conn->sndbuf_desc->len, &prep, copylen);
	smc_curs_write(&conn->tx_curs_prep,		smc_curs_copy(&conn->tx_curs_prep, &prep, conn);
	smc_curs_read(&prep, conn),
	conn);
	/* increased in send tasklet smc_cdc_tx_handler() */		/* increased in send tasklet smc_cdc_tx_handler() */
	smp_mb__before_atomic();		smp_mb__before_atomic();
	atomic_sub(copylen, &conn->sndbuf_space);		atomic_sub(copylen, &conn->sndbuf_space);
	@@ -417,8 +413,8 @@ static int smc_tx_rdma_writes(struct smc_connection *conn)
	int rc;		int rc;

	/* source: sndbuf */		/* source: sndbuf */
	smc_curs_write(&sent, smc_curs_read(&conn->tx_curs_sent, conn), conn);		smc_curs_copy(&sent, &conn->tx_curs_sent, conn);
	smc_curs_write(&prep, smc_curs_read(&conn->tx_curs_prep, conn), conn);		smc_curs_copy(&prep, &conn->tx_curs_prep, conn);
	/* cf. wmem_alloc - (snd_max - snd_una) */		/* cf. wmem_alloc - (snd_max - snd_una) */
	to_send = smc_curs_diff(conn->sndbuf_desc->len, &sent, &prep);		to_send = smc_curs_diff(conn->sndbuf_desc->len, &sent, &prep);
	if (to_send <= 0)		if (to_send <= 0)
	@@ -429,12 +425,8 @@ static int smc_tx_rdma_writes(struct smc_connection *conn)
	rmbespace = atomic_read(&conn->peer_rmbe_space);		rmbespace = atomic_read(&conn->peer_rmbe_space);
	if (rmbespace <= 0)		if (rmbespace <= 0)
	return 0;		return 0;
	smc_curs_write(&prod,		smc_curs_copy(&prod, &conn->local_tx_ctrl.prod, conn);
	smc_curs_read(&conn->local_tx_ctrl.prod, conn),		smc_curs_copy(&cons, &conn->local_rx_ctrl.cons, conn);
	conn);
	smc_curs_write(&cons,
	smc_curs_read(&conn->local_rx_ctrl.cons, conn),
	conn);

	/* if usable snd_wnd closes ask peer to advertise once it opens again */		/* if usable snd_wnd closes ask peer to advertise once it opens again */
	pflags = &conn->local_tx_ctrl.prod_flags;		pflags = &conn->local_tx_ctrl.prod_flags;
	@@ -481,14 +473,9 @@ static int smc_tx_rdma_writes(struct smc_connection *conn)
	pflags->urg_data_present = 1;		pflags->urg_data_present = 1;
	smc_tx_advance_cursors(conn, &prod, &sent, len);		smc_tx_advance_cursors(conn, &prod, &sent, len);
	/* update connection's cursors with advanced local cursors */		/* update connection's cursors with advanced local cursors */
	smc_curs_write(&conn->local_tx_ctrl.prod,		smc_curs_copy(&conn->local_tx_ctrl.prod, &prod, conn);
	smc_curs_read(&prod, conn),
	conn);
	/* dst: peer RMBE */		/* dst: peer RMBE */
	smc_curs_write(&conn->tx_curs_sent,		smc_curs_copy(&conn->tx_curs_sent, &sent, conn);/* src: local sndbuf */
	smc_curs_read(&sent, conn),
	conn);
	/* src: local sndbuf */

	return 0;		return 0;
	}		}
	@@ -606,17 +593,11 @@ void smc_tx_consumer_update(struct smc_connection *conn, bool force)
	int sender_free = conn->rmb_desc->len;		int sender_free = conn->rmb_desc->len;
	int to_confirm;		int to_confirm;

	smc_curs_write(&cons,		smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
	smc_curs_read(&conn->local_tx_ctrl.cons, conn),		smc_curs_copy(&cfed, &conn->rx_curs_confirmed, conn);
	conn);
	smc_curs_write(&cfed,
	smc_curs_read(&conn->rx_curs_confirmed, conn),
	conn);
	to_confirm = smc_curs_diff(conn->rmb_desc->len, &cfed, &cons);		to_confirm = smc_curs_diff(conn->rmb_desc->len, &cfed, &cons);
	if (to_confirm > conn->rmbe_update_limit) {		if (to_confirm > conn->rmbe_update_limit) {
	smc_curs_write(&prod,		smc_curs_copy(&prod, &conn->local_rx_ctrl.prod, conn);
	smc_curs_read(&conn->local_rx_ctrl.prod, conn),
	conn);
	sender_free = conn->rmb_desc->len -		sender_free = conn->rmb_desc->len -
	smc_curs_diff(conn->rmb_desc->len, &prod, &cfed);		smc_curs_diff(conn->rmb_desc->len, &prod, &cfed);
	}		}
	@@ -632,9 +613,8 @@ void smc_tx_consumer_update(struct smc_connection *conn, bool force)
	SMC_TX_WORK_DELAY);		SMC_TX_WORK_DELAY);
	return;		return;
	}		}
	smc_curs_write(&conn->rx_curs_confirmed,		smc_curs_copy(&conn->rx_curs_confirmed,
	smc_curs_read(&conn->local_tx_ctrl.cons, conn),		&conn->local_tx_ctrl.cons, conn);
	conn);
	conn->local_rx_ctrl.prod_flags.cons_curs_upd_req = 0;		conn->local_rx_ctrl.prod_flags.cons_curs_upd_req = 0;
	}		}
	if (conn->local_rx_ctrl.prod_flags.write_blocked &&		if (conn->local_rx_ctrl.prod_flags.write_blocked &&