Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e11ecddf authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller
Browse files

tcp: use TCP_SKB_CB(skb)->tcp_flags in input path 



Input path of TCP do not currently uses TCP_SKB_CB(skb)->tcp_flags,
which is only used in output path.

tcp_recvmsg(), looks at tcp_hdr(skb)->syn for every skb found in receive queue,
and its unfortunate because this bit is located in a cache line right before
the payload.

We can simplify TCP by copying tcp flags into TCP_SKB_CB(skb)->tcp_flags.

This patch does so, and avoids the cache line miss in tcp_recvmsg()

Following patches will
- allow a segment with FIN being coalesced in tcp_try_coalesce()
- simplify tcp_collapse() by not copying the headers.

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Acked-by: default avatarNeal Cardwell <ncardwell@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 13bb5180

net/ipv4/tcp.c

+10 −8
Original line number Original line Diff line number Diff line
@@ -1510,9 +1510,9 @@ static struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off) 




	while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) {

	while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) {

		offset = seq - TCP_SKB_CB(skb)->seq;

		offset = seq - TCP_SKB_CB(skb)->seq;

		if (tcp_hdr(skb)->syn)

		if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)

			offset--;

			offset--;

		if (offset < skb->len || tcp_hdr(skb)->fin) {

		if (offset < skb->len || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) {

			*off = offset;

			*off = offset;

			return skb;

			return skb;

		}

		}
@@ -1585,7 +1585,7 @@ int tcp_read_sock(struct sock *sk, read_descriptor_t *desc, 
			if (offset + 1 != skb->len)

			if (offset + 1 != skb->len)

				continue;

				continue;

		}

		}

		if (tcp_hdr(skb)->fin) {

		if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) {

			sk_eat_skb(sk, skb, false);

			sk_eat_skb(sk, skb, false);

			++seq;

			++seq;

			break;

			break;
@@ -1722,11 +1722,11 @@ int tcp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, 
				break;

				break;





			offset = *seq - TCP_SKB_CB(skb)->seq;

			offset = *seq - TCP_SKB_CB(skb)->seq;

			if (tcp_hdr(skb)->syn)

			if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)

				offset--;

				offset--;

			if (offset < skb->len)

			if (offset < skb->len)

				goto found_ok_skb;

				goto found_ok_skb;

			if (tcp_hdr(skb)->fin)

			if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)

				goto found_fin_ok;

				goto found_fin_ok;

			WARN(!(flags & MSG_PEEK),

			WARN(!(flags & MSG_PEEK),

			     "recvmsg bug 2: copied %X seq %X rcvnxt %X fl %X\n",

			     "recvmsg bug 2: copied %X seq %X rcvnxt %X fl %X\n",
@@ -1959,7 +1959,7 @@ int tcp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, 
		if (used + offset < skb->len)

		if (used + offset < skb->len)

			continue;

			continue;





		if (tcp_hdr(skb)->fin)

		if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)

			goto found_fin_ok;

			goto found_fin_ok;

		if (!(flags & MSG_PEEK)) {

		if (!(flags & MSG_PEEK)) {

			sk_eat_skb(sk, skb, copied_early);

			sk_eat_skb(sk, skb, copied_early);
@@ -2160,8 +2160,10 @@ void tcp_close(struct sock *sk, long timeout) 
	 *  reader process may not have drained the data yet!

	 *  reader process may not have drained the data yet!

	 */

	 */

	while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {

	while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {

		u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq -

		u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq;

			  tcp_hdr(skb)->fin;



		if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)

			len--;

		data_was_unread += len;

		data_was_unread += len;

		__kfree_skb(skb);

		__kfree_skb(skb);

	}

	}

net/ipv4/tcp_input.c

+5 −5
Original line number Original line Diff line number Diff line
@@ -4093,7 +4093,7 @@ static void tcp_ofo_queue(struct sock *sk) 
		__skb_unlink(skb, &tp->out_of_order_queue);

		__skb_unlink(skb, &tp->out_of_order_queue);

		__skb_queue_tail(&sk->sk_receive_queue, skb);

		__skb_queue_tail(&sk->sk_receive_queue, skb);

		tp->rcv_nxt = TCP_SKB_CB(skb)->end_seq;

		tp->rcv_nxt = TCP_SKB_CB(skb)->end_seq;

		if (tcp_hdr(skb)->fin)

		if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)

			tcp_fin(sk);

			tcp_fin(sk);

	}

	}

}

}
@@ -4513,7 +4513,7 @@ tcp_collapse(struct sock *sk, struct sk_buff_head *list, 
		 * - bloated or contains data before "start" or

		 * - bloated or contains data before "start" or

		 *   overlaps to the next one.

		 *   overlaps to the next one.

		 */

		 */

		if (!tcp_hdr(skb)->syn && !tcp_hdr(skb)->fin &&

		if (!(TCP_SKB_CB(skb)->tcp_flags & (TCPHDR_SYN | TCPHDR_FIN)) &&

		    (tcp_win_from_space(skb->truesize) > skb->len ||

		    (tcp_win_from_space(skb->truesize) > skb->len ||

		     before(TCP_SKB_CB(skb)->seq, start))) {

		     before(TCP_SKB_CB(skb)->seq, start))) {

			end_of_skbs = false;

			end_of_skbs = false;
@@ -4532,7 +4532,8 @@ tcp_collapse(struct sock *sk, struct sk_buff_head *list, 
		/* Decided to skip this, advance start seq. */

		/* Decided to skip this, advance start seq. */

		start = TCP_SKB_CB(skb)->end_seq;

		start = TCP_SKB_CB(skb)->end_seq;

	}

	}

	if (end_of_skbs || tcp_hdr(skb)->syn || tcp_hdr(skb)->fin)

	if (end_of_skbs ||

	    (TCP_SKB_CB(skb)->tcp_flags & (TCPHDR_SYN | TCPHDR_FIN)))

		return;

		return;





	while (before(start, end)) {

	while (before(start, end)) {
@@ -4579,8 +4580,7 @@ tcp_collapse(struct sock *sk, struct sk_buff_head *list, 
				skb = tcp_collapse_one(sk, skb, list);

				skb = tcp_collapse_one(sk, skb, list);

				if (!skb ||

				if (!skb ||

				    skb == tail ||

				    skb == tail ||

				    tcp_hdr(skb)->syn ||

				    (TCP_SKB_CB(skb)->tcp_flags & (TCPHDR_SYN | TCPHDR_FIN)))

				    tcp_hdr(skb)->fin)

					return;

					return;

			}

			}

		}

		}

net/ipv4/tcp_ipv4.c

+1 −0
Original line number Original line Diff line number Diff line
@@ -1638,6 +1638,7 @@ int tcp_v4_rcv(struct sk_buff *skb) 
	TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +

	TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +

				    skb->len - th->doff * 4);

				    skb->len - th->doff * 4);

	TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);

	TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);

	TCP_SKB_CB(skb)->tcp_flags = tcp_flag_byte(th);

	TCP_SKB_CB(skb)->tcp_tw_isn = 0;

	TCP_SKB_CB(skb)->tcp_tw_isn = 0;

	TCP_SKB_CB(skb)->ip_dsfield = ipv4_get_dsfield(iph);

	TCP_SKB_CB(skb)->ip_dsfield = ipv4_get_dsfield(iph);

	TCP_SKB_CB(skb)->sacked	 = 0;

	TCP_SKB_CB(skb)->sacked	 = 0;

net/ipv6/tcp_ipv6.c

+1 −0
Original line number Original line Diff line number Diff line
@@ -1415,6 +1415,7 @@ static int tcp_v6_rcv(struct sk_buff *skb) 
	TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +

	TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +

				    skb->len - th->doff*4);

				    skb->len - th->doff*4);

	TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);

	TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);

	TCP_SKB_CB(skb)->tcp_flags = tcp_flag_byte(th);

	TCP_SKB_CB(skb)->tcp_tw_isn = 0;

	TCP_SKB_CB(skb)->tcp_tw_isn = 0;

	TCP_SKB_CB(skb)->ip_dsfield = ipv6_get_dsfield(hdr);

	TCP_SKB_CB(skb)->ip_dsfield = ipv6_get_dsfield(hdr);

	TCP_SKB_CB(skb)->sacked = 0;

	TCP_SKB_CB(skb)->sacked = 0;