Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 3b24d854 authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller
Browse files

tcp/dccp: do not touch listener sk_refcnt under synflood 



When a SYNFLOOD targets a non SO_REUSEPORT listener, multiple
cpus contend on sk->sk_refcnt and sk->sk_wmem_alloc changes.

By letting listeners use SOCK_RCU_FREE infrastructure,
we can relax TCP_LISTEN lookup rules and avoid touching sk_refcnt

Note that we still use SLAB_DESTROY_BY_RCU rules for other sockets,
only listeners are impacted by this change.

Peak performance under SYNFLOOD is increased by ~33% :

On my test machine, I could process 3.2 Mpps instead of 2.4 Mpps

Most consuming functions are now skb_set_owner_w() and sock_wfree()
contending on sk->sk_wmem_alloc when cooking SYNACK and freeing them.

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 3a5d1c0e

include/net/inet6_hashtables.h

+8 −4
Original line number Diff line number Diff line
@@ -66,13 +66,15 @@ static inline struct sock *__inet6_lookup(struct net *net, 
					  const __be16 sport,

					  const struct in6_addr *daddr,

					  const u16 hnum,

					  const int dif)

					  const int dif,

					  bool *refcounted)

{

	struct sock *sk = __inet6_lookup_established(net, hashinfo, saddr,

						sport, daddr, hnum, dif);

	*refcounted = true;

	if (sk)

		return sk;



	*refcounted = false;

	return inet6_lookup_listener(net, hashinfo, skb, doff, saddr, sport,

				     daddr, hnum, dif);

}
@@ -81,17 +83,19 @@ static inline struct sock *__inet6_lookup_skb(struct inet_hashinfo *hashinfo, 
					      struct sk_buff *skb, int doff,

					      const __be16 sport,

					      const __be16 dport,

					      int iif)

					      int iif,

					      bool *refcounted)

{

	struct sock *sk = skb_steal_sock(skb);



	*refcounted = true;

	if (sk)

		return sk;



	return __inet6_lookup(dev_net(skb_dst(skb)->dev), hashinfo, skb,

			      doff, &ipv6_hdr(skb)->saddr, sport,

			      &ipv6_hdr(skb)->daddr, ntohs(dport),

			      iif);

			      iif, refcounted);

}



struct sock *inet6_lookup(struct net *net, struct inet_hashinfo *hashinfo,

include/net/inet_hashtables.h

+24 −16
Original line number Diff line number Diff line
@@ -100,14 +100,10 @@ struct inet_bind_hashbucket { 


/*

 * Sockets can be hashed in established or listening table

 * We must use different 'nulls' end-of-chain value for listening

 * hash table, or we might find a socket that was closed and

 * reallocated/inserted into established hash table

 */

#define LISTENING_NULLS_BASE (1U << 29)

struct inet_listen_hashbucket {

	spinlock_t		lock;

	struct hlist_nulls_head	head;

	struct hlist_head	head;

};



/* This is for listening sockets, thus all sockets which possess wildcards. */
@@ -304,13 +300,19 @@ static inline struct sock *__inet_lookup(struct net *net, 
					 struct sk_buff *skb, int doff,

					 const __be32 saddr, const __be16 sport,

					 const __be32 daddr, const __be16 dport,

					 const int dif)

					 const int dif,

					 bool *refcounted)

{

	u16 hnum = ntohs(dport);

	struct sock *sk = __inet_lookup_established(net, hashinfo,

				saddr, sport, daddr, hnum, dif);

	struct sock *sk;



	return sk ? : __inet_lookup_listener(net, hashinfo, skb, doff, saddr,

	sk = __inet_lookup_established(net, hashinfo, saddr, sport,

				       daddr, hnum, dif);

	*refcounted = true;

	if (sk)

		return sk;

	*refcounted = false;

	return __inet_lookup_listener(net, hashinfo, skb, doff, saddr,

				      sport, daddr, hnum, dif);

}
@@ -322,10 +324,13 @@ static inline struct sock *inet_lookup(struct net *net, 
				       const int dif)

{

	struct sock *sk;

	bool refcounted;



	sk = __inet_lookup(net, hashinfo, skb, doff, saddr, sport, daddr,

			   dport, dif);

			   dport, dif, &refcounted);



	if (sk && !refcounted && !atomic_inc_not_zero(&sk->sk_refcnt))

		sk = NULL;

	return sk;

}
@@ -333,17 +338,20 @@ static inline struct sock *__inet_lookup_skb(struct inet_hashinfo *hashinfo, 
					     struct sk_buff *skb,

					     int doff,

					     const __be16 sport,

					     const __be16 dport)

					     const __be16 dport,

					     bool *refcounted)

{

	struct sock *sk = skb_steal_sock(skb);

	const struct iphdr *iph = ip_hdr(skb);



	*refcounted = true;

	if (sk)

		return sk;

	else



	return __inet_lookup(dev_net(skb_dst(skb)->dev), hashinfo, skb,

			     doff, iph->saddr, sport,

				     iph->daddr, dport, inet_iif(skb));

			     iph->daddr, dport, inet_iif(skb),

			     refcounted);

}



u32 sk_ehashfn(const struct sock *sk);

net/dccp/ipv4.c

+5 −2
Original line number Diff line number Diff line
@@ -764,6 +764,7 @@ static int dccp_v4_rcv(struct sk_buff *skb) 
{

	const struct dccp_hdr *dh;

	const struct iphdr *iph;

	bool refcounted;

	struct sock *sk;

	int min_cov;
@@ -801,7 +802,7 @@ static int dccp_v4_rcv(struct sk_buff *skb) 


lookup:

	sk = __inet_lookup_skb(&dccp_hashinfo, skb, __dccp_hdr_len(dh),

			       dh->dccph_sport, dh->dccph_dport);

			       dh->dccph_sport, dh->dccph_dport, &refcounted);

	if (!sk) {

		dccp_pr_debug("failed to look up flow ID in table and "

			      "get corresponding socket\n");
@@ -830,6 +831,7 @@ static int dccp_v4_rcv(struct sk_buff *skb) 
			goto lookup;

		}

		sock_hold(sk);

		refcounted = true;

		nsk = dccp_check_req(sk, skb, req);

		if (!nsk) {

			reqsk_put(req);
@@ -886,6 +888,7 @@ static int dccp_v4_rcv(struct sk_buff *skb) 
	return 0;



discard_and_relse:

	if (refcounted)

		sock_put(sk);

	goto discard_it;

}

net/dccp/ipv6.c

+5 −2
Original line number Diff line number Diff line
@@ -642,6 +642,7 @@ static int dccp_v6_do_rcv(struct sock *sk, struct sk_buff *skb) 
static int dccp_v6_rcv(struct sk_buff *skb)

{

	const struct dccp_hdr *dh;

	bool refcounted;

	struct sock *sk;

	int min_cov;
@@ -670,7 +671,7 @@ static int dccp_v6_rcv(struct sk_buff *skb) 
lookup:

	sk = __inet6_lookup_skb(&dccp_hashinfo, skb, __dccp_hdr_len(dh),

			        dh->dccph_sport, dh->dccph_dport,

				inet6_iif(skb));

				inet6_iif(skb), &refcounted);

	if (!sk) {

		dccp_pr_debug("failed to look up flow ID in table and "

			      "get corresponding socket\n");
@@ -699,6 +700,7 @@ static int dccp_v6_rcv(struct sk_buff *skb) 
			goto lookup;

		}

		sock_hold(sk);

		refcounted = true;

		nsk = dccp_check_req(sk, skb, req);

		if (!nsk) {

			reqsk_put(req);
@@ -752,6 +754,7 @@ static int dccp_v6_rcv(struct sk_buff *skb) 
	return 0;



discard_and_relse:

	if (refcounted)

		sock_put(sk);

	goto discard_it;

}

net/ipv4/inet_diag.c

+1 −2
Original line number Diff line number Diff line
@@ -775,13 +775,12 @@ void inet_diag_dump_icsk(struct inet_hashinfo *hashinfo, struct sk_buff *skb, 


		for (i = s_i; i < INET_LHTABLE_SIZE; i++) {

			struct inet_listen_hashbucket *ilb;

			struct hlist_nulls_node *node;

			struct sock *sk;



			num = 0;

			ilb = &hashinfo->listening_hash[i];

			spin_lock_bh(&ilb->lock);

			sk_nulls_for_each(sk, node, &ilb->head) {

			sk_for_each(sk, &ilb->head) {

				struct inet_sock *inet = inet_sk(sk);



				if (!net_eq(sock_net(sk), net))