Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 05dbc7b5 authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller
Browse files

tcp/dccp: remove twchain 



TCP listener refactoring, part 3 :

Our goal is to hash SYN_RECV sockets into main ehash for fast lookup,
and parallel SYN processing.

Current inet_ehash_bucket contains two chains, one for ESTABLISH (and
friend states) sockets, another for TIME_WAIT sockets only.

As the hash table is sized to get at most one socket per bucket, it
makes little sense to have separate twchain, as it makes the lookup
slightly more complicated, and doubles hash table memory usage.

If we make sure all socket types have the lookup keys at the same
offsets, we can use a generic and faster lookup. It turns out TIME_WAIT
and ESTABLISHED sockets already have common lookup fields for IPv4.

[ INET_TW_MATCH() is no longer needed ]

I'll provide a follow-up to factorize IPv6 lookup as well, to remove
INET6_TW_MATCH()

This way, SYN_RECV pseudo sockets will be supported the same.

A new sock_gen_put() helper is added, doing either a sock_put() or
inet_twsk_put() [ and will support SYN_RECV later ].

Note this helper should only be called in real slow path, when rcu
lookup found a socket that was moved to another identity (freed/reused
immediately), but could eventually be used in other contexts, like
sock_edemux()

Before patch :

dmesg | grep "TCP established"

TCP established hash table entries: 524288 (order: 11, 8388608 bytes)

After patch :

TCP established hash table entries: 524288 (order: 10, 4194304 bytes)

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 53af53ae

include/net/inet_hashtables.h

+2 −7
Original line number Original line Diff line number Diff line
@@ -37,12 +37,11 @@ 
#include <asm/byteorder.h>

#include <asm/byteorder.h>





/* This is for all connections with a full identity, no wildcards.

/* This is for all connections with a full identity, no wildcards.

 * One chain is dedicated to TIME_WAIT sockets.

 * The 'e' prefix stands for Establish, but we really put all sockets

 * I'll experiment with dynamic table growth later.

 * but LISTEN ones.

 */

 */

struct inet_ehash_bucket {

struct inet_ehash_bucket {

	struct hlist_nulls_head chain;

	struct hlist_nulls_head chain;

	struct hlist_nulls_head twchain;

};

};





/* There are a few simple rules, which allow for local port reuse by

/* There are a few simple rules, which allow for local port reuse by
@@ -123,7 +122,6 @@ struct inet_hashinfo { 
	 *

	 *

	 *          TCP_ESTABLISHED <= sk->sk_state < TCP_CLOSE

	 *          TCP_ESTABLISHED <= sk->sk_state < TCP_CLOSE

	 *

	 *

	 * TIME_WAIT sockets use a separate chain (twchain).

	 */

	 */

	struct inet_ehash_bucket	*ehash;

	struct inet_ehash_bucket	*ehash;

	spinlock_t			*ehash_locks;

	spinlock_t			*ehash_locks;
@@ -318,9 +316,6 @@ static inline struct sock *inet_lookup_listener(struct net *net, 
	 net_eq(sock_net(__sk), (__net)))

	 net_eq(sock_net(__sk), (__net)))

#endif /* 64-bit arch */

#endif /* 64-bit arch */





#define INET_TW_MATCH(__sk, __net, __cookie, __saddr, __daddr, __ports, __dif)\

	INET_MATCH(__sk, __net, __cookie, __saddr, __daddr, __ports, __dif)



/*

/*

 * Sockets in TCP_CLOSE state are _always_ taken out of the hash, so we need

 * Sockets in TCP_CLOSE state are _always_ taken out of the hash, so we need

 * not check it for lookups anymore, thanks Alexey. -DaveM

 * not check it for lookups anymore, thanks Alexey. -DaveM

include/net/inet_timewait_sock.h

+1 −12
Original line number Original line Diff line number Diff line
@@ -141,18 +141,6 @@ struct inet_timewait_sock { 
};

};

#define tw_tclass tw_tos

#define tw_tclass tw_tos





static inline void inet_twsk_add_node_rcu(struct inet_timewait_sock *tw,

				      struct hlist_nulls_head *list)

{

	hlist_nulls_add_head_rcu(&tw->tw_node, list);

}



static inline void inet_twsk_add_bind_node(struct inet_timewait_sock *tw,

					   struct hlist_head *list)

{

	hlist_add_head(&tw->tw_bind_node, list);

}



static inline int inet_twsk_dead_hashed(const struct inet_timewait_sock *tw)

static inline int inet_twsk_dead_hashed(const struct inet_timewait_sock *tw)

{

{

	return !hlist_unhashed(&tw->tw_death_node);

	return !hlist_unhashed(&tw->tw_death_node);
@@ -192,6 +180,7 @@ static inline struct inet_timewait_sock *inet_twsk(const struct sock *sk) 
	return (struct inet_timewait_sock *)sk;

	return (struct inet_timewait_sock *)sk;

}

}





void inet_twsk_free(struct inet_timewait_sock *tw);

void inet_twsk_put(struct inet_timewait_sock *tw);

void inet_twsk_put(struct inet_timewait_sock *tw);





int inet_twsk_unhash(struct inet_timewait_sock *tw);

int inet_twsk_unhash(struct inet_timewait_sock *tw);

include/net/sock.h

+7 −1
Original line number Original line Diff line number Diff line
@@ -156,7 +156,7 @@ typedef __u64 __bitwise __addrpair; 
 */

 */

struct sock_common {

struct sock_common {

	/* skc_daddr and skc_rcv_saddr must be grouped on a 8 bytes aligned

	/* skc_daddr and skc_rcv_saddr must be grouped on a 8 bytes aligned

	 * address on 64bit arches : cf INET_MATCH() and INET_TW_MATCH()

	 * address on 64bit arches : cf INET_MATCH()

	 */

	 */

	union {

	union {

		__addrpair	skc_addrpair;

		__addrpair	skc_addrpair;
@@ -301,6 +301,8 @@ struct sock { 
#define sk_dontcopy_end		__sk_common.skc_dontcopy_end

#define sk_dontcopy_end		__sk_common.skc_dontcopy_end

#define sk_hash			__sk_common.skc_hash

#define sk_hash			__sk_common.skc_hash

#define sk_portpair		__sk_common.skc_portpair

#define sk_portpair		__sk_common.skc_portpair

#define sk_num			__sk_common.skc_num

#define sk_dport		__sk_common.skc_dport

#define sk_addrpair		__sk_common.skc_addrpair

#define sk_addrpair		__sk_common.skc_addrpair

#define sk_daddr		__sk_common.skc_daddr

#define sk_daddr		__sk_common.skc_daddr

#define sk_rcv_saddr		__sk_common.skc_rcv_saddr

#define sk_rcv_saddr		__sk_common.skc_rcv_saddr
@@ -1653,6 +1655,10 @@ static inline void sock_put(struct sock *sk) 
	if (atomic_dec_and_test(&sk->sk_refcnt))

	if (atomic_dec_and_test(&sk->sk_refcnt))

		sk_free(sk);

		sk_free(sk);

}

}

/* Generic version of sock_put(), dealing with all sockets

 * (TCP_TIMEWAIT, ESTABLISHED...)

 */

void sock_gen_put(struct sock *sk);





int sk_receive_skb(struct sock *sk, struct sk_buff *skb, const int nested);

int sk_receive_skb(struct sock *sk, struct sk_buff *skb, const int nested);

include/net/tcp.h

+0 −1
Original line number Original line Diff line number Diff line
@@ -1519,7 +1519,6 @@ enum tcp_seq_states { 
	TCP_SEQ_STATE_LISTENING,

	TCP_SEQ_STATE_LISTENING,

	TCP_SEQ_STATE_OPENREQ,

	TCP_SEQ_STATE_OPENREQ,

	TCP_SEQ_STATE_ESTABLISHED,

	TCP_SEQ_STATE_ESTABLISHED,

	TCP_SEQ_STATE_TIME_WAIT,

};

};





int tcp_seq_open(struct inode *inode, struct file *file);

int tcp_seq_open(struct inode *inode, struct file *file);

net/dccp/proto.c

+1 −3
Original line number Original line Diff line number Diff line
@@ -1158,10 +1158,8 @@ static int __init dccp_init(void) 
		goto out_free_bind_bucket_cachep;

		goto out_free_bind_bucket_cachep;

	}

	}





	for (i = 0; i <= dccp_hashinfo.ehash_mask; i++) {

	for (i = 0; i <= dccp_hashinfo.ehash_mask; i++)

		INIT_HLIST_NULLS_HEAD(&dccp_hashinfo.ehash[i].chain, i);

		INIT_HLIST_NULLS_HEAD(&dccp_hashinfo.ehash[i].chain, i);

		INIT_HLIST_NULLS_HEAD(&dccp_hashinfo.ehash[i].twchain, i);

	}





	if (inet_ehash_locks_alloc(&dccp_hashinfo))

	if (inet_ehash_locks_alloc(&dccp_hashinfo))

			goto out_free_dccp_ehash;

			goto out_free_dccp_ehash;