Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c6d3c96f authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'tcp-2nd-listener-hash' 



Martin KaFai Lau says:

====================
tcp: Add a 2nd listener hashtable (port+addr)

This patch set adds a 2nd listener hashtable.  It is to resolve
the performance issue when a process is listening at many IP
addresses with the same port (e.g. [IP1]:443, [IP2]:443... [IPN]:443)

v2:
- Move the new lhash2 and lhash2_mask before the existing
  listening_hash to avoid adding another cacheline
  to inet_hashinfo (Suggested by Eric Dumazet, Thanks!)
- I take this chance to plug an existing 4 bytes hole while
  adding 'unsigned int lhash2_mask'.
- Add some comments about lhash2 in inet_hashtables.h
====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents fb7516d4 27da6d37

include/net/inet_connection_sock.h

+2 −0
Original line number Diff line number Diff line
@@ -77,6 +77,7 @@ struct inet_connection_sock_af_ops { 
 * @icsk_af_ops		   Operations which are AF_INET{4,6} specific

 * @icsk_ulp_ops	   Pluggable ULP control hook

 * @icsk_ulp_data	   ULP private data

 * @icsk_listen_portaddr_node	hash to the portaddr listener hashtable

 * @icsk_ca_state:	   Congestion control state

 * @icsk_retransmits:	   Number of unrecovered [RTO] timeouts

 * @icsk_pending:	   Scheduled timer event
@@ -101,6 +102,7 @@ struct inet_connection_sock { 
	const struct inet_connection_sock_af_ops *icsk_af_ops;

	const struct tcp_ulp_ops  *icsk_ulp_ops;

	void			  *icsk_ulp_data;

	struct hlist_node         icsk_listen_portaddr_node;

	unsigned int		  (*icsk_sync_mss)(struct sock *sk, u32 pmtu);

	__u8			  icsk_ca_state:6,

				  icsk_ca_setsockopt:1,

include/net/inet_hashtables.h

+23 −6
Original line number Diff line number Diff line
@@ -111,6 +111,7 @@ struct inet_bind_hashbucket { 
 */

struct inet_listen_hashbucket {

	spinlock_t		lock;

	unsigned int		count;

	struct hlist_head	head;

};
@@ -132,12 +133,13 @@ struct inet_hashinfo { 
	/* Ok, let's try this, I give up, we do need a local binding

	 * TCP hash as well as the others for fast bind/connect.

	 */

	struct kmem_cache		*bind_bucket_cachep;

	struct inet_bind_hashbucket	*bhash;



	unsigned int			bhash_size;

	/* 4 bytes hole on 64 bit */



	struct kmem_cache		*bind_bucket_cachep;

	/* The 2nd listener table hashed by local port and address */

	unsigned int			lhash2_mask;

	struct inet_listen_hashbucket	*lhash2;



	/* All the above members are written once at bootup and

	 * never written again _or_ are predominantly read-access.
@@ -145,14 +147,25 @@ struct inet_hashinfo { 
	 * Now align to a new cache line as all the following members

	 * might be often dirty.

	 */

	/* All sockets in TCP_LISTEN state will be in here.  This is the only

	 * table where wildcard'd TCP sockets can exist.  Hash function here

	 * is just local port number.

	/* All sockets in TCP_LISTEN state will be in listening_hash.

	 * This is the only table where wildcard'd TCP sockets can

	 * exist.  listening_hash is only hashed by local port number.

	 * If lhash2 is initialized, the same socket will also be hashed

	 * to lhash2 by port and address.

	 */

	struct inet_listen_hashbucket	listening_hash[INET_LHTABLE_SIZE]

					____cacheline_aligned_in_smp;

};



#define inet_lhash2_for_each_icsk_rcu(__icsk, list) \

	hlist_for_each_entry_rcu(__icsk, list, icsk_listen_portaddr_node)



static inline struct inet_listen_hashbucket *

inet_lhash2_bucket(struct inet_hashinfo *h, u32 hash)

{

	return &h->lhash2[hash & h->lhash2_mask];

}



static inline struct inet_ehash_bucket *inet_ehash_bucket(

	struct inet_hashinfo *hashinfo,

	unsigned int hash)
@@ -208,6 +221,10 @@ int __inet_inherit_port(const struct sock *sk, struct sock *child); 
void inet_put_port(struct sock *sk);



void inet_hashinfo_init(struct inet_hashinfo *h);

void inet_hashinfo2_init(struct inet_hashinfo *h, const char *name,

			 unsigned long numentries, int scale,

			 unsigned long low_limit,

			 unsigned long high_limit);



bool inet_ehash_insert(struct sock *sk, struct sock *osk);

bool inet_ehash_nolisten(struct sock *sk, struct sock *osk);

include/net/ip.h

+9 −0
Original line number Diff line number Diff line
@@ -26,12 +26,14 @@ 
#include <linux/ip.h>

#include <linux/in.h>

#include <linux/skbuff.h>

#include <linux/jhash.h>



#include <net/inet_sock.h>

#include <net/route.h>

#include <net/snmp.h>

#include <net/flow.h>

#include <net/flow_dissector.h>

#include <net/netns/hash.h>



#define IPV4_MAX_PMTU		65535U		/* RFC 2675, Section 5.1 */
@@ -521,6 +523,13 @@ static inline unsigned int ipv4_addr_hash(__be32 ip) 
	return (__force unsigned int) ip;

}



static inline u32 ipv4_portaddr_hash(const struct net *net,

				     __be32 saddr,

				     unsigned int port)

{

	return jhash_1word((__force u32)saddr, net_hash_mix(net)) ^ port;

}



bool ip_call_ra_chain(struct sk_buff *skb);



/*

include/net/ipv6.h

+17 −0
Original line number Diff line number Diff line
@@ -22,6 +22,7 @@ 
#include <net/flow.h>

#include <net/flow_dissector.h>

#include <net/snmp.h>

#include <net/netns/hash.h>



#define SIN6_LEN_RFC2133	24
@@ -673,6 +674,22 @@ static inline bool ipv6_addr_v4mapped(const struct in6_addr *a) 
					cpu_to_be32(0x0000ffff))) == 0UL;

}



static inline u32 ipv6_portaddr_hash(const struct net *net,

				     const struct in6_addr *addr6,

				     unsigned int port)

{

	unsigned int hash, mix = net_hash_mix(net);



	if (ipv6_addr_any(addr6))

		hash = jhash_1word(0, mix);

	else if (ipv6_addr_v4mapped(addr6))

		hash = jhash_1word((__force u32)addr6->s6_addr32[3], mix);

	else

		hash = jhash2((__force u32 *)addr6->s6_addr32, 4, mix);



	return hash ^ port;

}



/*

 * Check for a RFC 4843 ORCHID address

 * (Overlay Routable Cryptographic Hash Identifiers)

net/ipv4/inet_hashtables.c

+165 −8
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ 
#include <linux/slab.h>

#include <linux/wait.h>

#include <linux/vmalloc.h>

#include <linux/bootmem.h>



#include <net/addrconf.h>

#include <net/inet_connection_sock.h>
@@ -168,6 +169,60 @@ int __inet_inherit_port(const struct sock *sk, struct sock *child) 
}

EXPORT_SYMBOL_GPL(__inet_inherit_port);



static struct inet_listen_hashbucket *

inet_lhash2_bucket_sk(struct inet_hashinfo *h, struct sock *sk)

{

	u32 hash;



#if IS_ENABLED(CONFIG_IPV6)

	if (sk->sk_family == AF_INET6)

		hash = ipv6_portaddr_hash(sock_net(sk),

					  &sk->sk_v6_rcv_saddr,

					  inet_sk(sk)->inet_num);

	else

#endif

		hash = ipv4_portaddr_hash(sock_net(sk),

					  inet_sk(sk)->inet_rcv_saddr,

					  inet_sk(sk)->inet_num);

	return inet_lhash2_bucket(h, hash);

}



static void inet_hash2(struct inet_hashinfo *h, struct sock *sk)

{

	struct inet_listen_hashbucket *ilb2;



	if (!h->lhash2)

		return;



	ilb2 = inet_lhash2_bucket_sk(h, sk);



	spin_lock(&ilb2->lock);

	if (sk->sk_reuseport && sk->sk_family == AF_INET6)

		hlist_add_tail_rcu(&inet_csk(sk)->icsk_listen_portaddr_node,

				   &ilb2->head);

	else

		hlist_add_head_rcu(&inet_csk(sk)->icsk_listen_portaddr_node,

				   &ilb2->head);

	ilb2->count++;

	spin_unlock(&ilb2->lock);

}



static void inet_unhash2(struct inet_hashinfo *h, struct sock *sk)

{

	struct inet_listen_hashbucket *ilb2;



	if (!h->lhash2 ||

	    WARN_ON_ONCE(hlist_unhashed(&inet_csk(sk)->icsk_listen_portaddr_node)))

		return;



	ilb2 = inet_lhash2_bucket_sk(h, sk);



	spin_lock(&ilb2->lock);

	hlist_del_init_rcu(&inet_csk(sk)->icsk_listen_portaddr_node);

	ilb2->count--;

	spin_unlock(&ilb2->lock);

}



static inline int compute_score(struct sock *sk, struct net *net,

				const unsigned short hnum, const __be32 daddr,

				const int dif, const int sdif, bool exact_dif)
@@ -207,6 +262,40 @@ static inline int compute_score(struct sock *sk, struct net *net, 
 */



/* called with rcu_read_lock() : No refcount taken on the socket */

static struct sock *inet_lhash2_lookup(struct net *net,

				struct inet_listen_hashbucket *ilb2,

				struct sk_buff *skb, int doff,

				const __be32 saddr, __be16 sport,

				const __be32 daddr, const unsigned short hnum,

				const int dif, const int sdif)

{

	bool exact_dif = inet_exact_dif_match(net, skb);

	struct inet_connection_sock *icsk;

	struct sock *sk, *result = NULL;

	int score, hiscore = 0;

	u32 phash = 0;



	inet_lhash2_for_each_icsk_rcu(icsk, &ilb2->head) {

		sk = (struct sock *)icsk;

		score = compute_score(sk, net, hnum, daddr,

				      dif, sdif, exact_dif);

		if (score > hiscore) {

			if (sk->sk_reuseport) {

				phash = inet_ehashfn(net, daddr, hnum,

						     saddr, sport);

				result = reuseport_select_sock(sk, phash,

							       skb, doff);

				if (result)

					return result;

			}

			result = sk;

			hiscore = score;

		}

	}



	return result;

}



struct sock *__inet_lookup_listener(struct net *net,

				    struct inet_hashinfo *hashinfo,

				    struct sk_buff *skb, int doff,
@@ -217,10 +306,42 @@ struct sock *__inet_lookup_listener(struct net *net, 
	unsigned int hash = inet_lhashfn(net, hnum);

	struct inet_listen_hashbucket *ilb = &hashinfo->listening_hash[hash];

	bool exact_dif = inet_exact_dif_match(net, skb);

	struct inet_listen_hashbucket *ilb2;

	struct sock *sk, *result = NULL;

	int score, hiscore = 0;

	unsigned int hash2;

	u32 phash = 0;



	if (ilb->count <= 10 || !hashinfo->lhash2)

		goto port_lookup;



	/* Too many sk in the ilb bucket (which is hashed by port alone).

	 * Try lhash2 (which is hashed by port and addr) instead.

	 */



	hash2 = ipv4_portaddr_hash(net, daddr, hnum);

	ilb2 = inet_lhash2_bucket(hashinfo, hash2);

	if (ilb2->count > ilb->count)

		goto port_lookup;



	result = inet_lhash2_lookup(net, ilb2, skb, doff,

				    saddr, sport, daddr, hnum,

				    dif, sdif);

	if (result)

		return result;



	/* Lookup lhash2 with INADDR_ANY */



	hash2 = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum);

	ilb2 = inet_lhash2_bucket(hashinfo, hash2);

	if (ilb2->count > ilb->count)

		goto port_lookup;



	return inet_lhash2_lookup(net, ilb2, skb, doff,

				  saddr, sport, daddr, hnum,

				  dif, sdif);



port_lookup:

	sk_for_each_rcu(sk, &ilb->head) {

		score = compute_score(sk, net, hnum, daddr,

				      dif, sdif, exact_dif);
@@ -476,6 +597,8 @@ int __inet_hash(struct sock *sk, struct sock *osk) 
		hlist_add_tail_rcu(&sk->sk_node, &ilb->head);

	else

		hlist_add_head_rcu(&sk->sk_node, &ilb->head);

	inet_hash2(hashinfo, sk);

	ilb->count++;

	sock_set_flag(sk, SOCK_RCU_FREE);

	sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);

unlock:
@@ -502,28 +625,35 @@ EXPORT_SYMBOL_GPL(inet_hash); 
void inet_unhash(struct sock *sk)

{

	struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;

	struct inet_listen_hashbucket *ilb;

	spinlock_t *lock;

	bool listener = false;

	int done;



	if (sk_unhashed(sk))

		return;



	if (sk->sk_state == TCP_LISTEN) {

		lock = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)].lock;

		ilb = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)];

		lock = &ilb->lock;

		listener = true;

	} else {

		lock = inet_ehash_lockp(hashinfo, sk->sk_hash);

	}

	spin_lock_bh(lock);

	if (sk_unhashed(sk))

		goto unlock;



	if (rcu_access_pointer(sk->sk_reuseport_cb))

		reuseport_detach_sock(sk);

	if (listener)

		done = __sk_del_node_init(sk);

	else

		done = __sk_nulls_del_node_init_rcu(sk);

	if (done)

	if (listener) {

		inet_unhash2(hashinfo, sk);

		 __sk_del_node_init(sk);

		 ilb->count--;

	} else {

		__sk_nulls_del_node_init_rcu(sk);

	}

	sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);

unlock:

	spin_unlock_bh(lock);

}

EXPORT_SYMBOL_GPL(inet_unhash);
@@ -658,10 +788,37 @@ void inet_hashinfo_init(struct inet_hashinfo *h) 
	for (i = 0; i < INET_LHTABLE_SIZE; i++) {

		spin_lock_init(&h->listening_hash[i].lock);

		INIT_HLIST_HEAD(&h->listening_hash[i].head);

		h->listening_hash[i].count = 0;

	}



	h->lhash2 = NULL;

}

EXPORT_SYMBOL_GPL(inet_hashinfo_init);



void __init inet_hashinfo2_init(struct inet_hashinfo *h, const char *name,

				unsigned long numentries, int scale,

				unsigned long low_limit,

				unsigned long high_limit)

{

	unsigned int i;



	h->lhash2 = alloc_large_system_hash(name,

					    sizeof(*h->lhash2),

					    numentries,

					    scale,

					    0,

					    NULL,

					    &h->lhash2_mask,

					    low_limit,

					    high_limit);



	for (i = 0; i <= h->lhash2_mask; i++) {

		spin_lock_init(&h->lhash2[i].lock);

		INIT_HLIST_HEAD(&h->lhash2[i].head);

		h->lhash2[i].count = 0;

	}

}



int inet_ehash_locks_alloc(struct inet_hashinfo *hashinfo)

{

	unsigned int locksz = sizeof(spinlock_t);