Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit b9948e11 authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'net-prefer-listeners-bound-to-an-address' 



Peter Oskolkov says:

====================
net: prefer listeners bound to an address

A relatively common use case is to have several IPs configured
on a host, and have different listeners for each of them. We would
like to add a "catch all" listener on addr_any, to match incoming
connections not served by any of the listeners bound to a specific
address.

However, port-only lookups can match addr_any sockets when sockets
listening on specific addresses are present if so_reuseport flag
is set. This patchset eliminates lookups into port-only hashtable,
as lookups by (addr,port) tuple are easily available.

In a future patchset I plan to explore whether it is possible
to remove port-only hashtables completely: additional refactoring
will be required, as some non-lookup code uses the hashtables.
====================

Reviewed-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 8e2ea53a 6254e5c6

net/ipv4/inet_hashtables.c

+8 −52
Original line number Diff line number Diff line
@@ -234,24 +234,16 @@ static inline int compute_score(struct sock *sk, struct net *net, 
				const int dif, const int sdif, bool exact_dif)

{

	int score = -1;

	struct inet_sock *inet = inet_sk(sk);

	bool dev_match;



	if (net_eq(sock_net(sk), net) && inet->inet_num == hnum &&

	if (net_eq(sock_net(sk), net) && sk->sk_num == hnum &&

			!ipv6_only_sock(sk)) {

		__be32 rcv_saddr = inet->inet_rcv_saddr;

		score = sk->sk_family == PF_INET ? 2 : 1;

		if (rcv_saddr) {

			if (rcv_saddr != daddr)

		if (sk->sk_rcv_saddr != daddr)

			return -1;

			score += 4;

		}

		dev_match = inet_sk_bound_dev_eq(net, sk->sk_bound_dev_if,

						 dif, sdif);

		if (!dev_match)



		if (!inet_sk_bound_dev_eq(net, sk->sk_bound_dev_if, dif, sdif))

			return -1;

		score += 4;



		score = sk->sk_family == PF_INET ? 2 : 1;

		if (sk->sk_incoming_cpu == raw_smp_processor_id())

			score++;

	}
@@ -307,26 +299,12 @@ struct sock *__inet_lookup_listener(struct net *net, 
				    const __be32 daddr, const unsigned short hnum,

				    const int dif, const int sdif)

{

	unsigned int hash = inet_lhashfn(net, hnum);

	struct inet_listen_hashbucket *ilb = &hashinfo->listening_hash[hash];

	bool exact_dif = inet_exact_dif_match(net, skb);

	struct inet_listen_hashbucket *ilb2;

	struct sock *sk, *result = NULL;

	int score, hiscore = 0;

	struct sock *result = NULL;

	unsigned int hash2;

	u32 phash = 0;



	if (ilb->count <= 10 || !hashinfo->lhash2)

		goto port_lookup;



	/* Too many sk in the ilb bucket (which is hashed by port alone).

	 * Try lhash2 (which is hashed by port and addr) instead.

	 */



	hash2 = ipv4_portaddr_hash(net, daddr, hnum);

	ilb2 = inet_lhash2_bucket(hashinfo, hash2);

	if (ilb2->count > ilb->count)

		goto port_lookup;



	result = inet_lhash2_lookup(net, ilb2, skb, doff,

				    saddr, sport, daddr, hnum,
@@ -335,34 +313,12 @@ struct sock *__inet_lookup_listener(struct net *net, 
		goto done;



	/* Lookup lhash2 with INADDR_ANY */



	hash2 = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum);

	ilb2 = inet_lhash2_bucket(hashinfo, hash2);

	if (ilb2->count > ilb->count)

		goto port_lookup;



	result = inet_lhash2_lookup(net, ilb2, skb, doff,

				    saddr, sport, daddr, hnum,

				    saddr, sport, htonl(INADDR_ANY), hnum,

				    dif, sdif);

	goto done;



port_lookup:

	sk_for_each_rcu(sk, &ilb->head) {

		score = compute_score(sk, net, hnum, daddr,

				      dif, sdif, exact_dif);

		if (score > hiscore) {

			if (sk->sk_reuseport) {

				phash = inet_ehashfn(net, daddr, hnum,

						     saddr, sport);

				result = reuseport_select_sock(sk, phash,

							       skb, doff);

				if (result)

					goto done;

			}

			result = sk;

			hiscore = score;

		}

	}

done:

	if (unlikely(IS_ERR(result)))

		return NULL;

net/ipv4/udp.c

+19 −57
Original line number Diff line number Diff line
@@ -380,15 +380,12 @@ static int compute_score(struct sock *sk, struct net *net, 
	    ipv6_only_sock(sk))

		return -1;



	score = (sk->sk_family == PF_INET) ? 2 : 1;

	inet = inet_sk(sk);



	if (inet->inet_rcv_saddr) {

		if (inet->inet_rcv_saddr != daddr)

	if (sk->sk_rcv_saddr != daddr)

		return -1;

		score += 4;

	}



	score = (sk->sk_family == PF_INET) ? 2 : 1;



	inet = inet_sk(sk);

	if (inet->inet_daddr) {

		if (inet->inet_daddr != saddr)

			return -1;
@@ -464,67 +461,32 @@ struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr, 
		__be16 sport, __be32 daddr, __be16 dport, int dif,

		int sdif, struct udp_table *udptable, struct sk_buff *skb)

{

	struct sock *sk, *result;

	struct sock *result;

	unsigned short hnum = ntohs(dport);

	unsigned int hash2, slot2, slot = udp_hashfn(net, hnum, udptable->mask);

	struct udp_hslot *hslot2, *hslot = &udptable->hash[slot];

	unsigned int hash2, slot2;

	struct udp_hslot *hslot2;

	bool exact_dif = udp_lib_exact_dif_match(net, skb);

	int score, badness;

	u32 hash = 0;



	if (hslot->count > 10) {

	hash2 = ipv4_portaddr_hash(net, daddr, hnum);

	slot2 = hash2 & udptable->mask;

	hslot2 = &udptable->hash2[slot2];

		if (hslot->count < hslot2->count)

			goto begin;



	result = udp4_lib_lookup2(net, saddr, sport,

				  daddr, hnum, dif, sdif,

				  exact_dif, hslot2, skb);

	if (!result) {

			unsigned int old_slot2 = slot2;

		hash2 = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum);

		slot2 = hash2 & udptable->mask;

			/* avoid searching the same slot again. */

			if (unlikely(slot2 == old_slot2))

				return result;



		hslot2 = &udptable->hash2[slot2];

			if (hslot->count < hslot2->count)

				goto begin;



		result = udp4_lib_lookup2(net, saddr, sport,

						  daddr, hnum, dif, sdif,

					  htonl(INADDR_ANY), hnum, dif, sdif,

					  exact_dif, hslot2, skb);

	}

	if (unlikely(IS_ERR(result)))

		return NULL;

	return result;

}

begin:

	result = NULL;

	badness = 0;

	sk_for_each_rcu(sk, &hslot->head) {

		score = compute_score(sk, net, saddr, sport,

				      daddr, hnum, dif, sdif, exact_dif);

		if (score > badness) {

			if (sk->sk_reuseport) {

				hash = udp_ehashfn(net, daddr, hnum,

						   saddr, sport);

				result = reuseport_select_sock(sk, hash, skb,

							sizeof(struct udphdr));

				if (unlikely(IS_ERR(result)))

					return NULL;

				if (result)

					return result;

			}

			result = sk;

			badness = score;

		}

	}

	return result;

}

EXPORT_SYMBOL_GPL(__udp4_lib_lookup);



static inline struct sock *__udp4_lib_lookup_skb(struct sk_buff *skb,

net/ipv6/inet6_hashtables.c

+6 −48
Original line number Diff line number Diff line
@@ -99,23 +99,16 @@ static inline int compute_score(struct sock *sk, struct net *net, 
				const int dif, const int sdif, bool exact_dif)

{

	int score = -1;

	bool dev_match;



	if (net_eq(sock_net(sk), net) && inet_sk(sk)->inet_num == hnum &&

	    sk->sk_family == PF_INET6) {



		score = 1;

		if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr)) {

		if (!ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr))

			return -1;

			score++;

		}

		dev_match = inet_sk_bound_dev_eq(net, sk->sk_bound_dev_if,

						 dif, sdif);

		if (!dev_match)



		if (!inet_sk_bound_dev_eq(net, sk->sk_bound_dev_if, dif, sdif))

			return -1;

		score++;



		score = 1;

		if (sk->sk_incoming_cpu == raw_smp_processor_id())

			score++;

	}
@@ -164,26 +157,12 @@ struct sock *inet6_lookup_listener(struct net *net, 
		const __be16 sport, const struct in6_addr *daddr,

		const unsigned short hnum, const int dif, const int sdif)

{

	unsigned int hash = inet_lhashfn(net, hnum);

	struct inet_listen_hashbucket *ilb = &hashinfo->listening_hash[hash];

	bool exact_dif = inet6_exact_dif_match(net, skb);

	struct inet_listen_hashbucket *ilb2;

	struct sock *sk, *result = NULL;

	int score, hiscore = 0;

	struct sock *result = NULL;

	unsigned int hash2;

	u32 phash = 0;



	if (ilb->count <= 10 || !hashinfo->lhash2)

		goto port_lookup;



	/* Too many sk in the ilb bucket (which is hashed by port alone).

	 * Try lhash2 (which is hashed by port and addr) instead.

	 */



	hash2 = ipv6_portaddr_hash(net, daddr, hnum);

	ilb2 = inet_lhash2_bucket(hashinfo, hash2);

	if (ilb2->count > ilb->count)

		goto port_lookup;



	result = inet6_lhash2_lookup(net, ilb2, skb, doff,

				     saddr, sport, daddr, hnum,
@@ -192,33 +171,12 @@ struct sock *inet6_lookup_listener(struct net *net, 
		goto done;



	/* Lookup lhash2 with in6addr_any */



	hash2 = ipv6_portaddr_hash(net, &in6addr_any, hnum);

	ilb2 = inet_lhash2_bucket(hashinfo, hash2);

	if (ilb2->count > ilb->count)

		goto port_lookup;



	result = inet6_lhash2_lookup(net, ilb2, skb, doff,

				     saddr, sport, daddr, hnum,

				     saddr, sport, &in6addr_any, hnum,

				     dif, sdif);

	goto done;



port_lookup:

	sk_for_each(sk, &ilb->head) {

		score = compute_score(sk, net, hnum, daddr, dif, sdif, exact_dif);

		if (score > hiscore) {

			if (sk->sk_reuseport) {

				phash = inet6_ehashfn(net, daddr, hnum,

						      saddr, sport);

				result = reuseport_select_sock(sk, phash,

							       skb, doff);

				if (result)

					goto done;

			}

			result = sk;

			hiscore = score;

		}

	}

done:

	if (unlikely(IS_ERR(result)))

		return NULL;

net/ipv6/udp.c

+21 −58
Original line number Diff line number Diff line
@@ -125,6 +125,9 @@ static int compute_score(struct sock *sk, struct net *net, 
	    sk->sk_family != PF_INET6)

		return -1;



	if (!ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr))

		return -1;



	score = 0;

	inet = inet_sk(sk);
@@ -134,12 +137,6 @@ static int compute_score(struct sock *sk, struct net *net, 
		score++;

	}



	if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr)) {

		if (!ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr))

			return -1;

		score++;

	}



	if (!ipv6_addr_any(&sk->sk_v6_daddr)) {

		if (!ipv6_addr_equal(&sk->sk_v6_daddr, saddr))

			return -1;
@@ -197,38 +194,27 @@ struct sock *__udp6_lib_lookup(struct net *net, 
			       int dif, int sdif, struct udp_table *udptable,

			       struct sk_buff *skb)

{

	struct sock *sk, *result;

	unsigned short hnum = ntohs(dport);

	unsigned int hash2, slot2, slot = udp_hashfn(net, hnum, udptable->mask);

	struct udp_hslot *hslot2, *hslot = &udptable->hash[slot];

	unsigned int hash2, slot2;

	struct udp_hslot *hslot2;

	struct sock *result;

	bool exact_dif = udp6_lib_exact_dif_match(net, skb);

	int score, badness;

	u32 hash = 0;



	if (hslot->count > 10) {

	hash2 = ipv6_portaddr_hash(net, daddr, hnum);

	slot2 = hash2 & udptable->mask;

	hslot2 = &udptable->hash2[slot2];

		if (hslot->count < hslot2->count)

			goto begin;



	result = udp6_lib_lookup2(net, saddr, sport,

				  daddr, hnum, dif, sdif, exact_dif,

				  hslot2, skb);

	if (!result) {

			unsigned int old_slot2 = slot2;

		hash2 = ipv6_portaddr_hash(net, &in6addr_any, hnum);

		slot2 = hash2 & udptable->mask;

			/* avoid searching the same slot again. */

			if (unlikely(slot2 == old_slot2))

				return result;



		hslot2 = &udptable->hash2[slot2];

			if (hslot->count < hslot2->count)

				goto begin;



		result = udp6_lib_lookup2(net, saddr, sport,

						  daddr, hnum, dif, sdif,

					  &in6addr_any, hnum, dif, sdif,

					  exact_dif, hslot2,

					  skb);

	}
@@ -236,29 +222,6 @@ struct sock *__udp6_lib_lookup(struct net *net, 
		return NULL;

	return result;

}

begin:

	result = NULL;

	badness = -1;

	sk_for_each_rcu(sk, &hslot->head) {

		score = compute_score(sk, net, saddr, sport, daddr, hnum, dif,

				      sdif, exact_dif);

		if (score > badness) {

			if (sk->sk_reuseport) {

				hash = udp6_ehashfn(net, daddr, hnum,

						    saddr, sport);

				result = reuseport_select_sock(sk, hash, skb,

							sizeof(struct udphdr));

				if (unlikely(IS_ERR(result)))

					return NULL;

				if (result)

					return result;

			}

			result = sk;

			badness = score;

		}

	}

	return result;

}

EXPORT_SYMBOL_GPL(__udp6_lib_lookup);



static struct sock *__udp6_lib_lookup_skb(struct sk_buff *skb,

tools/testing/selftests/net/.gitignore

+1 −0
Original line number Diff line number Diff line
@@ -3,6 +3,7 @@ socket 
psock_fanout

psock_snd

psock_tpacket

reuseport_addr_any

reuseport_bpf

reuseport_bpf_cpu

reuseport_bpf_numa