tcp/dccp: fix possible race __inet_lookup_established() (94671cf1) · Commits · e / devices / android_kernel_xiaomi_sm6125

include/linux/rculist_nulls.h

+37 −0

Original line number	Original line	Diff line number	Diff line
	@@ -100,6 +100,43 @@ static inline void hlist_nulls_add_head_rcu(struct hlist_nulls_node *n,
	first->pprev = &n->next;		first->pprev = &n->next;
	}		}

			/**
			* hlist_nulls_add_tail_rcu
			* @n: the element to add to the hash list.
			* @h: the list to add to.
			*
			* Description:
			* Adds the specified element to the specified hlist_nulls,
			* while permitting racing traversals.
			*
			* The caller must take whatever precautions are necessary
			* (such as holding appropriate locks) to avoid racing
			* with another list-mutation primitive, such as hlist_nulls_add_head_rcu()
			* or hlist_nulls_del_rcu(), running on this same list.
			* However, it is perfectly legal to run concurrently with
			* the _rcu list-traversal primitives, such as
			* hlist_nulls_for_each_entry_rcu(), used to prevent memory-consistency
			* problems on Alpha CPUs. Regardless of the type of CPU, the
			* list-traversal primitive must be guarded by rcu_read_lock().
			*/
			static inline void hlist_nulls_add_tail_rcu(struct hlist_nulls_node *n,
			struct hlist_nulls_head *h)
			{
			struct hlist_nulls_node i, last = NULL;

			/* Note: write side code, so rcu accessors are not needed. */
			for (i = h->first; !is_a_nulls(i); i = i->next)
			last = i;

			if (last) {
			n->next = last->next;
			n->pprev = &last->next;
			rcu_assign_pointer(hlist_next_rcu(last), n);
			} else {
			hlist_nulls_add_head_rcu(n, h);
			}
			}

	/**		/**
	* hlist_nulls_for_each_entry_rcu - iterate over rcu list of given type		* hlist_nulls_for_each_entry_rcu - iterate over rcu list of given type
	* @tpos: the type * to use as a loop cursor.		* @tpos: the type * to use as a loop cursor.

include/net/inet_hashtables.h

+9 −3

Original line number	Original line	Diff line number	Diff line
	@@ -106,12 +106,18 @@ struct inet_bind_hashbucket {
	struct hlist_head chain;		struct hlist_head chain;
	};		};

	/*		/* Sockets can be hashed in established or listening table.
	* Sockets can be hashed in established or listening table		* We must use different 'nulls' end-of-chain value for all hash buckets :
			* A socket might transition from ESTABLISH to LISTEN state without
			* RCU grace period. A lookup in ehash table needs to handle this case.
	*/		*/
			#define LISTENING_NULLS_BASE (1U << 29)
	struct inet_listen_hashbucket {		struct inet_listen_hashbucket {
	spinlock_t lock;		spinlock_t lock;
			union {
	struct hlist_head head;		struct hlist_head head;
			struct hlist_nulls_head nulls_head;
			};
	};		};

	/* This is for listening sockets, thus all sockets which possess wildcards. */		/* This is for listening sockets, thus all sockets which possess wildcards. */

include/net/sock.h

+5 −0

Original line number	Original line	Diff line number	Diff line
	@@ -693,6 +693,11 @@ static inline void __sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_h
	hlist_nulls_add_head_rcu(&sk->sk_nulls_node, list);		hlist_nulls_add_head_rcu(&sk->sk_nulls_node, list);
	}		}

			static inline void __sk_nulls_add_node_tail_rcu(struct sock sk, struct hlist_nulls_head list)
			{
			hlist_nulls_add_tail_rcu(&sk->sk_nulls_node, list);
			}

	static inline void sk_nulls_add_node_rcu(struct sock sk, struct hlist_nulls_head list)		static inline void sk_nulls_add_node_rcu(struct sock sk, struct hlist_nulls_head list)
	{		{
	sock_hold(sk);		sock_hold(sk);

net/ipv4/inet_diag.c

+2 −1

Original line number	Original line	Diff line number	Diff line
	@@ -911,11 +911,12 @@ void inet_diag_dump_icsk(struct inet_hashinfo hashinfo, struct sk_buff skb,

	for (i = s_i; i < INET_LHTABLE_SIZE; i++) {		for (i = s_i; i < INET_LHTABLE_SIZE; i++) {
	struct inet_listen_hashbucket *ilb;		struct inet_listen_hashbucket *ilb;
			struct hlist_nulls_node *node;

	num = 0;		num = 0;
	ilb = &hashinfo->listening_hash[i];		ilb = &hashinfo->listening_hash[i];
	spin_lock(&ilb->lock);		spin_lock(&ilb->lock);
	sk_for_each(sk, &ilb->head) {		sk_nulls_for_each(sk, node, &ilb->nulls_head) {
	struct inet_sock *inet = inet_sk(sk);		struct inet_sock *inet = inet_sk(sk);

	if (!net_eq(sock_net(sk), net))		if (!net_eq(sock_net(sk), net))

net/ipv4/inet_hashtables.c

+9 −9

Original line number	Original line	Diff line number	Diff line
	@@ -219,9 +219,10 @@ struct sock __inet_lookup_listener(struct net net,
	int score, hiscore = 0, matches = 0, reuseport = 0;		int score, hiscore = 0, matches = 0, reuseport = 0;
	bool exact_dif = inet_exact_dif_match(net, skb);		bool exact_dif = inet_exact_dif_match(net, skb);
	struct sock sk, result = NULL;		struct sock sk, result = NULL;
			struct hlist_nulls_node *node;
	u32 phash = 0;		u32 phash = 0;

	sk_for_each_rcu(sk, &ilb->head) {		sk_nulls_for_each_rcu(sk, node, &ilb->nulls_head) {
	score = compute_score(sk, net, hnum, daddr,		score = compute_score(sk, net, hnum, daddr,
	dif, sdif, exact_dif);		dif, sdif, exact_dif);
	if (score > hiscore) {		if (score > hiscore) {
	@@ -442,10 +443,11 @@ static int inet_reuseport_add_sock(struct sock *sk,
	struct inet_listen_hashbucket *ilb)		struct inet_listen_hashbucket *ilb)
	{		{
	struct inet_bind_bucket *tb = inet_csk(sk)->icsk_bind_hash;		struct inet_bind_bucket *tb = inet_csk(sk)->icsk_bind_hash;
			const struct hlist_nulls_node *node;
	struct sock *sk2;		struct sock *sk2;
	kuid_t uid = sock_i_uid(sk);		kuid_t uid = sock_i_uid(sk);

	sk_for_each_rcu(sk2, &ilb->head) {		sk_nulls_for_each_rcu(sk2, node, &ilb->nulls_head) {
	if (sk2 != sk &&		if (sk2 != sk &&
	sk2->sk_family == sk->sk_family &&		sk2->sk_family == sk->sk_family &&
	ipv6_only_sock(sk2) == ipv6_only_sock(sk) &&		ipv6_only_sock(sk2) == ipv6_only_sock(sk) &&
	@@ -480,9 +482,9 @@ int __inet_hash(struct sock sk, struct sock osk)
	}		}
	if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport &&		if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport &&
	sk->sk_family == AF_INET6)		sk->sk_family == AF_INET6)
	hlist_add_tail_rcu(&sk->sk_node, &ilb->head);		__sk_nulls_add_node_tail_rcu(sk, &ilb->nulls_head);
	else		else
	hlist_add_head_rcu(&sk->sk_node, &ilb->head);		__sk_nulls_add_node_rcu(sk, &ilb->nulls_head);
	sock_set_flag(sk, SOCK_RCU_FREE);		sock_set_flag(sk, SOCK_RCU_FREE);
	sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);		sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
	unlock:		unlock:
	@@ -525,9 +527,6 @@ void inet_unhash(struct sock *sk)
	spin_lock_bh(lock);		spin_lock_bh(lock);
	if (rcu_access_pointer(sk->sk_reuseport_cb))		if (rcu_access_pointer(sk->sk_reuseport_cb))
	reuseport_detach_sock(sk);		reuseport_detach_sock(sk);
	if (listener)
	done = __sk_del_node_init(sk);
	else
	done = __sk_nulls_del_node_init_rcu(sk);		done = __sk_nulls_del_node_init_rcu(sk);
	if (done)		if (done)
	sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);		sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
	@@ -664,7 +663,8 @@ void inet_hashinfo_init(struct inet_hashinfo *h)

	for (i = 0; i < INET_LHTABLE_SIZE; i++) {		for (i = 0; i < INET_LHTABLE_SIZE; i++) {
	spin_lock_init(&h->listening_hash[i].lock);		spin_lock_init(&h->listening_hash[i].lock);
	INIT_HLIST_HEAD(&h->listening_hash[i].head);		INIT_HLIST_NULLS_HEAD(&h->listening_hash[i].nulls_head,
			i + LISTENING_NULLS_BASE);
	}		}
	}		}
	EXPORT_SYMBOL_GPL(inet_hashinfo_init);		EXPORT_SYMBOL_GPL(inet_hashinfo_init);