Merge "udp: add support for UDP_GRO cmsg" (22370f2a) · Commits · e / devices / android_kernel_oneplus_sm7250

include/linux/udp.h

+18 −1

Original line number	Diff line number	Diff line
		@@ -49,7 +49,13 @@ struct udp_sock {
		unsigned int corkflag; /* Cork is required */
		__u8 encap_type; /* Is this an Encapsulation socket? */
		unsigned char no_check6_tx:1,/* Send zero UDP6 checksums on TX? */
		no_check6_rx:1;/* Allow zero UDP6 checksums on RX? */
		no_check6_rx:1,/* Allow zero UDP6 checksums on RX? */
		encap_enabled:1, /* This socket enabled encap
		* processing; UDP tunnels and
		* different encapsulation layer set
		* this
		*/
		gro_enabled:1; /* Can accept GRO packets */
		/*
		* Following member retains the information to create a UDP header
		* when the socket is uncorked.
		@@ -115,6 +121,17 @@ static inline bool udp_get_no_check6_rx(struct sock *sk)
		return udp_sk(sk)->no_check6_rx;
		}

		static inline void udp_cmsg_recv(struct msghdr msg, struct sock sk,
		struct sk_buff *skb)
		{
		int gso_size;

		if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
		gso_size = skb_shinfo(skb)->gso_size;
		put_cmsg(msg, SOL_UDP, UDP_GRO, sizeof(gso_size), &gso_size);
		}
		}

		#define udp_portaddr_for_each_entry(__sk, list) \
		hlist_for_each_entry(__sk, list, __sk_common.skc_portaddr_node)

include/net/udp_tunnel.h

+6 −0

Original line number	Diff line number	Diff line
		@@ -165,6 +165,12 @@ static inline int udp_tunnel_handle_offloads(struct sk_buff *skb, bool udp_csum)

		static inline void udp_tunnel_encap_enable(struct socket *sock)
		{
		struct udp_sock *up = udp_sk(sock->sk);

		if (up->encap_enabled)
		return;

		up->encap_enabled = 1;
		#if IS_ENABLED(CONFIG_IPV6)
		if (sock->sk->sk_family == PF_INET6)
		ipv6_stub->udpv6_encap_enable();

include/uapi/linux/udp.h

+1 −0

Original line number	Diff line number	Diff line
		@@ -33,6 +33,7 @@ struct udphdr {
		#define UDP_NO_CHECK6_TX 101 /* Disable sending checksum for UDP6X */
		#define UDP_NO_CHECK6_RX 102 /* Disable accpeting checksum for UDP6 */
		#define UDP_SEGMENT 103 /* Set GSO segmentation size */
		#define UDP_GRO 104 /* This socket can receive UDP GRO packets */

		/* UDP encapsulation types */
		#define UDP_ENCAP_ESPINUDP_NON_IKE 1 /* draft-ietf-ipsec-nat-t-ike-00/01 */

net/ipv4/udp.c

+25 −6

Original line number	Diff line number	Diff line
		@@ -115,6 +115,7 @@
		#include "udp_impl.h"
		#include <net/sock_reuseport.h>
		#include <net/addrconf.h>
		#include <net/udp_tunnel.h>

		struct udp_table udp_table __read_mostly;
		EXPORT_SYMBOL(udp_table);
		@@ -1718,6 +1719,10 @@ int udp_recvmsg(struct sock sk, struct msghdr msg, size_t len, int noblock,
		memset(sin->sin_zero, 0, sizeof(sin->sin_zero));
		addr_len = sizeof(sin);
		}

		if (udp_sk(sk)->gro_enabled)
		udp_cmsg_recv(msg, sk, skb);

		if (inet->cmsg_flags)
		ip_cmsg_recv_offset(msg, sk, skb, sizeof(struct udphdr), off);

		@@ -2403,12 +2408,16 @@ void udp_destroy_sock(struct sock *sk)
		bool slow = lock_sock_fast(sk);
		udp_flush_pending_frames(sk);
		unlock_sock_fast(sk, slow);
		if (static_branch_unlikely(&udp_encap_needed_key) && up->encap_type) {
		if (static_branch_unlikely(&udp_encap_needed_key)) {
		if (up->encap_type) {
		void (encap_destroy)(struct sock sk);
		encap_destroy = READ_ONCE(up->encap_destroy);
		if (encap_destroy)
		encap_destroy(sk);
		}
		if (up->encap_enabled)
		static_branch_disable(&udp_encap_needed_key);
		}
		}

		/*
		@@ -2452,7 +2461,9 @@ int udp_lib_setsockopt(struct sock *sk, int level, int optname,
		/* FALLTHROUGH */
		case UDP_ENCAP_L2TPINUDP:
		up->encap_type = val;
		udp_encap_enable();
		lock_sock(sk);
		udp_tunnel_encap_enable(sk->sk_socket);
		release_sock(sk);
		break;
		default:
		err = -ENOPROTOOPT;
		@@ -2474,6 +2485,14 @@ int udp_lib_setsockopt(struct sock *sk, int level, int optname,
		up->gso_size = val;
		break;

		case UDP_GRO:
		lock_sock(sk);
		if (valbool)
		udp_tunnel_encap_enable(sk->sk_socket);
		up->gro_enabled = valbool;
		release_sock(sk);
		break;

		/*
		* UDP-Lite's partial checksum coverage (RFC 3828).
		*/

net/ipv4/udp_offload.c

+87 −22

Original line number	Diff line number	Diff line
		@@ -343,6 +343,54 @@ static struct sk_buff udp4_ufo_fragment(struct sk_buff skb,
		return segs;
		}

		#define UDP_GRO_CNT_MAX 64
		static struct sk_buff udp_gro_receive_segment(struct list_head head,
		struct sk_buff *skb)
		{
		struct udphdr *uh = udp_hdr(skb);
		struct sk_buff *pp = NULL;
		struct udphdr *uh2;
		struct sk_buff *p;

		/* requires non zero csum, for symmetry with GSO */
		if (!uh->check) {
		NAPI_GRO_CB(skb)->flush = 1;
		return NULL;
		}

		/* pull encapsulating udp header */
		skb_gro_pull(skb, sizeof(struct udphdr));
		skb_gro_postpull_rcsum(skb, uh, sizeof(struct udphdr));

		list_for_each_entry(p, head, list) {
		if (!NAPI_GRO_CB(p)->same_flow)
		continue;

		uh2 = udp_hdr(p);

		/* Match ports only, as csum is always non zero */
		if (((u32 )&uh->source != (u32 )&uh2->source)) {
		NAPI_GRO_CB(p)->same_flow = 0;
		continue;
		}

		/* Terminate the flow on len mismatch or if it grow "too much".
		* Under small packet flood GRO count could elsewhere grow a lot
		* leading to execessive truesize values
		*/
		if (!skb_gro_receive(p, skb) &&
		NAPI_GRO_CB(p)->count >= UDP_GRO_CNT_MAX)
		pp = p;
		else if (uh->len != uh2->len)
		pp = p;

		return pp;
		}

		/* mismatch, but we never need to flush */
		return NULL;
		}

		struct sk_buff udp_gro_receive(struct list_head head, struct sk_buff *skb,
		struct udphdr *uh, udp_lookup_t lookup)
		{
		@@ -353,23 +401,27 @@ struct sk_buff udp_gro_receive(struct list_head head, struct sk_buff *skb,
		int flush = 1;
		struct sock *sk;

		rcu_read_lock();
		sk = (*lookup)(skb, uh->source, uh->dest);
		if (!sk)
		goto out_unlock;

		if (udp_sk(sk)->gro_enabled) {
		pp = call_gro_receive(udp_gro_receive_segment, head, skb);
		rcu_read_unlock();
		return pp;
		}

		if (NAPI_GRO_CB(skb)->encap_mark \|\|
		(skb->ip_summed != CHECKSUM_PARTIAL &&
		NAPI_GRO_CB(skb)->csum_cnt == 0 &&
		!NAPI_GRO_CB(skb)->csum_valid))
		goto out;
		!NAPI_GRO_CB(skb)->csum_valid) \|\|
		!udp_sk(sk)->gro_receive)
		goto out_unlock;

		/* mark that this skb passed once through the tunnel gro layer */
		NAPI_GRO_CB(skb)->encap_mark = 1;

		rcu_read_lock();
		sk = (*lookup)(skb, uh->source, uh->dest);

		if (sk && udp_sk(sk)->gro_receive)
		goto unflush;
		goto out_unlock;

		unflush:
		flush = 0;

		list_for_each_entry(p, head, list) {
		@@ -394,7 +446,6 @@ struct sk_buff udp_gro_receive(struct list_head head, struct sk_buff *skb,

		out_unlock:
		rcu_read_unlock();
		out:
		skb_gro_flush_final(skb, pp, flush);
		return pp;
		}
		@@ -427,6 +478,19 @@ static struct sk_buff udp4_gro_receive(struct list_head head,
		return NULL;
		}

		static int udp_gro_complete_segment(struct sk_buff *skb)
		{
		struct udphdr *uh = udp_hdr(skb);

		skb->csum_start = (unsigned char *)uh - skb->head;
		skb->csum_offset = offsetof(struct udphdr, check);
		skb->ip_summed = CHECKSUM_PARTIAL;

		skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
		skb_shinfo(skb)->gso_type \|= SKB_GSO_UDP_L4;
		return 0;
		}

		int udp_gro_complete(struct sk_buff *skb, int nhoff,
		udp_lookup_t lookup)
		{
		@@ -437,16 +501,21 @@ int udp_gro_complete(struct sk_buff *skb, int nhoff,

		uh->len = newlen;

		/* Set encapsulation before calling into inner gro_complete() functions
		* to make them set up the inner offsets.
		*/
		skb->encapsulation = 1;

		rcu_read_lock();
		sk = (*lookup)(skb, uh->source, uh->dest);
		if (sk && udp_sk(sk)->gro_complete)
		if (sk && udp_sk(sk)->gro_enabled) {
		err = udp_gro_complete_segment(skb);
		} else if (sk && udp_sk(sk)->gro_complete) {
		skb_shinfo(skb)->gso_type = uh->check ? SKB_GSO_UDP_TUNNEL_CSUM
		: SKB_GSO_UDP_TUNNEL;

		/* Set encapsulation before calling into inner gro_complete()
		* functions to make them set up the inner offsets.
		*/
		skb->encapsulation = 1;
		err = udp_sk(sk)->gro_complete(sk, skb,
		nhoff + sizeof(struct udphdr));
		}
		rcu_read_unlock();

		if (skb->remcsum_offload)
		@@ -461,13 +530,9 @@ static int udp4_gro_complete(struct sk_buff *skb, int nhoff)
		const struct iphdr *iph = ip_hdr(skb);
		struct udphdr uh = (struct udphdr )(skb->data + nhoff);

		if (uh->check) {
		skb_shinfo(skb)->gso_type \|= SKB_GSO_UDP_TUNNEL_CSUM;
		if (uh->check)
		uh->check = ~udp_v4_check(skb->len - nhoff, iph->saddr,
		iph->daddr, 0);
		} else {
		skb_shinfo(skb)->gso_type \|= SKB_GSO_UDP_TUNNEL;
		}

		return udp_gro_complete(skb, nhoff, udp4_lib_lookup_skb);
		}