Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 15e36f5b authored by Willem de Bruijn's avatar Willem de Bruijn Committed by David S. Miller
Browse files

udp: paged allocation with gso 



When sending large datagrams that are later segmented, store data in
page frags to avoid copying from linear in skb_segment.

Signed-off-by: default avatarWillem de Bruijn <willemb@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent ad405857

net/ipv4/ip_output.c

+11 −4
Original line number Diff line number Diff line
@@ -878,11 +878,13 @@ static int __ip_append_data(struct sock *sk, 
	struct rtable *rt = (struct rtable *)cork->dst;

	unsigned int wmem_alloc_delta = 0;

	u32 tskey = 0;

	bool paged;



	skb = skb_peek_tail(queue);



	exthdrlen = !skb ? rt->dst.header_len : 0;

	mtu = cork->gso_size ? IP_MAX_MTU : cork->fragsize;

	paged = !!cork->gso_size;



	if (cork->tx_flags & SKBTX_ANY_SW_TSTAMP &&

	    sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)
@@ -934,6 +936,7 @@ static int __ip_append_data(struct sock *sk, 
			unsigned int fraglen;

			unsigned int fraggap;

			unsigned int alloclen;

			unsigned int pagedlen = 0;

			struct sk_buff *skb_prev;

alloc_new_skb:

			skb_prev = skb;
@@ -954,8 +957,12 @@ static int __ip_append_data(struct sock *sk, 
			if ((flags & MSG_MORE) &&

			    !(rt->dst.dev->features&NETIF_F_SG))

				alloclen = mtu;

			else

			else if (!paged)

				alloclen = fraglen;

			else {

				alloclen = min_t(int, fraglen, MAX_HEADER);

				pagedlen = fraglen - alloclen;

			}



			alloclen += exthdrlen;
@@ -999,7 +1006,7 @@ static int __ip_append_data(struct sock *sk, 
			/*

			 *	Find where to start putting bytes.

			 */

			data = skb_put(skb, fraglen + exthdrlen);

			data = skb_put(skb, fraglen + exthdrlen - pagedlen);

			skb_set_network_header(skb, exthdrlen);

			skb->transport_header = (skb->network_header +

						 fragheaderlen);
@@ -1015,7 +1022,7 @@ static int __ip_append_data(struct sock *sk, 
				pskb_trim_unique(skb_prev, maxfraglen);

			}



			copy = datalen - transhdrlen - fraggap;

			copy = datalen - transhdrlen - fraggap - pagedlen;

			if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, fraggap, skb) < 0) {

				err = -EFAULT;

				kfree_skb(skb);
@@ -1023,7 +1030,7 @@ static int __ip_append_data(struct sock *sk, 
			}



			offset += copy;

			length -= datalen - fraggap;

			length -= copy + transhdrlen;

			transhdrlen = 0;

			exthdrlen = 0;

			csummode = CHECKSUM_NONE;

net/ipv6/ip6_output.c

+14 −5
Original line number Diff line number Diff line
@@ -1276,6 +1276,7 @@ static int __ip6_append_data(struct sock *sk, 
	int csummode = CHECKSUM_NONE;

	unsigned int maxnonfragsize, headersize;

	unsigned int wmem_alloc_delta = 0;

	bool paged;



	skb = skb_peek_tail(queue);

	if (!skb) {
@@ -1283,6 +1284,7 @@ static int __ip6_append_data(struct sock *sk, 
		dst_exthdrlen = rt->dst.header_len - rt->rt6i_nfheader_len;

	}



	paged = !!cork->gso_size;

	mtu = cork->gso_size ? IP6_MAX_MTU : cork->fragsize;

	orig_mtu = mtu;
@@ -1374,6 +1376,7 @@ static int __ip6_append_data(struct sock *sk, 
			unsigned int fraglen;

			unsigned int fraggap;

			unsigned int alloclen;

			unsigned int pagedlen = 0;

alloc_new_skb:

			/* There's no room in the current skb */

			if (skb)
@@ -1396,11 +1399,17 @@ static int __ip6_append_data(struct sock *sk, 


			if (datalen > (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - fragheaderlen)

				datalen = maxfraglen - fragheaderlen - rt->dst.trailer_len;

			fraglen = datalen + fragheaderlen;



			if ((flags & MSG_MORE) &&

			    !(rt->dst.dev->features&NETIF_F_SG))

				alloclen = mtu;

			else

				alloclen = datalen + fragheaderlen;

			else if (!paged)

				alloclen = fraglen;

			else {

				alloclen = min_t(int, fraglen, MAX_HEADER);

				pagedlen = fraglen - alloclen;

			}



			alloclen += dst_exthdrlen;
@@ -1422,7 +1431,7 @@ static int __ip6_append_data(struct sock *sk, 
			 */

			alloclen += sizeof(struct frag_hdr);



			copy = datalen - transhdrlen - fraggap;

			copy = datalen - transhdrlen - fraggap - pagedlen;

			if (copy < 0) {

				err = -EINVAL;

				goto error;
@@ -1461,7 +1470,7 @@ static int __ip6_append_data(struct sock *sk, 
			/*

			 *	Find where to start putting bytes

			 */

			data = skb_put(skb, fraglen);

			data = skb_put(skb, fraglen - pagedlen);

			skb_set_network_header(skb, exthdrlen);

			data += fragheaderlen;

			skb->transport_header = (skb->network_header +
@@ -1484,7 +1493,7 @@ static int __ip6_append_data(struct sock *sk, 
			}



			offset += copy;

			length -= datalen - fraggap;

			length -= copy + transhdrlen;

			transhdrlen = 0;

			exthdrlen = 0;

			dst_exthdrlen = 0;