Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 9c55e01c authored by Jens Axboe's avatar Jens Axboe Committed by David S. Miller
Browse files

[TCP]: Splice receive support.



Support for network splice receive.

Signed-off-by: default avatarJens Axboe <jens.axboe@oracle.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent bbdfc2f7
Loading
Loading
Loading
Loading
+3 −0
Original line number Original line Diff line number Diff line
@@ -22,6 +22,7 @@
#include <asm/socket.h>
#include <asm/socket.h>


struct poll_table_struct;
struct poll_table_struct;
struct pipe_inode_info;
struct inode;
struct inode;
struct net;
struct net;


@@ -172,6 +173,8 @@ struct proto_ops {
				      struct vm_area_struct * vma);
				      struct vm_area_struct * vma);
	ssize_t		(*sendpage)  (struct socket *sock, struct page *page,
	ssize_t		(*sendpage)  (struct socket *sock, struct page *page,
				      int offset, size_t size, int flags);
				      int offset, size_t size, int flags);
	ssize_t 	(*splice_read)(struct socket *sock,  loff_t *ppos,
				       struct pipe_inode_info *pipe, size_t len, unsigned int flags);
};
};


struct net_proto_family {
struct net_proto_family {
+6 −0
Original line number Original line Diff line number Diff line
@@ -95,6 +95,7 @@


struct net_device;
struct net_device;
struct scatterlist;
struct scatterlist;
struct pipe_inode_info;


#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
struct nf_conntrack {
struct nf_conntrack {
@@ -1559,6 +1560,11 @@ extern int skb_store_bits(struct sk_buff *skb, int offset,
extern __wsum	       skb_copy_and_csum_bits(const struct sk_buff *skb,
extern __wsum	       skb_copy_and_csum_bits(const struct sk_buff *skb,
					      int offset, u8 *to, int len,
					      int offset, u8 *to, int len,
					      __wsum csum);
					      __wsum csum);
extern int             skb_splice_bits(struct sk_buff *skb,
						unsigned int offset,
						struct pipe_inode_info *pipe,
						unsigned int len,
						unsigned int flags);
extern void	       skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to);
extern void	       skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to);
extern void	       skb_split(struct sk_buff *skb,
extern void	       skb_split(struct sk_buff *skb,
				 struct sk_buff *skb1, const u32 len);
				 struct sk_buff *skb1, const u32 len);
+3 −0
Original line number Original line Diff line number Diff line
@@ -309,6 +309,9 @@ extern int tcp_twsk_unique(struct sock *sk,


extern void			tcp_twsk_destructor(struct sock *sk);
extern void			tcp_twsk_destructor(struct sock *sk);


extern ssize_t			tcp_splice_read(struct socket *sk, loff_t *ppos,
					        struct pipe_inode_info *pipe, size_t len, unsigned int flags);

static inline void tcp_dec_quickack_mode(struct sock *sk,
static inline void tcp_dec_quickack_mode(struct sock *sk,
					 const unsigned int pkts)
					 const unsigned int pkts)
{
{
+246 −0
Original line number Original line Diff line number Diff line
@@ -52,6 +52,7 @@
#endif
#endif
#include <linux/string.h>
#include <linux/string.h>
#include <linux/skbuff.h>
#include <linux/skbuff.h>
#include <linux/splice.h>
#include <linux/cache.h>
#include <linux/cache.h>
#include <linux/rtnetlink.h>
#include <linux/rtnetlink.h>
#include <linux/init.h>
#include <linux/init.h>
@@ -71,6 +72,40 @@
static struct kmem_cache *skbuff_head_cache __read_mostly;
static struct kmem_cache *skbuff_head_cache __read_mostly;
static struct kmem_cache *skbuff_fclone_cache __read_mostly;
static struct kmem_cache *skbuff_fclone_cache __read_mostly;


static void sock_pipe_buf_release(struct pipe_inode_info *pipe,
				  struct pipe_buffer *buf)
{
	struct sk_buff *skb = (struct sk_buff *) buf->private;

	kfree_skb(skb);
}

static void sock_pipe_buf_get(struct pipe_inode_info *pipe,
				struct pipe_buffer *buf)
{
	struct sk_buff *skb = (struct sk_buff *) buf->private;

	skb_get(skb);
}

static int sock_pipe_buf_steal(struct pipe_inode_info *pipe,
			       struct pipe_buffer *buf)
{
	return 1;
}


/* Pipe buffer operations for a socket. */
static struct pipe_buf_operations sock_pipe_buf_ops = {
	.can_merge = 0,
	.map = generic_pipe_buf_map,
	.unmap = generic_pipe_buf_unmap,
	.confirm = generic_pipe_buf_confirm,
	.release = sock_pipe_buf_release,
	.steal = sock_pipe_buf_steal,
	.get = sock_pipe_buf_get,
};

/*
/*
 *	Keep out-of-line to prevent kernel bloat.
 *	Keep out-of-line to prevent kernel bloat.
 *	__builtin_return_address is not used because it is not always
 *	__builtin_return_address is not used because it is not always
@@ -1122,6 +1157,217 @@ int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
	return -EFAULT;
	return -EFAULT;
}
}


/*
 * Callback from splice_to_pipe(), if we need to release some pages
 * at the end of the spd in case we error'ed out in filling the pipe.
 */
static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
{
	struct sk_buff *skb = (struct sk_buff *) spd->partial[i].private;

	kfree_skb(skb);
}

/*
 * Fill page/offset/length into spd, if it can hold more pages.
 */
static inline int spd_fill_page(struct splice_pipe_desc *spd, struct page *page,
				unsigned int len, unsigned int offset,
				struct sk_buff *skb)
{
	if (unlikely(spd->nr_pages == PIPE_BUFFERS))
		return 1;

	spd->pages[spd->nr_pages] = page;
	spd->partial[spd->nr_pages].len = len;
	spd->partial[spd->nr_pages].offset = offset;
	spd->partial[spd->nr_pages].private = (unsigned long) skb_get(skb);
	spd->nr_pages++;
	return 0;
}

/*
 * Map linear and fragment data from the skb to spd. Returns number of
 * pages mapped.
 */
static int __skb_splice_bits(struct sk_buff *skb, unsigned int *offset,
			     unsigned int *total_len,
			     struct splice_pipe_desc *spd)
{
	unsigned int nr_pages = spd->nr_pages;
	unsigned int poff, plen, len, toff, tlen;
	int headlen, seg;

	toff = *offset;
	tlen = *total_len;
	if (!tlen)
		goto err;

	/*
	 * if the offset is greater than the linear part, go directly to
	 * the fragments.
	 */
	headlen = skb_headlen(skb);
	if (toff >= headlen) {
		toff -= headlen;
		goto map_frag;
	}

	/*
	 * first map the linear region into the pages/partial map, skipping
	 * any potential initial offset.
	 */
	len = 0;
	while (len < headlen) {
		void *p = skb->data + len;

		poff = (unsigned long) p & (PAGE_SIZE - 1);
		plen = min_t(unsigned int, headlen - len, PAGE_SIZE - poff);
		len += plen;

		if (toff) {
			if (plen <= toff) {
				toff -= plen;
				continue;
			}
			plen -= toff;
			poff += toff;
			toff = 0;
		}

		plen = min(plen, tlen);
		if (!plen)
			break;

		/*
		 * just jump directly to update and return, no point
		 * in going over fragments when the output is full.
		 */
		if (spd_fill_page(spd, virt_to_page(p), plen, poff, skb))
			goto done;

		tlen -= plen;
	}

	/*
	 * then map the fragments
	 */
map_frag:
	for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) {
		const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];

		plen = f->size;
		poff = f->page_offset;

		if (toff) {
			if (plen <= toff) {
				toff -= plen;
				continue;
			}
			plen -= toff;
			poff += toff;
			toff = 0;
		}

		plen = min(plen, tlen);
		if (!plen)
			break;

		if (spd_fill_page(spd, f->page, plen, poff, skb))
			break;

		tlen -= plen;
	}

done:
	if (spd->nr_pages - nr_pages) {
		*offset = 0;
		*total_len = tlen;
		return 0;
	}
err:
	return 1;
}

/*
 * Map data from the skb to a pipe. Should handle both the linear part,
 * the fragments, and the frag list. It does NOT handle frag lists within
 * the frag list, if such a thing exists. We'd probably need to recurse to
 * handle that cleanly.
 */
int skb_splice_bits(struct sk_buff *__skb, unsigned int offset,
		    struct pipe_inode_info *pipe, unsigned int tlen,
		    unsigned int flags)
{
	struct partial_page partial[PIPE_BUFFERS];
	struct page *pages[PIPE_BUFFERS];
	struct splice_pipe_desc spd = {
		.pages = pages,
		.partial = partial,
		.flags = flags,
		.ops = &sock_pipe_buf_ops,
		.spd_release = sock_spd_release,
	};
	struct sk_buff *skb;

	/*
	 * I'd love to avoid the clone here, but tcp_read_sock()
	 * ignores reference counts and unconditonally kills the sk_buff
	 * on return from the actor.
	 */
	skb = skb_clone(__skb, GFP_KERNEL);
	if (unlikely(!skb))
		return -ENOMEM;

	/*
	 * __skb_splice_bits() only fails if the output has no room left,
	 * so no point in going over the frag_list for the error case.
	 */
	if (__skb_splice_bits(skb, &offset, &tlen, &spd))
		goto done;
	else if (!tlen)
		goto done;

	/*
	 * now see if we have a frag_list to map
	 */
	if (skb_shinfo(skb)->frag_list) {
		struct sk_buff *list = skb_shinfo(skb)->frag_list;

		for (; list && tlen; list = list->next) {
			if (__skb_splice_bits(list, &offset, &tlen, &spd))
				break;
		}
	}

done:
	/*
	 * drop our reference to the clone, the pipe consumption will
	 * drop the rest.
	 */
	kfree_skb(skb);

	if (spd.nr_pages) {
		int ret;

		/*
		 * Drop the socket lock, otherwise we have reverse
		 * locking dependencies between sk_lock and i_mutex
		 * here as compared to sendfile(). We enter here
		 * with the socket lock held, and splice_to_pipe() will
		 * grab the pipe inode lock. For sendfile() emulation,
		 * we call into ->sendpage() with the i_mutex lock held
		 * and networking will grab the socket lock.
		 */
		release_sock(__skb->sk);
		ret = splice_to_pipe(pipe, &spd);
		lock_sock(__skb->sk);
		return ret;
	}

	return 0;
}

/**
/**
 *	skb_store_bits - store bits from kernel buffer to skb
 *	skb_store_bits - store bits from kernel buffer to skb
 *	@skb: destination buffer
 *	@skb: destination buffer
+1 −0
Original line number Original line Diff line number Diff line
@@ -838,6 +838,7 @@ const struct proto_ops inet_stream_ops = {
	.recvmsg	   = sock_common_recvmsg,
	.recvmsg	   = sock_common_recvmsg,
	.mmap		   = sock_no_mmap,
	.mmap		   = sock_no_mmap,
	.sendpage	   = tcp_sendpage,
	.sendpage	   = tcp_sendpage,
	.splice_read	   = tcp_splice_read,
#ifdef CONFIG_COMPAT
#ifdef CONFIG_COMPAT
	.compat_setsockopt = compat_sock_common_setsockopt,
	.compat_setsockopt = compat_sock_common_setsockopt,
	.compat_getsockopt = compat_sock_common_getsockopt,
	.compat_getsockopt = compat_sock_common_getsockopt,
Loading