Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit d7e8883c authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller
Browse files

net: make GRO aware of skb->head_frag 



GRO can check if skb to be merged has its skb->head mapped to a page
fragment, instead of a kmalloc() area.

We 'upgrade' skb->head as a fragment in itself

This avoids the frag_list fallback, and permits to build true GRO skb
(one sk_buff and up to 16 fragments), using less memory.

This reduces number of cache misses when user makes its copy, since a
single sk_buff is fetched.

This is a followup of patch "net: allow skb->head to be a page fragment"

Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Cc: Ilpo Järvinen <ilpo.jarvinen@helsinki.fi>
Cc: Herbert Xu <herbert@gondor.apana.org.au>
Cc: Maciej Żenczykowski <maze@google.com>
Cc: Neal Cardwell <ncardwell@google.com>
Cc: Tom Herbert <therbert@google.com>
Cc: Jeff Kirsher <jeffrey.t.kirsher@intel.com>
Cc: Ben Hutchings <bhutchings@solarflare.com>
Cc: Matt Carlson <mcarlson@broadcom.com>
Cc: Michael Chan <mchan@broadcom.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 8d4057a9

include/linux/netdevice.h

+2 −0
Original line number Original line Diff line number Diff line
@@ -1509,6 +1509,8 @@ struct napi_gro_cb { 




	/* Free the skb? */

	/* Free the skb? */

	int free;

	int free;

#define NAPI_GRO_FREE		  1

#define NAPI_GRO_FREE_STOLEN_HEAD 2

};

};





#define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb)

#define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb)

include/linux/skbuff.h

+1 −0
Original line number Original line Diff line number Diff line
@@ -561,6 +561,7 @@ static inline struct rtable *skb_rtable(const struct sk_buff *skb) 
extern void kfree_skb(struct sk_buff *skb);

extern void kfree_skb(struct sk_buff *skb);

extern void consume_skb(struct sk_buff *skb);

extern void consume_skb(struct sk_buff *skb);

extern void	       __kfree_skb(struct sk_buff *skb);

extern void	       __kfree_skb(struct sk_buff *skb);

extern struct kmem_cache *skbuff_head_cache;

extern struct sk_buff *__alloc_skb(unsigned int size,

extern struct sk_buff *__alloc_skb(unsigned int size,

				   gfp_t priority, int fclone, int node);

				   gfp_t priority, int fclone, int node);

extern struct sk_buff *build_skb(void *data, unsigned int frag_size);

extern struct sk_buff *build_skb(void *data, unsigned int frag_size);

net/core/dev.c

+4 −1
Original line number Original line Diff line number Diff line
@@ -3546,7 +3546,10 @@ gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb) 
		break;

		break;





	case GRO_MERGED_FREE:

	case GRO_MERGED_FREE:

		consume_skb(skb);

		if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)

			kmem_cache_free(skbuff_head_cache, skb);

		else

			__kfree_skb(skb);

		break;

		break;





	case GRO_HELD:

	case GRO_HELD:

net/core/skbuff.c

+26 −1
Original line number Original line Diff line number Diff line
@@ -69,7 +69,7 @@ 
#include <trace/events/skb.h>

#include <trace/events/skb.h>

#include <linux/highmem.h>

#include <linux/highmem.h>





static struct kmem_cache *skbuff_head_cache __read_mostly;

struct kmem_cache *skbuff_head_cache __read_mostly;

static struct kmem_cache *skbuff_fclone_cache __read_mostly;

static struct kmem_cache *skbuff_fclone_cache __read_mostly;





static void sock_pipe_buf_release(struct pipe_inode_info *pipe,

static void sock_pipe_buf_release(struct pipe_inode_info *pipe,
@@ -2901,6 +2901,31 @@ int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb) 




		NAPI_GRO_CB(skb)->free = 1;

		NAPI_GRO_CB(skb)->free = 1;

		goto done;

		goto done;

	} else if (skb->head_frag) {

		int nr_frags = pinfo->nr_frags;

		skb_frag_t *frag = pinfo->frags + nr_frags;

		struct page *page = virt_to_head_page(skb->head);

		unsigned int first_size = headlen - offset;

		unsigned int first_offset;



		if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS)

			return -E2BIG;



		first_offset = skb->data -

			       (unsigned char *)page_address(page) +

			       offset;



		pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags;



		frag->page.p	  = page;

		frag->page_offset = first_offset;

		skb_frag_size_set(frag, first_size);



		memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags);

		/* We dont need to clear skbinfo->nr_frags here */



		NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD;

		goto done;

	} else if (skb_gro_len(p) != pinfo->gso_size)

	} else if (skb_gro_len(p) != pinfo->gso_size)

		return -E2BIG;

		return -E2BIG;