Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit fd11a83d authored by Alexander Duyck's avatar Alexander Duyck Committed by David S. Miller
Browse files

net: Pull out core bits of __netdev_alloc_skb and add __napi_alloc_skb 



This change pulls the core functionality out of __netdev_alloc_skb and
places them in a new function named __alloc_rx_skb.  The reason for doing
this is to make these bits accessible to a new function __napi_alloc_skb.
In addition __alloc_rx_skb now has a new flags value that is used to
determine which page frag pool to allocate from.  If the SKB_ALLOC_NAPI
flag is set then the NAPI pool is used.  The advantage of this is that we
do not have to use local_irq_save/restore when accessing the NAPI pool from
NAPI context.

In my test setup I saw at least 11ns of savings using the napi_alloc_skb
function versus the netdev_alloc_skb function, most of this being due to
the fact that we didn't have to call local_irq_save/restore.

The main use case for napi_alloc_skb would be for things such as copybreak
or page fragment based receive paths where an skb is allocated after the
data has been received instead of before.

Signed-off-by: default avatarAlexander Duyck <alexander.h.duyck@redhat.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent ffde7328

include/linux/skbuff.h

+9 −0
Original line number Original line Diff line number Diff line
@@ -151,6 +151,7 @@ struct net_device; 
struct scatterlist;

struct scatterlist;

struct pipe_inode_info;

struct pipe_inode_info;

struct iov_iter;

struct iov_iter;

struct napi_struct;





#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)

#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)

struct nf_conntrack {

struct nf_conntrack {
@@ -673,6 +674,7 @@ struct sk_buff { 




#define SKB_ALLOC_FCLONE	0x01

#define SKB_ALLOC_FCLONE	0x01

#define SKB_ALLOC_RX		0x02

#define SKB_ALLOC_RX		0x02

#define SKB_ALLOC_NAPI		0x04





/* Returns true if the skb was allocated from PFMEMALLOC reserves */

/* Returns true if the skb was allocated from PFMEMALLOC reserves */

static inline bool skb_pfmemalloc(const struct sk_buff *skb)

static inline bool skb_pfmemalloc(const struct sk_buff *skb)
@@ -2165,6 +2167,13 @@ static inline struct sk_buff *netdev_alloc_skb_ip_align(struct net_device *dev, 
}

}





void *napi_alloc_frag(unsigned int fragsz);

void *napi_alloc_frag(unsigned int fragsz);

struct sk_buff *__napi_alloc_skb(struct napi_struct *napi,

				 unsigned int length, gfp_t gfp_mask);

static inline struct sk_buff *napi_alloc_skb(struct napi_struct *napi,

					     unsigned int length)

{

	return __napi_alloc_skb(napi, length, GFP_ATOMIC);

}





/**

/**

 * __dev_alloc_pages - allocate page for network Rx

 * __dev_alloc_pages - allocate page for network Rx

net/core/dev.c

+1 −1
Original line number Original line Diff line number Diff line
@@ -4172,7 +4172,7 @@ struct sk_buff *napi_get_frags(struct napi_struct *napi) 
	struct sk_buff *skb = napi->skb;

	struct sk_buff *skb = napi->skb;





	if (!skb) {

	if (!skb) {

		skb = netdev_alloc_skb_ip_align(napi->dev, GRO_MAX_HEAD);

		skb = napi_alloc_skb(napi, GRO_MAX_HEAD);

		napi->skb = skb;

		napi->skb = skb;

	}

	}

	return skb;

	return skb;

net/core/skbuff.c

+67 −7
Original line number Original line Diff line number Diff line
@@ -444,10 +444,13 @@ void *napi_alloc_frag(unsigned int fragsz) 
EXPORT_SYMBOL(napi_alloc_frag);

EXPORT_SYMBOL(napi_alloc_frag);





/**

/**

 *	__netdev_alloc_skb - allocate an skbuff for rx on a specific device

 *	__alloc_rx_skb - allocate an skbuff for rx

 *	@dev: network device to receive on

 *	@length: length to allocate

 *	@length: length to allocate

 *	@gfp_mask: get_free_pages mask, passed to alloc_skb

 *	@gfp_mask: get_free_pages mask, passed to alloc_skb

 *	@flags:	If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for

 *		allocations in case we have to fallback to __alloc_skb()

 *		If SKB_ALLOC_NAPI is set, page fragment will be allocated

 *		from napi_cache instead of netdev_cache.

 *

 *

 *	Allocate a new &sk_buff and assign it a usage count of one. The

 *	Allocate a new &sk_buff and assign it a usage count of one. The

 *	buffer has unspecified headroom built in. Users should allocate

 *	buffer has unspecified headroom built in. Users should allocate
@@ -456,11 +459,11 @@ EXPORT_SYMBOL(napi_alloc_frag); 
 *

 *

 *	%NULL is returned if there is no free memory.

 *	%NULL is returned if there is no free memory.

 */

 */

struct sk_buff *__netdev_alloc_skb(struct net_device *dev,

static struct sk_buff *__alloc_rx_skb(unsigned int length, gfp_t gfp_mask,

				   unsigned int length, gfp_t gfp_mask)

				      int flags)

{

{

	struct sk_buff *skb = NULL;

	struct sk_buff *skb = NULL;

	unsigned int fragsz = SKB_DATA_ALIGN(length + NET_SKB_PAD) +

	unsigned int fragsz = SKB_DATA_ALIGN(length) +

			      SKB_DATA_ALIGN(sizeof(struct skb_shared_info));

			      SKB_DATA_ALIGN(sizeof(struct skb_shared_info));





	if (fragsz <= PAGE_SIZE && !(gfp_mask & (__GFP_WAIT | GFP_DMA))) {

	if (fragsz <= PAGE_SIZE && !(gfp_mask & (__GFP_WAIT | GFP_DMA))) {
@@ -469,7 +472,9 @@ struct sk_buff *__netdev_alloc_skb(struct net_device *dev, 
		if (sk_memalloc_socks())

		if (sk_memalloc_socks())

			gfp_mask |= __GFP_MEMALLOC;

			gfp_mask |= __GFP_MEMALLOC;





		data = __netdev_alloc_frag(fragsz, gfp_mask);

		data = (flags & SKB_ALLOC_NAPI) ?

			__napi_alloc_frag(fragsz, gfp_mask) :

			__netdev_alloc_frag(fragsz, gfp_mask);





		if (likely(data)) {

		if (likely(data)) {

			skb = build_skb(data, fragsz);

			skb = build_skb(data, fragsz);
@@ -477,17 +482,72 @@ struct sk_buff *__netdev_alloc_skb(struct net_device *dev, 
				put_page(virt_to_head_page(data));

				put_page(virt_to_head_page(data));

		}

		}

	} else {

	} else {

		skb = __alloc_skb(length + NET_SKB_PAD, gfp_mask,

		skb = __alloc_skb(length, gfp_mask,

				  SKB_ALLOC_RX, NUMA_NO_NODE);

				  SKB_ALLOC_RX, NUMA_NO_NODE);

	}

	}

	return skb;

}



/**

 *	__netdev_alloc_skb - allocate an skbuff for rx on a specific device

 *	@dev: network device to receive on

 *	@length: length to allocate

 *	@gfp_mask: get_free_pages mask, passed to alloc_skb

 *

 *	Allocate a new &sk_buff and assign it a usage count of one. The

 *	buffer has NET_SKB_PAD headroom built in. Users should allocate

 *	the headroom they think they need without accounting for the

 *	built in space. The built in space is used for optimisations.

 *

 *	%NULL is returned if there is no free memory.

 */

struct sk_buff *__netdev_alloc_skb(struct net_device *dev,

				   unsigned int length, gfp_t gfp_mask)

{

	struct sk_buff *skb;



	length += NET_SKB_PAD;

	skb = __alloc_rx_skb(length, gfp_mask, 0);



	if (likely(skb)) {

	if (likely(skb)) {

		skb_reserve(skb, NET_SKB_PAD);

		skb_reserve(skb, NET_SKB_PAD);

		skb->dev = dev;

		skb->dev = dev;

	}

	}



	return skb;

	return skb;

}

}

EXPORT_SYMBOL(__netdev_alloc_skb);

EXPORT_SYMBOL(__netdev_alloc_skb);





/**

 *	__napi_alloc_skb - allocate skbuff for rx in a specific NAPI instance

 *	@napi: napi instance this buffer was allocated for

 *	@length: length to allocate

 *	@gfp_mask: get_free_pages mask, passed to alloc_skb and alloc_pages

 *

 *	Allocate a new sk_buff for use in NAPI receive.  This buffer will

 *	attempt to allocate the head from a special reserved region used

 *	only for NAPI Rx allocation.  By doing this we can save several

 *	CPU cycles by avoiding having to disable and re-enable IRQs.

 *

 *	%NULL is returned if there is no free memory.

 */

struct sk_buff *__napi_alloc_skb(struct napi_struct *napi,

				 unsigned int length, gfp_t gfp_mask)

{

	struct sk_buff *skb;



	length += NET_SKB_PAD + NET_IP_ALIGN;

	skb = __alloc_rx_skb(length, gfp_mask, SKB_ALLOC_NAPI);



	if (likely(skb)) {

		skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);

		skb->dev = napi->dev;

	}



	return skb;

}

EXPORT_SYMBOL(__napi_alloc_skb);



void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,

void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,

		     int size, unsigned int truesize)

		     int size, unsigned int truesize)

{

{