Loading drivers/net/virtio_net.c +43 −46 Original line number Diff line number Diff line Loading @@ -250,24 +250,6 @@ static void skb_xmit_done(struct virtqueue *vq) netif_wake_subqueue(vi->dev, vq2txq(vq)); } static unsigned int mergeable_ctx_to_buf_truesize(unsigned long mrg_ctx) { unsigned int truesize = mrg_ctx & (MERGEABLE_BUFFER_ALIGN - 1); return (truesize + 1) * MERGEABLE_BUFFER_ALIGN; } static void *mergeable_ctx_to_buf_address(unsigned long mrg_ctx) { return (void *)(mrg_ctx & -MERGEABLE_BUFFER_ALIGN); } static unsigned long mergeable_buf_to_ctx(void *buf, unsigned int truesize) { unsigned int size = truesize / MERGEABLE_BUFFER_ALIGN; return (unsigned long)buf | (size - 1); } /* Called from bottom half context */ static struct sk_buff *page_to_skb(struct virtnet_info *vi, struct receive_queue *rq, Loading Loading @@ -511,15 +493,13 @@ static struct page *xdp_linearize_page(struct receive_queue *rq, while (--*num_buf) { unsigned int buflen; unsigned long ctx; void *buf; int off; ctx = (unsigned long)virtqueue_get_buf(rq->vq, &buflen); if (unlikely(!ctx)) buf = virtqueue_get_buf(rq->vq, &buflen); if (unlikely(!buf)) goto err_buf; buf = mergeable_ctx_to_buf_address(ctx); p = virt_to_head_page(buf); off = buf - page_address(p); Loading Loading @@ -548,10 +528,10 @@ static struct page *xdp_linearize_page(struct receive_queue *rq, static struct sk_buff *receive_mergeable(struct net_device *dev, struct virtnet_info *vi, struct receive_queue *rq, unsigned long ctx, void *buf, void *ctx, unsigned int len) { void *buf = mergeable_ctx_to_buf_address(ctx); struct virtio_net_hdr_mrg_rxbuf *hdr = buf; u16 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers); struct page *page = virt_to_head_page(buf); Loading Loading @@ -639,7 +619,13 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, } rcu_read_unlock(); truesize = max(len, mergeable_ctx_to_buf_truesize(ctx)); if (unlikely(len > (unsigned long)ctx)) { pr_debug("%s: rx error: len %u exceeds truesize 0x%lu\n", dev->name, len, (unsigned long)ctx); dev->stats.rx_length_errors++; goto err_skb; } truesize = (unsigned long)ctx; head_skb = page_to_skb(vi, rq, page, offset, len, truesize); curr_skb = head_skb; Loading @@ -648,7 +634,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, while (--num_buf) { int num_skb_frags; ctx = (unsigned long)virtqueue_get_buf(rq->vq, &len); buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx); if (unlikely(!ctx)) { pr_debug("%s: rx error: %d buffers out of %d missing\n", dev->name, num_buf, Loading @@ -658,8 +644,14 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, goto err_buf; } buf = mergeable_ctx_to_buf_address(ctx); page = virt_to_head_page(buf); if (unlikely(len > (unsigned long)ctx)) { pr_debug("%s: rx error: len %u exceeds truesize 0x%lu\n", dev->name, len, (unsigned long)ctx); dev->stats.rx_length_errors++; goto err_skb; } truesize = (unsigned long)ctx; num_skb_frags = skb_shinfo(curr_skb)->nr_frags; if (unlikely(num_skb_frags == MAX_SKB_FRAGS)) { Loading @@ -675,7 +667,6 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, head_skb->truesize += nskb->truesize; num_skb_frags = 0; } truesize = max(len, mergeable_ctx_to_buf_truesize(ctx)); if (curr_skb != head_skb) { head_skb->data_len += len; head_skb->len += len; Loading @@ -700,14 +691,14 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, err_skb: put_page(page); while (--num_buf) { ctx = (unsigned long)virtqueue_get_buf(rq->vq, &len); if (unlikely(!ctx)) { buf = virtqueue_get_buf(rq->vq, &len); if (unlikely(!buf)) { pr_debug("%s: rx error: %d buffers missing\n", dev->name, num_buf); dev->stats.rx_length_errors++; break; } page = virt_to_head_page(mergeable_ctx_to_buf_address(ctx)); page = virt_to_head_page(buf); put_page(page); } err_buf: Loading @@ -718,7 +709,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, } static int receive_buf(struct virtnet_info *vi, struct receive_queue *rq, void *buf, unsigned int len) void *buf, unsigned int len, void **ctx) { struct net_device *dev = vi->dev; struct sk_buff *skb; Loading @@ -729,9 +720,7 @@ static int receive_buf(struct virtnet_info *vi, struct receive_queue *rq, pr_debug("%s: short packet %i\n", dev->name, len); dev->stats.rx_length_errors++; if (vi->mergeable_rx_bufs) { unsigned long ctx = (unsigned long)buf; void *base = mergeable_ctx_to_buf_address(ctx); put_page(virt_to_head_page(base)); put_page(virt_to_head_page(buf)); } else if (vi->big_packets) { give_pages(rq, buf); } else { Loading @@ -741,7 +730,7 @@ static int receive_buf(struct virtnet_info *vi, struct receive_queue *rq, } if (vi->mergeable_rx_bufs) skb = receive_mergeable(dev, vi, rq, (unsigned long)buf, len); skb = receive_mergeable(dev, vi, rq, buf, ctx, len); else if (vi->big_packets) skb = receive_big(dev, vi, rq, buf, len); else Loading Loading @@ -869,7 +858,7 @@ static int add_recvbuf_mergeable(struct virtnet_info *vi, struct page_frag *alloc_frag = &rq->alloc_frag; unsigned int headroom = virtnet_get_headroom(vi); char *buf; unsigned long ctx; void *ctx; int err; unsigned int len, hole; Loading @@ -879,7 +868,7 @@ static int add_recvbuf_mergeable(struct virtnet_info *vi, buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset; buf += headroom; /* advance address leaving hole at front of pkt */ ctx = mergeable_buf_to_ctx(buf, len); ctx = (void *)(unsigned long)len; get_page(alloc_frag->page); alloc_frag->offset += len + headroom; hole = alloc_frag->size - alloc_frag->offset; Loading @@ -894,7 +883,7 @@ static int add_recvbuf_mergeable(struct virtnet_info *vi, } sg_init_one(rq->sg, buf, len); err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, (void *)ctx, gfp); err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp); if (err < 0) put_page(virt_to_head_page(buf)); Loading Loading @@ -988,11 +977,21 @@ static int virtnet_receive(struct receive_queue *rq, int budget) void *buf; struct virtnet_stats *stats = this_cpu_ptr(vi->stats); if (vi->mergeable_rx_bufs) { void *ctx; while (received < budget && (buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx))) { bytes += receive_buf(vi, rq, buf, len, ctx); received++; } } else { while (received < budget && (buf = virtqueue_get_buf(rq->vq, &len)) != NULL) { bytes += receive_buf(vi, rq, buf, len); bytes += receive_buf(vi, rq, buf, len, NULL); received++; } } if (rq->vq->num_free > virtqueue_get_vring_size(rq->vq) / 2) { if (!try_fill_recv(vi, rq, GFP_ATOMIC)) Loading Loading @@ -2014,9 +2013,7 @@ static void free_unused_bufs(struct virtnet_info *vi) while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) { if (vi->mergeable_rx_bufs) { unsigned long ctx = (unsigned long)buf; void *base = mergeable_ctx_to_buf_address(ctx); put_page(virt_to_head_page(base)); put_page(virt_to_head_page(buf)); } else if (vi->big_packets) { give_pages(&vi->rq[i], buf); } else { Loading Loading
drivers/net/virtio_net.c +43 −46 Original line number Diff line number Diff line Loading @@ -250,24 +250,6 @@ static void skb_xmit_done(struct virtqueue *vq) netif_wake_subqueue(vi->dev, vq2txq(vq)); } static unsigned int mergeable_ctx_to_buf_truesize(unsigned long mrg_ctx) { unsigned int truesize = mrg_ctx & (MERGEABLE_BUFFER_ALIGN - 1); return (truesize + 1) * MERGEABLE_BUFFER_ALIGN; } static void *mergeable_ctx_to_buf_address(unsigned long mrg_ctx) { return (void *)(mrg_ctx & -MERGEABLE_BUFFER_ALIGN); } static unsigned long mergeable_buf_to_ctx(void *buf, unsigned int truesize) { unsigned int size = truesize / MERGEABLE_BUFFER_ALIGN; return (unsigned long)buf | (size - 1); } /* Called from bottom half context */ static struct sk_buff *page_to_skb(struct virtnet_info *vi, struct receive_queue *rq, Loading Loading @@ -511,15 +493,13 @@ static struct page *xdp_linearize_page(struct receive_queue *rq, while (--*num_buf) { unsigned int buflen; unsigned long ctx; void *buf; int off; ctx = (unsigned long)virtqueue_get_buf(rq->vq, &buflen); if (unlikely(!ctx)) buf = virtqueue_get_buf(rq->vq, &buflen); if (unlikely(!buf)) goto err_buf; buf = mergeable_ctx_to_buf_address(ctx); p = virt_to_head_page(buf); off = buf - page_address(p); Loading Loading @@ -548,10 +528,10 @@ static struct page *xdp_linearize_page(struct receive_queue *rq, static struct sk_buff *receive_mergeable(struct net_device *dev, struct virtnet_info *vi, struct receive_queue *rq, unsigned long ctx, void *buf, void *ctx, unsigned int len) { void *buf = mergeable_ctx_to_buf_address(ctx); struct virtio_net_hdr_mrg_rxbuf *hdr = buf; u16 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers); struct page *page = virt_to_head_page(buf); Loading Loading @@ -639,7 +619,13 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, } rcu_read_unlock(); truesize = max(len, mergeable_ctx_to_buf_truesize(ctx)); if (unlikely(len > (unsigned long)ctx)) { pr_debug("%s: rx error: len %u exceeds truesize 0x%lu\n", dev->name, len, (unsigned long)ctx); dev->stats.rx_length_errors++; goto err_skb; } truesize = (unsigned long)ctx; head_skb = page_to_skb(vi, rq, page, offset, len, truesize); curr_skb = head_skb; Loading @@ -648,7 +634,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, while (--num_buf) { int num_skb_frags; ctx = (unsigned long)virtqueue_get_buf(rq->vq, &len); buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx); if (unlikely(!ctx)) { pr_debug("%s: rx error: %d buffers out of %d missing\n", dev->name, num_buf, Loading @@ -658,8 +644,14 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, goto err_buf; } buf = mergeable_ctx_to_buf_address(ctx); page = virt_to_head_page(buf); if (unlikely(len > (unsigned long)ctx)) { pr_debug("%s: rx error: len %u exceeds truesize 0x%lu\n", dev->name, len, (unsigned long)ctx); dev->stats.rx_length_errors++; goto err_skb; } truesize = (unsigned long)ctx; num_skb_frags = skb_shinfo(curr_skb)->nr_frags; if (unlikely(num_skb_frags == MAX_SKB_FRAGS)) { Loading @@ -675,7 +667,6 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, head_skb->truesize += nskb->truesize; num_skb_frags = 0; } truesize = max(len, mergeable_ctx_to_buf_truesize(ctx)); if (curr_skb != head_skb) { head_skb->data_len += len; head_skb->len += len; Loading @@ -700,14 +691,14 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, err_skb: put_page(page); while (--num_buf) { ctx = (unsigned long)virtqueue_get_buf(rq->vq, &len); if (unlikely(!ctx)) { buf = virtqueue_get_buf(rq->vq, &len); if (unlikely(!buf)) { pr_debug("%s: rx error: %d buffers missing\n", dev->name, num_buf); dev->stats.rx_length_errors++; break; } page = virt_to_head_page(mergeable_ctx_to_buf_address(ctx)); page = virt_to_head_page(buf); put_page(page); } err_buf: Loading @@ -718,7 +709,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, } static int receive_buf(struct virtnet_info *vi, struct receive_queue *rq, void *buf, unsigned int len) void *buf, unsigned int len, void **ctx) { struct net_device *dev = vi->dev; struct sk_buff *skb; Loading @@ -729,9 +720,7 @@ static int receive_buf(struct virtnet_info *vi, struct receive_queue *rq, pr_debug("%s: short packet %i\n", dev->name, len); dev->stats.rx_length_errors++; if (vi->mergeable_rx_bufs) { unsigned long ctx = (unsigned long)buf; void *base = mergeable_ctx_to_buf_address(ctx); put_page(virt_to_head_page(base)); put_page(virt_to_head_page(buf)); } else if (vi->big_packets) { give_pages(rq, buf); } else { Loading @@ -741,7 +730,7 @@ static int receive_buf(struct virtnet_info *vi, struct receive_queue *rq, } if (vi->mergeable_rx_bufs) skb = receive_mergeable(dev, vi, rq, (unsigned long)buf, len); skb = receive_mergeable(dev, vi, rq, buf, ctx, len); else if (vi->big_packets) skb = receive_big(dev, vi, rq, buf, len); else Loading Loading @@ -869,7 +858,7 @@ static int add_recvbuf_mergeable(struct virtnet_info *vi, struct page_frag *alloc_frag = &rq->alloc_frag; unsigned int headroom = virtnet_get_headroom(vi); char *buf; unsigned long ctx; void *ctx; int err; unsigned int len, hole; Loading @@ -879,7 +868,7 @@ static int add_recvbuf_mergeable(struct virtnet_info *vi, buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset; buf += headroom; /* advance address leaving hole at front of pkt */ ctx = mergeable_buf_to_ctx(buf, len); ctx = (void *)(unsigned long)len; get_page(alloc_frag->page); alloc_frag->offset += len + headroom; hole = alloc_frag->size - alloc_frag->offset; Loading @@ -894,7 +883,7 @@ static int add_recvbuf_mergeable(struct virtnet_info *vi, } sg_init_one(rq->sg, buf, len); err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, (void *)ctx, gfp); err = virtqueue_add_inbuf_ctx(rq->vq, rq->sg, 1, buf, ctx, gfp); if (err < 0) put_page(virt_to_head_page(buf)); Loading Loading @@ -988,11 +977,21 @@ static int virtnet_receive(struct receive_queue *rq, int budget) void *buf; struct virtnet_stats *stats = this_cpu_ptr(vi->stats); if (vi->mergeable_rx_bufs) { void *ctx; while (received < budget && (buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx))) { bytes += receive_buf(vi, rq, buf, len, ctx); received++; } } else { while (received < budget && (buf = virtqueue_get_buf(rq->vq, &len)) != NULL) { bytes += receive_buf(vi, rq, buf, len); bytes += receive_buf(vi, rq, buf, len, NULL); received++; } } if (rq->vq->num_free > virtqueue_get_vring_size(rq->vq) / 2) { if (!try_fill_recv(vi, rq, GFP_ATOMIC)) Loading Loading @@ -2014,9 +2013,7 @@ static void free_unused_bufs(struct virtnet_info *vi) while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) { if (vi->mergeable_rx_bufs) { unsigned long ctx = (unsigned long)buf; void *base = mergeable_ctx_to_buf_address(ctx); put_page(virt_to_head_page(base)); put_page(virt_to_head_page(buf)); } else if (vi->big_packets) { give_pages(&vi->rq[i], buf); } else { Loading