macvtap: macvtapTX zero-copy support (97bc3633) · Commits · e / devices / android_kernel_teracube_emerald

drivers/net/macvtap.c

+121 −11

Original line number	Original line	Diff line number	Diff line
	@@ -60,6 +60,7 @@ static struct proto macvtap_proto = {
	*/		*/
	static dev_t macvtap_major;		static dev_t macvtap_major;
	#define MACVTAP_NUM_DEVS 65536		#define MACVTAP_NUM_DEVS 65536
			#define GOODCOPY_LEN 128
	static struct class *macvtap_class;		static struct class *macvtap_class;
	static struct cdev macvtap_cdev;		static struct cdev macvtap_cdev;

	@@ -340,6 +341,7 @@ static int macvtap_open(struct inode inode, struct file file)
	{		{
	struct net *net = current->nsproxy->net_ns;		struct net *net = current->nsproxy->net_ns;
	struct net_device *dev = dev_get_by_index(net, iminor(inode));		struct net_device *dev = dev_get_by_index(net, iminor(inode));
			struct macvlan_dev *vlan = netdev_priv(dev);
	struct macvtap_queue *q;		struct macvtap_queue *q;
	int err;		int err;

	@@ -369,6 +371,16 @@ static int macvtap_open(struct inode inode, struct file file)
	q->flags = IFF_VNET_HDR \| IFF_NO_PI \| IFF_TAP;		q->flags = IFF_VNET_HDR \| IFF_NO_PI \| IFF_TAP;
	q->vnet_hdr_sz = sizeof(struct virtio_net_hdr);		q->vnet_hdr_sz = sizeof(struct virtio_net_hdr);

			/*
			* so far only KVM virtio_net uses macvtap, enable zero copy between
			* guest kernel and host kernel when lower device supports zerocopy
			*/
			if (vlan) {
			if ((vlan->lowerdev->features & NETIF_F_HIGHDMA) &&
			(vlan->lowerdev->features & NETIF_F_SG))
			sock_set_flag(&q->sk, SOCK_ZEROCOPY);
			}

	err = macvtap_set_queue(dev, file, q);		err = macvtap_set_queue(dev, file, q);
	if (err)		if (err)
	sock_put(&q->sk);		sock_put(&q->sk);
	@@ -433,6 +445,80 @@ static inline struct sk_buff macvtap_alloc_skb(struct sock sk, size_t prepad,
	return skb;		return skb;
	}		}

			/* set skb frags from iovec, this can move to core network code for reuse */
			static int zerocopy_sg_from_iovec(struct sk_buff skb, const struct iovec from,
			int offset, size_t count)
			{
			int len = iov_length(from, count) - offset;
			int copy = skb_headlen(skb);
			int size, offset1 = 0;
			int i = 0;
			skb_frag_t *f;

			/* Skip over from offset */
			while (count && (offset >= from->iov_len)) {
			offset -= from->iov_len;
			++from;
			--count;
			}

			/* copy up to skb headlen */
			while (count && (copy > 0)) {
			size = min_t(unsigned int, copy, from->iov_len - offset);
			if (copy_from_user(skb->data + offset1, from->iov_base + offset,
			size))
			return -EFAULT;
			if (copy > size) {
			++from;
			--count;
			}
			copy -= size;
			offset1 += size;
			offset = 0;
			}

			if (len == offset1)
			return 0;

			while (count--) {
			struct page *page[MAX_SKB_FRAGS];
			int num_pages;
			unsigned long base;

			len = from->iov_len - offset1;
			if (!len) {
			offset1 = 0;
			++from;
			continue;
			}
			base = (unsigned long)from->iov_base + offset1;
			size = ((base & ~PAGE_MASK) + len + ~PAGE_MASK) >> PAGE_SHIFT;
			num_pages = get_user_pages_fast(base, size, 0, &page[i]);
			if ((num_pages != size) \|\|
			(num_pages > MAX_SKB_FRAGS - skb_shinfo(skb)->nr_frags))
			/* put_page is in skb free */
			return -EFAULT;
			skb->data_len += len;
			skb->len += len;
			skb->truesize += len;
			atomic_add(len, &skb->sk->sk_wmem_alloc);
			while (len) {
			f = &skb_shinfo(skb)->frags[i];
			f->page = page[i];
			f->page_offset = base & ~PAGE_MASK;
			f->size = min_t(int, len, PAGE_SIZE - f->page_offset);
			skb_shinfo(skb)->nr_frags++;
			/* increase sk_wmem_alloc */
			base += f->size;
			len -= f->size;
			i++;
			}
			offset1 = 0;
			++from;
			}
			return 0;
			}

	/*		/*
	* macvtap_skb_from_vnet_hdr and macvtap_skb_to_vnet_hdr should		* macvtap_skb_from_vnet_hdr and macvtap_skb_to_vnet_hdr should
	* be shared with the tun/tap driver.		* be shared with the tun/tap driver.
	@@ -517,16 +603,18 @@ static int macvtap_skb_to_vnet_hdr(const struct sk_buff *skb,


	/* Get packet from user space buffer */		/* Get packet from user space buffer */
	static ssize_t macvtap_get_user(struct macvtap_queue *q,		static ssize_t macvtap_get_user(struct macvtap_queue q, struct msghdr m,
	const struct iovec *iv, size_t count,		const struct iovec *iv, unsigned long total_len,
	int noblock)		size_t count, int noblock)
	{		{
	struct sk_buff *skb;		struct sk_buff *skb;
	struct macvlan_dev *vlan;		struct macvlan_dev *vlan;
	size_t len = count;		unsigned long len = total_len;
	int err;		int err;
	struct virtio_net_hdr vnet_hdr = { 0 };		struct virtio_net_hdr vnet_hdr = { 0 };
	int vnet_hdr_len = 0;		int vnet_hdr_len = 0;
			int copylen;
			bool zerocopy = false;

	if (q->flags & IFF_VNET_HDR) {		if (q->flags & IFF_VNET_HDR) {
	vnet_hdr_len = q->vnet_hdr_sz;		vnet_hdr_len = q->vnet_hdr_sz;
	@@ -554,12 +642,31 @@ static ssize_t macvtap_get_user(struct macvtap_queue *q,
	if (unlikely(len < ETH_HLEN))		if (unlikely(len < ETH_HLEN))
	goto err;		goto err;

	skb = macvtap_alloc_skb(&q->sk, NET_IP_ALIGN, len, vnet_hdr.hdr_len,		if (m && m->msg_control && sock_flag(&q->sk, SOCK_ZEROCOPY))
	noblock, &err);		zerocopy = true;

			if (zerocopy) {
			/* There are 256 bytes to be copied in skb, so there is enough
			* room for skb expand head in case it is used.
			* The rest buffer is mapped from userspace.
			*/
			copylen = vnet_hdr.hdr_len;
			if (!copylen)
			copylen = GOODCOPY_LEN;
			} else
			copylen = len;

			skb = macvtap_alloc_skb(&q->sk, NET_IP_ALIGN, copylen,
			vnet_hdr.hdr_len, noblock, &err);
	if (!skb)		if (!skb)
	goto err;		goto err;

	err = skb_copy_datagram_from_iovec(skb, 0, iv, vnet_hdr_len, len);		if (zerocopy) {
			err = zerocopy_sg_from_iovec(skb, iv, vnet_hdr_len, count);
			skb_shinfo(skb)->tx_flags \|= SKBTX_DEV_ZEROCOPY;
			} else
			err = skb_copy_datagram_from_iovec(skb, 0, iv, vnet_hdr_len,
			len);
	if (err)		if (err)
	goto err_kfree;		goto err_kfree;

	@@ -575,13 +682,16 @@ static ssize_t macvtap_get_user(struct macvtap_queue *q,

	rcu_read_lock_bh();		rcu_read_lock_bh();
	vlan = rcu_dereference_bh(q->vlan);		vlan = rcu_dereference_bh(q->vlan);
			/* copy skb_ubuf_info for callback when skb has no error */
			if (zerocopy)
			skb_shinfo(skb)->destructor_arg = m->msg_control;
	if (vlan)		if (vlan)
	macvlan_start_xmit(skb, vlan->dev);		macvlan_start_xmit(skb, vlan->dev);
	else		else
	kfree_skb(skb);		kfree_skb(skb);
	rcu_read_unlock_bh();		rcu_read_unlock_bh();

	return count;		return total_len;

	err_kfree:		err_kfree:
	kfree_skb(skb);		kfree_skb(skb);
	@@ -603,7 +713,7 @@ static ssize_t macvtap_aio_write(struct kiocb iocb, const struct iovec iv,
	ssize_t result = -ENOLINK;		ssize_t result = -ENOLINK;
	struct macvtap_queue *q = file->private_data;		struct macvtap_queue *q = file->private_data;

	result = macvtap_get_user(q, iv, iov_length(iv, count),		result = macvtap_get_user(q, NULL, iv, iov_length(iv, count), count,
	file->f_flags & O_NONBLOCK);		file->f_flags & O_NONBLOCK);
	return result;		return result;
	}		}
	@@ -817,7 +927,7 @@ static int macvtap_sendmsg(struct kiocb iocb, struct socket sock,
	struct msghdr *m, size_t total_len)		struct msghdr *m, size_t total_len)
	{		{
	struct macvtap_queue *q = container_of(sock, struct macvtap_queue, sock);		struct macvtap_queue *q = container_of(sock, struct macvtap_queue, sock);
	return macvtap_get_user(q, m->msg_iov, total_len,		return macvtap_get_user(q, m, m->msg_iov, total_len, m->msg_iovlen,
	m->msg_flags & MSG_DONTWAIT);		m->msg_flags & MSG_DONTWAIT);
	}		}