Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 608b9977 authored by Paolo Abeni's avatar Paolo Abeni Committed by David S. Miller
Browse files

tun: use per cpu variables for stats accounting 



Currently the tun device accounting uses dev->stats without applying any
kind of protection, regardless that accounting happens in preemptible
process context.
This patch move the tun stats to a per cpu data structure, and protect
the updates with  u64_stats_update_begin()/u64_stats_update_end() or
this_cpu_inc according to the stat type. The per cpu stats are
aggregated by the newly added ndo_get_stats64 ops.

Signed-off-by: default avatarPaolo Abeni <pabeni@redhat.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 548aacdd

drivers/net/tun.c

+83 −12
Original line number Original line Diff line number Diff line
@@ -131,6 +131,17 @@ struct tap_filter { 




#define TUN_FLOW_EXPIRE (3 * HZ)

#define TUN_FLOW_EXPIRE (3 * HZ)





struct tun_pcpu_stats {

	u64 rx_packets;

	u64 rx_bytes;

	u64 tx_packets;

	u64 tx_bytes;

	struct u64_stats_sync syncp;

	u32 rx_dropped;

	u32 tx_dropped;

	u32 rx_frame_errors;

};



/* A tun_file connects an open character device to a tuntap netdevice. It

/* A tun_file connects an open character device to a tuntap netdevice. It

 * also contains all socket related structures (except sock_fprog and tap_filter)

 * also contains all socket related structures (except sock_fprog and tap_filter)

 * to serve as one transmit queue for tuntap device. The sock_fprog and

 * to serve as one transmit queue for tuntap device. The sock_fprog and
@@ -205,6 +216,7 @@ struct tun_struct { 
	struct list_head disabled;

	struct list_head disabled;

	void *security;

	void *security;

	u32 flow_count;

	u32 flow_count;

	struct tun_pcpu_stats __percpu *pcpu_stats;

};

};





#ifdef CONFIG_TUN_VNET_CROSS_LE

#ifdef CONFIG_TUN_VNET_CROSS_LE
@@ -886,7 +898,7 @@ static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev) 
	return NETDEV_TX_OK;

	return NETDEV_TX_OK;





drop:

drop:

	dev->stats.tx_dropped++;

	this_cpu_inc(tun->pcpu_stats->tx_dropped);

	skb_tx_error(skb);

	skb_tx_error(skb);

	kfree_skb(skb);

	kfree_skb(skb);

	rcu_read_unlock();

	rcu_read_unlock();
@@ -949,6 +961,43 @@ static void tun_set_headroom(struct net_device *dev, int new_hr) 
	tun->align = new_hr;

	tun->align = new_hr;

}

}





static struct rtnl_link_stats64 *

tun_net_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)

{

	u32 rx_dropped = 0, tx_dropped = 0, rx_frame_errors = 0;

	struct tun_struct *tun = netdev_priv(dev);

	struct tun_pcpu_stats *p;

	int i;



	for_each_possible_cpu(i) {

		u64 rxpackets, rxbytes, txpackets, txbytes;

		unsigned int start;



		p = per_cpu_ptr(tun->pcpu_stats, i);

		do {

			start = u64_stats_fetch_begin(&p->syncp);

			rxpackets	= p->rx_packets;

			rxbytes		= p->rx_bytes;

			txpackets	= p->tx_packets;

			txbytes		= p->tx_bytes;

		} while (u64_stats_fetch_retry(&p->syncp, start));



		stats->rx_packets	+= rxpackets;

		stats->rx_bytes		+= rxbytes;

		stats->tx_packets	+= txpackets;

		stats->tx_bytes		+= txbytes;



		/* u32 counters */

		rx_dropped	+= p->rx_dropped;

		rx_frame_errors	+= p->rx_frame_errors;

		tx_dropped	+= p->tx_dropped;

	}

	stats->rx_dropped  = rx_dropped;

	stats->rx_frame_errors = rx_frame_errors;

	stats->tx_dropped = tx_dropped;

	return stats;

}



static const struct net_device_ops tun_netdev_ops = {

static const struct net_device_ops tun_netdev_ops = {

	.ndo_uninit		= tun_net_uninit,

	.ndo_uninit		= tun_net_uninit,

	.ndo_open		= tun_net_open,

	.ndo_open		= tun_net_open,
@@ -961,6 +1010,7 @@ static const struct net_device_ops tun_netdev_ops = { 
	.ndo_poll_controller	= tun_poll_controller,

	.ndo_poll_controller	= tun_poll_controller,

#endif

#endif

	.ndo_set_rx_headroom	= tun_set_headroom,

	.ndo_set_rx_headroom	= tun_set_headroom,

	.ndo_get_stats64	= tun_net_get_stats64,

};

};





static const struct net_device_ops tap_netdev_ops = {

static const struct net_device_ops tap_netdev_ops = {
@@ -979,6 +1029,7 @@ static const struct net_device_ops tap_netdev_ops = { 
#endif

#endif

	.ndo_features_check	= passthru_features_check,

	.ndo_features_check	= passthru_features_check,

	.ndo_set_rx_headroom	= tun_set_headroom,

	.ndo_set_rx_headroom	= tun_set_headroom,

	.ndo_get_stats64	= tun_net_get_stats64,

};

};





static void tun_flow_init(struct tun_struct *tun)

static void tun_flow_init(struct tun_struct *tun)
@@ -1103,6 +1154,7 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile, 
	size_t total_len = iov_iter_count(from);

	size_t total_len = iov_iter_count(from);

	size_t len = total_len, align = tun->align, linear;

	size_t len = total_len, align = tun->align, linear;

	struct virtio_net_hdr gso = { 0 };

	struct virtio_net_hdr gso = { 0 };

	struct tun_pcpu_stats *stats;

	int good_linear;

	int good_linear;

	int copylen;

	int copylen;

	bool zerocopy = false;

	bool zerocopy = false;
@@ -1177,7 +1229,7 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile, 
	skb = tun_alloc_skb(tfile, align, copylen, linear, noblock);

	skb = tun_alloc_skb(tfile, align, copylen, linear, noblock);

	if (IS_ERR(skb)) {

	if (IS_ERR(skb)) {

		if (PTR_ERR(skb) != -EAGAIN)

		if (PTR_ERR(skb) != -EAGAIN)

			tun->dev->stats.rx_dropped++;

			this_cpu_inc(tun->pcpu_stats->rx_dropped);

		return PTR_ERR(skb);

		return PTR_ERR(skb);

	}

	}
@@ -1192,7 +1244,7 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile, 
	}

	}





	if (err) {

	if (err) {

		tun->dev->stats.rx_dropped++;

		this_cpu_inc(tun->pcpu_stats->rx_dropped);

		kfree_skb(skb);

		kfree_skb(skb);

		return -EFAULT;

		return -EFAULT;

	}

	}
@@ -1200,7 +1252,7 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile, 
	if (gso.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {

	if (gso.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {

		if (!skb_partial_csum_set(skb, tun16_to_cpu(tun, gso.csum_start),

		if (!skb_partial_csum_set(skb, tun16_to_cpu(tun, gso.csum_start),

					  tun16_to_cpu(tun, gso.csum_offset))) {

					  tun16_to_cpu(tun, gso.csum_offset))) {

			tun->dev->stats.rx_frame_errors++;

			this_cpu_inc(tun->pcpu_stats->rx_frame_errors);

			kfree_skb(skb);

			kfree_skb(skb);

			return -EINVAL;

			return -EINVAL;

		}

		}
@@ -1217,7 +1269,7 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile, 
				pi.proto = htons(ETH_P_IPV6);

				pi.proto = htons(ETH_P_IPV6);

				break;

				break;

			default:

			default:

				tun->dev->stats.rx_dropped++;

				this_cpu_inc(tun->pcpu_stats->rx_dropped);

				kfree_skb(skb);

				kfree_skb(skb);

				return -EINVAL;

				return -EINVAL;

			}

			}
@@ -1245,7 +1297,7 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile, 
			skb_shinfo(skb)->gso_type = SKB_GSO_UDP;

			skb_shinfo(skb)->gso_type = SKB_GSO_UDP;

			break;

			break;

		default:

		default:

			tun->dev->stats.rx_frame_errors++;

			this_cpu_inc(tun->pcpu_stats->rx_frame_errors);

			kfree_skb(skb);

			kfree_skb(skb);

			return -EINVAL;

			return -EINVAL;

		}

		}
@@ -1255,7 +1307,7 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile, 




		skb_shinfo(skb)->gso_size = tun16_to_cpu(tun, gso.gso_size);

		skb_shinfo(skb)->gso_size = tun16_to_cpu(tun, gso.gso_size);

		if (skb_shinfo(skb)->gso_size == 0) {

		if (skb_shinfo(skb)->gso_size == 0) {

			tun->dev->stats.rx_frame_errors++;

			this_cpu_inc(tun->pcpu_stats->rx_frame_errors);

			kfree_skb(skb);

			kfree_skb(skb);

			return -EINVAL;

			return -EINVAL;

		}

		}
@@ -1278,8 +1330,12 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile, 
	rxhash = skb_get_hash(skb);

	rxhash = skb_get_hash(skb);

	netif_rx_ni(skb);

	netif_rx_ni(skb);





	tun->dev->stats.rx_packets++;

	stats = get_cpu_ptr(tun->pcpu_stats);

	tun->dev->stats.rx_bytes += len;

	u64_stats_update_begin(&stats->syncp);

	stats->rx_packets++;

	stats->rx_bytes += len;

	u64_stats_update_end(&stats->syncp);

	put_cpu_ptr(stats);





	tun_flow_update(tun, rxhash, tfile);

	tun_flow_update(tun, rxhash, tfile);

	return total_len;

	return total_len;
@@ -1308,6 +1364,7 @@ static ssize_t tun_put_user(struct tun_struct *tun, 
			    struct iov_iter *iter)

			    struct iov_iter *iter)

{

{

	struct tun_pi pi = { 0, skb->protocol };

	struct tun_pi pi = { 0, skb->protocol };

	struct tun_pcpu_stats *stats;

	ssize_t total;

	ssize_t total;

	int vlan_offset = 0;

	int vlan_offset = 0;

	int vlan_hlen = 0;

	int vlan_hlen = 0;
@@ -1408,8 +1465,13 @@ static ssize_t tun_put_user(struct tun_struct *tun, 
	skb_copy_datagram_iter(skb, vlan_offset, iter, skb->len - vlan_offset);

	skb_copy_datagram_iter(skb, vlan_offset, iter, skb->len - vlan_offset);





done:

done:

	tun->dev->stats.tx_packets++;

	/* caller is in process context, */

	tun->dev->stats.tx_bytes += skb->len + vlan_hlen;

	stats = get_cpu_ptr(tun->pcpu_stats);

	u64_stats_update_begin(&stats->syncp);

	stats->tx_packets++;

	stats->tx_bytes += skb->len + vlan_hlen;

	u64_stats_update_end(&stats->syncp);

	put_cpu_ptr(tun->pcpu_stats);





	return total;

	return total;

}

}
@@ -1467,6 +1529,7 @@ static void tun_free_netdev(struct net_device *dev) 
	struct tun_struct *tun = netdev_priv(dev);

	struct tun_struct *tun = netdev_priv(dev);





	BUG_ON(!(list_empty(&tun->disabled)));

	BUG_ON(!(list_empty(&tun->disabled)));

	free_percpu(tun->pcpu_stats);

	tun_flow_uninit(tun);

	tun_flow_uninit(tun);

	security_tun_dev_free_security(tun->security);

	security_tun_dev_free_security(tun->security);

	free_netdev(dev);

	free_netdev(dev);
@@ -1715,11 +1778,17 @@ static int tun_set_iff(struct net *net, struct file *file, struct ifreq *ifr) 
		tun->filter_attached = false;

		tun->filter_attached = false;

		tun->sndbuf = tfile->socket.sk->sk_sndbuf;

		tun->sndbuf = tfile->socket.sk->sk_sndbuf;





		tun->pcpu_stats = netdev_alloc_pcpu_stats(struct tun_pcpu_stats);

		if (!tun->pcpu_stats) {

			err = -ENOMEM;

			goto err_free_dev;

		}



		spin_lock_init(&tun->lock);

		spin_lock_init(&tun->lock);





		err = security_tun_dev_alloc_security(&tun->security);

		err = security_tun_dev_alloc_security(&tun->security);

		if (err < 0)

		if (err < 0)

			goto err_free_dev;

			goto err_free_stat;





		tun_net_init(dev);

		tun_net_init(dev);

		tun_flow_init(tun);

		tun_flow_init(tun);
@@ -1763,6 +1832,8 @@ static int tun_set_iff(struct net *net, struct file *file, struct ifreq *ifr) 
err_free_flow:

err_free_flow:

	tun_flow_uninit(tun);

	tun_flow_uninit(tun);

	security_tun_dev_free_security(tun->security);

	security_tun_dev_free_security(tun->security);

err_free_stat:

	free_percpu(tun->pcpu_stats);

err_free_dev:

err_free_dev:

	free_netdev(dev);

	free_netdev(dev);

	return err;

	return err;