Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 8e845f4c authored by David L Stevens's avatar David L Stevens Committed by David S. Miller
Browse files

sunvnet: make transmit path zero-copy in the kernel 



This patch removes pre-allocated transmit buffers and instead directly maps
pending packets on demand. This saves O(n^2) maximum-sized transmit buffers,
for n hosts on a vswitch, as well as a copy to those buffers.

Single-stream TCP throughput linux-solaris dropped ~5% for 1500-byte MTU,
but linux-linux at 1500-bytes increased ~20%.

Signed-off-by: default avatarDavid L Stevens <david.stevens@oracle.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent e4defc77

drivers/net/ethernet/sun/sunvnet.c

+174 −44
Original line number Diff line number Diff line
@@ -780,6 +780,117 @@ struct vnet_port *tx_port_find(struct vnet *vp, struct sk_buff *skb) 
	return ret;

}



static struct sk_buff *vnet_clean_tx_ring(struct vnet_port *port,

					  unsigned *pending)

{

	struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING];

	struct sk_buff *skb = NULL;

	int i, txi;



	*pending = 0;



	txi = dr->prod-1;

	if (txi < 0)

		txi = VNET_TX_RING_SIZE-1;



	for (i = 0; i < VNET_TX_RING_SIZE; ++i) {

		struct vio_net_desc *d;



		d = vio_dring_entry(dr, txi);



		if (d->hdr.state == VIO_DESC_DONE) {

			if (port->tx_bufs[txi].skb) {

				BUG_ON(port->tx_bufs[txi].skb->next);



				port->tx_bufs[txi].skb->next = skb;

				skb = port->tx_bufs[txi].skb;

				port->tx_bufs[txi].skb = NULL;



				ldc_unmap(port->vio.lp,

					  port->tx_bufs[txi].cookies,

					  port->tx_bufs[txi].ncookies);

			}

			d->hdr.state = VIO_DESC_FREE;

		} else if (d->hdr.state == VIO_DESC_READY) {

			(*pending)++;

		} else if (d->hdr.state == VIO_DESC_FREE) {

			break;

		}

		--txi;

		if (txi < 0)

			txi = VNET_TX_RING_SIZE-1;

	}

	return skb;

}



static inline void vnet_free_skbs(struct sk_buff *skb)

{

	struct sk_buff *next;



	while (skb) {

		next = skb->next;

		skb->next = NULL;

		dev_kfree_skb(skb);

		skb = next;

	}

}



static void vnet_clean_timer_expire(unsigned long port0)

{

	struct vnet_port *port = (struct vnet_port *)port0;

	struct sk_buff *freeskbs;

	unsigned pending;

	unsigned long flags;



	spin_lock_irqsave(&port->vio.lock, flags);

	freeskbs = vnet_clean_tx_ring(port, &pending);

	spin_unlock_irqrestore(&port->vio.lock, flags);



	vnet_free_skbs(freeskbs);



	if (pending)

		(void)mod_timer(&port->clean_timer,

				jiffies + VNET_CLEAN_TIMEOUT);

	 else

		del_timer(&port->clean_timer);

}



static inline struct sk_buff *vnet_skb_shape(struct sk_buff *skb, void **pstart,

					     int *plen)

{

	struct sk_buff *nskb;

	int len, pad;



	len = skb->len;

	pad = 0;

	if (len < ETH_ZLEN) {

		pad += ETH_ZLEN - skb->len;

		len += pad;

	}

	len += VNET_PACKET_SKIP;

	pad += 8 - (len & 7);

	len += 8 - (len & 7);



	if (((unsigned long)skb->data & 7) != VNET_PACKET_SKIP ||

	    skb_tailroom(skb) < pad ||

	    skb_headroom(skb) < VNET_PACKET_SKIP) {

		nskb = alloc_and_align_skb(skb->dev, skb->len);

		skb_reserve(nskb, VNET_PACKET_SKIP);

		if (skb_copy_bits(skb, 0, nskb->data, skb->len)) {

			dev_kfree_skb(nskb);

			dev_kfree_skb(skb);

			return NULL;

		}

		(void)skb_put(nskb, skb->len);

		dev_kfree_skb(skb);

		skb = nskb;

	}



	*pstart = skb->data - VNET_PACKET_SKIP;

	*plen = len;

	return skb;

}



static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev)

{

	struct vnet *vp = netdev_priv(dev);
@@ -788,12 +899,20 @@ static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev) 
	struct vio_net_desc *d;

	unsigned long flags;

	unsigned int len;

	void *tx_buf;

	int i, err;

	struct sk_buff *freeskbs = NULL;

	int i, err, txi;

	void *start = NULL;

	int nlen = 0;

	unsigned pending = 0;



	if (unlikely(!port))

		goto out_dropped;



	skb = vnet_skb_shape(skb, &start, &nlen);



	if (unlikely(!skb))

		goto out_dropped;



	spin_lock_irqsave(&port->vio.lock, flags);



	dr = &port->vio.drings[VIO_DRIVER_TX_RING];
@@ -811,14 +930,27 @@ static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev) 


	d = vio_dring_cur(dr);



	tx_buf = port->tx_bufs[dr->prod].buf;

	skb_copy_from_linear_data(skb, tx_buf + VNET_PACKET_SKIP, skb->len);

	txi = dr->prod;



	freeskbs = vnet_clean_tx_ring(port, &pending);



	BUG_ON(port->tx_bufs[txi].skb);



	len = skb->len;

	if (len < ETH_ZLEN) {

	if (len < ETH_ZLEN)

		len = ETH_ZLEN;

		memset(tx_buf+VNET_PACKET_SKIP+skb->len, 0, len - skb->len);



	port->tx_bufs[txi].skb = skb;

	skb = NULL;



	err = ldc_map_single(port->vio.lp, start, nlen,

			     port->tx_bufs[txi].cookies, 2,

			     (LDC_MAP_SHADOW | LDC_MAP_DIRECT | LDC_MAP_RW));

	if (err < 0) {

		netdev_info(dev, "tx buffer map error %d\n", err);

		goto out_dropped_unlock;

	}

	port->tx_bufs[txi].ncookies = err;



	/* We don't rely on the ACKs to free the skb in vnet_start_xmit(),

	 * thus it is safe to not set VIO_ACK_ENABLE for each transmission:
@@ -830,9 +962,9 @@ static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev) 
	 */

	d->hdr.ack = VIO_ACK_DISABLE;

	d->size = len;

	d->ncookies = port->tx_bufs[dr->prod].ncookies;

	d->ncookies = port->tx_bufs[txi].ncookies;

	for (i = 0; i < d->ncookies; i++)

		d->cookies[i] = port->tx_bufs[dr->prod].cookies[i];

		d->cookies[i] = port->tx_bufs[txi].cookies[i];



	/* This has to be a non-SMP write barrier because we are writing

	 * to memory which is shared with the peer LDOM.
@@ -876,7 +1008,7 @@ static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev) 
	port->start_cons = false;



	dev->stats.tx_packets++;

	dev->stats.tx_bytes += skb->len;

	dev->stats.tx_bytes += port->tx_bufs[txi].skb->len;



	dr->prod = (dr->prod + 1) & (VNET_TX_RING_SIZE - 1);

	if (unlikely(vnet_tx_dring_avail(dr) < 2)) {
@@ -887,7 +1019,9 @@ static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev) 


	spin_unlock_irqrestore(&port->vio.lock, flags);



	dev_kfree_skb(skb);

	vnet_free_skbs(freeskbs);



	(void)mod_timer(&port->clean_timer, jiffies + VNET_CLEAN_TIMEOUT);



	return NETDEV_TX_OK;
@@ -895,7 +1029,14 @@ static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev) 
	spin_unlock_irqrestore(&port->vio.lock, flags);



out_dropped:

	if (skb)

		dev_kfree_skb(skb);

	vnet_free_skbs(freeskbs);

	if (pending)

		(void)mod_timer(&port->clean_timer,

				jiffies + VNET_CLEAN_TIMEOUT);

	else

		del_timer(&port->clean_timer);

	dev->stats.tx_dropped++;

	return NETDEV_TX_OK;

}
@@ -1097,17 +1238,22 @@ static void vnet_port_free_tx_bufs(struct vnet_port *port) 
	}



	for (i = 0; i < VNET_TX_RING_SIZE; i++) {

		void *buf = port->tx_bufs[i].buf;

		struct vio_net_desc *d;

		void *skb = port->tx_bufs[i].skb;



		if (!buf)

		if (!skb)

			continue;



		d = vio_dring_entry(dr, i);

		if (d->hdr.state == VIO_DESC_READY)

			pr_warn("active transmit buffers freed\n");



		ldc_unmap(port->vio.lp,

			  port->tx_bufs[i].cookies,

			  port->tx_bufs[i].ncookies);



		kfree(buf);

		port->tx_bufs[i].buf = NULL;

		dev_kfree_skb(skb);

		port->tx_bufs[i].skb = NULL;

		d->hdr.state = VIO_DESC_FREE;

	}

}
@@ -1118,34 +1264,6 @@ static int vnet_port_alloc_tx_bufs(struct vnet_port *port) 
	int i, err, ncookies;

	void *dring;



	for (i = 0; i < VNET_TX_RING_SIZE; i++) {

		void *buf = kzalloc(VNET_MAXPACKET + 8, GFP_KERNEL);

		int map_len = (VNET_MAXPACKET + 7) & ~7;



		err = -ENOMEM;

		if (!buf)

			goto err_out;



		err = -EFAULT;

		if ((unsigned long)buf & (8UL - 1)) {

			pr_err("TX buffer misaligned\n");

			kfree(buf);

			goto err_out;

		}



		err = ldc_map_single(port->vio.lp, buf, map_len,

				     port->tx_bufs[i].cookies, 2,

				     (LDC_MAP_SHADOW |

				      LDC_MAP_DIRECT |

				      LDC_MAP_RW));

		if (err < 0) {

			kfree(buf);

			goto err_out;

		}

		port->tx_bufs[i].buf = buf;

		port->tx_bufs[i].ncookies = err;

	}



	dr = &port->vio.drings[VIO_DRIVER_TX_RING];



	len = (VNET_TX_RING_SIZE *
@@ -1172,6 +1290,12 @@ static int vnet_port_alloc_tx_bufs(struct vnet_port *port) 
	dr->pending = VNET_TX_RING_SIZE;

	dr->ncookies = ncookies;



	for (i = 0; i < VNET_TX_RING_SIZE; ++i) {

		struct vio_net_desc *d;



		d = vio_dring_entry(dr, i);

		d->hdr.state = VIO_DESC_FREE;

	}

	return 0;



err_out:
@@ -1203,6 +1327,8 @@ static struct vnet *vnet_new(const u64 *local_mac) 
	dev = alloc_etherdev(sizeof(*vp));

	if (!dev)

		return ERR_PTR(-ENOMEM);

	dev->needed_headroom = VNET_PACKET_SKIP + 8;

	dev->needed_tailroom = 8;



	for (i = 0; i < ETH_ALEN; i++)

		dev->dev_addr[i] = (*local_mac >> (5 - i) * 8) & 0xff;
@@ -1397,6 +1523,9 @@ static int vnet_port_probe(struct vio_dev *vdev, const struct vio_device_id *id) 
	pr_info("%s: PORT ( remote-mac %pM%s )\n",

		vp->dev->name, port->raddr, switch_port ? " switch-port" : "");



	setup_timer(&port->clean_timer, vnet_clean_timer_expire,

		    (unsigned long)port);



	vio_port_up(&port->vio);



	mdesc_release(hp);
@@ -1423,6 +1552,7 @@ static int vnet_port_remove(struct vio_dev *vdev) 
		unsigned long flags;



		del_timer_sync(&port->vio.timer);

		del_timer_sync(&port->clean_timer);



		spin_lock_irqsave(&vp->lock, flags);

		list_del(&port->list);

drivers/net/ethernet/sun/sunvnet.h

+8 −1
Original line number Diff line number Diff line
@@ -11,6 +11,11 @@ 
 */

#define VNET_TX_TIMEOUT			(5 * HZ)



/* length of time (or less) we expect pending descriptors to be marked

 * as VIO_DESC_DONE and skbs ready to be freed

 */

#define	VNET_CLEAN_TIMEOUT		((HZ/100)+1)



#define VNET_MAXPACKET			1518ULL /* ETH_FRAMELEN + VLAN_HDR */

#define VNET_TX_RING_SIZE		512

#define VNET_TX_WAKEUP_THRESH(dr)	((dr)->pending / 4)
@@ -22,7 +27,7 @@ 
#define VNET_PACKET_SKIP		6



struct vnet_tx_entry {

	void			*buf;

	struct sk_buff		*skb;

	unsigned int		ncookies;

	struct ldc_trans_cookie	cookies[2];

};
@@ -46,6 +51,8 @@ struct vnet_port { 
	bool			stop_rx;

	bool			start_cons;



	struct timer_list	clean_timer;



	u64			rmtu;

};