Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 9227dc5e authored by Lendacky, Thomas's avatar Lendacky, Thomas Committed by David S. Miller
Browse files

amd-xgbe: Add support for per DMA channel interrupts 



This patch provides support for interrupts that are generated by the
Tx/Rx DMA channel pairs of the device.  This allows for Tx and Rx
processing to run across multiple processsors.

Signed-off-by: default avatarTom Lendacky <thomas.lendacky@amd.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 174fd259

Documentation/devicetree/bindings/net/amd-xgbe.txt

+10 −2
Original line number Diff line number Diff line
@@ -7,7 +7,10 @@ Required properties: 
   - PCS registers

- interrupt-parent: Should be the phandle for the interrupt controller

  that services interrupts for this device

- interrupts: Should contain the amd-xgbe interrupt

- interrupts: Should contain the amd-xgbe interrupt(s). The first interrupt

  listed is required and is the general device interrupt. If the optional

  amd,per-channel-interrupt property is specified, then one additional

  interrupt for each DMA channel supported by the device should be specified

- clocks:

   - DMA clock for the amd-xgbe device (used for calculating the

     correct Rx interrupt watchdog timer value on a DMA channel
@@ -23,6 +26,9 @@ Optional properties: 
- mac-address: mac address to be assigned to the device. Can be overridden

  by UEFI.

- dma-coherent: Present if dma operations are coherent

- amd,per-channel-interrupt: Indicates that Rx and Tx complete will generate

  a unique interrupt for each DMA channel - this requires an additional

  interrupt be configured for each DMA channel



Example:

	xgbe@e0700000 {
@@ -30,7 +36,9 @@ Example: 
		reg = <0 0xe0700000 0 0x80000>,

		      <0 0xe0780000 0 0x80000>;

		interrupt-parent = <&gic>;

		interrupts = <0 325 4>;

		interrupts = <0 325 4>,

			     <0 326 1>, <0 327 1>, <0 328 1>, <0 329 1>;

		amd,per-channel-interrupt;

		clocks = <&xgbe_dma_clk>, <&xgbe_ptp_clk>;

		clock-names = "dma_clk", "ptp_clk";

		phy-handle = <&phy>;

drivers/net/ethernet/amd/xgbe/xgbe-dev.c

+8 −4
Original line number Diff line number Diff line
@@ -481,16 +481,20 @@ static void xgbe_enable_dma_interrupts(struct xgbe_prv_data *pdata) 


		if (channel->tx_ring) {

			/* Enable the following Tx interrupts

			 *   TIE  - Transmit Interrupt Enable (unless polling)

			 *   TIE  - Transmit Interrupt Enable (unless using

			 *          per channel interrupts)

			 */

			if (!pdata->per_channel_irq)

				XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, TIE, 1);

		}

		if (channel->rx_ring) {

			/* Enable following Rx interrupts

			 *   RBUE - Receive Buffer Unavailable Enable

			 *   RIE  - Receive Interrupt Enable

			 *   RIE  - Receive Interrupt Enable (unless using

			 *          per channel interrupts)

			 */

			XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, RBUE, 1);

			if (!pdata->per_channel_irq)

				XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, RIE, 1);

		}

drivers/net/ethernet/amd/xgbe/xgbe-drv.c

+184 −42
Original line number Diff line number Diff line
@@ -114,6 +114,7 @@ 
 *     THE POSSIBILITY OF SUCH DAMAGE.

 */



#include <linux/platform_device.h>

#include <linux/spinlock.h>

#include <linux/tcp.h>

#include <linux/if_vlan.h>
@@ -126,7 +127,8 @@ 
#include "xgbe.h"

#include "xgbe-common.h"



static int xgbe_poll(struct napi_struct *, int);

static int xgbe_one_poll(struct napi_struct *, int);

static int xgbe_all_poll(struct napi_struct *, int);

static void xgbe_set_rx_mode(struct net_device *);



static int xgbe_alloc_channels(struct xgbe_prv_data *pdata)
@@ -134,6 +136,7 @@ static int xgbe_alloc_channels(struct xgbe_prv_data *pdata) 
	struct xgbe_channel *channel_mem, *channel;

	struct xgbe_ring *tx_ring, *rx_ring;

	unsigned int count, i;

	int ret = -ENOMEM;



	count = max_t(unsigned int, pdata->tx_ring_count, pdata->rx_ring_count);
@@ -158,6 +161,19 @@ static int xgbe_alloc_channels(struct xgbe_prv_data *pdata) 
		channel->dma_regs = pdata->xgmac_regs + DMA_CH_BASE +

				    (DMA_CH_INC * i);



		if (pdata->per_channel_irq) {

			/* Get the DMA interrupt (offset 1) */

			ret = platform_get_irq(pdata->pdev, i + 1);

			if (ret < 0) {

				netdev_err(pdata->netdev,

					   "platform_get_irq %u failed\n",

					   i + 1);

				goto err_irq;

			}



			channel->dma_irq = ret;

		}



		if (i < pdata->tx_ring_count) {

			spin_lock_init(&tx_ring->lock);

			channel->tx_ring = tx_ring++;
@@ -168,9 +184,9 @@ static int xgbe_alloc_channels(struct xgbe_prv_data *pdata) 
			channel->rx_ring = rx_ring++;

		}



		DBGPR("  %s - queue_index=%u, dma_regs=%p, tx=%p, rx=%p\n",

		DBGPR("  %s: queue=%u, dma_regs=%p, dma_irq=%d, tx=%p, rx=%p\n",

		      channel->name, channel->queue_index, channel->dma_regs,

		      channel->tx_ring, channel->rx_ring);

		      channel->dma_irq, channel->tx_ring, channel->rx_ring);

	}



	pdata->channel = channel_mem;
@@ -178,6 +194,9 @@ static int xgbe_alloc_channels(struct xgbe_prv_data *pdata) 


	return 0;



err_irq:

	kfree(rx_ring);



err_rx_ring:

	kfree(tx_ring);
@@ -185,9 +204,7 @@ static int xgbe_alloc_channels(struct xgbe_prv_data *pdata) 
	kfree(channel_mem);



err_channel:

	netdev_err(pdata->netdev, "channel allocation failed\n");



	return -ENOMEM;

	return ret;

}



static void xgbe_free_channels(struct xgbe_prv_data *pdata)
@@ -287,11 +304,7 @@ static irqreturn_t xgbe_isr(int irq, void *data) 
	if (!dma_isr)

		goto isr_done;



	DBGPR("-->xgbe_isr\n");



	DBGPR("  DMA_ISR = %08x\n", dma_isr);

	DBGPR("  DMA_DS0 = %08x\n", XGMAC_IOREAD(pdata, DMA_DSR0));

	DBGPR("  DMA_DS1 = %08x\n", XGMAC_IOREAD(pdata, DMA_DSR1));



	for (i = 0; i < pdata->channel_count; i++) {

		if (!(dma_isr & (1 << i)))
@@ -302,6 +315,10 @@ static irqreturn_t xgbe_isr(int irq, void *data) 
		dma_ch_isr = XGMAC_DMA_IOREAD(channel, DMA_CH_SR);

		DBGPR("  DMA_CH%u_ISR = %08x\n", i, dma_ch_isr);



		/* If we get a TI or RI interrupt that means per channel DMA

		 * interrupts are not enabled, so we use the private data napi

		 * structure, not the per channel napi structure

		 */

		if (XGMAC_GET_BITS(dma_ch_isr, DMA_CH_SR, TI) ||

		    XGMAC_GET_BITS(dma_ch_isr, DMA_CH_SR, RI)) {

			if (napi_schedule_prep(&pdata->napi)) {
@@ -344,12 +361,28 @@ static irqreturn_t xgbe_isr(int irq, void *data) 


	DBGPR("  DMA_ISR = %08x\n", XGMAC_IOREAD(pdata, DMA_ISR));



	DBGPR("<--xgbe_isr\n");



isr_done:

	return IRQ_HANDLED;

}



static irqreturn_t xgbe_dma_isr(int irq, void *data)

{

	struct xgbe_channel *channel = data;



	/* Per channel DMA interrupts are enabled, so we use the per

	 * channel napi structure and not the private data napi structure

	 */

	if (napi_schedule_prep(&channel->napi)) {

		/* Disable Tx and Rx interrupts */

		disable_irq(channel->dma_irq);



		/* Turn on polling */

		__napi_schedule(&channel->napi);

	}



	return IRQ_HANDLED;

}



static enum hrtimer_restart xgbe_tx_timer(struct hrtimer *timer)

{

	struct xgbe_channel *channel = container_of(timer,
@@ -357,18 +390,24 @@ static enum hrtimer_restart xgbe_tx_timer(struct hrtimer *timer) 
						    tx_timer);

	struct xgbe_ring *ring = channel->tx_ring;

	struct xgbe_prv_data *pdata = channel->pdata;

	struct napi_struct *napi;

	unsigned long flags;



	DBGPR("-->xgbe_tx_timer\n");



	napi = (pdata->per_channel_irq) ? &channel->napi : &pdata->napi;



	spin_lock_irqsave(&ring->lock, flags);



	if (napi_schedule_prep(&pdata->napi)) {

	if (napi_schedule_prep(napi)) {

		/* Disable Tx and Rx interrupts */

		if (pdata->per_channel_irq)

			disable_irq(channel->dma_irq);

		else

			xgbe_disable_rx_tx_ints(pdata);



		/* Turn on polling */

		__napi_schedule(&pdata->napi);

		__napi_schedule(napi);

	}



	channel->tx_timer_active = 0;
@@ -504,19 +543,47 @@ void xgbe_get_all_hw_features(struct xgbe_prv_data *pdata) 


static void xgbe_napi_enable(struct xgbe_prv_data *pdata, unsigned int add)

{

	struct xgbe_channel *channel;

	unsigned int i;



	if (pdata->per_channel_irq) {

		channel = pdata->channel;

		for (i = 0; i < pdata->channel_count; i++, channel++) {

			if (add)

		netif_napi_add(pdata->netdev, &pdata->napi, xgbe_poll,

			       NAPI_POLL_WEIGHT);

				netif_napi_add(pdata->netdev, &channel->napi,

					       xgbe_one_poll, NAPI_POLL_WEIGHT);



			napi_enable(&channel->napi);

		}

	} else {

		if (add)

			netif_napi_add(pdata->netdev, &pdata->napi,

				       xgbe_all_poll, NAPI_POLL_WEIGHT);



		napi_enable(&pdata->napi);

	}

}



static void xgbe_napi_disable(struct xgbe_prv_data *pdata, unsigned int del)

{

	struct xgbe_channel *channel;

	unsigned int i;



	if (pdata->per_channel_irq) {

		channel = pdata->channel;

		for (i = 0; i < pdata->channel_count; i++, channel++) {

			napi_disable(&channel->napi);



			if (del)

				netif_napi_del(&channel->napi);

		}

	} else {

		napi_disable(&pdata->napi);



		if (del)

			netif_napi_del(&pdata->napi);

	}

}



void xgbe_init_tx_coalesce(struct xgbe_prv_data *pdata)

{
@@ -828,7 +895,9 @@ static void xgbe_stop(struct xgbe_prv_data *pdata) 


static void xgbe_restart_dev(struct xgbe_prv_data *pdata, unsigned int reset)

{

	struct xgbe_channel *channel;

	struct xgbe_hw_if *hw_if = &pdata->hw_if;

	unsigned int i;



	DBGPR("-->xgbe_restart_dev\n");
@@ -837,7 +906,12 @@ static void xgbe_restart_dev(struct xgbe_prv_data *pdata, unsigned int reset) 
		return;



	xgbe_stop(pdata);

	synchronize_irq(pdata->irq_number);

	synchronize_irq(pdata->dev_irq);

	if (pdata->per_channel_irq) {

		channel = pdata->channel;

		for (i = 0; i < pdata->channel_count; i++, channel++)

			synchronize_irq(channel->dma_irq);

	}



	xgbe_free_tx_data(pdata);

	xgbe_free_rx_data(pdata);
@@ -1165,6 +1239,9 @@ static int xgbe_open(struct net_device *netdev) 
	struct xgbe_prv_data *pdata = netdev_priv(netdev);

	struct xgbe_hw_if *hw_if = &pdata->hw_if;

	struct xgbe_desc_if *desc_if = &pdata->desc_if;

	struct xgbe_channel *channel = NULL;

	char dma_irq_name[IFNAMSIZ + 32];

	unsigned int i = 0;

	int ret;



	DBGPR("-->xgbe_open\n");
@@ -1208,14 +1285,32 @@ static int xgbe_open(struct net_device *netdev) 
	INIT_WORK(&pdata->tx_tstamp_work, xgbe_tx_tstamp);



	/* Request interrupts */

	ret = devm_request_irq(pdata->dev, netdev->irq, xgbe_isr, 0,

	ret = devm_request_irq(pdata->dev, pdata->dev_irq, xgbe_isr, 0,

			       netdev->name, pdata);

	if (ret) {

		netdev_alert(netdev, "error requesting irq %d\n",

			     pdata->irq_number);

			     pdata->dev_irq);

		goto err_rings;

	}

	pdata->irq_number = netdev->irq;



	if (pdata->per_channel_irq) {

		channel = pdata->channel;

		for (i = 0; i < pdata->channel_count; i++, channel++) {

			snprintf(dma_irq_name, sizeof(dma_irq_name) - 1,

				 "%s-TxRx-%u", netdev_name(netdev),

				 channel->queue_index);



			ret = devm_request_irq(pdata->dev, channel->dma_irq,

					       xgbe_dma_isr, 0, dma_irq_name,

					       channel);

			if (ret) {

				netdev_alert(netdev,

					     "error requesting irq %d\n",

					     channel->dma_irq);

				goto err_irq;

			}

		}

	}



	ret = xgbe_start(pdata);

	if (ret)
@@ -1228,8 +1323,14 @@ static int xgbe_open(struct net_device *netdev) 
err_start:

	hw_if->exit(pdata);



	devm_free_irq(pdata->dev, pdata->irq_number, pdata);

	pdata->irq_number = 0;

err_irq:

	if (pdata->per_channel_irq) {

		/* Using an unsigned int, 'i' will go to UINT_MAX and exit */

		for (i--, channel--; i < pdata->channel_count; i--, channel--)

			devm_free_irq(pdata->dev, channel->dma_irq, channel);

	}



	devm_free_irq(pdata->dev, pdata->dev_irq, pdata);



err_rings:

	desc_if->free_ring_resources(pdata);
@@ -1254,6 +1355,8 @@ static int xgbe_close(struct net_device *netdev) 
	struct xgbe_prv_data *pdata = netdev_priv(netdev);

	struct xgbe_hw_if *hw_if = &pdata->hw_if;

	struct xgbe_desc_if *desc_if = &pdata->desc_if;

	struct xgbe_channel *channel;

	unsigned int i;



	DBGPR("-->xgbe_close\n");
@@ -1269,10 +1372,12 @@ static int xgbe_close(struct net_device *netdev) 
	/* Free the channel and ring structures */

	xgbe_free_channels(pdata);



	/* Release the interrupt */

	if (pdata->irq_number != 0) {

		devm_free_irq(pdata->dev, pdata->irq_number, pdata);

		pdata->irq_number = 0;

	/* Release the interrupts */

	devm_free_irq(pdata->dev, pdata->dev_irq, pdata);

	if (pdata->per_channel_irq) {

		channel = pdata->channel;

		for (i = 0; i < pdata->channel_count; i++, channel++)

			devm_free_irq(pdata->dev, channel->dma_irq, channel);

	}



	/* Disable the clocks */
@@ -1505,14 +1610,20 @@ static int xgbe_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, 
static void xgbe_poll_controller(struct net_device *netdev)

{

	struct xgbe_prv_data *pdata = netdev_priv(netdev);

	struct xgbe_channel *channel;

	unsigned int i;



	DBGPR("-->xgbe_poll_controller\n");



	disable_irq(pdata->irq_number);



	xgbe_isr(pdata->irq_number, pdata);



	enable_irq(pdata->irq_number);

	if (pdata->per_channel_irq) {

		channel = pdata->channel;

		for (i = 0; i < pdata->channel_count; i++, channel++)

			xgbe_dma_isr(channel->dma_irq, channel);

	} else {

		disable_irq(pdata->dev_irq);

		xgbe_isr(pdata->dev_irq, pdata);

		enable_irq(pdata->dev_irq);

	}



	DBGPR("<--xgbe_poll_controller\n");

}
@@ -1704,6 +1815,7 @@ static int xgbe_rx_poll(struct xgbe_channel *channel, int budget) 
	struct xgbe_ring_data *rdata;

	struct xgbe_packet_data *packet;

	struct net_device *netdev = pdata->netdev;

	struct napi_struct *napi;

	struct sk_buff *skb;

	struct skb_shared_hwtstamps *hwtstamps;

	unsigned int incomplete, error, context_next, context;
@@ -1717,6 +1829,8 @@ static int xgbe_rx_poll(struct xgbe_channel *channel, int budget) 
	if (!ring)

		return 0;



	napi = (pdata->per_channel_irq) ? &channel->napi : &pdata->napi;



	rdata = XGBE_GET_DESC_DATA(ring, ring->cur);

	packet = &ring->packet_data;

	while (packet_count < budget) {
@@ -1849,10 +1963,10 @@ static int xgbe_rx_poll(struct xgbe_channel *channel, int budget) 
		skb->dev = netdev;

		skb->protocol = eth_type_trans(skb, netdev);

		skb_record_rx_queue(skb, channel->queue_index);

		skb_mark_napi_id(skb, &pdata->napi);

		skb_mark_napi_id(skb, napi);



		netdev->last_rx = jiffies;

		napi_gro_receive(&pdata->napi, skb);

		napi_gro_receive(napi, skb);



next_packet:

		packet_count++;
@@ -1874,7 +1988,35 @@ static int xgbe_rx_poll(struct xgbe_channel *channel, int budget) 
	return packet_count;

}



static int xgbe_poll(struct napi_struct *napi, int budget)

static int xgbe_one_poll(struct napi_struct *napi, int budget)

{

	struct xgbe_channel *channel = container_of(napi, struct xgbe_channel,

						    napi);

	int processed = 0;



	DBGPR("-->xgbe_one_poll: budget=%d\n", budget);



	/* Cleanup Tx ring first */

	xgbe_tx_poll(channel);



	/* Process Rx ring next */

	processed = xgbe_rx_poll(channel, budget);



	/* If we processed everything, we are done */

	if (processed < budget) {

		/* Turn off polling */

		napi_complete(napi);



		/* Enable Tx and Rx interrupts */

		enable_irq(channel->dma_irq);

	}



	DBGPR("<--xgbe_one_poll: received = %d\n", processed);



	return processed;

}



static int xgbe_all_poll(struct napi_struct *napi, int budget)

{

	struct xgbe_prv_data *pdata = container_of(napi, struct xgbe_prv_data,

						   napi);
@@ -1883,7 +2025,7 @@ static int xgbe_poll(struct napi_struct *napi, int budget) 
	int processed, last_processed;

	unsigned int i;



	DBGPR("-->xgbe_poll: budget=%d\n", budget);

	DBGPR("-->xgbe_all_poll: budget=%d\n", budget);



	processed = 0;

	ring_budget = budget / pdata->rx_ring_count;
@@ -1911,7 +2053,7 @@ static int xgbe_poll(struct napi_struct *napi, int budget) 
		xgbe_enable_rx_tx_ints(pdata);

	}



	DBGPR("<--xgbe_poll: received = %d\n", processed);

	DBGPR("<--xgbe_all_poll: received = %d\n", processed);



	return processed;

}

drivers/net/ethernet/amd/xgbe/xgbe-main.c

+8 −2
Original line number Diff line number Diff line
@@ -264,12 +264,18 @@ static int xgbe_probe(struct platform_device *pdev) 
		pdata->awcache = XGBE_DMA_SYS_AWCACHE;

	}



	/* Check for per channel interrupt support */

	if (of_property_read_bool(dev->of_node, XGBE_DMA_IRQS))

		pdata->per_channel_irq = 1;



	ret = platform_get_irq(pdev, 0);

	if (ret < 0) {

		dev_err(dev, "platform_get_irq failed\n");

		dev_err(dev, "platform_get_irq 0 failed\n");

		goto err_io;

	}

	netdev->irq = ret;

	pdata->dev_irq = ret;



	netdev->irq = pdata->dev_irq;

	netdev->base_addr = (unsigned long)pdata->xgmac_regs;



	/* Set all the function pointers */

drivers/net/ethernet/amd/xgbe/xgbe.h

+9 −1
Original line number Diff line number Diff line
@@ -173,6 +173,7 @@ 
/* Device-tree clock names */

#define XGBE_DMA_CLOCK		"dma_clk"

#define XGBE_PTP_CLOCK		"ptp_clk"

#define XGBE_DMA_IRQS		"amd,per-channel-interrupt"



/* Timestamp support - values based on 50MHz PTP clock

 *   50MHz => 20 nsec
@@ -359,6 +360,12 @@ struct xgbe_channel { 
	unsigned int queue_index;

	void __iomem *dma_regs;



	/* Per channel interrupt irq number */

	int dma_irq;



	/* Netdev related settings */

	struct napi_struct napi;



	unsigned int saved_ier;



	unsigned int tx_timer_active;
@@ -609,7 +616,8 @@ struct xgbe_prv_data { 
	/* XPCS indirect addressing mutex */

	struct mutex xpcs_mutex;



	int irq_number;

	int dev_irq;

	unsigned int per_channel_irq;



	struct xgbe_hw_if hw_if;

	struct xgbe_desc_if desc_if;