Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit da87ca4d authored by Dan Williams's avatar Dan Williams
Browse files

ioat: fix tasklet tear down 

Since commit 77873803 "net_dma: mark broken" we no longer pin dma
engines active for the network-receive-offload use case.  As a result
the ->free_chan_resources() that occurs after the driver self test no
longer has a NET_DMA induced ->alloc_chan_resources() to back it up.  A
late firing irq can lead to ksoftirqd spinning indefinitely due to the
tasklet_disable() performed by ->free_chan_resources().  Only
->alloc_chan_resources() can clear this condition in affected kernels.

This problem has been present since commit 3e037454 "I/OAT: Add
support for MSI and MSI-X" in 2.6.24, but is now exposed. Given the
NET_DMA use case is deprecated we can revisit moving the driver to use
threaded irqs.  For now, just tear down the irq and tasklet properly by:

1/ Disable the irq from triggering the tasklet

2/ Disable the irq from re-arming

3/ Flush inflight interrupts

4/ Flush the timer

5/ Flush inflight tasklets

References:
https://lkml.org/lkml/2014/1/27/282
https://lkml.org/lkml/2014/2/19/672



Cc: Ingo Molnar <mingo@elte.hu>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: <stable@vger.kernel.org>
Reported-by: default avatarMike Galbraith <bitbucket@online.de>
Reported-by: default avatarStanislav Fomichev <stfomichev@yandex-team.ru>
Tested-by: default avatarMike Galbraith <bitbucket@online.de>
Tested-by: default avatarStanislav Fomichev <stfomichev@yandex-team.ru>
Reviewed-by: default avatarThomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarDan Williams <dan.j.williams@intel.com>
parent 6d0abeca

drivers/dma/ioat/dma.c

+45 −7
Original line number Original line Diff line number Diff line
@@ -77,6 +77,7 @@ static irqreturn_t ioat_dma_do_interrupt(int irq, void *data) 
	attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);

	attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);

	for_each_set_bit(bit, &attnstatus, BITS_PER_LONG) {

	for_each_set_bit(bit, &attnstatus, BITS_PER_LONG) {

		chan = ioat_chan_by_index(instance, bit);

		chan = ioat_chan_by_index(instance, bit);

		if (test_bit(IOAT_RUN, &chan->state))

			tasklet_schedule(&chan->cleanup_task);

			tasklet_schedule(&chan->cleanup_task);

	}

	}
@@ -93,6 +94,7 @@ static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data) 
{

{

	struct ioat_chan_common *chan = data;

	struct ioat_chan_common *chan = data;





	if (test_bit(IOAT_RUN, &chan->state))

		tasklet_schedule(&chan->cleanup_task);

		tasklet_schedule(&chan->cleanup_task);





	return IRQ_HANDLED;

	return IRQ_HANDLED;
@@ -116,7 +118,6 @@ void ioat_init_channel(struct ioatdma_device *device, struct ioat_chan_common *c 
	chan->timer.function = device->timer_fn;

	chan->timer.function = device->timer_fn;

	chan->timer.data = data;

	chan->timer.data = data;

	tasklet_init(&chan->cleanup_task, device->cleanup_fn, data);

	tasklet_init(&chan->cleanup_task, device->cleanup_fn, data);

	tasklet_disable(&chan->cleanup_task);

}

}





/**

/**
@@ -354,13 +355,49 @@ static int ioat1_dma_alloc_chan_resources(struct dma_chan *c) 
	writel(((u64) chan->completion_dma) >> 32,

	writel(((u64) chan->completion_dma) >> 32,

	       chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);

	       chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);





	tasklet_enable(&chan->cleanup_task);

	set_bit(IOAT_RUN, &chan->state);

	ioat1_dma_start_null_desc(ioat);  /* give chain to dma device */

	ioat1_dma_start_null_desc(ioat);  /* give chain to dma device */

	dev_dbg(to_dev(chan), "%s: allocated %d descriptors\n",

	dev_dbg(to_dev(chan), "%s: allocated %d descriptors\n",

		__func__, ioat->desccount);

		__func__, ioat->desccount);

	return ioat->desccount;

	return ioat->desccount;

}

}





void ioat_stop(struct ioat_chan_common *chan)

{

	struct ioatdma_device *device = chan->device;

	struct pci_dev *pdev = device->pdev;

	int chan_id = chan_num(chan);

	struct msix_entry *msix;



	/* 1/ stop irq from firing tasklets

	 * 2/ stop the tasklet from re-arming irqs

	 */

	clear_bit(IOAT_RUN, &chan->state);



	/* flush inflight interrupts */

	switch (device->irq_mode) {

	case IOAT_MSIX:

		msix = &device->msix_entries[chan_id];

		synchronize_irq(msix->vector);

		break;

	case IOAT_MSI:

	case IOAT_INTX:

		synchronize_irq(pdev->irq);

		break;

	default:

		break;

	}



	/* flush inflight timers */

	del_timer_sync(&chan->timer);



	/* flush inflight tasklet runs */

	tasklet_kill(&chan->cleanup_task);



	/* final cleanup now that everything is quiesced and can't re-arm */

	device->cleanup_fn((unsigned long) &chan->common);

}



/**

/**

 * ioat1_dma_free_chan_resources - release all the descriptors

 * ioat1_dma_free_chan_resources - release all the descriptors

 * @chan: the channel to be cleaned

 * @chan: the channel to be cleaned
@@ -379,9 +416,7 @@ static void ioat1_dma_free_chan_resources(struct dma_chan *c) 
	if (ioat->desccount == 0)

	if (ioat->desccount == 0)

		return;

		return;





	tasklet_disable(&chan->cleanup_task);

	ioat_stop(chan);

	del_timer_sync(&chan->timer);

	ioat1_cleanup(ioat);





	/* Delay 100ms after reset to allow internal DMA logic to quiesce

	/* Delay 100ms after reset to allow internal DMA logic to quiesce

	 * before removing DMA descriptor resources.

	 * before removing DMA descriptor resources.
@@ -526,8 +561,11 @@ ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest, 
static void ioat1_cleanup_event(unsigned long data)

static void ioat1_cleanup_event(unsigned long data)

{

{

	struct ioat_dma_chan *ioat = to_ioat_chan((void *) data);

	struct ioat_dma_chan *ioat = to_ioat_chan((void *) data);

	struct ioat_chan_common *chan = &ioat->base;





	ioat1_cleanup(ioat);

	ioat1_cleanup(ioat);

	if (!test_bit(IOAT_RUN, &chan->state))

		return;

	writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);

	writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);

}

}

drivers/dma/ioat/dma.h

+1 −0
Original line number Original line Diff line number Diff line
@@ -356,6 +356,7 @@ bool ioat_cleanup_preamble(struct ioat_chan_common *chan, 
void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type);

void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type);

void ioat_kobject_del(struct ioatdma_device *device);

void ioat_kobject_del(struct ioatdma_device *device);

int ioat_dma_setup_interrupts(struct ioatdma_device *device);

int ioat_dma_setup_interrupts(struct ioatdma_device *device);

void ioat_stop(struct ioat_chan_common *chan);

extern const struct sysfs_ops ioat_sysfs_ops;

extern const struct sysfs_ops ioat_sysfs_ops;

extern struct ioat_sysfs_entry ioat_version_attr;

extern struct ioat_sysfs_entry ioat_version_attr;

extern struct ioat_sysfs_entry ioat_cap_attr;

extern struct ioat_sysfs_entry ioat_cap_attr;

drivers/dma/ioat/dma_v2.c

+5 −6
Original line number Original line Diff line number Diff line
@@ -190,8 +190,11 @@ static void ioat2_cleanup(struct ioat2_dma_chan *ioat) 
void ioat2_cleanup_event(unsigned long data)

void ioat2_cleanup_event(unsigned long data)

{

{

	struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data);

	struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data);

	struct ioat_chan_common *chan = &ioat->base;





	ioat2_cleanup(ioat);

	ioat2_cleanup(ioat);

	if (!test_bit(IOAT_RUN, &chan->state))

		return;

	writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);

	writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);

}

}
@@ -553,10 +556,10 @@ int ioat2_alloc_chan_resources(struct dma_chan *c) 
	ioat->issued = 0;

	ioat->issued = 0;

	ioat->tail = 0;

	ioat->tail = 0;

	ioat->alloc_order = order;

	ioat->alloc_order = order;

	set_bit(IOAT_RUN, &chan->state);

	spin_unlock_bh(&ioat->prep_lock);

	spin_unlock_bh(&ioat->prep_lock);

	spin_unlock_bh(&chan->cleanup_lock);

	spin_unlock_bh(&chan->cleanup_lock);





	tasklet_enable(&chan->cleanup_task);

	ioat2_start_null_desc(ioat);

	ioat2_start_null_desc(ioat);





	/* check that we got off the ground */

	/* check that we got off the ground */
@@ -566,7 +569,6 @@ int ioat2_alloc_chan_resources(struct dma_chan *c) 
	} while (i++ < 20 && !is_ioat_active(status) && !is_ioat_idle(status));

	} while (i++ < 20 && !is_ioat_active(status) && !is_ioat_idle(status));





	if (is_ioat_active(status) || is_ioat_idle(status)) {

	if (is_ioat_active(status) || is_ioat_idle(status)) {

		set_bit(IOAT_RUN, &chan->state);

		return 1 << ioat->alloc_order;

		return 1 << ioat->alloc_order;

	} else {

	} else {

		u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);

		u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
@@ -809,11 +811,8 @@ void ioat2_free_chan_resources(struct dma_chan *c) 
	if (!ioat->ring)

	if (!ioat->ring)

		return;

		return;





	tasklet_disable(&chan->cleanup_task);

	ioat_stop(chan);

	del_timer_sync(&chan->timer);

	device->cleanup_fn((unsigned long) c);

	device->reset_hw(chan);

	device->reset_hw(chan);

	clear_bit(IOAT_RUN, &chan->state);





	spin_lock_bh(&chan->cleanup_lock);

	spin_lock_bh(&chan->cleanup_lock);

	spin_lock_bh(&ioat->prep_lock);

	spin_lock_bh(&ioat->prep_lock);

drivers/dma/ioat/dma_v3.c

+3 −0
Original line number Original line Diff line number Diff line
@@ -464,8 +464,11 @@ static void ioat3_cleanup(struct ioat2_dma_chan *ioat) 
static void ioat3_cleanup_event(unsigned long data)

static void ioat3_cleanup_event(unsigned long data)

{

{

	struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data);

	struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data);

	struct ioat_chan_common *chan = &ioat->base;





	ioat3_cleanup(ioat);

	ioat3_cleanup(ioat);

	if (!test_bit(IOAT_RUN, &chan->state))

		return;

	writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);

	writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);

}

}