Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 885b2010 authored by Dave Jiang's avatar Dave Jiang Committed by Vinod Koul
Browse files

dmaengine: ioatdma: remove dma_v2.* 



Clean out dma_v2 and remove ioat2 calls since we are moving everything
to just ioat.

Signed-off-by: default avatarDave Jiang <dave.jiang@intel.com>
Acked-by: default avatarDan Williams <dan.j.williams@intel.com>
Signed-off-by: default avatarVinod Koul <vinod.koul@intel.com>
parent 55f878ec

drivers/dma/ioat/Makefile

+1 −1
Original line number Diff line number Diff line 
obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o

ioatdma-y := pci.o dma.o dma_v2.o dma_v3.o dca.o
 
ioatdma-y := pci.o dma.o dma_v3.o dca.o

drivers/dma/ioat/dca.c

+0 −1
Original line number Diff line number Diff line
@@ -31,7 +31,6 @@ 


#include "dma.h"

#include "registers.h"

#include "dma_v2.h"



/*

 * Bit 7 of a tag map entry is the "valid" bit, if it is set then bits 0:6

drivers/dma/ioat/dma.c

+661 −6
Original line number Diff line number Diff line
@@ -41,6 +41,19 @@ int ioat_pending_level = 4; 
module_param(ioat_pending_level, int, 0644);

MODULE_PARM_DESC(ioat_pending_level,

		 "high-water mark for pushing ioat descriptors (default: 4)");

int ioat_ring_alloc_order = 8;

module_param(ioat_ring_alloc_order, int, 0644);

MODULE_PARM_DESC(ioat_ring_alloc_order,

		 "ioat+: allocate 2^n descriptors per channel (default: 8 max: 16)");

static int ioat_ring_max_alloc_order = IOAT_MAX_ORDER;

module_param(ioat_ring_max_alloc_order, int, 0644);

MODULE_PARM_DESC(ioat_ring_max_alloc_order,

		 "ioat+: upper limit for ring size (default: 16)");

static char ioat_interrupt_style[32] = "msix";

module_param_string(ioat_interrupt_style, ioat_interrupt_style,

		    sizeof(ioat_interrupt_style), 0644);

MODULE_PARM_DESC(ioat_interrupt_style,

		 "set ioat interrupt style: msix (default), msi, intx");



/**

 * ioat_dma_do_interrupt - handler used for single vector interrupt mode
@@ -314,12 +327,6 @@ int ioat_dma_self_test(struct ioatdma_device *ioat_dma) 
	return err;

}



static char ioat_interrupt_style[32] = "msix";

module_param_string(ioat_interrupt_style, ioat_interrupt_style,

		    sizeof(ioat_interrupt_style), 0644);

MODULE_PARM_DESC(ioat_interrupt_style,

		 "set ioat interrupt style: msix (default), msi, intx");



/**

 * ioat_dma_setup_interrupts - setup interrupt handler

 * @ioat_dma: ioat dma device
@@ -577,3 +584,651 @@ void ioat_dma_remove(struct ioatdma_device *ioat_dma) 


	INIT_LIST_HEAD(&dma->channels);

}



void __ioat_issue_pending(struct ioatdma_chan *ioat_chan)

{

	ioat_chan->dmacount += ioat_ring_pending(ioat_chan);

	ioat_chan->issued = ioat_chan->head;

	writew(ioat_chan->dmacount,

	       ioat_chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET);

	dev_dbg(to_dev(ioat_chan),

		"%s: head: %#x tail: %#x issued: %#x count: %#x\n",

		__func__, ioat_chan->head, ioat_chan->tail,

		ioat_chan->issued, ioat_chan->dmacount);

}



void ioat_issue_pending(struct dma_chan *c)

{

	struct ioatdma_chan *ioat_chan = to_ioat_chan(c);



	if (ioat_ring_pending(ioat_chan)) {

		spin_lock_bh(&ioat_chan->prep_lock);

		__ioat_issue_pending(ioat_chan);

		spin_unlock_bh(&ioat_chan->prep_lock);

	}

}



/**

 * ioat_update_pending - log pending descriptors

 * @ioat: ioat+ channel

 *

 * Check if the number of unsubmitted descriptors has exceeded the

 * watermark.  Called with prep_lock held

 */

static void ioat_update_pending(struct ioatdma_chan *ioat_chan)

{

	if (ioat_ring_pending(ioat_chan) > ioat_pending_level)

		__ioat_issue_pending(ioat_chan);

}



static void __ioat_start_null_desc(struct ioatdma_chan *ioat_chan)

{

	struct ioat_ring_ent *desc;

	struct ioat_dma_descriptor *hw;



	if (ioat_ring_space(ioat_chan) < 1) {

		dev_err(to_dev(ioat_chan),

			"Unable to start null desc - ring full\n");

		return;

	}



	dev_dbg(to_dev(ioat_chan),

		"%s: head: %#x tail: %#x issued: %#x\n",

		__func__, ioat_chan->head, ioat_chan->tail, ioat_chan->issued);

	desc = ioat_get_ring_ent(ioat_chan, ioat_chan->head);



	hw = desc->hw;

	hw->ctl = 0;

	hw->ctl_f.null = 1;

	hw->ctl_f.int_en = 1;

	hw->ctl_f.compl_write = 1;

	/* set size to non-zero value (channel returns error when size is 0) */

	hw->size = NULL_DESC_BUFFER_SIZE;

	hw->src_addr = 0;

	hw->dst_addr = 0;

	async_tx_ack(&desc->txd);

	ioat_set_chainaddr(ioat_chan, desc->txd.phys);

	dump_desc_dbg(ioat_chan, desc);

	/* make sure descriptors are written before we submit */

	wmb();

	ioat_chan->head += 1;

	__ioat_issue_pending(ioat_chan);

}



static void ioat_start_null_desc(struct ioatdma_chan *ioat_chan)

{

	spin_lock_bh(&ioat_chan->prep_lock);

	__ioat_start_null_desc(ioat_chan);

	spin_unlock_bh(&ioat_chan->prep_lock);

}



void __ioat_restart_chan(struct ioatdma_chan *ioat_chan)

{

	/* set the tail to be re-issued */

	ioat_chan->issued = ioat_chan->tail;

	ioat_chan->dmacount = 0;

	set_bit(IOAT_COMPLETION_PENDING, &ioat_chan->state);

	mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);



	dev_dbg(to_dev(ioat_chan),

		"%s: head: %#x tail: %#x issued: %#x count: %#x\n",

		__func__, ioat_chan->head, ioat_chan->tail,

		ioat_chan->issued, ioat_chan->dmacount);



	if (ioat_ring_pending(ioat_chan)) {

		struct ioat_ring_ent *desc;



		desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail);

		ioat_set_chainaddr(ioat_chan, desc->txd.phys);

		__ioat_issue_pending(ioat_chan);

	} else

		__ioat_start_null_desc(ioat_chan);

}



int ioat_quiesce(struct ioatdma_chan *ioat_chan, unsigned long tmo)

{

	unsigned long end = jiffies + tmo;

	int err = 0;

	u32 status;



	status = ioat_chansts(ioat_chan);

	if (is_ioat_active(status) || is_ioat_idle(status))

		ioat_suspend(ioat_chan);

	while (is_ioat_active(status) || is_ioat_idle(status)) {

		if (tmo && time_after(jiffies, end)) {

			err = -ETIMEDOUT;

			break;

		}

		status = ioat_chansts(ioat_chan);

		cpu_relax();

	}



	return err;

}



int ioat_reset_sync(struct ioatdma_chan *ioat_chan, unsigned long tmo)

{

	unsigned long end = jiffies + tmo;

	int err = 0;



	ioat_reset(ioat_chan);

	while (ioat_reset_pending(ioat_chan)) {

		if (end && time_after(jiffies, end)) {

			err = -ETIMEDOUT;

			break;

		}

		cpu_relax();

	}



	return err;

}



/**

 * ioat_enumerate_channels - find and initialize the device's channels

 * @ioat_dma: the ioat dma device to be enumerated

 */

int ioat_enumerate_channels(struct ioatdma_device *ioat_dma)

{

	struct ioatdma_chan *ioat_chan;

	struct device *dev = &ioat_dma->pdev->dev;

	struct dma_device *dma = &ioat_dma->dma_dev;

	u8 xfercap_log;

	int i;



	INIT_LIST_HEAD(&dma->channels);

	dma->chancnt = readb(ioat_dma->reg_base + IOAT_CHANCNT_OFFSET);

	dma->chancnt &= 0x1f; /* bits [4:0] valid */

	if (dma->chancnt > ARRAY_SIZE(ioat_dma->idx)) {

		dev_warn(dev, "(%d) exceeds max supported channels (%zu)\n",

			 dma->chancnt, ARRAY_SIZE(ioat_dma->idx));

		dma->chancnt = ARRAY_SIZE(ioat_dma->idx);

	}

	xfercap_log = readb(ioat_dma->reg_base + IOAT_XFERCAP_OFFSET);

	xfercap_log &= 0x1f; /* bits [4:0] valid */

	if (xfercap_log == 0)

		return 0;

	dev_dbg(dev, "%s: xfercap = %d\n", __func__, 1 << xfercap_log);



	for (i = 0; i < dma->chancnt; i++) {

		ioat_chan = devm_kzalloc(dev, sizeof(*ioat_chan), GFP_KERNEL);

		if (!ioat_chan)

			break;



		ioat_init_channel(ioat_dma, ioat_chan, i);

		ioat_chan->xfercap_log = xfercap_log;

		spin_lock_init(&ioat_chan->prep_lock);

		if (ioat_dma->reset_hw(ioat_chan)) {

			i = 0;

			break;

		}

	}

	dma->chancnt = i;

	return i;

}



static dma_cookie_t ioat_tx_submit_unlock(struct dma_async_tx_descriptor *tx)

{

	struct dma_chan *c = tx->chan;

	struct ioatdma_chan *ioat_chan = to_ioat_chan(c);

	dma_cookie_t cookie;



	cookie = dma_cookie_assign(tx);

	dev_dbg(to_dev(ioat_chan), "%s: cookie: %d\n", __func__, cookie);



	if (!test_and_set_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state))

		mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);



	/* make descriptor updates visible before advancing ioat->head,

	 * this is purposefully not smp_wmb() since we are also

	 * publishing the descriptor updates to a dma device

	 */

	wmb();



	ioat_chan->head += ioat_chan->produce;



	ioat_update_pending(ioat_chan);

	spin_unlock_bh(&ioat_chan->prep_lock);



	return cookie;

}



static struct ioat_ring_ent *

ioat_alloc_ring_ent(struct dma_chan *chan, gfp_t flags)

{

	struct ioat_dma_descriptor *hw;

	struct ioat_ring_ent *desc;

	struct ioatdma_device *ioat_dma;

	dma_addr_t phys;



	ioat_dma = to_ioatdma_device(chan->device);

	hw = pci_pool_alloc(ioat_dma->dma_pool, flags, &phys);

	if (!hw)

		return NULL;

	memset(hw, 0, sizeof(*hw));



	desc = kmem_cache_zalloc(ioat_cache, flags);

	if (!desc) {

		pci_pool_free(ioat_dma->dma_pool, hw, phys);

		return NULL;

	}



	dma_async_tx_descriptor_init(&desc->txd, chan);

	desc->txd.tx_submit = ioat_tx_submit_unlock;

	desc->hw = hw;

	desc->txd.phys = phys;

	return desc;

}



static void

ioat_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan)

{

	struct ioatdma_device *ioat_dma;



	ioat_dma = to_ioatdma_device(chan->device);

	pci_pool_free(ioat_dma->dma_pool, desc->hw, desc->txd.phys);

	kmem_cache_free(ioat_cache, desc);

}



static struct ioat_ring_ent **

ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags)

{

	struct ioat_ring_ent **ring;

	int descs = 1 << order;

	int i;



	if (order > ioat_get_max_alloc_order())

		return NULL;



	/* allocate the array to hold the software ring */

	ring = kcalloc(descs, sizeof(*ring), flags);

	if (!ring)

		return NULL;

	for (i = 0; i < descs; i++) {

		ring[i] = ioat_alloc_ring_ent(c, flags);

		if (!ring[i]) {

			while (i--)

				ioat_free_ring_ent(ring[i], c);

			kfree(ring);

			return NULL;

		}

		set_desc_id(ring[i], i);

	}



	/* link descs */

	for (i = 0; i < descs-1; i++) {

		struct ioat_ring_ent *next = ring[i+1];

		struct ioat_dma_descriptor *hw = ring[i]->hw;



		hw->next = next->txd.phys;

	}

	ring[i]->hw->next = ring[0]->txd.phys;



	return ring;

}



/**

 * ioat_free_chan_resources - release all the descriptors

 * @chan: the channel to be cleaned

 */

void ioat_free_chan_resources(struct dma_chan *c)

{

	struct ioatdma_chan *ioat_chan = to_ioat_chan(c);

	struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;

	struct ioat_ring_ent *desc;

	const int total_descs = 1 << ioat_chan->alloc_order;

	int descs;

	int i;



	/* Before freeing channel resources first check

	 * if they have been previously allocated for this channel.

	 */

	if (!ioat_chan->ring)

		return;



	ioat_stop(ioat_chan);

	ioat_dma->reset_hw(ioat_chan);



	spin_lock_bh(&ioat_chan->cleanup_lock);

	spin_lock_bh(&ioat_chan->prep_lock);

	descs = ioat_ring_space(ioat_chan);

	dev_dbg(to_dev(ioat_chan), "freeing %d idle descriptors\n", descs);

	for (i = 0; i < descs; i++) {

		desc = ioat_get_ring_ent(ioat_chan, ioat_chan->head + i);

		ioat_free_ring_ent(desc, c);

	}



	if (descs < total_descs)

		dev_err(to_dev(ioat_chan), "Freeing %d in use descriptors!\n",

			total_descs - descs);



	for (i = 0; i < total_descs - descs; i++) {

		desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail + i);

		dump_desc_dbg(ioat_chan, desc);

		ioat_free_ring_ent(desc, c);

	}



	kfree(ioat_chan->ring);

	ioat_chan->ring = NULL;

	ioat_chan->alloc_order = 0;

	pci_pool_free(ioat_dma->completion_pool, ioat_chan->completion,

		      ioat_chan->completion_dma);

	spin_unlock_bh(&ioat_chan->prep_lock);

	spin_unlock_bh(&ioat_chan->cleanup_lock);



	ioat_chan->last_completion = 0;

	ioat_chan->completion_dma = 0;

	ioat_chan->dmacount = 0;

}



/* ioat_alloc_chan_resources - allocate/initialize ioat descriptor ring

 * @chan: channel to be initialized

 */

int ioat_alloc_chan_resources(struct dma_chan *c)

{

	struct ioatdma_chan *ioat_chan = to_ioat_chan(c);

	struct ioat_ring_ent **ring;

	u64 status;

	int order;

	int i = 0;

	u32 chanerr;



	/* have we already been set up? */

	if (ioat_chan->ring)

		return 1 << ioat_chan->alloc_order;



	/* Setup register to interrupt and write completion status on error */

	writew(IOAT_CHANCTRL_RUN, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);



	/* allocate a completion writeback area */

	/* doing 2 32bit writes to mmio since 1 64b write doesn't work */

	ioat_chan->completion =

		pci_pool_alloc(ioat_chan->ioat_dma->completion_pool,

			       GFP_KERNEL, &ioat_chan->completion_dma);

	if (!ioat_chan->completion)

		return -ENOMEM;



	memset(ioat_chan->completion, 0, sizeof(*ioat_chan->completion));

	writel(((u64)ioat_chan->completion_dma) & 0x00000000FFFFFFFF,

	       ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);

	writel(((u64)ioat_chan->completion_dma) >> 32,

	       ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);



	order = ioat_get_alloc_order();

	ring = ioat_alloc_ring(c, order, GFP_KERNEL);

	if (!ring)

		return -ENOMEM;



	spin_lock_bh(&ioat_chan->cleanup_lock);

	spin_lock_bh(&ioat_chan->prep_lock);

	ioat_chan->ring = ring;

	ioat_chan->head = 0;

	ioat_chan->issued = 0;

	ioat_chan->tail = 0;

	ioat_chan->alloc_order = order;

	set_bit(IOAT_RUN, &ioat_chan->state);

	spin_unlock_bh(&ioat_chan->prep_lock);

	spin_unlock_bh(&ioat_chan->cleanup_lock);



	ioat_start_null_desc(ioat_chan);



	/* check that we got off the ground */

	do {

		udelay(1);

		status = ioat_chansts(ioat_chan);

	} while (i++ < 20 && !is_ioat_active(status) && !is_ioat_idle(status));



	if (is_ioat_active(status) || is_ioat_idle(status))

		return 1 << ioat_chan->alloc_order;



	chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);



	dev_WARN(to_dev(ioat_chan),

		"failed to start channel chanerr: %#x\n", chanerr);

	ioat_free_chan_resources(c);

	return -EFAULT;

}



bool reshape_ring(struct ioatdma_chan *ioat_chan, int order)

{

	/* reshape differs from normal ring allocation in that we want

	 * to allocate a new software ring while only

	 * extending/truncating the hardware ring

	 */

	struct dma_chan *c = &ioat_chan->dma_chan;

	const u32 curr_size = ioat_ring_size(ioat_chan);

	const u16 active = ioat_ring_active(ioat_chan);

	const u32 new_size = 1 << order;

	struct ioat_ring_ent **ring;

	u32 i;



	if (order > ioat_get_max_alloc_order())

		return false;



	/* double check that we have at least 1 free descriptor */

	if (active == curr_size)

		return false;



	/* when shrinking, verify that we can hold the current active

	 * set in the new ring

	 */

	if (active >= new_size)

		return false;



	/* allocate the array to hold the software ring */

	ring = kcalloc(new_size, sizeof(*ring), GFP_NOWAIT);

	if (!ring)

		return false;



	/* allocate/trim descriptors as needed */

	if (new_size > curr_size) {

		/* copy current descriptors to the new ring */

		for (i = 0; i < curr_size; i++) {

			u16 curr_idx = (ioat_chan->tail+i) & (curr_size-1);

			u16 new_idx = (ioat_chan->tail+i) & (new_size-1);



			ring[new_idx] = ioat_chan->ring[curr_idx];

			set_desc_id(ring[new_idx], new_idx);

		}



		/* add new descriptors to the ring */

		for (i = curr_size; i < new_size; i++) {

			u16 new_idx = (ioat_chan->tail+i) & (new_size-1);



			ring[new_idx] = ioat_alloc_ring_ent(c, GFP_NOWAIT);

			if (!ring[new_idx]) {

				while (i--) {

					u16 new_idx = (ioat_chan->tail+i) &

						       (new_size-1);



					ioat_free_ring_ent(ring[new_idx], c);

				}

				kfree(ring);

				return false;

			}

			set_desc_id(ring[new_idx], new_idx);

		}



		/* hw link new descriptors */

		for (i = curr_size-1; i < new_size; i++) {

			u16 new_idx = (ioat_chan->tail+i) & (new_size-1);

			struct ioat_ring_ent *next =

				ring[(new_idx+1) & (new_size-1)];

			struct ioat_dma_descriptor *hw = ring[new_idx]->hw;



			hw->next = next->txd.phys;

		}

	} else {

		struct ioat_dma_descriptor *hw;

		struct ioat_ring_ent *next;



		/* copy current descriptors to the new ring, dropping the

		 * removed descriptors

		 */

		for (i = 0; i < new_size; i++) {

			u16 curr_idx = (ioat_chan->tail+i) & (curr_size-1);

			u16 new_idx = (ioat_chan->tail+i) & (new_size-1);



			ring[new_idx] = ioat_chan->ring[curr_idx];

			set_desc_id(ring[new_idx], new_idx);

		}



		/* free deleted descriptors */

		for (i = new_size; i < curr_size; i++) {

			struct ioat_ring_ent *ent;



			ent = ioat_get_ring_ent(ioat_chan, ioat_chan->tail+i);

			ioat_free_ring_ent(ent, c);

		}



		/* fix up hardware ring */

		hw = ring[(ioat_chan->tail+new_size-1) & (new_size-1)]->hw;

		next = ring[(ioat_chan->tail+new_size) & (new_size-1)];

		hw->next = next->txd.phys;

	}



	dev_dbg(to_dev(ioat_chan), "%s: allocated %d descriptors\n",

		__func__, new_size);



	kfree(ioat_chan->ring);

	ioat_chan->ring = ring;

	ioat_chan->alloc_order = order;



	return true;

}



/**

 * ioat_check_space_lock - verify space and grab ring producer lock

 * @ioat: ioat,3 channel (ring) to operate on

 * @num_descs: allocation length

 */

int ioat_check_space_lock(struct ioatdma_chan *ioat_chan, int num_descs)

{

	bool retry;



 retry:

	spin_lock_bh(&ioat_chan->prep_lock);

	/* never allow the last descriptor to be consumed, we need at

	 * least one free at all times to allow for on-the-fly ring

	 * resizing.

	 */

	if (likely(ioat_ring_space(ioat_chan) > num_descs)) {

		dev_dbg(to_dev(ioat_chan), "%s: num_descs: %d (%x:%x:%x)\n",

			__func__, num_descs, ioat_chan->head,

			ioat_chan->tail, ioat_chan->issued);

		ioat_chan->produce = num_descs;

		return 0;  /* with ioat->prep_lock held */

	}

	retry = test_and_set_bit(IOAT_RESHAPE_PENDING, &ioat_chan->state);

	spin_unlock_bh(&ioat_chan->prep_lock);



	/* is another cpu already trying to expand the ring? */

	if (retry)

		goto retry;



	spin_lock_bh(&ioat_chan->cleanup_lock);

	spin_lock_bh(&ioat_chan->prep_lock);

	retry = reshape_ring(ioat_chan, ioat_chan->alloc_order + 1);

	clear_bit(IOAT_RESHAPE_PENDING, &ioat_chan->state);

	spin_unlock_bh(&ioat_chan->prep_lock);

	spin_unlock_bh(&ioat_chan->cleanup_lock);



	/* if we were able to expand the ring retry the allocation */

	if (retry)

		goto retry;



	dev_dbg_ratelimited(to_dev(ioat_chan),

			    "%s: ring full! num_descs: %d (%x:%x:%x)\n",

			    __func__, num_descs, ioat_chan->head,

			    ioat_chan->tail, ioat_chan->issued);



	/* progress reclaim in the allocation failure case we may be

	 * called under bh_disabled so we need to trigger the timer

	 * event directly

	 */

	if (time_is_before_jiffies(ioat_chan->timer.expires)

	    && timer_pending(&ioat_chan->timer)) {

		struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;



		mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);

		ioat_dma->timer_fn((unsigned long)ioat_chan);

	}



	return -ENOMEM;

}



struct dma_async_tx_descriptor *

ioat_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,

			   dma_addr_t dma_src, size_t len, unsigned long flags)

{

	struct ioatdma_chan *ioat_chan = to_ioat_chan(c);

	struct ioat_dma_descriptor *hw;

	struct ioat_ring_ent *desc;

	dma_addr_t dst = dma_dest;

	dma_addr_t src = dma_src;

	size_t total_len = len;

	int num_descs, idx, i;



	num_descs = ioat_xferlen_to_descs(ioat_chan, len);

	if (likely(num_descs) &&

	    ioat_check_space_lock(ioat_chan, num_descs) == 0)

		idx = ioat_chan->head;

	else

		return NULL;

	i = 0;

	do {

		size_t copy = min_t(size_t, len, 1 << ioat_chan->xfercap_log);



		desc = ioat_get_ring_ent(ioat_chan, idx + i);

		hw = desc->hw;



		hw->size = copy;

		hw->ctl = 0;

		hw->src_addr = src;

		hw->dst_addr = dst;



		len -= copy;

		dst += copy;

		src += copy;

		dump_desc_dbg(ioat_chan, desc);

	} while (++i < num_descs);



	desc->txd.flags = flags;

	desc->len = total_len;

	hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);

	hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE);

	hw->ctl_f.compl_write = 1;

	dump_desc_dbg(ioat_chan, desc);

	/* we leave the channel locked to ensure in order submission */



	return &desc->txd;

}



static ssize_t ring_size_show(struct dma_chan *c, char *page)

{

	struct ioatdma_chan *ioat_chan = to_ioat_chan(c);



	return sprintf(page, "%d\n", (1 << ioat_chan->alloc_order) & ~1);

}

static struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size);



static ssize_t ring_active_show(struct dma_chan *c, char *page)

{

	struct ioatdma_chan *ioat_chan = to_ioat_chan(c);



	/* ...taken outside the lock, no need to be precise */

	return sprintf(page, "%d\n", ioat_ring_active(ioat_chan));

}

static struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active);



static struct attribute *ioat_attrs[] = {

	&ring_size_attr.attr,

	&ring_active_attr.attr,

	&ioat_cap_attr.attr,

	&ioat_version_attr.attr,

	NULL,

};



struct kobj_type ioat_ktype = {

	.sysfs_ops = &ioat_sysfs_ops,

	.default_attrs = ioat_attrs,

};

drivers/dma/ioat/dma.h

+116 −3
Original line number Diff line number Diff line
@@ -18,13 +18,14 @@ 
#define IOATDMA_H



#include <linux/dmaengine.h>

#include "hw.h"

#include "registers.h"

#include <linux/init.h>

#include <linux/dmapool.h>

#include <linux/cache.h>

#include <linux/pci_ids.h>

#include <net/tcp.h>

#include <linux/circ_buf.h>

#include <linux/interrupt.h>

#include "registers.h"

#include "hw.h"



#define IOAT_DMA_VERSION  "4.00"
@@ -154,6 +155,41 @@ struct ioat_sed_ent { 
	unsigned int hw_pool;

};



/**

 * struct ioat_ring_ent - wrapper around hardware descriptor

 * @hw: hardware DMA descriptor (for memcpy)

 * @fill: hardware fill descriptor

 * @xor: hardware xor descriptor

 * @xor_ex: hardware xor extension descriptor

 * @pq: hardware pq descriptor

 * @pq_ex: hardware pq extension descriptor

 * @pqu: hardware pq update descriptor

 * @raw: hardware raw (un-typed) descriptor

 * @txd: the generic software descriptor for all engines

 * @len: total transaction length for unmap

 * @result: asynchronous result of validate operations

 * @id: identifier for debug

 */



struct ioat_ring_ent {

	union {

		struct ioat_dma_descriptor *hw;

		struct ioat_xor_descriptor *xor;

		struct ioat_xor_ext_descriptor *xor_ex;

		struct ioat_pq_descriptor *pq;

		struct ioat_pq_ext_descriptor *pq_ex;

		struct ioat_pq_update_descriptor *pqu;

		struct ioat_raw_descriptor *raw;

	};

	size_t len;

	struct dma_async_tx_descriptor txd;

	enum sum_check_flags *result;

	#ifdef DEBUG

	int id;

	#endif

	struct ioat_sed_ent *sed;

};



static inline struct ioatdma_chan *to_ioat_chan(struct dma_chan *c)

{

	return container_of(c, struct ioatdma_chan, dma_chan);
@@ -291,6 +327,60 @@ static inline bool is_ioat_bug(unsigned long err) 
	return !!err;

}



#define IOAT_MAX_ORDER 16

#define ioat_get_alloc_order() \

	(min(ioat_ring_alloc_order, IOAT_MAX_ORDER))

#define ioat_get_max_alloc_order() \

	(min(ioat_ring_max_alloc_order, IOAT_MAX_ORDER))



static inline u32 ioat_ring_size(struct ioatdma_chan *ioat_chan)

{

	return 1 << ioat_chan->alloc_order;

}



/* count of descriptors in flight with the engine */

static inline u16 ioat_ring_active(struct ioatdma_chan *ioat_chan)

{

	return CIRC_CNT(ioat_chan->head, ioat_chan->tail,

			ioat_ring_size(ioat_chan));

}



/* count of descriptors pending submission to hardware */

static inline u16 ioat_ring_pending(struct ioatdma_chan *ioat_chan)

{

	return CIRC_CNT(ioat_chan->head, ioat_chan->issued,

			ioat_ring_size(ioat_chan));

}



static inline u32 ioat_ring_space(struct ioatdma_chan *ioat_chan)

{

	return ioat_ring_size(ioat_chan) - ioat_ring_active(ioat_chan);

}



static inline u16

ioat_xferlen_to_descs(struct ioatdma_chan *ioat_chan, size_t len)

{

	u16 num_descs = len >> ioat_chan->xfercap_log;



	num_descs += !!(len & ((1 << ioat_chan->xfercap_log) - 1));

	return num_descs;

}



static inline struct ioat_ring_ent *

ioat_get_ring_ent(struct ioatdma_chan *ioat_chan, u16 idx)

{

	return ioat_chan->ring[idx & (ioat_ring_size(ioat_chan) - 1)];

}



static inline void

ioat_set_chainaddr(struct ioatdma_chan *ioat_chan, u64 addr)

{

	writel(addr & 0x00000000FFFFFFFF,

	       ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);

	writel(addr >> 32,

	       ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);

}



int ioat_probe(struct ioatdma_device *ioat_dma);

int ioat_register(struct ioatdma_device *ioat_dma);

int ioat_dma_self_test(struct ioatdma_device *ioat_dma);
@@ -306,7 +396,30 @@ void ioat_kobject_add(struct ioatdma_device *ioat_dma, struct kobj_type *type); 
void ioat_kobject_del(struct ioatdma_device *ioat_dma);

int ioat_dma_setup_interrupts(struct ioatdma_device *ioat_dma);

void ioat_stop(struct ioatdma_chan *ioat_chan);

int ioat_dma_probe(struct ioatdma_device *ioat_dma, int dca);

int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca);

struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase);

int ioat_check_space_lock(struct ioatdma_chan *ioat_chan, int num_descs);

int ioat_enumerate_channels(struct ioatdma_device *ioat_dma);

struct dma_async_tx_descriptor *

ioat_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,

			   dma_addr_t dma_src, size_t len, unsigned long flags);

void ioat_issue_pending(struct dma_chan *chan);

int ioat_alloc_chan_resources(struct dma_chan *c);

void ioat_free_chan_resources(struct dma_chan *c);

void __ioat_restart_chan(struct ioatdma_chan *ioat_chan);

bool reshape_ring(struct ioatdma_chan *ioat, int order);

void __ioat_issue_pending(struct ioatdma_chan *ioat_chan);

void ioat_timer_event(unsigned long data);

int ioat_quiesce(struct ioatdma_chan *ioat_chan, unsigned long tmo);

int ioat_reset_sync(struct ioatdma_chan *ioat_chan, unsigned long tmo);



extern const struct sysfs_ops ioat_sysfs_ops;

extern struct ioat_sysfs_entry ioat_version_attr;

extern struct ioat_sysfs_entry ioat_cap_attr;

extern int ioat_pending_level;

extern int ioat_ring_alloc_order;

extern struct kobj_type ioat_ktype;

extern struct kmem_cache *ioat_cache;



#endif /* IOATDMA_H */

drivers/dma/ioat/dma_v2.c

deleted100644 → 0
+0 −695

File deleted.

Preview size limit exceeded, changes collapsed.