Merge branch 'iop-raid6' into async-tx-next (3e48e656) · Commits · e / devices / android_kernel_fairphone_FP3

arch/arm/include/asm/hardware/iop3xx-adma.h

+76 −0

Original line number	Diff line number	Diff line
		@@ -187,11 +187,74 @@ union iop3xx_desc {
		void *ptr;
		};

		/* No support for p+q operations */
		static inline int
		iop_chan_pq_slot_count(size_t len, int src_cnt, int *slots_per_op)
		{
		BUG();
		return 0;
		}

		static inline void
		iop_desc_init_pq(struct iop_adma_desc_slot *desc, int src_cnt,
		unsigned long flags)
		{
		BUG();
		}

		static inline void
		iop_desc_set_pq_addr(struct iop_adma_desc_slot desc, dma_addr_t addr)
		{
		BUG();
		}

		static inline void
		iop_desc_set_pq_src_addr(struct iop_adma_desc_slot *desc, int src_idx,
		dma_addr_t addr, unsigned char coef)
		{
		BUG();
		}

		static inline int
		iop_chan_pq_zero_sum_slot_count(size_t len, int src_cnt, int *slots_per_op)
		{
		BUG();
		return 0;
		}

		static inline void
		iop_desc_init_pq_zero_sum(struct iop_adma_desc_slot *desc, int src_cnt,
		unsigned long flags)
		{
		BUG();
		}

		static inline void
		iop_desc_set_pq_zero_sum_byte_count(struct iop_adma_desc_slot *desc, u32 len)
		{
		BUG();
		}

		#define iop_desc_set_pq_zero_sum_src_addr iop_desc_set_pq_src_addr

		static inline void
		iop_desc_set_pq_zero_sum_addr(struct iop_adma_desc_slot *desc, int pq_idx,
		dma_addr_t *src)
		{
		BUG();
		}

		static inline int iop_adma_get_max_xor(void)
		{
		return 32;
		}

		static inline int iop_adma_get_max_pq(void)
		{
		BUG();
		return 0;
		}

		static inline u32 iop_chan_get_current_descriptor(struct iop_adma_chan *chan)
		{
		int id = chan->device->id;
		@@ -332,6 +395,11 @@ static inline int iop_chan_zero_sum_slot_count(size_t len, int src_cnt,
		return slot_cnt;
		}

		static inline int iop_desc_is_pq(struct iop_adma_desc_slot *desc)
		{
		return 0;
		}

		static inline u32 iop_desc_get_dest_addr(struct iop_adma_desc_slot *desc,
		struct iop_adma_chan *chan)
		{
		@@ -349,6 +417,14 @@ static inline u32 iop_desc_get_dest_addr(struct iop_adma_desc_slot *desc,
		return 0;
		}


		static inline u32 iop_desc_get_qdest_addr(struct iop_adma_desc_slot *desc,
		struct iop_adma_chan *chan)
		{
		BUG();
		return 0;
		}

		static inline u32 iop_desc_get_byte_count(struct iop_adma_desc_slot *desc,
		struct iop_adma_chan *chan)
		{

arch/arm/include/asm/hardware/iop_adma.h

+1 −0

Original line number	Diff line number	Diff line
		@@ -106,6 +106,7 @@ struct iop_adma_desc_slot {
		union {
		u32 *xor_check_result;
		u32 *crc32_result;
		u32 *pq_check_result;
		};
		};

arch/arm/mach-iop13xx/include/mach/adma.h

+107 −0

Original line number	Diff line number	Diff line
		@@ -150,6 +150,8 @@ static inline int iop_adma_get_max_xor(void)
		return 16;
		}

		#define iop_adma_get_max_pq iop_adma_get_max_xor

		static inline u32 iop_chan_get_current_descriptor(struct iop_adma_chan *chan)
		{
		return __raw_readl(ADMA_ADAR(chan));
		@@ -211,7 +213,10 @@ iop_chan_xor_slot_count(size_t len, int src_cnt, int *slots_per_op)
		#define IOP_ADMA_MAX_BYTE_COUNT ADMA_MAX_BYTE_COUNT
		#define IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT ADMA_MAX_BYTE_COUNT
		#define IOP_ADMA_XOR_MAX_BYTE_COUNT ADMA_MAX_BYTE_COUNT
		#define IOP_ADMA_PQ_MAX_BYTE_COUNT ADMA_MAX_BYTE_COUNT
		#define iop_chan_zero_sum_slot_count(l, s, o) iop_chan_xor_slot_count(l, s, o)
		#define iop_chan_pq_slot_count iop_chan_xor_slot_count
		#define iop_chan_pq_zero_sum_slot_count iop_chan_xor_slot_count

		static inline u32 iop_desc_get_dest_addr(struct iop_adma_desc_slot *desc,
		struct iop_adma_chan *chan)
		@@ -220,6 +225,13 @@ static inline u32 iop_desc_get_dest_addr(struct iop_adma_desc_slot *desc,
		return hw_desc->dest_addr;
		}

		static inline u32 iop_desc_get_qdest_addr(struct iop_adma_desc_slot *desc,
		struct iop_adma_chan *chan)
		{
		struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
		return hw_desc->q_dest_addr;
		}

		static inline u32 iop_desc_get_byte_count(struct iop_adma_desc_slot *desc,
		struct iop_adma_chan *chan)
		{
		@@ -319,6 +331,58 @@ iop_desc_init_zero_sum(struct iop_adma_desc_slot *desc, int src_cnt,
		return 1;
		}

		static inline void
		iop_desc_init_pq(struct iop_adma_desc_slot *desc, int src_cnt,
		unsigned long flags)
		{
		struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
		union {
		u32 value;
		struct iop13xx_adma_desc_ctrl field;
		} u_desc_ctrl;

		u_desc_ctrl.value = 0;
		u_desc_ctrl.field.src_select = src_cnt - 1;
		u_desc_ctrl.field.xfer_dir = 3; /* local to internal bus */
		u_desc_ctrl.field.pq_xfer_en = 1;
		u_desc_ctrl.field.p_xfer_dis = !!(flags & DMA_PREP_PQ_DISABLE_P);
		u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT;
		hw_desc->desc_ctrl = u_desc_ctrl.value;
		}

		static inline int iop_desc_is_pq(struct iop_adma_desc_slot *desc)
		{
		struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
		union {
		u32 value;
		struct iop13xx_adma_desc_ctrl field;
		} u_desc_ctrl;

		u_desc_ctrl.value = hw_desc->desc_ctrl;
		return u_desc_ctrl.field.pq_xfer_en;
		}

		static inline void
		iop_desc_init_pq_zero_sum(struct iop_adma_desc_slot *desc, int src_cnt,
		unsigned long flags)
		{
		struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
		union {
		u32 value;
		struct iop13xx_adma_desc_ctrl field;
		} u_desc_ctrl;

		u_desc_ctrl.value = 0;
		u_desc_ctrl.field.src_select = src_cnt - 1;
		u_desc_ctrl.field.xfer_dir = 3; /* local to internal bus */
		u_desc_ctrl.field.zero_result = 1;
		u_desc_ctrl.field.status_write_back_en = 1;
		u_desc_ctrl.field.pq_xfer_en = 1;
		u_desc_ctrl.field.p_xfer_dis = !!(flags & DMA_PREP_PQ_DISABLE_P);
		u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT;
		hw_desc->desc_ctrl = u_desc_ctrl.value;
		}

		static inline void iop_desc_set_byte_count(struct iop_adma_desc_slot *desc,
		struct iop_adma_chan *chan,
		u32 byte_count)
		@@ -351,6 +415,7 @@ iop_desc_set_zero_sum_byte_count(struct iop_adma_desc_slot *desc, u32 len)
		}
		}

		#define iop_desc_set_pq_zero_sum_byte_count iop_desc_set_zero_sum_byte_count

		static inline void iop_desc_set_dest_addr(struct iop_adma_desc_slot *desc,
		struct iop_adma_chan *chan,
		@@ -361,6 +426,16 @@ static inline void iop_desc_set_dest_addr(struct iop_adma_desc_slot *desc,
		hw_desc->upper_dest_addr = 0;
		}

		static inline void
		iop_desc_set_pq_addr(struct iop_adma_desc_slot desc, dma_addr_t addr)
		{
		struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;

		hw_desc->dest_addr = addr[0];
		hw_desc->q_dest_addr = addr[1];
		hw_desc->upper_dest_addr = 0;
		}

		static inline void iop_desc_set_memcpy_src_addr(struct iop_adma_desc_slot *desc,
		dma_addr_t addr)
		{
		@@ -388,6 +463,29 @@ static inline void iop_desc_set_xor_src_addr(struct iop_adma_desc_slot *desc,
		} while (slot_cnt);
		}

		static inline void
		iop_desc_set_pq_src_addr(struct iop_adma_desc_slot *desc, int src_idx,
		dma_addr_t addr, unsigned char coef)
		{
		int slot_cnt = desc->slot_cnt, slots_per_op = desc->slots_per_op;
		struct iop13xx_adma_desc_hw hw_desc = desc->hw_desc, iter;
		struct iop13xx_adma_src *src;
		int i = 0;

		do {
		iter = iop_hw_desc_slot_idx(hw_desc, i);
		src = &iter->src[src_idx];
		src->src_addr = addr;
		src->pq_upper_src_addr = 0;
		src->pq_dmlt = coef;
		slot_cnt -= slots_per_op;
		if (slot_cnt) {
		i += slots_per_op;
		addr += IOP_ADMA_PQ_MAX_BYTE_COUNT;
		}
		} while (slot_cnt);
		}

		static inline void
		iop_desc_init_interrupt(struct iop_adma_desc_slot *desc,
		struct iop_adma_chan *chan)
		@@ -399,6 +497,15 @@ iop_desc_init_interrupt(struct iop_adma_desc_slot *desc,
		}

		#define iop_desc_set_zero_sum_src_addr iop_desc_set_xor_src_addr
		#define iop_desc_set_pq_zero_sum_src_addr iop_desc_set_pq_src_addr

		static inline void
		iop_desc_set_pq_zero_sum_addr(struct iop_adma_desc_slot *desc, int pq_idx,
		dma_addr_t *src)
		{
		iop_desc_set_xor_src_addr(desc, pq_idx, src[pq_idx]);
		iop_desc_set_xor_src_addr(desc, pq_idx+1, src[pq_idx+1]);
		}

		static inline void iop_desc_set_next_desc(struct iop_adma_desc_slot *desc,
		u32 next_desc_addr)

drivers/dma/iop-adma.c

+393 −44

Original line number	Diff line number	Diff line
		@@ -31,6 +31,7 @@
		#include <linux/platform_device.h>
		#include <linux/memory.h>
		#include <linux/ioport.h>
		#include <linux/raid/pq.h>

		#include <mach/adma.h>

		@@ -57,31 +58,14 @@ static void iop_adma_free_slots(struct iop_adma_desc_slot *slot)
		}
		}

		static dma_cookie_t
		iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc,
		struct iop_adma_chan *iop_chan, dma_cookie_t cookie)
		static void
		iop_desc_unmap(struct iop_adma_chan iop_chan, struct iop_adma_desc_slot desc)
		{
		BUG_ON(desc->async_tx.cookie < 0);
		if (desc->async_tx.cookie > 0) {
		cookie = desc->async_tx.cookie;
		desc->async_tx.cookie = 0;

		/* call the callback (must not sleep or submit new
		* operations to this channel)
		*/
		if (desc->async_tx.callback)
		desc->async_tx.callback(
		desc->async_tx.callback_param);

		/* unmap dma addresses
		* (unmap_single vs unmap_page?)
		*/
		if (desc->group_head && desc->unmap_len) {
		struct dma_async_tx_descriptor *tx = &desc->async_tx;
		struct iop_adma_desc_slot *unmap = desc->group_head;
		struct device *dev =
		&iop_chan->device->pdev->dev;
		struct device *dev = &iop_chan->device->pdev->dev;
		u32 len = unmap->unmap_len;
		enum dma_ctrl_flags flags = desc->async_tx.flags;
		enum dma_ctrl_flags flags = tx->flags;
		u32 src_cnt;
		dma_addr_t addr;
		dma_addr_t dest;
		@@ -101,21 +85,83 @@ iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc,

		if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
		while (src_cnt--) {
		addr = iop_desc_get_src_addr(unmap,
		iop_chan,
		src_cnt);
		addr = iop_desc_get_src_addr(unmap, iop_chan, src_cnt);
		if (addr == dest)
		continue;
		dma_unmap_page(dev, addr, len,
		DMA_TO_DEVICE);
		dma_unmap_page(dev, addr, len, DMA_TO_DEVICE);
		}
		}
		desc->group_head = NULL;
		}

		static void
		iop_desc_unmap_pq(struct iop_adma_chan iop_chan, struct iop_adma_desc_slot desc)
		{
		struct dma_async_tx_descriptor *tx = &desc->async_tx;
		struct iop_adma_desc_slot *unmap = desc->group_head;
		struct device *dev = &iop_chan->device->pdev->dev;
		u32 len = unmap->unmap_len;
		enum dma_ctrl_flags flags = tx->flags;
		u32 src_cnt = unmap->unmap_src_cnt;
		dma_addr_t pdest = iop_desc_get_dest_addr(unmap, iop_chan);
		dma_addr_t qdest = iop_desc_get_qdest_addr(unmap, iop_chan);
		int i;

		if (tx->flags & DMA_PREP_CONTINUE)
		src_cnt -= 3;

		if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP) && !desc->pq_check_result) {
		dma_unmap_page(dev, pdest, len, DMA_BIDIRECTIONAL);
		dma_unmap_page(dev, qdest, len, DMA_BIDIRECTIONAL);
		}

		if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
		dma_addr_t addr;

		for (i = 0; i < src_cnt; i++) {
		addr = iop_desc_get_src_addr(unmap, iop_chan, i);
		dma_unmap_page(dev, addr, len, DMA_TO_DEVICE);
		}
		if (desc->pq_check_result) {
		dma_unmap_page(dev, pdest, len, DMA_TO_DEVICE);
		dma_unmap_page(dev, qdest, len, DMA_TO_DEVICE);
		}
		}

		desc->group_head = NULL;
		}


		static dma_cookie_t
		iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc,
		struct iop_adma_chan *iop_chan, dma_cookie_t cookie)
		{
		struct dma_async_tx_descriptor *tx = &desc->async_tx;

		BUG_ON(tx->cookie < 0);
		if (tx->cookie > 0) {
		cookie = tx->cookie;
		tx->cookie = 0;

		/* call the callback (must not sleep or submit new
		* operations to this channel)
		*/
		if (tx->callback)
		tx->callback(tx->callback_param);

		/* unmap dma addresses
		* (unmap_single vs unmap_page?)
		*/
		if (desc->group_head && desc->unmap_len) {
		if (iop_desc_is_pq(desc))
		iop_desc_unmap_pq(iop_chan, desc);
		else
		iop_desc_unmap(iop_chan, desc);
		}
		}

		/* run dependent operations */
		dma_run_dependencies(&desc->async_tx);
		dma_run_dependencies(tx);

		return cookie;
		}
		@@ -287,7 +333,12 @@ static void iop_adma_tasklet(unsigned long data)
		{
		struct iop_adma_chan iop_chan = (struct iop_adma_chan ) data;

		spin_lock(&iop_chan->lock);
		/* lockdep will flag depedency submissions as potentially
		* recursive locking, this is not the case as a dependency
		* submission will never recurse a channels submit routine.
		* There are checks in async_tx.c to prevent this.
		*/
		spin_lock_nested(&iop_chan->lock, SINGLE_DEPTH_NESTING);
		__iop_adma_slot_cleanup(iop_chan);
		spin_unlock(&iop_chan->lock);
		}
		@@ -696,6 +747,118 @@ iop_adma_prep_dma_xor_val(struct dma_chan chan, dma_addr_t dma_src,
		return sw_desc ? &sw_desc->async_tx : NULL;
		}

		static struct dma_async_tx_descriptor *
		iop_adma_prep_dma_pq(struct dma_chan chan, dma_addr_t dst, dma_addr_t *src,
		unsigned int src_cnt, const unsigned char *scf, size_t len,
		unsigned long flags)
		{
		struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
		struct iop_adma_desc_slot sw_desc, g;
		int slot_cnt, slots_per_op;
		int continue_srcs;

		if (unlikely(!len))
		return NULL;
		BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);

		dev_dbg(iop_chan->device->common.dev,
		"%s src_cnt: %d len: %u flags: %lx\n",
		__func__, src_cnt, len, flags);

		if (dmaf_p_disabled_continue(flags))
		continue_srcs = 1+src_cnt;
		else if (dmaf_continue(flags))
		continue_srcs = 3+src_cnt;
		else
		continue_srcs = 0+src_cnt;

		spin_lock_bh(&iop_chan->lock);
		slot_cnt = iop_chan_pq_slot_count(len, continue_srcs, &slots_per_op);
		sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
		if (sw_desc) {
		int i;

		g = sw_desc->group_head;
		iop_desc_set_byte_count(g, iop_chan, len);

		/* even if P is disabled its destination address (bits
		* [3:0]) must match Q. It is ok if P points to an
		* invalid address, it won't be written.
		*/
		if (flags & DMA_PREP_PQ_DISABLE_P)
		dst[0] = dst[1] & 0x7;

		iop_desc_set_pq_addr(g, dst);
		sw_desc->unmap_src_cnt = src_cnt;
		sw_desc->unmap_len = len;
		sw_desc->async_tx.flags = flags;
		for (i = 0; i < src_cnt; i++)
		iop_desc_set_pq_src_addr(g, i, src[i], scf[i]);

		/* if we are continuing a previous operation factor in
		* the old p and q values, see the comment for dma_maxpq
		* in include/linux/dmaengine.h
		*/
		if (dmaf_p_disabled_continue(flags))
		iop_desc_set_pq_src_addr(g, i++, dst[1], 1);
		else if (dmaf_continue(flags)) {
		iop_desc_set_pq_src_addr(g, i++, dst[0], 0);
		iop_desc_set_pq_src_addr(g, i++, dst[1], 1);
		iop_desc_set_pq_src_addr(g, i++, dst[1], 0);
		}
		iop_desc_init_pq(g, i, flags);
		}
		spin_unlock_bh(&iop_chan->lock);

		return sw_desc ? &sw_desc->async_tx : NULL;
		}

		static struct dma_async_tx_descriptor *
		iop_adma_prep_dma_pq_val(struct dma_chan chan, dma_addr_t pq, dma_addr_t *src,
		unsigned int src_cnt, const unsigned char *scf,
		size_t len, enum sum_check_flags *pqres,
		unsigned long flags)
		{
		struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
		struct iop_adma_desc_slot sw_desc, g;
		int slot_cnt, slots_per_op;

		if (unlikely(!len))
		return NULL;
		BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);

		dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n",
		__func__, src_cnt, len);

		spin_lock_bh(&iop_chan->lock);
		slot_cnt = iop_chan_pq_zero_sum_slot_count(len, src_cnt + 2, &slots_per_op);
		sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
		if (sw_desc) {
		/* for validate operations p and q are tagged onto the
		* end of the source list
		*/
		int pq_idx = src_cnt;

		g = sw_desc->group_head;
		iop_desc_init_pq_zero_sum(g, src_cnt+2, flags);
		iop_desc_set_pq_zero_sum_byte_count(g, len);
		g->pq_check_result = pqres;
		pr_debug("\t%s: g->pq_check_result: %p\n",
		__func__, g->pq_check_result);
		sw_desc->unmap_src_cnt = src_cnt+2;
		sw_desc->unmap_len = len;
		sw_desc->async_tx.flags = flags;
		while (src_cnt--)
		iop_desc_set_pq_zero_sum_src_addr(g, src_cnt,
		src[src_cnt],
		scf[src_cnt]);
		iop_desc_set_pq_zero_sum_addr(g, pq_idx, src);
		}
		spin_unlock_bh(&iop_chan->lock);

		return sw_desc ? &sw_desc->async_tx : NULL;
		}

		static void iop_adma_free_chan_resources(struct dma_chan *chan)
		{
		struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
		@@ -1105,6 +1268,170 @@ iop_adma_xor_val_self_test(struct iop_adma_device *device)
		return err;
		}

		#ifdef CONFIG_MD_RAID6_PQ
		static int __devinit
		iop_adma_pq_zero_sum_self_test(struct iop_adma_device *device)
		{
		/* combined sources, software pq results, and extra hw pq results */
		struct page *pq[IOP_ADMA_NUM_SRC_TEST+2+2];
		/* ptr to the extra hw pq buffers defined above */
		struct page **pq_hw = &pq[IOP_ADMA_NUM_SRC_TEST+2];
		/* address conversion buffers (dma_map / page_address) */
		void *pq_sw[IOP_ADMA_NUM_SRC_TEST+2];
		dma_addr_t pq_src[IOP_ADMA_NUM_SRC_TEST];
		dma_addr_t pq_dest[2];

		int i;
		struct dma_async_tx_descriptor *tx;
		struct dma_chan *dma_chan;
		dma_cookie_t cookie;
		u32 zero_sum_result;
		int err = 0;
		struct device *dev;

		dev_dbg(device->common.dev, "%s\n", __func__);

		for (i = 0; i < ARRAY_SIZE(pq); i++) {
		pq[i] = alloc_page(GFP_KERNEL);
		if (!pq[i]) {
		while (i--)
		__free_page(pq[i]);
		return -ENOMEM;
		}
		}

		/* Fill in src buffers */
		for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) {
		pq_sw[i] = page_address(pq[i]);
		memset(pq_sw[i], 0x11111111 * (1<<i), PAGE_SIZE);
		}
		pq_sw[i] = page_address(pq[i]);
		pq_sw[i+1] = page_address(pq[i+1]);

		dma_chan = container_of(device->common.channels.next,
		struct dma_chan,
		device_node);
		if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
		err = -ENODEV;
		goto out;
		}

		dev = dma_chan->device->dev;

		/* initialize the dests */
		memset(page_address(pq_hw[0]), 0 , PAGE_SIZE);
		memset(page_address(pq_hw[1]), 0 , PAGE_SIZE);

		/* test pq */
		pq_dest[0] = dma_map_page(dev, pq_hw[0], 0, PAGE_SIZE, DMA_FROM_DEVICE);
		pq_dest[1] = dma_map_page(dev, pq_hw[1], 0, PAGE_SIZE, DMA_FROM_DEVICE);
		for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
		pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
		DMA_TO_DEVICE);

		tx = iop_adma_prep_dma_pq(dma_chan, pq_dest, pq_src,
		IOP_ADMA_NUM_SRC_TEST, (u8 *)raid6_gfexp,
		PAGE_SIZE,
		DMA_PREP_INTERRUPT \|
		DMA_CTRL_ACK);

		cookie = iop_adma_tx_submit(tx);
		iop_adma_issue_pending(dma_chan);
		msleep(8);

		if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) !=
		DMA_SUCCESS) {
		dev_err(dev, "Self-test pq timed out, disabling\n");
		err = -ENODEV;
		goto free_resources;
		}

		raid6_call.gen_syndrome(IOP_ADMA_NUM_SRC_TEST+2, PAGE_SIZE, pq_sw);

		if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST],
		page_address(pq_hw[0]), PAGE_SIZE) != 0) {
		dev_err(dev, "Self-test p failed compare, disabling\n");
		err = -ENODEV;
		goto free_resources;
		}
		if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST+1],
		page_address(pq_hw[1]), PAGE_SIZE) != 0) {
		dev_err(dev, "Self-test q failed compare, disabling\n");
		err = -ENODEV;
		goto free_resources;
		}

		/* test correct zero sum using the software generated pq values */
		for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++)
		pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
		DMA_TO_DEVICE);

		zero_sum_result = ~0;
		tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST],
		pq_src, IOP_ADMA_NUM_SRC_TEST,
		raid6_gfexp, PAGE_SIZE, &zero_sum_result,
		DMA_PREP_INTERRUPT\|DMA_CTRL_ACK);

		cookie = iop_adma_tx_submit(tx);
		iop_adma_issue_pending(dma_chan);
		msleep(8);

		if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) !=
		DMA_SUCCESS) {
		dev_err(dev, "Self-test pq-zero-sum timed out, disabling\n");
		err = -ENODEV;
		goto free_resources;
		}

		if (zero_sum_result != 0) {
		dev_err(dev, "Self-test pq-zero-sum failed to validate: %x\n",
		zero_sum_result);
		err = -ENODEV;
		goto free_resources;
		}

		/* test incorrect zero sum */
		i = IOP_ADMA_NUM_SRC_TEST;
		memset(pq_sw[i] + 100, 0, 100);
		memset(pq_sw[i+1] + 200, 0, 200);
		for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++)
		pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
		DMA_TO_DEVICE);

		zero_sum_result = 0;
		tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST],
		pq_src, IOP_ADMA_NUM_SRC_TEST,
		raid6_gfexp, PAGE_SIZE, &zero_sum_result,
		DMA_PREP_INTERRUPT\|DMA_CTRL_ACK);

		cookie = iop_adma_tx_submit(tx);
		iop_adma_issue_pending(dma_chan);
		msleep(8);

		if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) !=
		DMA_SUCCESS) {
		dev_err(dev, "Self-test !pq-zero-sum timed out, disabling\n");
		err = -ENODEV;
		goto free_resources;
		}

		if (zero_sum_result != (SUM_CHECK_P_RESULT \| SUM_CHECK_Q_RESULT)) {
		dev_err(dev, "Self-test !pq-zero-sum failed to validate: %x\n",
		zero_sum_result);
		err = -ENODEV;
		goto free_resources;
		}

		free_resources:
		iop_adma_free_chan_resources(dma_chan);
		out:
		i = ARRAY_SIZE(pq);
		while (i--)
		__free_page(pq[i]);
		return err;
		}
		#endif

		static int __devexit iop_adma_remove(struct platform_device *dev)
		{
		struct iop_adma_device *device = platform_get_drvdata(dev);
		@@ -1195,6 +1522,13 @@ static int __devinit iop_adma_probe(struct platform_device *pdev)
		if (dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask))
		dma_dev->device_prep_dma_xor_val =
		iop_adma_prep_dma_xor_val;
		if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
		dma_set_maxpq(dma_dev, iop_adma_get_max_pq(), 0);
		dma_dev->device_prep_dma_pq = iop_adma_prep_dma_pq;
		}
		if (dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask))
		dma_dev->device_prep_dma_pq_val =
		iop_adma_prep_dma_pq_val;
		if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
		dma_dev->device_prep_dma_interrupt =
		iop_adma_prep_dma_interrupt;
		@@ -1255,6 +1589,21 @@ static int __devinit iop_adma_probe(struct platform_device *pdev)
		goto err_free_iop_chan;
		}

		if (dma_has_cap(DMA_PQ, dma_dev->cap_mask) &&
		dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) {
		#ifdef CONFIG_MD_RAID6_PQ
		ret = iop_adma_pq_zero_sum_self_test(adev);
		dev_dbg(&pdev->dev, "pq self test returned %d\n", ret);
		#else
		/* can not test raid6, so do not publish capability */
		dma_cap_clear(DMA_PQ, dma_dev->cap_mask);
		dma_cap_clear(DMA_PQ_VAL, dma_dev->cap_mask);
		ret = 0;
		#endif
		if (ret)
		goto err_free_iop_chan;
		}

		dev_printk(KERN_INFO, &pdev->dev, "Intel(R) IOP: "
		"( %s%s%s%s%s%s%s)\n",
		dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : "",