Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 614049a8 authored by Boris Brezillon's avatar Boris Brezillon
Browse files

mtd: nand: sunxi: add support for DMA assisted operations 



The sunxi NAND controller is able to pipeline ECC operations only when
operated in DMA mode, which improves a lot NAND throughput while keeping
CPU usage low.

Signed-off-by: default avatarBoris Brezillon <boris.brezillon@free-electrons.com>
parent decba6d4

drivers/mtd/nand/sunxi_nand.c

+323 −7
Original line number Diff line number Diff line
@@ -153,6 +153,7 @@ 


/* define bit use in NFC_ECC_ST */

#define NFC_ECC_ERR(x)		BIT(x)

#define NFC_ECC_ERR_MSK		GENMASK(15, 0)

#define NFC_ECC_PAT_FOUND(x)	BIT(x + 16)

#define NFC_ECC_ERR_CNT(b, x)	(((x) >> (((b) % 4) * 8)) & 0xff)
@@ -273,6 +274,7 @@ struct sunxi_nfc { 
	unsigned long clk_rate;

	struct list_head chips;

	struct completion complete;

	struct dma_chan *dmac;

};



static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_hw_control *ctrl)
@@ -365,6 +367,67 @@ static int sunxi_nfc_rst(struct sunxi_nfc *nfc) 
	return ret;

}



static int sunxi_nfc_dma_op_prepare(struct mtd_info *mtd, const void *buf,

				    int chunksize, int nchunks,

				    enum dma_data_direction ddir,

				    struct scatterlist *sg)

{

	struct nand_chip *nand = mtd_to_nand(mtd);

	struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);

	struct dma_async_tx_descriptor *dmad;

	enum dma_transfer_direction tdir;

	dma_cookie_t dmat;

	int ret;



	if (ddir == DMA_FROM_DEVICE)

		tdir = DMA_DEV_TO_MEM;

	else

		tdir = DMA_MEM_TO_DEV;



	sg_init_one(sg, buf, nchunks * chunksize);

	ret = dma_map_sg(nfc->dev, sg, 1, ddir);

	if (!ret)

		return -ENOMEM;



	dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK);

	if (IS_ERR(dmad)) {

		ret = PTR_ERR(dmad);

		goto err_unmap_buf;

	}



	writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD,

	       nfc->regs + NFC_REG_CTL);

	writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM);

	writel(chunksize, nfc->regs + NFC_REG_CNT);

	dmat = dmaengine_submit(dmad);



	ret = dma_submit_error(dmat);

	if (ret)

		goto err_clr_dma_flag;



	return 0;



err_clr_dma_flag:

	writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,

	       nfc->regs + NFC_REG_CTL);



err_unmap_buf:

	dma_unmap_sg(nfc->dev, sg, 1, ddir);

	return ret;

}



static void sunxi_nfc_dma_op_cleanup(struct mtd_info *mtd,

				     enum dma_data_direction ddir,

				     struct scatterlist *sg)

{

	struct nand_chip *nand = mtd_to_nand(mtd);

	struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);



	dma_unmap_sg(nfc->dev, sg, 1, ddir);

	writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,

	       nfc->regs + NFC_REG_CTL);

}



static int sunxi_nfc_dev_ready(struct mtd_info *mtd)

{

	struct nand_chip *nand = mtd_to_nand(mtd);
@@ -822,17 +885,15 @@ static void sunxi_nfc_hw_ecc_update_stats(struct mtd_info *mtd, 
}



static int sunxi_nfc_hw_ecc_correct(struct mtd_info *mtd, u8 *data, u8 *oob,

				    int step, bool *erased)

				    int step, u32 status, bool *erased)

{

	struct nand_chip *nand = mtd_to_nand(mtd);

	struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);

	struct nand_ecc_ctrl *ecc = &nand->ecc;

	u32 status, tmp;

	u32 tmp;



	*erased = false;



	status = readl(nfc->regs + NFC_REG_ECC_ST);



	if (status & NFC_ECC_ERR(step))

		return -EBADMSG;
@@ -898,6 +959,7 @@ static int sunxi_nfc_hw_ecc_read_chunk(struct mtd_info *mtd, 
	*cur_off = oob_off + ecc->bytes + 4;



	ret = sunxi_nfc_hw_ecc_correct(mtd, data, oob_required ? oob : NULL, 0,

				       readl(nfc->regs + NFC_REG_ECC_ST),

				       &erased);

	if (erased)

		return 1;
@@ -967,6 +1029,128 @@ static void sunxi_nfc_hw_ecc_read_extra_oob(struct mtd_info *mtd, 
		*cur_off = mtd->oobsize + mtd->writesize;

}



static int sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info *mtd, uint8_t *buf,

					    int oob_required, int page,

					    int nchunks)

{

	struct nand_chip *nand = mtd_to_nand(mtd);

	bool randomized = nand->options & NAND_NEED_SCRAMBLING;

	struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);

	struct nand_ecc_ctrl *ecc = &nand->ecc;

	unsigned int max_bitflips = 0;

	int ret, i, raw_mode = 0;

	struct scatterlist sg;

	u32 status;



	ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);

	if (ret)

		return ret;



	ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, nchunks,

				       DMA_FROM_DEVICE, &sg);

	if (ret)

		return ret;



	sunxi_nfc_hw_ecc_enable(mtd);

	sunxi_nfc_randomizer_config(mtd, page, false);

	sunxi_nfc_randomizer_enable(mtd);



	writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) |

	       NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET);



	dma_async_issue_pending(nfc->dmac);



	writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS,

	       nfc->regs + NFC_REG_CMD);



	ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);

	if (ret)

		dmaengine_terminate_all(nfc->dmac);



	sunxi_nfc_randomizer_disable(mtd);

	sunxi_nfc_hw_ecc_disable(mtd);



	sunxi_nfc_dma_op_cleanup(mtd, DMA_FROM_DEVICE, &sg);



	if (ret)

		return ret;



	status = readl(nfc->regs + NFC_REG_ECC_ST);



	for (i = 0; i < nchunks; i++) {

		int data_off = i * ecc->size;

		int oob_off = i * (ecc->bytes + 4);

		u8 *data = buf + data_off;

		u8 *oob = nand->oob_poi + oob_off;

		bool erased;



		ret = sunxi_nfc_hw_ecc_correct(mtd, randomized ? data : NULL,

					       oob_required ? oob : NULL,

					       i, status, &erased);



		/* ECC errors are handled in the second loop. */

		if (ret < 0)

			continue;



		if (oob_required && !erased) {

			/* TODO: use DMA to retrieve OOB */

			nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);

			nand->read_buf(mtd, oob, ecc->bytes + 4);



			sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, i,

							    !i, page);

		}



		if (erased)

			raw_mode = 1;



		sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);

	}



	if (status & NFC_ECC_ERR_MSK) {

		for (i = 0; i < nchunks; i++) {

			int data_off = i * ecc->size;

			int oob_off = i * (ecc->bytes + 4);

			u8 *data = buf + data_off;

			u8 *oob = nand->oob_poi + oob_off;



			if (!(status & NFC_ECC_ERR(i)))

				continue;



			/*

			 * Re-read the data with the randomizer disabled to

			 * identify bitflips in erased pages.

			 */

			if (randomized) {

				/* TODO: use DMA to read page in raw mode */

				nand->cmdfunc(mtd, NAND_CMD_RNDOUT,

					      data_off, -1);

				nand->read_buf(mtd, data, ecc->size);

			}



			/* TODO: use DMA to retrieve OOB */

			nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);

			nand->read_buf(mtd, oob, ecc->bytes + 4);



			ret = nand_check_erased_ecc_chunk(data,	ecc->size,

							  oob, ecc->bytes + 4,

							  NULL, 0,

							  ecc->strength);

			if (ret >= 0)

				raw_mode = 1;



			sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);

		}

	}



	if (oob_required)

		sunxi_nfc_hw_ecc_read_extra_oob(mtd, nand->oob_poi,

						NULL, !raw_mode,

						page);



	return max_bitflips;

}



static int sunxi_nfc_hw_ecc_write_chunk(struct mtd_info *mtd,

					const u8 *data, int data_off,

					const u8 *oob, int oob_off,
@@ -1065,6 +1249,23 @@ static int sunxi_nfc_hw_ecc_read_page(struct mtd_info *mtd, 
	return max_bitflips;

}



static int sunxi_nfc_hw_ecc_read_page_dma(struct mtd_info *mtd,

					  struct nand_chip *chip, u8 *buf,

					  int oob_required, int page)

{

	int ret;



	ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, oob_required, page,

					       chip->ecc.steps);

	if (ret >= 0)

		return ret;



	/* Fallback to PIO mode */

	chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1);



	return sunxi_nfc_hw_ecc_read_page(mtd, chip, buf, oob_required, page);

}



static int sunxi_nfc_hw_ecc_read_subpage(struct mtd_info *mtd,

					 struct nand_chip *chip,

					 u32 data_offs, u32 readlen,
@@ -1098,6 +1299,25 @@ static int sunxi_nfc_hw_ecc_read_subpage(struct mtd_info *mtd, 
	return max_bitflips;

}



static int sunxi_nfc_hw_ecc_read_subpage_dma(struct mtd_info *mtd,

					     struct nand_chip *chip,

					     u32 data_offs, u32 readlen,

					     u8 *buf, int page)

{

	int nchunks = DIV_ROUND_UP(data_offs + readlen, chip->ecc.size);

	int ret;



	ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, false, page, nchunks);

	if (ret >= 0)

		return ret;



	/* Fallback to PIO mode */

	chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1);



	return sunxi_nfc_hw_ecc_read_subpage(mtd, chip, data_offs, readlen,

					     buf, page);

}



static int sunxi_nfc_hw_ecc_write_page(struct mtd_info *mtd,

				       struct nand_chip *chip,

				       const uint8_t *buf, int oob_required,
@@ -1130,6 +1350,69 @@ static int sunxi_nfc_hw_ecc_write_page(struct mtd_info *mtd, 
	return 0;

}



static int sunxi_nfc_hw_ecc_write_page_dma(struct mtd_info *mtd,

					   struct nand_chip *chip,

					   const u8 *buf,

					   int oob_required,

					   int page)

{

	struct nand_chip *nand = mtd_to_nand(mtd);

	struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);

	struct nand_ecc_ctrl *ecc = &nand->ecc;

	struct scatterlist sg;

	int ret, i;



	ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);

	if (ret)

		return ret;



	ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, ecc->steps,

				       DMA_TO_DEVICE, &sg);

	if (ret)

		goto pio_fallback;



	for (i = 0; i < ecc->steps; i++) {

		const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4));



		sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, i, !i, page);

	}



	sunxi_nfc_hw_ecc_enable(mtd);

	sunxi_nfc_randomizer_config(mtd, page, false);

	sunxi_nfc_randomizer_enable(mtd);



	writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG,

	       nfc->regs + NFC_REG_RCMD_SET);



	dma_async_issue_pending(nfc->dmac);



	writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD |

	       NFC_DATA_TRANS | NFC_ACCESS_DIR,

	       nfc->regs + NFC_REG_CMD);



	ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);

	if (ret)

		dmaengine_terminate_all(nfc->dmac);



	sunxi_nfc_randomizer_disable(mtd);

	sunxi_nfc_hw_ecc_disable(mtd);



	sunxi_nfc_dma_op_cleanup(mtd, DMA_TO_DEVICE, &sg);



	if (ret)

		return ret;



	if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))

		/* TODO: use DMA to transfer extra OOB bytes ? */

		sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,

						 NULL, page);



	return 0;



pio_fallback:

	return sunxi_nfc_hw_ecc_write_page(mtd, chip, buf, oob_required, page);

}



static int sunxi_nfc_hw_syndrome_ecc_read_page(struct mtd_info *mtd,

					       struct nand_chip *chip,

					       uint8_t *buf, int oob_required,
@@ -1550,14 +1833,27 @@ static int sunxi_nand_hw_ecc_ctrl_init(struct mtd_info *mtd, 
				       struct nand_ecc_ctrl *ecc,

				       struct device_node *np)

{

	struct nand_chip *nand = mtd_to_nand(mtd);

	struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);

	struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);

	int ret;



	ret = sunxi_nand_hw_common_ecc_ctrl_init(mtd, ecc, np);

	if (ret)

		return ret;



	if (nfc->dmac) {

		ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma;

		ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma;

		ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma;

		nand->options |= NAND_USE_BOUNCE_BUFFER;

	} else {

		ecc->read_page = sunxi_nfc_hw_ecc_read_page;

		ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;

		ecc->write_page = sunxi_nfc_hw_ecc_write_page;

	}



	/* TODO: support DMA for raw accesses */

	ecc->read_oob_raw = nand_read_oob_std;

	ecc->write_oob_raw = nand_write_oob_std;

	ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
@@ -1881,16 +2177,34 @@ static int sunxi_nfc_probe(struct platform_device *pdev) 
	if (ret)

		goto out_mod_clk_unprepare;



	nfc->dmac = dma_request_slave_channel(dev, "rxtx");

	if (nfc->dmac) {

		struct dma_slave_config dmac_cfg = { };



		dmac_cfg.src_addr = r->start + NFC_REG_IO_DATA;

		dmac_cfg.dst_addr = dmac_cfg.src_addr;

		dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;

		dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width;

		dmac_cfg.src_maxburst = 4;

		dmac_cfg.dst_maxburst = 4;

		dmaengine_slave_config(nfc->dmac, &dmac_cfg);

	} else {

		dev_warn(dev, "failed to request rxtx DMA channel\n");

	}



	platform_set_drvdata(pdev, nfc);



	ret = sunxi_nand_chips_init(dev, nfc);

	if (ret) {

		dev_err(dev, "failed to init nand chips\n");

		goto out_mod_clk_unprepare;

		goto out_release_dmac;

	}



	return 0;



out_release_dmac:

	if (nfc->dmac)

		dma_release_channel(nfc->dmac);

out_mod_clk_unprepare:

	clk_disable_unprepare(nfc->mod_clk);

out_ahb_clk_unprepare:
@@ -1904,6 +2218,8 @@ static int sunxi_nfc_remove(struct platform_device *pdev) 
	struct sunxi_nfc *nfc = platform_get_drvdata(pdev);



	sunxi_nand_chips_cleanup(nfc);

	if (nfc->dmac)

		dma_release_channel(nfc->dmac);

	clk_disable_unprepare(nfc->mod_clk);

	clk_disable_unprepare(nfc->ahb_clk);