Merge "mmc: block: add discard and secdiscard support for CMDQ mode" (1cc64d67) · Commits · e / devices / android_kernel_xiaomi_markw

drivers/mmc/card/block.c

+165 −2

Original line number	Diff line number	Diff line
		@@ -89,6 +89,8 @@ MODULE_ALIAS("mmc:block");
		#define PCKD_TRGR_LOWER_BOUND 5
		#define PCKD_TRGR_PRECISION_MULTIPLIER 100

		static struct mmc_cmdq_req *mmc_cmdq_prep_dcmd(
		struct mmc_queue_req mqrq, struct mmc_queue mq);
		static DEFINE_MUTEX(block_mutex);

		/*
		@@ -1400,6 +1402,87 @@ static inline void mmc_blk_reset_success(struct mmc_blk_data *md, int type)
		md->reset_done &= ~type;
		}

		static struct mmc_cmdq_req mmc_blk_cmdq_prep_discard_req(struct mmc_queue mq,
		struct request *req)
		{
		struct mmc_blk_data *md = mq->data;
		struct mmc_card *card = md->queue.card;
		struct mmc_host *host = card->host;
		struct mmc_cmdq_context_info *ctx_info = &host->cmdq_ctx;
		struct mmc_cmdq_req *cmdq_req;
		struct mmc_queue_req *active_mqrq;

		BUG_ON(req->tag > card->ext_csd.cmdq_depth);
		BUG_ON(test_and_set_bit(req->tag, &host->cmdq_ctx.active_reqs));

		set_bit(CMDQ_STATE_DCMD_ACTIVE, &ctx_info->curr_state);

		active_mqrq = &mq->mqrq_cmdq[req->tag];
		active_mqrq->req = req;

		cmdq_req = mmc_cmdq_prep_dcmd(active_mqrq, mq);
		cmdq_req->cmdq_req_flags \|= QBR;
		cmdq_req->mrq.cmd = &cmdq_req->cmd;
		cmdq_req->tag = req->tag;
		return cmdq_req;
		}

		static int mmc_blk_cmdq_issue_discard_rq(struct mmc_queue *mq,
		struct request *req)
		{
		struct mmc_blk_data *md = mq->data;
		struct mmc_card *card = md->queue.card;
		struct mmc_cmdq_req *cmdq_req = NULL;
		struct mmc_host *host = card->host;
		struct mmc_cmdq_context_info *ctx_info = &host->cmdq_ctx;
		unsigned int from, nr, arg;
		int err = 0;

		if (!mmc_can_erase(card)) {
		err = -EOPNOTSUPP;
		goto out;
		}

		from = blk_rq_pos(req);
		nr = blk_rq_sectors(req);

		if (mmc_can_discard(card))
		arg = MMC_DISCARD_ARG;
		else if (mmc_can_trim(card))
		arg = MMC_TRIM_ARG;
		else
		arg = MMC_ERASE_ARG;

		cmdq_req = mmc_blk_cmdq_prep_discard_req(mq, req);
		if (card->quirks & MMC_QUIRK_INAND_CMD38) {
		__mmc_switch_cmdq_mode(cmdq_req->mrq.cmd,
		EXT_CSD_CMD_SET_NORMAL,
		INAND_CMD38_ARG_EXT_CSD,
		arg == MMC_TRIM_ARG ?
		INAND_CMD38_ARG_TRIM :
		INAND_CMD38_ARG_ERASE,
		0, true, false);
		err = mmc_cmdq_wait_for_dcmd(card->host, cmdq_req);
		if (err)
		goto clear_dcmd;
		}
		err = mmc_cmdq_erase(cmdq_req, card, from, nr, arg);
		clear_dcmd:
		/* clear pending request */
		if (cmdq_req) {
		BUG_ON(!test_and_clear_bit(cmdq_req->tag,
		&ctx_info->active_reqs));
		clear_bit(CMDQ_STATE_DCMD_ACTIVE, &ctx_info->curr_state);
		}
		out:
		blk_end_request(req, err, blk_rq_bytes(req));

		if (test_and_clear_bit(0, &ctx_info->req_starved))
		blk_run_queue(mq->queue);
		mmc_release_host(host);
		return err ? 1 : 0;
		}

		static int mmc_blk_issue_discard_rq(struct mmc_queue mq, struct request req)
		{
		struct mmc_blk_data *md = mq->data;
		@@ -1443,6 +1526,79 @@ out:
		return err ? 0 : 1;
		}

		static int mmc_blk_cmdq_issue_secdiscard_rq(struct mmc_queue *mq,
		struct request *req)
		{
		struct mmc_blk_data *md = mq->data;
		struct mmc_card *card = md->queue.card;
		struct mmc_cmdq_req *cmdq_req = NULL;
		unsigned int from, nr, arg;
		struct mmc_host *host = card->host;
		struct mmc_cmdq_context_info *ctx_info = &host->cmdq_ctx;
		int err = 0;

		if (!(mmc_can_secure_erase_trim(card))) {
		err = -EOPNOTSUPP;
		goto out;
		}

		from = blk_rq_pos(req);
		nr = blk_rq_sectors(req);

		if (mmc_can_trim(card) && !mmc_erase_group_aligned(card, from, nr))
		arg = MMC_SECURE_TRIM1_ARG;
		else
		arg = MMC_SECURE_ERASE_ARG;

		cmdq_req = mmc_blk_cmdq_prep_discard_req(mq, req);
		if (card->quirks & MMC_QUIRK_INAND_CMD38) {
		__mmc_switch_cmdq_mode(cmdq_req->mrq.cmd,
		EXT_CSD_CMD_SET_NORMAL,
		INAND_CMD38_ARG_EXT_CSD,
		arg == MMC_SECURE_TRIM1_ARG ?
		INAND_CMD38_ARG_SECTRIM1 :
		INAND_CMD38_ARG_SECERASE,
		0, true, false);
		err = mmc_cmdq_wait_for_dcmd(card->host, cmdq_req);
		if (err)
		goto clear_dcmd;
		}

		err = mmc_cmdq_erase(cmdq_req, card, from, nr, arg);
		if (err)
		goto clear_dcmd;

		if (arg == MMC_SECURE_TRIM1_ARG) {
		if (card->quirks & MMC_QUIRK_INAND_CMD38) {
		__mmc_switch_cmdq_mode(cmdq_req->mrq.cmd,
		EXT_CSD_CMD_SET_NORMAL,
		INAND_CMD38_ARG_EXT_CSD,
		INAND_CMD38_ARG_SECTRIM2,
		0, true, false);
		err = mmc_cmdq_wait_for_dcmd(card->host, cmdq_req);
		if (err)
		goto clear_dcmd;
		}

		err = mmc_cmdq_erase(cmdq_req, card, from, nr,
		MMC_SECURE_TRIM2_ARG);
		}
		clear_dcmd:
		/* clear pending request */
		if (cmdq_req) {
		BUG_ON(!test_and_clear_bit(cmdq_req->tag,
		&ctx_info->active_reqs));
		clear_bit(CMDQ_STATE_DCMD_ACTIVE, &ctx_info->curr_state);
		}
		out:
		blk_end_request(req, err, blk_rq_bytes(req));

		if (test_and_clear_bit(0, &ctx_info->req_starved))
		blk_run_queue(mq->queue);
		mmc_release_host(host);
		return err ? 1 : 0;
		}

		static int mmc_blk_issue_secdiscard_rq(struct mmc_queue *mq,
		struct request *req)
		{
		@@ -3086,11 +3242,18 @@ static int mmc_blk_cmdq_issue_rq(struct mmc_queue mq, struct request req)
		}

		if (req) {
		if (cmd_flags & REQ_FLUSH)
		ret = mmc_blk_cmdq_issue_flush_rq(mq, req);
		if (cmd_flags & REQ_DISCARD) {
		if (cmd_flags & REQ_SECURE &&
		!(card->quirks & MMC_QUIRK_SEC_ERASE_TRIM_BROKEN))
		ret = mmc_blk_cmdq_issue_secdiscard_rq(mq, req);
		else
		ret = mmc_blk_cmdq_issue_discard_rq(mq, req);
		} else if (cmd_flags & REQ_FLUSH) {
		ret = mmc_blk_cmdq_issue_flush_rq(mq, req);
		} else {
		ret = mmc_blk_cmdq_issue_rw_rq(mq, req);
		}
		}

		switch_failure:
		return ret;

drivers/mmc/core/core.c

+204 −25

Original line number	Diff line number	Diff line
		@@ -1198,6 +1198,35 @@ int mmc_cmdq_start_req(struct mmc_host host, struct mmc_cmdq_req cmdq_req)
		}
		EXPORT_SYMBOL(mmc_cmdq_start_req);

		static void mmc_cmdq_dcmd_req_done(struct mmc_request *mrq)
		{
		complete(&mrq->completion);
		}

		int mmc_cmdq_wait_for_dcmd(struct mmc_host *host,
		struct mmc_cmdq_req *cmdq_req)
		{
		struct mmc_request *mrq = &cmdq_req->mrq;
		struct mmc_command *cmd = mrq->cmd;
		int err = 0;

		init_completion(&mrq->completion);
		mrq->done = mmc_cmdq_dcmd_req_done;
		err = mmc_cmdq_start_req(host, cmdq_req);
		if (err)
		return err;

		wait_for_completion_io(&mrq->completion);
		if (cmd->error) {
		pr_err("%s: DCMD %d failed with err %d\n",
		mmc_hostname(host), cmd->opcode,
		cmd->error);
		err = cmd->error;
		}
		return err;
		}
		EXPORT_SYMBOL(mmc_cmdq_wait_for_dcmd);

		int mmc_cmdq_prepare_flush(struct mmc_command *cmd)
		{
		return __mmc_switch_cmdq_mode(cmd, EXT_CSD_CMD_SET_NORMAL,
		@@ -2631,19 +2660,9 @@ static unsigned int mmc_erase_timeout(struct mmc_card *card,
		return mmc_mmc_erase_timeout(card, arg, qty);
		}

		static int mmc_do_erase(struct mmc_card *card, unsigned int from,
		unsigned int to, unsigned int arg)
		static u32 mmc_get_erase_qty(struct mmc_card *card, u32 from, u32 to)
		{
		struct mmc_command cmd = {0};
		unsigned int qty = 0;
		unsigned long timeout;
		unsigned int fr, nr;
		int err;

		fr = from;
		nr = to - from + 1;
		trace_mmc_blk_erase_start(arg, fr, nr);

		u32 qty = 0;
		/*
		* qty is used to calculate the erase timeout which depends on how many
		* erase groups (or allocation units in SD terminology) are affected.
		@@ -2668,6 +2687,115 @@ static int mmc_do_erase(struct mmc_card *card, unsigned int from,
		else
		qty += ((to / card->erase_size) -
		(from / card->erase_size)) + 1;
		return qty;
		}

		static int mmc_cmdq_send_erase_cmd(struct mmc_cmdq_req *cmdq_req,
		struct mmc_card *card, u32 opcode, u32 arg, u32 qty)
		{
		struct mmc_command *cmd = cmdq_req->mrq.cmd;
		int err;

		memset(cmd, 0, sizeof(struct mmc_command));

		cmd->opcode = opcode;
		cmd->arg = arg;
		if (cmd->opcode == MMC_ERASE) {
		cmd->flags = MMC_RSP_SPI_R1B \| MMC_RSP_R1B \| MMC_CMD_AC;
		cmd->busy_timeout = mmc_erase_timeout(card, arg, qty);
		} else {
		cmd->flags = MMC_RSP_SPI_R1 \| MMC_RSP_R1 \| MMC_CMD_AC;
		}

		err = mmc_cmdq_wait_for_dcmd(card->host, cmdq_req);
		if (err) {
		pr_err("mmc_erase: group start error %d, status %#x\n",
		err, cmd->resp[0]);
		return -EIO;
		}
		return 0;
		}

		static int mmc_cmdq_do_erase(struct mmc_cmdq_req *cmdq_req,
		struct mmc_card *card, unsigned int from,
		unsigned int to, unsigned int arg)
		{
		struct mmc_command *cmd = cmdq_req->mrq.cmd;
		unsigned int qty = 0;
		unsigned long timeout;
		unsigned int fr, nr;
		int err;

		fr = from;
		nr = to - from + 1;
		trace_mmc_blk_erase_start(arg, fr, nr);

		qty = mmc_get_erase_qty(card, from, to);

		if (!mmc_card_blockaddr(card)) {
		from <<= 9;
		to <<= 9;
		}

		err = mmc_cmdq_send_erase_cmd(cmdq_req, card, MMC_ERASE_GROUP_START,
		from, qty);
		if (err)
		goto out;

		err = mmc_cmdq_send_erase_cmd(cmdq_req, card, MMC_ERASE_GROUP_END,
		to, qty);
		if (err)
		goto out;

		err = mmc_cmdq_send_erase_cmd(cmdq_req, card, MMC_ERASE,
		arg, qty);
		if (err)
		goto out;

		timeout = jiffies + msecs_to_jiffies(MMC_CORE_TIMEOUT_MS);
		do {
		memset(cmd, 0, sizeof(struct mmc_command));
		cmd->opcode = MMC_SEND_STATUS;
		cmd->arg = card->rca << 16;
		cmd->flags = MMC_RSP_R1 \| MMC_CMD_AC;
		/* Do not retry else we can't see errors */
		err = mmc_cmdq_wait_for_dcmd(card->host, cmdq_req);
		if (err \|\| (cmd->resp[0] & 0xFDF92000)) {
		pr_err("error %d requesting status %#x\n",
		err, cmd->resp[0]);
		err = -EIO;
		goto out;
		}
		/* Timeout if the device never becomes ready for data and
		* never leaves the program state.
		*/
		if (time_after(jiffies, timeout)) {
		pr_err("%s: Card stuck in programming state! %s\n",
		mmc_hostname(card->host), __func__);
		err = -EIO;
		goto out;
		}
		} while (!(cmd->resp[0] & R1_READY_FOR_DATA) \|\|
		(R1_CURRENT_STATE(cmd->resp[0]) == R1_STATE_PRG));
		out:
		trace_mmc_blk_erase_end(arg, fr, nr);
		return err;
		}

		static int mmc_do_erase(struct mmc_card *card, unsigned int from,
		unsigned int to, unsigned int arg)
		{
		struct mmc_command cmd = {0};
		unsigned int qty = 0;
		unsigned long timeout;
		unsigned int fr, nr;
		int err;

		fr = from;
		nr = to - from + 1;
		trace_mmc_blk_erase_start(arg, fr, nr);

		qty = mmc_get_erase_qty(card, from, to);

		if (!mmc_card_blockaddr(card)) {
		from <<= 9;
		@@ -2752,20 +2880,9 @@ out:
		return err;
		}

		/**
		* mmc_erase - erase sectors.
		* @card: card to erase
		* @from: first sector to erase
		* @nr: number of sectors to erase
		* @arg: erase command argument (SD supports only %MMC_ERASE_ARG)
		*
		* Caller must claim host before calling this function.
		*/
		int mmc_erase(struct mmc_card *card, unsigned int from, unsigned int nr,
		unsigned int arg)
		int mmc_erase_sanity_check(struct mmc_card *card, unsigned int from,
		unsigned int nr, unsigned int arg)
		{
		unsigned int rem, to = from + nr;

		if (!(card->host->caps & MMC_CAP_ERASE) \|\|
		!(card->csd.cmdclass & CCC_ERASE))
		return -EOPNOTSUPP;
		@@ -2788,6 +2905,68 @@ int mmc_erase(struct mmc_card *card, unsigned int from, unsigned int nr,
		if (from % card->erase_size \|\| nr % card->erase_size)
		return -EINVAL;
		}
		return 0;
		}

		int mmc_cmdq_erase(struct mmc_cmdq_req *cmdq_req,
		struct mmc_card *card, unsigned int from, unsigned int nr,
		unsigned int arg)
		{
		unsigned int rem, to = from + nr;
		int ret;

		ret = mmc_erase_sanity_check(card, from, nr, arg);
		if (ret)
		return ret;

		if (arg == MMC_ERASE_ARG) {
		rem = from % card->erase_size;
		if (rem) {
		rem = card->erase_size - rem;
		from += rem;
		if (nr > rem)
		nr -= rem;
		else
		return 0;
		}
		rem = nr % card->erase_size;
		if (rem)
		nr -= rem;
		}

		if (nr == 0)
		return 0;

		to = from + nr;

		if (to <= from)
		return -EINVAL;

		/* 'from' and 'to' are inclusive */
		to -= 1;

		return mmc_cmdq_do_erase(cmdq_req, card, from, to, arg);
		}
		EXPORT_SYMBOL(mmc_cmdq_erase);

		/**
		* mmc_erase - erase sectors.
		* @card: card to erase
		* @from: first sector to erase
		* @nr: number of sectors to erase
		* @arg: erase command argument (SD supports only %MMC_ERASE_ARG)
		*
		* Caller must claim host before calling this function.
		*/
		int mmc_erase(struct mmc_card *card, unsigned int from, unsigned int nr,
		unsigned int arg)
		{
		unsigned int rem, to = from + nr;
		int ret;

		ret = mmc_erase_sanity_check(card, from, nr, arg);
		if (ret)
		return ret;

		if (arg == MMC_ERASE_ARG) {
		rem = from % card->erase_size;

drivers/mmc/host/cmdq_hci.c

+3 −0

Original line number	Diff line number	Diff line
		@@ -597,6 +597,9 @@ static void cmdq_finish_data(struct mmc_host *mmc, unsigned int tag)
		struct cmdq_host cq_host = (struct cmdq_host )mmc_cmdq_private(mmc);

		mrq = get_req_by_tag(cq_host, tag);
		if (tag == cq_host->dcmd_slot)
		mrq->cmd->resp[0] = cmdq_readl(cq_host, CQCRDCT);

		mrq->done(mrq);
		}

include/linux/mmc/core.h

+5 −0

Original line number	Diff line number	Diff line
		@@ -122,6 +122,11 @@ extern void mmc_cmdq_post_req(struct mmc_host host, struct mmc_request mrq,
		extern int mmc_cmdq_start_req(struct mmc_host *host,
		struct mmc_cmdq_req *cmdq_req);
		extern int mmc_cmdq_prepare_flush(struct mmc_command *cmd);
		extern int mmc_cmdq_wait_for_dcmd(struct mmc_host *host,
		struct mmc_cmdq_req *cmdq_req);
		extern int mmc_cmdq_erase(struct mmc_cmdq_req *cmdq_req,
		struct mmc_card *card, unsigned int from, unsigned int nr,
		unsigned int arg);

		extern int mmc_stop_bkops(struct mmc_card *);
		extern int mmc_read_bkops_status(struct mmc_card *);