Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 498c4394 authored by Jon Derrick's avatar Jon Derrick Committed by Jens Axboe
Browse files

NVMe: Unify SQ entry writing and doorbell ringing 



This patch changes sq_cmd writers to instead create their command on
the stack. __nvme_submit_cmd copies the sq entry to the queue and writes
the doorbell.

Signed-off-by: default avatarJon Derrick <jonathan.derrick@intel.com>
Reviewed-by: default avatarChristoph Hellwig <hch@lst.de>
Signed-off-by: default avatarJens Axboe <axboe@fb.com>
parent 0034af03

drivers/block/nvme-core.c

+35 −45
Original line number Diff line number Diff line
@@ -730,18 +730,16 @@ static int nvme_setup_prps(struct nvme_dev *dev, struct nvme_iod *iod, 
static void nvme_submit_priv(struct nvme_queue *nvmeq, struct request *req,

		struct nvme_iod *iod)

{

	struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];

	struct nvme_command cmnd;



	memcpy(cmnd, req->cmd, sizeof(struct nvme_command));

	cmnd->rw.command_id = req->tag;

	memcpy(&cmnd, req->cmd, sizeof(cmnd));

	cmnd.rw.command_id = req->tag;

	if (req->nr_phys_segments) {

		cmnd->rw.prp1 = cpu_to_le64(sg_dma_address(iod->sg));

		cmnd->rw.prp2 = cpu_to_le64(iod->first_dma);

		cmnd.rw.prp1 = cpu_to_le64(sg_dma_address(iod->sg));

		cmnd.rw.prp2 = cpu_to_le64(iod->first_dma);

	}



	if (++nvmeq->sq_tail == nvmeq->q_depth)

		nvmeq->sq_tail = 0;

	writel(nvmeq->sq_tail, nvmeq->q_db);

	__nvme_submit_cmd(nvmeq, &cmnd);

}



/*
@@ -754,45 +752,41 @@ static void nvme_submit_discard(struct nvme_queue *nvmeq, struct nvme_ns *ns, 
{

	struct nvme_dsm_range *range =

				(struct nvme_dsm_range *)iod_list(iod)[0];

	struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];

	struct nvme_command cmnd;



	range->cattr = cpu_to_le32(0);

	range->nlb = cpu_to_le32(blk_rq_bytes(req) >> ns->lba_shift);

	range->slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));



	memset(cmnd, 0, sizeof(*cmnd));

	cmnd->dsm.opcode = nvme_cmd_dsm;

	cmnd->dsm.command_id = req->tag;

	cmnd->dsm.nsid = cpu_to_le32(ns->ns_id);

	cmnd->dsm.prp1 = cpu_to_le64(iod->first_dma);

	cmnd->dsm.nr = 0;

	cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);

	memset(&cmnd, 0, sizeof(cmnd));

	cmnd.dsm.opcode = nvme_cmd_dsm;

	cmnd.dsm.command_id = req->tag;

	cmnd.dsm.nsid = cpu_to_le32(ns->ns_id);

	cmnd.dsm.prp1 = cpu_to_le64(iod->first_dma);

	cmnd.dsm.nr = 0;

	cmnd.dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);



	if (++nvmeq->sq_tail == nvmeq->q_depth)

		nvmeq->sq_tail = 0;

	writel(nvmeq->sq_tail, nvmeq->q_db);

	__nvme_submit_cmd(nvmeq, &cmnd);

}



static void nvme_submit_flush(struct nvme_queue *nvmeq, struct nvme_ns *ns,

								int cmdid)

{

	struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];

	struct nvme_command cmnd;



	memset(cmnd, 0, sizeof(*cmnd));

	cmnd->common.opcode = nvme_cmd_flush;

	cmnd->common.command_id = cmdid;

	cmnd->common.nsid = cpu_to_le32(ns->ns_id);

	memset(&cmnd, 0, sizeof(cmnd));

	cmnd.common.opcode = nvme_cmd_flush;

	cmnd.common.command_id = cmdid;

	cmnd.common.nsid = cpu_to_le32(ns->ns_id);



	if (++nvmeq->sq_tail == nvmeq->q_depth)

		nvmeq->sq_tail = 0;

	writel(nvmeq->sq_tail, nvmeq->q_db);

	__nvme_submit_cmd(nvmeq, &cmnd);

}



static int nvme_submit_iod(struct nvme_queue *nvmeq, struct nvme_iod *iod,

							struct nvme_ns *ns)

{

	struct request *req = iod_get_private(iod);

	struct nvme_command *cmnd;

	struct nvme_command cmnd;

	u16 control = 0;

	u32 dsmgmt = 0;
@@ -804,19 +798,17 @@ static int nvme_submit_iod(struct nvme_queue *nvmeq, struct nvme_iod *iod, 
	if (req->cmd_flags & REQ_RAHEAD)

		dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;



	cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];

	memset(cmnd, 0, sizeof(*cmnd));



	cmnd->rw.opcode = (rq_data_dir(req) ? nvme_cmd_write : nvme_cmd_read);

	cmnd->rw.command_id = req->tag;

	cmnd->rw.nsid = cpu_to_le32(ns->ns_id);

	cmnd->rw.prp1 = cpu_to_le64(sg_dma_address(iod->sg));

	cmnd->rw.prp2 = cpu_to_le64(iod->first_dma);

	cmnd->rw.slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));

	cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);

	memset(&cmnd, 0, sizeof(cmnd));

	cmnd.rw.opcode = (rq_data_dir(req) ? nvme_cmd_write : nvme_cmd_read);

	cmnd.rw.command_id = req->tag;

	cmnd.rw.nsid = cpu_to_le32(ns->ns_id);

	cmnd.rw.prp1 = cpu_to_le64(sg_dma_address(iod->sg));

	cmnd.rw.prp2 = cpu_to_le64(iod->first_dma);

	cmnd.rw.slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));

	cmnd.rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);



	if (blk_integrity_rq(req)) {

		cmnd->rw.metadata = cpu_to_le64(sg_dma_address(iod->meta_sg));

		cmnd.rw.metadata = cpu_to_le64(sg_dma_address(iod->meta_sg));

		switch (ns->pi_type) {

		case NVME_NS_DPS_PI_TYPE3:

			control |= NVME_RW_PRINFO_PRCHK_GUARD;
@@ -825,19 +817,17 @@ static int nvme_submit_iod(struct nvme_queue *nvmeq, struct nvme_iod *iod, 
		case NVME_NS_DPS_PI_TYPE2:

			control |= NVME_RW_PRINFO_PRCHK_GUARD |

					NVME_RW_PRINFO_PRCHK_REF;

			cmnd->rw.reftag = cpu_to_le32(

			cmnd.rw.reftag = cpu_to_le32(

					nvme_block_nr(ns, blk_rq_pos(req)));

			break;

		}

	} else if (ns->ms)

		control |= NVME_RW_PRINFO_PRACT;



	cmnd->rw.control = cpu_to_le16(control);

	cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);

	cmnd.rw.control = cpu_to_le16(control);

	cmnd.rw.dsmgmt = cpu_to_le32(dsmgmt);



	if (++nvmeq->sq_tail == nvmeq->q_depth)

		nvmeq->sq_tail = 0;

	writel(nvmeq->sq_tail, nvmeq->q_db);

	__nvme_submit_cmd(nvmeq, &cmnd);



	return 0;

}