Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 1ab0cd69 authored by Jens Axboe's avatar Jens Axboe Committed by Keith Busch
Browse files

nvme-pci: split the nvme queue lock into submission and completion locks 



This is now feasible. We protect the submission queue ring with
->sq_lock, and the completion side with ->cq_lock.

Reviewed-by: default avatarChristoph Hellwig <hch@lst.de>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
parent 5cb525c8

drivers/nvme/host/pci.c

+23 −21
Original line number Original line Diff line number Diff line
@@ -147,9 +147,10 @@ static inline struct nvme_dev *to_nvme_dev(struct nvme_ctrl *ctrl) 
struct nvme_queue {

struct nvme_queue {

	struct device *q_dmadev;

	struct device *q_dmadev;

	struct nvme_dev *dev;

	struct nvme_dev *dev;

	spinlock_t q_lock;

	spinlock_t sq_lock;

	struct nvme_command *sq_cmds;

	struct nvme_command *sq_cmds;

	struct nvme_command __iomem *sq_cmds_io;

	struct nvme_command __iomem *sq_cmds_io;

	spinlock_t cq_lock ____cacheline_aligned_in_smp;

	volatile struct nvme_completion *cqes;

	volatile struct nvme_completion *cqes;

	struct blk_mq_tags **tags;

	struct blk_mq_tags **tags;

	dma_addr_t sq_dma_addr;

	dma_addr_t sq_dma_addr;
@@ -894,9 +895,9 @@ static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx, 




	blk_mq_start_request(req);

	blk_mq_start_request(req);





	spin_lock_irq(&nvmeq->q_lock);

	spin_lock_irq(&nvmeq->sq_lock);

	__nvme_submit_cmd(nvmeq, &cmnd);

	__nvme_submit_cmd(nvmeq, &cmnd);

	spin_unlock_irq(&nvmeq->q_lock);

	spin_unlock_irq(&nvmeq->sq_lock);

	return BLK_STS_OK;

	return BLK_STS_OK;

out_cleanup_iod:

out_cleanup_iod:

	nvme_free_iod(dev, req);

	nvme_free_iod(dev, req);
@@ -1000,9 +1001,9 @@ static irqreturn_t nvme_irq(int irq, void *data) 
	struct nvme_queue *nvmeq = data;

	struct nvme_queue *nvmeq = data;

	u16 start, end;

	u16 start, end;





	spin_lock(&nvmeq->q_lock);

	spin_lock(&nvmeq->cq_lock);

	nvme_process_cq(nvmeq, &start, &end, -1);

	nvme_process_cq(nvmeq, &start, &end, -1);

	spin_unlock(&nvmeq->q_lock);

	spin_unlock(&nvmeq->cq_lock);





	if (start == end)

	if (start == end)

		return IRQ_NONE;

		return IRQ_NONE;
@@ -1026,9 +1027,9 @@ static int __nvme_poll(struct nvme_queue *nvmeq, unsigned int tag) 
	if (!nvme_cqe_pending(nvmeq))

	if (!nvme_cqe_pending(nvmeq))

		return 0;

		return 0;





	spin_lock_irq(&nvmeq->q_lock);

	spin_lock_irq(&nvmeq->cq_lock);

	found = nvme_process_cq(nvmeq, &start, &end, tag);

	found = nvme_process_cq(nvmeq, &start, &end, tag);

	spin_unlock_irq(&nvmeq->q_lock);

	spin_unlock_irq(&nvmeq->cq_lock);





	nvme_complete_cqes(nvmeq, start, end);

	nvme_complete_cqes(nvmeq, start, end);

	return found;

	return found;
@@ -1051,9 +1052,9 @@ static void nvme_pci_submit_async_event(struct nvme_ctrl *ctrl) 
	c.common.opcode = nvme_admin_async_event;

	c.common.opcode = nvme_admin_async_event;

	c.common.command_id = NVME_AQ_BLK_MQ_DEPTH;

	c.common.command_id = NVME_AQ_BLK_MQ_DEPTH;





	spin_lock_irq(&nvmeq->q_lock);

	spin_lock_irq(&nvmeq->sq_lock);

	__nvme_submit_cmd(nvmeq, &c);

	__nvme_submit_cmd(nvmeq, &c);

	spin_unlock_irq(&nvmeq->q_lock);

	spin_unlock_irq(&nvmeq->sq_lock);

}

}





static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)

static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
@@ -1310,15 +1311,15 @@ static int nvme_suspend_queue(struct nvme_queue *nvmeq) 
{

{

	int vector;

	int vector;





	spin_lock_irq(&nvmeq->q_lock);

	spin_lock_irq(&nvmeq->cq_lock);

	if (nvmeq->cq_vector == -1) {

	if (nvmeq->cq_vector == -1) {

		spin_unlock_irq(&nvmeq->q_lock);

		spin_unlock_irq(&nvmeq->cq_lock);

		return 1;

		return 1;

	}

	}

	vector = nvmeq->cq_vector;

	vector = nvmeq->cq_vector;

	nvmeq->dev->online_queues--;

	nvmeq->dev->online_queues--;

	nvmeq->cq_vector = -1;

	nvmeq->cq_vector = -1;

	spin_unlock_irq(&nvmeq->q_lock);

	spin_unlock_irq(&nvmeq->cq_lock);





	/*

	/*

	 * Ensure that nvme_queue_rq() sees it ->cq_vector == -1 without

	 * Ensure that nvme_queue_rq() sees it ->cq_vector == -1 without
@@ -1344,9 +1345,9 @@ static void nvme_disable_admin_queue(struct nvme_dev *dev, bool shutdown) 
	else

	else

		nvme_disable_ctrl(&dev->ctrl, dev->ctrl.cap);

		nvme_disable_ctrl(&dev->ctrl, dev->ctrl.cap);





	spin_lock_irq(&nvmeq->q_lock);

	spin_lock_irq(&nvmeq->cq_lock);

	nvme_process_cq(nvmeq, &start, &end, -1);

	nvme_process_cq(nvmeq, &start, &end, -1);

	spin_unlock_irq(&nvmeq->q_lock);

	spin_unlock_irq(&nvmeq->cq_lock);





	nvme_complete_cqes(nvmeq, start, end);

	nvme_complete_cqes(nvmeq, start, end);

}

}
@@ -1406,7 +1407,8 @@ static int nvme_alloc_queue(struct nvme_dev *dev, int qid, int depth) 




	nvmeq->q_dmadev = dev->dev;

	nvmeq->q_dmadev = dev->dev;

	nvmeq->dev = dev;

	nvmeq->dev = dev;

	spin_lock_init(&nvmeq->q_lock);

	spin_lock_init(&nvmeq->sq_lock);

	spin_lock_init(&nvmeq->cq_lock);

	nvmeq->cq_head = 0;

	nvmeq->cq_head = 0;

	nvmeq->cq_phase = 1;

	nvmeq->cq_phase = 1;

	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];

	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
@@ -1442,7 +1444,7 @@ static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid) 
{

{

	struct nvme_dev *dev = nvmeq->dev;

	struct nvme_dev *dev = nvmeq->dev;





	spin_lock_irq(&nvmeq->q_lock);

	spin_lock_irq(&nvmeq->cq_lock);

	nvmeq->sq_tail = 0;

	nvmeq->sq_tail = 0;

	nvmeq->cq_head = 0;

	nvmeq->cq_head = 0;

	nvmeq->cq_phase = 1;

	nvmeq->cq_phase = 1;
@@ -1450,7 +1452,7 @@ static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid) 
	memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth));

	memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth));

	nvme_dbbuf_init(dev, nvmeq, qid);

	nvme_dbbuf_init(dev, nvmeq, qid);

	dev->online_queues++;

	dev->online_queues++;

	spin_unlock_irq(&nvmeq->q_lock);

	spin_unlock_irq(&nvmeq->cq_lock);

}

}





static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)

static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
@@ -2001,14 +2003,14 @@ static void nvme_del_cq_end(struct request *req, blk_status_t error) 
		unsigned long flags;

		unsigned long flags;





		/*

		/*

		 * We might be called with the AQ q_lock held

		 * We might be called with the AQ cq_lock held

		 * and the I/O queue q_lock should always

		 * and the I/O queue cq_lock should always

		 * nest inside the AQ one.

		 * nest inside the AQ one.

		 */

		 */

		spin_lock_irqsave_nested(&nvmeq->q_lock, flags,

		spin_lock_irqsave_nested(&nvmeq->cq_lock, flags,

					SINGLE_DEPTH_NESTING);

					SINGLE_DEPTH_NESTING);

		nvme_process_cq(nvmeq, &start, &end, -1);

		nvme_process_cq(nvmeq, &start, &end, -1);

		spin_unlock_irqrestore(&nvmeq->q_lock, flags);

		spin_unlock_irqrestore(&nvmeq->cq_lock, flags);





		nvme_complete_cqes(nvmeq, start, end);

		nvme_complete_cqes(nvmeq, start, end);

	}

	}