Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 459bd0dc authored by Jens Axboe's avatar Jens Axboe
Browse files

Merge branch 'nvme-4.13' of git://git.infradead.org/nvme into for-linus 

Pull followup NVMe (mostly) changes from Sagi:

I added the quiesce/unquiesce patches in here as it's
easy for me easily apply changes on top. It has accumulated
reviews and includes mostly nvme anyway, please tell me if
you don't want to take them with this.

This includes:
- quiesce/unquiesce fixes in nvme and others from me
- nvme-fc add create association padding spec updates from James
- some more quirking from MKP
- nvmet nit cleanup from Max
- Fix nvme-rdma racy RDMA completion signalling from Marta
- some centralization patches from me
- add tagset nr_hw_queues updates on controller resets in
  nvme drivers from me
- nvme-rdma fix resources recycling when doing error recovery from me
- minor cleanups in nvme-fc from me
parents 56c76417 c2f30f08

drivers/block/mtip32xx/mtip32xx.c

+15 −4
Original line number Diff line number Diff line
@@ -950,7 +950,7 @@ static int mtip_quiesce_io(struct mtip_port *port, unsigned long timeout) 
	unsigned long to;

	bool active = true;



	blk_mq_stop_hw_queues(port->dd->queue);

	blk_mq_quiesce_queue(port->dd->queue);



	to = jiffies + msecs_to_jiffies(timeout);

	do {
@@ -970,10 +970,10 @@ static int mtip_quiesce_io(struct mtip_port *port, unsigned long timeout) 
			break;

	} while (time_before(jiffies, to));



	blk_mq_start_stopped_hw_queues(port->dd->queue, true);

	blk_mq_unquiesce_queue(port->dd->queue);

	return active ? -EBUSY : 0;

err_fault:

	blk_mq_start_stopped_hw_queues(port->dd->queue, true);

	blk_mq_unquiesce_queue(port->dd->queue);

	return -EFAULT;

}
@@ -2737,6 +2737,9 @@ static void mtip_abort_cmd(struct request *req, void *data, 
	struct mtip_cmd *cmd = blk_mq_rq_to_pdu(req);

	struct driver_data *dd = data;



	if (!blk_mq_request_started(req))

		return;



	dbg_printk(MTIP_DRV_NAME " Aborting request, tag = %d\n", req->tag);



	clear_bit(req->tag, dd->port->cmds_to_issue);
@@ -2749,6 +2752,9 @@ static void mtip_queue_cmd(struct request *req, void *data, 
{

	struct driver_data *dd = data;



	if (!blk_mq_request_started(req))

		return;



	set_bit(req->tag, dd->port->cmds_to_issue);

	blk_abort_request(req);

}
@@ -2814,6 +2820,8 @@ static int mtip_service_thread(void *data) 
				dev_warn(&dd->pdev->dev,

					"Completion workers still active!");



			blk_mq_quiesce_queue(dd->queue);



			spin_lock(dd->queue->queue_lock);

			blk_mq_tagset_busy_iter(&dd->tags,

							mtip_queue_cmd, dd);
@@ -2826,6 +2834,8 @@ static int mtip_service_thread(void *data) 
							mtip_abort_cmd, dd);



			clear_bit(MTIP_PF_TO_ACTIVE_BIT, &dd->port->flags);



			blk_mq_unquiesce_queue(dd->queue);

		}



		if (test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) {
@@ -3995,8 +4005,9 @@ static int mtip_block_remove(struct driver_data *dd) 
						dd->disk->disk_name);



	blk_freeze_queue_start(dd->queue);

	blk_mq_stop_hw_queues(dd->queue);

	blk_mq_quiesce_queue(dd->queue);

	blk_mq_tagset_busy_iter(&dd->tags, mtip_no_dev_cleanup, dd);

	blk_mq_unquiesce_queue(dd->queue);



	/*

	 * Delete our gendisk structure. This also removes the device

drivers/block/nbd.c

+2 −2
Original line number Diff line number Diff line
@@ -661,9 +661,9 @@ static void nbd_clear_req(struct request *req, void *data, bool reserved) 


static void nbd_clear_que(struct nbd_device *nbd)

{

	blk_mq_stop_hw_queues(nbd->disk->queue);

	blk_mq_quiesce_queue(nbd->disk->queue);

	blk_mq_tagset_busy_iter(&nbd->tag_set, nbd_clear_req, NULL);

	blk_mq_start_hw_queues(nbd->disk->queue);

	blk_mq_unquiesce_queue(nbd->disk->queue);

	dev_dbg(disk_to_dev(nbd->disk), "queue cleared\n");

}

drivers/block/virtio_blk.c

+2 −2
Original line number Diff line number Diff line
@@ -840,7 +840,7 @@ static int virtblk_freeze(struct virtio_device *vdev) 
	/* Make sure no work handler is accessing the device. */

	flush_work(&vblk->config_work);



	blk_mq_stop_hw_queues(vblk->disk->queue);

	blk_mq_quiesce_queue(vblk->disk->queue);



	vdev->config->del_vqs(vdev);

	return 0;
@@ -857,7 +857,7 @@ static int virtblk_restore(struct virtio_device *vdev) 


	virtio_device_ready(vdev);



	blk_mq_start_stopped_hw_queues(vblk->disk->queue, true);

	blk_mq_unquiesce_queue(vblk->disk->queue);

	return 0;

}

#endif

drivers/nvme/host/core.c

+21 −19
Original line number Diff line number Diff line
@@ -131,7 +131,7 @@ void nvme_complete_rq(struct request *req) 
{

	if (unlikely(nvme_req(req)->status && nvme_req_needs_retry(req))) {

		nvme_req(req)->retries++;

		blk_mq_requeue_request(req, !blk_mq_queue_stopped(req->q));

		blk_mq_requeue_request(req, true);

		return;

	}
@@ -2591,12 +2591,29 @@ static void nvme_release_instance(struct nvme_ctrl *ctrl) 
	spin_unlock(&dev_list_lock);

}



void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)

void nvme_stop_ctrl(struct nvme_ctrl *ctrl)

{

	nvme_stop_keep_alive(ctrl);

	flush_work(&ctrl->async_event_work);

	flush_work(&ctrl->scan_work);

	nvme_remove_namespaces(ctrl);

}

EXPORT_SYMBOL_GPL(nvme_stop_ctrl);



void nvme_start_ctrl(struct nvme_ctrl *ctrl)

{

	if (ctrl->kato)

		nvme_start_keep_alive(ctrl);



	if (ctrl->queue_count > 1) {

		nvme_queue_scan(ctrl);

		nvme_queue_async_events(ctrl);

		nvme_start_queues(ctrl);

	}

}

EXPORT_SYMBOL_GPL(nvme_start_ctrl);



void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)

{

	device_destroy(nvme_class, MKDEV(nvme_char_major, ctrl->instance));



	spin_lock(&dev_list_lock);
@@ -2694,9 +2711,6 @@ void nvme_kill_queues(struct nvme_ctrl *ctrl) 
	/* Forcibly unquiesce queues to avoid blocking dispatch */

	blk_mq_unquiesce_queue(ctrl->admin_q);



	/* Forcibly start all queues to avoid having stuck requests */

	blk_mq_start_hw_queues(ctrl->admin_q);



	list_for_each_entry(ns, &ctrl->namespaces, list) {

		/*

		 * Revalidating a dead namespace sets capacity to 0. This will
@@ -2709,16 +2723,6 @@ void nvme_kill_queues(struct nvme_ctrl *ctrl) 


		/* Forcibly unquiesce queues to avoid blocking dispatch */

		blk_mq_unquiesce_queue(ns->queue);



		/*

		 * Forcibly start all queues to avoid having stuck requests.

		 * Note that we must ensure the queues are not stopped

		 * when the final removal happens.

		 */

		blk_mq_start_hw_queues(ns->queue);



		/* draining requests in requeue list */

		blk_mq_kick_requeue_list(ns->queue);

	}

	mutex_unlock(&ctrl->namespaces_mutex);

}
@@ -2787,10 +2791,8 @@ void nvme_start_queues(struct nvme_ctrl *ctrl) 
	struct nvme_ns *ns;



	mutex_lock(&ctrl->namespaces_mutex);

	list_for_each_entry(ns, &ctrl->namespaces, list) {

	list_for_each_entry(ns, &ctrl->namespaces, list)

		blk_mq_unquiesce_queue(ns->queue);

		blk_mq_kick_requeue_list(ns->queue);

	}

	mutex_unlock(&ctrl->namespaces_mutex);

}

EXPORT_SYMBOL_GPL(nvme_start_queues);

drivers/nvme/host/fc.c

+40 −43
Original line number Diff line number Diff line
@@ -148,13 +148,10 @@ struct nvme_fc_ctrl { 
	struct device		*dev;

	struct nvme_fc_lport	*lport;

	struct nvme_fc_rport	*rport;

	u32			queue_count;

	u32			cnum;



	u64			association_id;



	u64			cap;



	struct list_head	ctrl_list;	/* rport->ctrl_list */



	struct blk_mq_tag_set	admin_tag_set;
@@ -1614,7 +1611,7 @@ nvme_fc_free_io_queues(struct nvme_fc_ctrl *ctrl) 
{

	int i;



	for (i = 1; i < ctrl->queue_count; i++)

	for (i = 1; i < ctrl->ctrl.queue_count; i++)

		nvme_fc_free_queue(&ctrl->queues[i]);

}
@@ -1635,10 +1632,10 @@ __nvme_fc_create_hw_queue(struct nvme_fc_ctrl *ctrl, 
static void

nvme_fc_delete_hw_io_queues(struct nvme_fc_ctrl *ctrl)

{

	struct nvme_fc_queue *queue = &ctrl->queues[ctrl->queue_count - 1];

	struct nvme_fc_queue *queue = &ctrl->queues[ctrl->ctrl.queue_count - 1];

	int i;



	for (i = ctrl->queue_count - 1; i >= 1; i--, queue--)

	for (i = ctrl->ctrl.queue_count - 1; i >= 1; i--, queue--)

		__nvme_fc_delete_hw_queue(ctrl, queue, i);

}
@@ -1648,7 +1645,7 @@ nvme_fc_create_hw_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize) 
	struct nvme_fc_queue *queue = &ctrl->queues[1];

	int i, ret;



	for (i = 1; i < ctrl->queue_count; i++, queue++) {

	for (i = 1; i < ctrl->ctrl.queue_count; i++, queue++) {

		ret = __nvme_fc_create_hw_queue(ctrl, queue, i, qsize);

		if (ret)

			goto delete_queues;
@@ -1667,7 +1664,7 @@ nvme_fc_connect_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize) 
{

	int i, ret = 0;



	for (i = 1; i < ctrl->queue_count; i++) {

	for (i = 1; i < ctrl->ctrl.queue_count; i++) {

		ret = nvme_fc_connect_queue(ctrl, &ctrl->queues[i], qsize,

					(qsize / 5));

		if (ret)
@@ -1685,7 +1682,7 @@ nvme_fc_init_io_queues(struct nvme_fc_ctrl *ctrl) 
{

	int i;



	for (i = 1; i < ctrl->queue_count; i++)

	for (i = 1; i < ctrl->ctrl.queue_count; i++)

		nvme_fc_init_queue(ctrl, i, ctrl->ctrl.sqsize);

}
@@ -1706,6 +1703,7 @@ nvme_fc_ctrl_free(struct kref *ref) 
	list_del(&ctrl->ctrl_list);

	spin_unlock_irqrestore(&ctrl->rport->lock, flags);



	blk_mq_unquiesce_queue(ctrl->ctrl.admin_q);

	blk_cleanup_queue(ctrl->ctrl.admin_q);

	blk_mq_free_tag_set(&ctrl->admin_tag_set);
@@ -1969,10 +1967,9 @@ nvme_fc_start_fcp_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue, 
		if (ret != -EBUSY)

			return BLK_STS_IOERR;



		if (op->rq) {

			blk_mq_stop_hw_queues(op->rq->q);

			blk_mq_delay_queue(queue->hctx, NVMEFC_QUEUE_DELAY);

		}

		if (op->rq)

			blk_mq_delay_run_hw_queue(queue->hctx, NVMEFC_QUEUE_DELAY);



		return BLK_STS_RESOURCE;

	}
@@ -2178,17 +2175,20 @@ static int 
nvme_fc_create_io_queues(struct nvme_fc_ctrl *ctrl)

{

	struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;

	unsigned int nr_io_queues;

	int ret;



	ret = nvme_set_queue_count(&ctrl->ctrl, &opts->nr_io_queues);

	nr_io_queues = min(min(opts->nr_io_queues, num_online_cpus()),

				ctrl->lport->ops->max_hw_queues);

	ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);

	if (ret) {

		dev_info(ctrl->ctrl.device,

			"set_queue_count failed: %d\n", ret);

		return ret;

	}



	ctrl->queue_count = opts->nr_io_queues + 1;

	if (!opts->nr_io_queues)

	ctrl->ctrl.queue_count = nr_io_queues + 1;

	if (!nr_io_queues)

		return 0;



	nvme_fc_init_io_queues(ctrl);
@@ -2204,7 +2204,7 @@ nvme_fc_create_io_queues(struct nvme_fc_ctrl *ctrl) 
						sizeof(struct scatterlist)) +

					ctrl->lport->ops->fcprqst_priv_sz;

	ctrl->tag_set.driver_data = ctrl;

	ctrl->tag_set.nr_hw_queues = ctrl->queue_count - 1;

	ctrl->tag_set.nr_hw_queues = ctrl->ctrl.queue_count - 1;

	ctrl->tag_set.timeout = NVME_IO_TIMEOUT;



	ret = blk_mq_alloc_tag_set(&ctrl->tag_set);
@@ -2232,7 +2232,6 @@ nvme_fc_create_io_queues(struct nvme_fc_ctrl *ctrl) 
out_delete_hw_queues:

	nvme_fc_delete_hw_io_queues(ctrl);

out_cleanup_blk_queue:

	nvme_stop_keep_alive(&ctrl->ctrl);

	blk_cleanup_queue(ctrl->ctrl.connect_q);

out_free_tag_set:

	blk_mq_free_tag_set(&ctrl->tag_set);
@@ -2248,17 +2247,21 @@ static int 
nvme_fc_reinit_io_queues(struct nvme_fc_ctrl *ctrl)

{

	struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;

	unsigned int nr_io_queues;

	int ret;



	ret = nvme_set_queue_count(&ctrl->ctrl, &opts->nr_io_queues);

	nr_io_queues = min(min(opts->nr_io_queues, num_online_cpus()),

				ctrl->lport->ops->max_hw_queues);

	ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);

	if (ret) {

		dev_info(ctrl->ctrl.device,

			"set_queue_count failed: %d\n", ret);

		return ret;

	}



	ctrl->ctrl.queue_count = nr_io_queues + 1;

	/* check for io queues existing */

	if (ctrl->queue_count == 1)

	if (ctrl->ctrl.queue_count == 1)

		return 0;



	nvme_fc_init_io_queues(ctrl);
@@ -2275,6 +2278,8 @@ nvme_fc_reinit_io_queues(struct nvme_fc_ctrl *ctrl) 
	if (ret)

		goto out_delete_hw_queues;



	blk_mq_update_nr_hw_queues(&ctrl->tag_set, nr_io_queues);



	return 0;



out_delete_hw_queues:
@@ -2316,7 +2321,7 @@ nvme_fc_create_association(struct nvme_fc_ctrl *ctrl) 
		goto out_delete_hw_queue;



	if (ctrl->ctrl.state != NVME_CTRL_NEW)

		blk_mq_start_stopped_hw_queues(ctrl->ctrl.admin_q, true);

		blk_mq_unquiesce_queue(ctrl->ctrl.admin_q);



	ret = nvmf_connect_admin_queue(&ctrl->ctrl);

	if (ret)
@@ -2329,7 +2334,7 @@ nvme_fc_create_association(struct nvme_fc_ctrl *ctrl) 
	 * prior connection values

	 */



	ret = nvmf_reg_read64(&ctrl->ctrl, NVME_REG_CAP, &ctrl->cap);

	ret = nvmf_reg_read64(&ctrl->ctrl, NVME_REG_CAP, &ctrl->ctrl.cap);

	if (ret) {

		dev_err(ctrl->ctrl.device,

			"prop_get NVME_REG_CAP failed\n");
@@ -2337,9 +2342,9 @@ nvme_fc_create_association(struct nvme_fc_ctrl *ctrl) 
	}



	ctrl->ctrl.sqsize =

		min_t(int, NVME_CAP_MQES(ctrl->cap) + 1, ctrl->ctrl.sqsize);

		min_t(int, NVME_CAP_MQES(ctrl->ctrl.cap) + 1, ctrl->ctrl.sqsize);



	ret = nvme_enable_ctrl(&ctrl->ctrl, ctrl->cap);

	ret = nvme_enable_ctrl(&ctrl->ctrl, ctrl->ctrl.cap);

	if (ret)

		goto out_disconnect_admin_queue;
@@ -2360,8 +2365,6 @@ nvme_fc_create_association(struct nvme_fc_ctrl *ctrl) 
		goto out_disconnect_admin_queue;

	}



	nvme_start_keep_alive(&ctrl->ctrl);



	/* FC-NVME supports normal SGL Data Block Descriptors */



	if (opts->queue_size > ctrl->ctrl.maxcmd) {
@@ -2381,7 +2384,7 @@ nvme_fc_create_association(struct nvme_fc_ctrl *ctrl) 
	 * Create the io queues

	 */



	if (ctrl->queue_count > 1) {

	if (ctrl->ctrl.queue_count > 1) {

		if (ctrl->ctrl.state == NVME_CTRL_NEW)

			ret = nvme_fc_create_io_queues(ctrl);

		else
@@ -2395,17 +2398,12 @@ nvme_fc_create_association(struct nvme_fc_ctrl *ctrl) 


	ctrl->ctrl.nr_reconnects = 0;



	if (ctrl->queue_count > 1) {

		nvme_start_queues(&ctrl->ctrl);

		nvme_queue_scan(&ctrl->ctrl);

		nvme_queue_async_events(&ctrl->ctrl);

	}

	nvme_start_ctrl(&ctrl->ctrl);



	return 0;	/* Success */



out_term_aen_ops:

	nvme_fc_term_aen_ops(ctrl);

	nvme_stop_keep_alive(&ctrl->ctrl);

out_disconnect_admin_queue:

	/* send a Disconnect(association) LS to fc-nvme target */

	nvme_fc_xmt_disconnect_assoc(ctrl);
@@ -2428,8 +2426,6 @@ nvme_fc_delete_association(struct nvme_fc_ctrl *ctrl) 
{

	unsigned long flags;



	nvme_stop_keep_alive(&ctrl->ctrl);



	spin_lock_irqsave(&ctrl->lock, flags);

	ctrl->flags |= FCCTRL_TERMIO;

	ctrl->iocnt = 0;
@@ -2447,7 +2443,7 @@ nvme_fc_delete_association(struct nvme_fc_ctrl *ctrl) 
	 * io requests back to the block layer as part of normal completions

	 * (but with error status).

	 */

	if (ctrl->queue_count > 1) {

	if (ctrl->ctrl.queue_count > 1) {

		nvme_stop_queues(&ctrl->ctrl);

		blk_mq_tagset_busy_iter(&ctrl->tag_set,

				nvme_fc_terminate_exchange, &ctrl->ctrl);
@@ -2470,7 +2466,7 @@ nvme_fc_delete_association(struct nvme_fc_ctrl *ctrl) 
	 * use blk_mq_tagset_busy_itr() and the transport routine to

	 * terminate the exchanges.

	 */

	blk_mq_stop_hw_queues(ctrl->ctrl.admin_q);

	blk_mq_quiesce_queue(ctrl->ctrl.admin_q);

	blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,

				nvme_fc_terminate_exchange, &ctrl->ctrl);
@@ -2511,7 +2507,8 @@ nvme_fc_delete_ctrl_work(struct work_struct *work) 


	cancel_work_sync(&ctrl->ctrl.reset_work);

	cancel_delayed_work_sync(&ctrl->connect_work);



	nvme_stop_ctrl(&ctrl->ctrl);

	nvme_remove_namespaces(&ctrl->ctrl);

	/*

	 * kill the association on the link side.  this will block

	 * waiting for io to terminate
@@ -2606,6 +2603,7 @@ nvme_fc_reset_ctrl_work(struct work_struct *work) 
		container_of(work, struct nvme_fc_ctrl, ctrl.reset_work);

	int ret;



	nvme_stop_ctrl(&ctrl->ctrl);

	/* will block will waiting for io to terminate */

	nvme_fc_delete_association(ctrl);
@@ -2702,18 +2700,17 @@ nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts, 
	spin_lock_init(&ctrl->lock);



	/* io queue count */

	ctrl->queue_count = min_t(unsigned int,

	ctrl->ctrl.queue_count = min_t(unsigned int,

				opts->nr_io_queues,

				lport->ops->max_hw_queues);

	opts->nr_io_queues = ctrl->queue_count;	/* so opts has valid value */

	ctrl->queue_count++;	/* +1 for admin queue */

	ctrl->ctrl.queue_count++;	/* +1 for admin queue */



	ctrl->ctrl.sqsize = opts->queue_size - 1;

	ctrl->ctrl.kato = opts->kato;



	ret = -ENOMEM;

	ctrl->queues = kcalloc(ctrl->queue_count, sizeof(struct nvme_fc_queue),

				GFP_KERNEL);

	ctrl->queues = kcalloc(ctrl->ctrl.queue_count,

				sizeof(struct nvme_fc_queue), GFP_KERNEL);

	if (!ctrl->queues)

		goto out_free_ida;