Merge branch 'nvme-4.18' of git://git.infradead.org/nvme into for-linus (f9da9d07) · Commits · e / devices / android_kernel_fairphone_FP5

drivers/nvme/host/core.c

+1 −0

Original line number	Original line	Diff line number	Diff line
	@@ -1808,6 +1808,7 @@ static void nvme_set_queue_limits(struct nvme_ctrl *ctrl,
	u32 max_segments =		u32 max_segments =
	(ctrl->max_hw_sectors / (ctrl->page_size >> 9)) + 1;		(ctrl->max_hw_sectors / (ctrl->page_size >> 9)) + 1;

			max_segments = min_not_zero(max_segments, ctrl->max_segments);
	blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors);		blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors);
	blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX));		blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX));
	}		}

drivers/nvme/host/fc.c

+3 −3

Original line number	Original line	Diff line number	Diff line
	@@ -2790,6 +2790,9 @@ nvme_fc_delete_association(struct nvme_fc_ctrl *ctrl)
	/* re-enable the admin_q so anything new can fast fail */		/* re-enable the admin_q so anything new can fast fail */
	blk_mq_unquiesce_queue(ctrl->ctrl.admin_q);		blk_mq_unquiesce_queue(ctrl->ctrl.admin_q);

			/* resume the io queues so that things will fast fail */
			nvme_start_queues(&ctrl->ctrl);

	nvme_fc_ctlr_inactive_on_rport(ctrl);		nvme_fc_ctlr_inactive_on_rport(ctrl);
	}		}

	@@ -2804,9 +2807,6 @@ nvme_fc_delete_ctrl(struct nvme_ctrl *nctrl)
	* waiting for io to terminate		* waiting for io to terminate
	*/		*/
	nvme_fc_delete_association(ctrl);		nvme_fc_delete_association(ctrl);

	/* resume the io queues so that things will fast fail */
	nvme_start_queues(nctrl);
	}		}

	static void		static void

drivers/nvme/host/nvme.h

+1 −0

Original line number	Original line	Diff line number	Diff line
	@@ -170,6 +170,7 @@ struct nvme_ctrl {
	u64 cap;		u64 cap;
	u32 page_size;		u32 page_size;
	u32 max_hw_sectors;		u32 max_hw_sectors;
			u32 max_segments;
	u16 oncs;		u16 oncs;
	u16 oacs;		u16 oacs;
	u16 nssa;		u16 nssa;

drivers/nvme/host/pci.c

+38 −6

Original line number	Original line	Diff line number	Diff line
	@@ -38,6 +38,13 @@

	#define SGES_PER_PAGE (PAGE_SIZE / sizeof(struct nvme_sgl_desc))		#define SGES_PER_PAGE (PAGE_SIZE / sizeof(struct nvme_sgl_desc))

			/*
			* These can be higher, but we need to ensure that any command doesn't
			* require an sg allocation that needs more than a page of data.
			*/
			#define NVME_MAX_KB_SZ 4096
			#define NVME_MAX_SEGS 127

	static int use_threaded_interrupts;		static int use_threaded_interrupts;
	module_param(use_threaded_interrupts, int, 0);		module_param(use_threaded_interrupts, int, 0);

	@@ -100,6 +107,8 @@ struct nvme_dev {
	struct nvme_ctrl ctrl;		struct nvme_ctrl ctrl;
	struct completion ioq_wait;		struct completion ioq_wait;

			mempool_t *iod_mempool;

	/* shadow doorbell buffer support: */		/* shadow doorbell buffer support: */
	u32 *dbbuf_dbs;		u32 *dbbuf_dbs;
	dma_addr_t dbbuf_dbs_dma_addr;		dma_addr_t dbbuf_dbs_dma_addr;
	@@ -477,10 +486,7 @@ static blk_status_t nvme_init_iod(struct request rq, struct nvme_dev dev)
	iod->use_sgl = nvme_pci_use_sgls(dev, rq);		iod->use_sgl = nvme_pci_use_sgls(dev, rq);

	if (nseg > NVME_INT_PAGES \|\| size > NVME_INT_BYTES(dev)) {		if (nseg > NVME_INT_PAGES \|\| size > NVME_INT_BYTES(dev)) {
	size_t alloc_size = nvme_pci_iod_alloc_size(dev, size, nseg,		iod->sg = mempool_alloc(dev->iod_mempool, GFP_ATOMIC);
	iod->use_sgl);

	iod->sg = kmalloc(alloc_size, GFP_ATOMIC);
	if (!iod->sg)		if (!iod->sg)
	return BLK_STS_RESOURCE;		return BLK_STS_RESOURCE;
	} else {		} else {
	@@ -526,7 +532,7 @@ static void nvme_free_iod(struct nvme_dev dev, struct request req)
	}		}

	if (iod->sg != iod->inline_sg)		if (iod->sg != iod->inline_sg)
	kfree(iod->sg);		mempool_free(iod->sg, dev->iod_mempool);
	}		}

	#ifdef CONFIG_BLK_DEV_INTEGRITY		#ifdef CONFIG_BLK_DEV_INTEGRITY
	@@ -2280,6 +2286,7 @@ static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl)
	blk_put_queue(dev->ctrl.admin_q);		blk_put_queue(dev->ctrl.admin_q);
	kfree(dev->queues);		kfree(dev->queues);
	free_opal_dev(dev->ctrl.opal_dev);		free_opal_dev(dev->ctrl.opal_dev);
			mempool_destroy(dev->iod_mempool);
	kfree(dev);		kfree(dev);
	}		}

	@@ -2289,6 +2296,7 @@ static void nvme_remove_dead_ctrl(struct nvme_dev *dev, int status)

	nvme_get_ctrl(&dev->ctrl);		nvme_get_ctrl(&dev->ctrl);
	nvme_dev_disable(dev, false);		nvme_dev_disable(dev, false);
			nvme_kill_queues(&dev->ctrl);
	if (!queue_work(nvme_wq, &dev->remove_work))		if (!queue_work(nvme_wq, &dev->remove_work))
	nvme_put_ctrl(&dev->ctrl);		nvme_put_ctrl(&dev->ctrl);
	}		}
	@@ -2333,6 +2341,13 @@ static void nvme_reset_work(struct work_struct *work)
	if (result)		if (result)
	goto out;		goto out;

			/*
			* Limit the max command size to prevent iod->sg allocations going
			* over a single page.
			*/
			dev->ctrl.max_hw_sectors = NVME_MAX_KB_SZ << 1;
			dev->ctrl.max_segments = NVME_MAX_SEGS;

	result = nvme_init_identify(&dev->ctrl);		result = nvme_init_identify(&dev->ctrl);
	if (result)		if (result)
	goto out;		goto out;
	@@ -2405,7 +2420,6 @@ static void nvme_remove_dead_ctrl_work(struct work_struct *work)
	struct nvme_dev *dev = container_of(work, struct nvme_dev, remove_work);		struct nvme_dev *dev = container_of(work, struct nvme_dev, remove_work);
	struct pci_dev *pdev = to_pci_dev(dev->dev);		struct pci_dev *pdev = to_pci_dev(dev->dev);

	nvme_kill_queues(&dev->ctrl);
	if (pci_get_drvdata(pdev))		if (pci_get_drvdata(pdev))
	device_release_driver(&pdev->dev);		device_release_driver(&pdev->dev);
	nvme_put_ctrl(&dev->ctrl);		nvme_put_ctrl(&dev->ctrl);
	@@ -2509,6 +2523,7 @@ static int nvme_probe(struct pci_dev pdev, const struct pci_device_id id)
	int node, result = -ENOMEM;		int node, result = -ENOMEM;
	struct nvme_dev *dev;		struct nvme_dev *dev;
	unsigned long quirks = id->driver_data;		unsigned long quirks = id->driver_data;
			size_t alloc_size;

	node = dev_to_node(&pdev->dev);		node = dev_to_node(&pdev->dev);
	if (node == NUMA_NO_NODE)		if (node == NUMA_NO_NODE)
	@@ -2546,6 +2561,23 @@ static int nvme_probe(struct pci_dev pdev, const struct pci_device_id id)
	if (result)		if (result)
	goto release_pools;		goto release_pools;

			/*
			* Double check that our mempool alloc size will cover the biggest
			* command we support.
			*/
			alloc_size = nvme_pci_iod_alloc_size(dev, NVME_MAX_KB_SZ,
			NVME_MAX_SEGS, true);
			WARN_ON_ONCE(alloc_size > PAGE_SIZE);

			dev->iod_mempool = mempool_create_node(1, mempool_kmalloc,
			mempool_kfree,
			(void *) alloc_size,
			GFP_KERNEL, node);
			if (!dev->iod_mempool) {
			result = -ENOMEM;
			goto release_pools;
			}

	dev_info(dev->ctrl.device, "pci function %s\n", dev_name(&pdev->dev));		dev_info(dev->ctrl.device, "pci function %s\n", dev_name(&pdev->dev));

	nvme_get_ctrl(&dev->ctrl);		nvme_get_ctrl(&dev->ctrl);

drivers/nvme/host/rdma.c

+37 −36

Original line number	Original line	Diff line number	Diff line
	@@ -560,12 +560,6 @@ static void nvme_rdma_free_queue(struct nvme_rdma_queue *queue)
	if (!test_and_clear_bit(NVME_RDMA_Q_ALLOCATED, &queue->flags))		if (!test_and_clear_bit(NVME_RDMA_Q_ALLOCATED, &queue->flags))
	return;		return;

	if (nvme_rdma_queue_idx(queue) == 0) {
	nvme_rdma_free_qe(queue->device->dev,
	&queue->ctrl->async_event_sqe,
	sizeof(struct nvme_command), DMA_TO_DEVICE);
	}

	nvme_rdma_destroy_queue_ib(queue);		nvme_rdma_destroy_queue_ib(queue);
	rdma_destroy_id(queue->cm_id);		rdma_destroy_id(queue->cm_id);
	}		}
	@@ -698,7 +692,7 @@ static struct blk_mq_tag_set nvme_rdma_alloc_tagset(struct nvme_ctrl nctrl,
	set = &ctrl->tag_set;		set = &ctrl->tag_set;
	memset(set, 0, sizeof(*set));		memset(set, 0, sizeof(*set));
	set->ops = &nvme_rdma_mq_ops;		set->ops = &nvme_rdma_mq_ops;
	set->queue_depth = nctrl->opts->queue_size;		set->queue_depth = nctrl->sqsize + 1;
	set->reserved_tags = 1; /* fabric connect */		set->reserved_tags = 1; /* fabric connect */
	set->numa_node = NUMA_NO_NODE;		set->numa_node = NUMA_NO_NODE;
	set->flags = BLK_MQ_F_SHOULD_MERGE;		set->flags = BLK_MQ_F_SHOULD_MERGE;
	@@ -734,11 +728,12 @@ static struct blk_mq_tag_set nvme_rdma_alloc_tagset(struct nvme_ctrl nctrl,
	static void nvme_rdma_destroy_admin_queue(struct nvme_rdma_ctrl *ctrl,		static void nvme_rdma_destroy_admin_queue(struct nvme_rdma_ctrl *ctrl,
	bool remove)		bool remove)
	{		{
	nvme_rdma_stop_queue(&ctrl->queues[0]);
	if (remove) {		if (remove) {
	blk_cleanup_queue(ctrl->ctrl.admin_q);		blk_cleanup_queue(ctrl->ctrl.admin_q);
	nvme_rdma_free_tagset(&ctrl->ctrl, ctrl->ctrl.admin_tagset);		nvme_rdma_free_tagset(&ctrl->ctrl, ctrl->ctrl.admin_tagset);
	}		}
			nvme_rdma_free_qe(ctrl->device->dev, &ctrl->async_event_sqe,
			sizeof(struct nvme_command), DMA_TO_DEVICE);
	nvme_rdma_free_queue(&ctrl->queues[0]);		nvme_rdma_free_queue(&ctrl->queues[0]);
	}		}

	@@ -755,11 +750,16 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,

	ctrl->max_fr_pages = nvme_rdma_get_max_fr_pages(ctrl->device->dev);		ctrl->max_fr_pages = nvme_rdma_get_max_fr_pages(ctrl->device->dev);

			error = nvme_rdma_alloc_qe(ctrl->device->dev, &ctrl->async_event_sqe,
			sizeof(struct nvme_command), DMA_TO_DEVICE);
			if (error)
			goto out_free_queue;

	if (new) {		if (new) {
	ctrl->ctrl.admin_tagset = nvme_rdma_alloc_tagset(&ctrl->ctrl, true);		ctrl->ctrl.admin_tagset = nvme_rdma_alloc_tagset(&ctrl->ctrl, true);
	if (IS_ERR(ctrl->ctrl.admin_tagset)) {		if (IS_ERR(ctrl->ctrl.admin_tagset)) {
	error = PTR_ERR(ctrl->ctrl.admin_tagset);		error = PTR_ERR(ctrl->ctrl.admin_tagset);
	goto out_free_queue;		goto out_free_async_qe;
	}		}

	ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set);		ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set);
	@@ -795,12 +795,6 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,
	if (error)		if (error)
	goto out_stop_queue;		goto out_stop_queue;

	error = nvme_rdma_alloc_qe(ctrl->queues[0].device->dev,
	&ctrl->async_event_sqe, sizeof(struct nvme_command),
	DMA_TO_DEVICE);
	if (error)
	goto out_stop_queue;

	return 0;		return 0;

	out_stop_queue:		out_stop_queue:
	@@ -811,6 +805,9 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,
	out_free_tagset:		out_free_tagset:
	if (new)		if (new)
	nvme_rdma_free_tagset(&ctrl->ctrl, ctrl->ctrl.admin_tagset);		nvme_rdma_free_tagset(&ctrl->ctrl, ctrl->ctrl.admin_tagset);
			out_free_async_qe:
			nvme_rdma_free_qe(ctrl->device->dev, &ctrl->async_event_sqe,
			sizeof(struct nvme_command), DMA_TO_DEVICE);
	out_free_queue:		out_free_queue:
	nvme_rdma_free_queue(&ctrl->queues[0]);		nvme_rdma_free_queue(&ctrl->queues[0]);
	return error;		return error;
	@@ -819,7 +816,6 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,
	static void nvme_rdma_destroy_io_queues(struct nvme_rdma_ctrl *ctrl,		static void nvme_rdma_destroy_io_queues(struct nvme_rdma_ctrl *ctrl,
	bool remove)		bool remove)
	{		{
	nvme_rdma_stop_io_queues(ctrl);
	if (remove) {		if (remove) {
	blk_cleanup_queue(ctrl->ctrl.connect_q);		blk_cleanup_queue(ctrl->ctrl.connect_q);
	nvme_rdma_free_tagset(&ctrl->ctrl, ctrl->ctrl.tagset);		nvme_rdma_free_tagset(&ctrl->ctrl, ctrl->ctrl.tagset);
	@@ -888,9 +884,9 @@ static void nvme_rdma_free_ctrl(struct nvme_ctrl *nctrl)
	list_del(&ctrl->list);		list_del(&ctrl->list);
	mutex_unlock(&nvme_rdma_ctrl_mutex);		mutex_unlock(&nvme_rdma_ctrl_mutex);

	kfree(ctrl->queues);
	nvmf_free_options(nctrl->opts);		nvmf_free_options(nctrl->opts);
	free_ctrl:		free_ctrl:
			kfree(ctrl->queues);
	kfree(ctrl);		kfree(ctrl);
	}		}

	@@ -949,6 +945,7 @@ static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work)
	return;		return;

	destroy_admin:		destroy_admin:
			nvme_rdma_stop_queue(&ctrl->queues[0]);
	nvme_rdma_destroy_admin_queue(ctrl, false);		nvme_rdma_destroy_admin_queue(ctrl, false);
	requeue:		requeue:
	dev_info(ctrl->ctrl.device, "Failed reconnect attempt %d\n",		dev_info(ctrl->ctrl.device, "Failed reconnect attempt %d\n",
	@@ -965,12 +962,14 @@ static void nvme_rdma_error_recovery_work(struct work_struct *work)

	if (ctrl->ctrl.queue_count > 1) {		if (ctrl->ctrl.queue_count > 1) {
	nvme_stop_queues(&ctrl->ctrl);		nvme_stop_queues(&ctrl->ctrl);
			nvme_rdma_stop_io_queues(ctrl);
	blk_mq_tagset_busy_iter(&ctrl->tag_set,		blk_mq_tagset_busy_iter(&ctrl->tag_set,
	nvme_cancel_request, &ctrl->ctrl);		nvme_cancel_request, &ctrl->ctrl);
	nvme_rdma_destroy_io_queues(ctrl, false);		nvme_rdma_destroy_io_queues(ctrl, false);
	}		}

	blk_mq_quiesce_queue(ctrl->ctrl.admin_q);		blk_mq_quiesce_queue(ctrl->ctrl.admin_q);
			nvme_rdma_stop_queue(&ctrl->queues[0]);
	blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,		blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,
	nvme_cancel_request, &ctrl->ctrl);		nvme_cancel_request, &ctrl->ctrl);
	nvme_rdma_destroy_admin_queue(ctrl, false);		nvme_rdma_destroy_admin_queue(ctrl, false);
	@@ -1736,6 +1735,7 @@ static void nvme_rdma_shutdown_ctrl(struct nvme_rdma_ctrl *ctrl, bool shutdown)
	{		{
	if (ctrl->ctrl.queue_count > 1) {		if (ctrl->ctrl.queue_count > 1) {
	nvme_stop_queues(&ctrl->ctrl);		nvme_stop_queues(&ctrl->ctrl);
			nvme_rdma_stop_io_queues(ctrl);
	blk_mq_tagset_busy_iter(&ctrl->tag_set,		blk_mq_tagset_busy_iter(&ctrl->tag_set,
	nvme_cancel_request, &ctrl->ctrl);		nvme_cancel_request, &ctrl->ctrl);
	nvme_rdma_destroy_io_queues(ctrl, shutdown);		nvme_rdma_destroy_io_queues(ctrl, shutdown);
	@@ -1747,6 +1747,7 @@ static void nvme_rdma_shutdown_ctrl(struct nvme_rdma_ctrl *ctrl, bool shutdown)
	nvme_disable_ctrl(&ctrl->ctrl, ctrl->ctrl.cap);		nvme_disable_ctrl(&ctrl->ctrl, ctrl->ctrl.cap);

	blk_mq_quiesce_queue(ctrl->ctrl.admin_q);		blk_mq_quiesce_queue(ctrl->ctrl.admin_q);
			nvme_rdma_stop_queue(&ctrl->queues[0]);
	blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,		blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,
	nvme_cancel_request, &ctrl->ctrl);		nvme_cancel_request, &ctrl->ctrl);
	blk_mq_unquiesce_queue(ctrl->ctrl.admin_q);		blk_mq_unquiesce_queue(ctrl->ctrl.admin_q);
	@@ -1932,11 +1933,6 @@ static struct nvme_ctrl nvme_rdma_create_ctrl(struct device dev,
	goto out_free_ctrl;		goto out_free_ctrl;
	}		}

	ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_rdma_ctrl_ops,
	0 /* no quirks, we're perfect! */);
	if (ret)
	goto out_free_ctrl;

	INIT_DELAYED_WORK(&ctrl->reconnect_work,		INIT_DELAYED_WORK(&ctrl->reconnect_work,
	nvme_rdma_reconnect_ctrl_work);		nvme_rdma_reconnect_ctrl_work);
	INIT_WORK(&ctrl->err_work, nvme_rdma_error_recovery_work);		INIT_WORK(&ctrl->err_work, nvme_rdma_error_recovery_work);
	@@ -1950,14 +1946,19 @@ static struct nvme_ctrl nvme_rdma_create_ctrl(struct device dev,
	ctrl->queues = kcalloc(ctrl->ctrl.queue_count, sizeof(*ctrl->queues),		ctrl->queues = kcalloc(ctrl->ctrl.queue_count, sizeof(*ctrl->queues),
	GFP_KERNEL);		GFP_KERNEL);
	if (!ctrl->queues)		if (!ctrl->queues)
	goto out_uninit_ctrl;		goto out_free_ctrl;

			ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_rdma_ctrl_ops,
			0 /* no quirks, we're perfect! */);
			if (ret)
			goto out_kfree_queues;

	changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING);		changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING);
	WARN_ON_ONCE(!changed);		WARN_ON_ONCE(!changed);

	ret = nvme_rdma_configure_admin_queue(ctrl, true);		ret = nvme_rdma_configure_admin_queue(ctrl, true);
	if (ret)		if (ret)
	goto out_kfree_queues;		goto out_uninit_ctrl;

	/* sanity check icdoff */		/* sanity check icdoff */
	if (ctrl->ctrl.icdoff) {		if (ctrl->ctrl.icdoff) {
	@@ -1974,20 +1975,19 @@ static struct nvme_ctrl nvme_rdma_create_ctrl(struct device dev,
	goto out_remove_admin_queue;		goto out_remove_admin_queue;
	}		}

	if (opts->queue_size > ctrl->ctrl.maxcmd) {		/* only warn if argument is too large here, will clamp later */
	/* warn if maxcmd is lower than queue_size */
	dev_warn(ctrl->ctrl.device,
	"queue_size %zu > ctrl maxcmd %u, clamping down\n",
	opts->queue_size, ctrl->ctrl.maxcmd);
	opts->queue_size = ctrl->ctrl.maxcmd;
	}

	if (opts->queue_size > ctrl->ctrl.sqsize + 1) {		if (opts->queue_size > ctrl->ctrl.sqsize + 1) {
	/* warn if sqsize is lower than queue_size */
	dev_warn(ctrl->ctrl.device,		dev_warn(ctrl->ctrl.device,
	"queue_size %zu > ctrl sqsize %u, clamping down\n",		"queue_size %zu > ctrl sqsize %u, clamping down\n",
	opts->queue_size, ctrl->ctrl.sqsize + 1);		opts->queue_size, ctrl->ctrl.sqsize + 1);
	opts->queue_size = ctrl->ctrl.sqsize + 1;		}

			/* warn if maxcmd is lower than sqsize+1 */
			if (ctrl->ctrl.sqsize + 1 > ctrl->ctrl.maxcmd) {
			dev_warn(ctrl->ctrl.device,
			"sqsize %u > ctrl maxcmd %u, clamping down\n",
			ctrl->ctrl.sqsize + 1, ctrl->ctrl.maxcmd);
			ctrl->ctrl.sqsize = ctrl->ctrl.maxcmd - 1;
	}		}

	if (opts->nr_io_queues) {		if (opts->nr_io_queues) {
	@@ -2013,15 +2013,16 @@ static struct nvme_ctrl nvme_rdma_create_ctrl(struct device dev,
	return &ctrl->ctrl;		return &ctrl->ctrl;

	out_remove_admin_queue:		out_remove_admin_queue:
			nvme_rdma_stop_queue(&ctrl->queues[0]);
	nvme_rdma_destroy_admin_queue(ctrl, true);		nvme_rdma_destroy_admin_queue(ctrl, true);
	out_kfree_queues:
	kfree(ctrl->queues);
	out_uninit_ctrl:		out_uninit_ctrl:
	nvme_uninit_ctrl(&ctrl->ctrl);		nvme_uninit_ctrl(&ctrl->ctrl);
	nvme_put_ctrl(&ctrl->ctrl);		nvme_put_ctrl(&ctrl->ctrl);
	if (ret > 0)		if (ret > 0)
	ret = -EIO;		ret = -EIO;
	return ERR_PTR(ret);		return ERR_PTR(ret);
			out_kfree_queues:
			kfree(ctrl->queues);
	out_free_ctrl:		out_free_ctrl:
	kfree(ctrl);		kfree(ctrl);
	return ERR_PTR(ret);		return ERR_PTR(ret);