scsi: ufs: add cpu-dma latency PM QOS request (695e8559) · Commits · e / devices / android_kernel_sony_msm8998

Documentation/devicetree/bindings/ufs/ufshcd-pltfrm.txt

+14 −0

Original line number	Diff line number	Diff line
		@@ -113,6 +113,20 @@ Note: The instantaneous bandwidth (IB) value in the vectors-KBps field should
		- qcom,bus-vector-names: specifies string IDs for the corresponding
		bus vectors in the same order as qcom,msm-bus,vectors-KBps property.

		- qcom,cpu-dma-latency-us: optional parameter specifying the allowed CPU-DMA
		latency parameter for PM QOS, in units of microseconds. If this parameter is
		not specified a default of 200us is used.
		- qcom,cpu-affinity: this is a string that specifies the pm QoS request type.
		The supported cpu affinity modes are:
		"all_cores" - PM_QOS_REQ_ALL_CORES is applicable to all CPU cores that are
		online and this would have a power impact when there are more number of CPUs.
		"affine_irq" - PM_QOS_REQ_AFFINE_IRQ request type shall update/apply the vote
		only to that CPU to which this IRQ's affinity is set to.
		"affine_cores" - PM_QOS_REQ_AFFINE_CORES request type is used for targets that
		have little cluster and will update/apply the vote to all the cores in the
		little cluster.
		The default CPU affinity mode is PM_QOS_REQ_AFFINE_IRQ.

		Example:
		ufshc@0xfc598000 {
		...

drivers/scsi/ufs/ufshcd-pltfrm.c

+26 −1

Original line number	Diff line number	Diff line
		@@ -235,7 +235,31 @@ static void ufshcd_parse_pm_levels(struct ufs_hba *hba)
		}
		}

		#ifdef CONFIG_PM
		#ifdef CONFIG_SMP
		static void ufshcd_parse_pm_qos(struct ufs_hba *hba)
		{
		const char *cpu_affinity = NULL;

		hba->pm_qos.cpu_dma_latency_us = UFS_DEFAULT_CPU_DMA_LATENCY_US;
		of_property_read_u32(hba->dev->of_node, "qcom,cpu-dma-latency-us",
		&hba->pm_qos.cpu_dma_latency_us);
		dev_dbg(hba->dev, "cpu_dma_latency_us = %u\n",
		hba->pm_qos.cpu_dma_latency_us);

		hba->pm_qos.req.type = PM_QOS_REQ_AFFINE_IRQ;
		if (!of_property_read_string(hba->dev->of_node, "qcom,cpu-affinity",
		&cpu_affinity)) {
		if (!strcmp(cpu_affinity, "all_cores"))
		hba->pm_qos.req.type = PM_QOS_REQ_ALL_CORES;
		else if (!strcmp(cpu_affinity, "affine_cores"))
		hba->pm_qos.req.type = PM_QOS_REQ_AFFINE_CORES;
		else if (!strcmp(cpu_affinity, "affine_irq"))
		hba->pm_qos.req.type = PM_QOS_REQ_AFFINE_IRQ;
		}
		dev_dbg(hba->dev, "hba->pm_qos.pm_qos_req.type = %u\n",
		hba->pm_qos.req.type);
		}

		/**
		* ufshcd_pltfrm_suspend - suspend power management function
		* @dev: pointer to device handle
		@@ -339,6 +363,7 @@ int ufshcd_pltfrm_init(struct platform_device *pdev,
		goto dealloc_host;
		}

		ufshcd_parse_pm_qos(hba);
		ufshcd_parse_pm_levels(hba);
		pm_runtime_set_active(&pdev->dev);
		pm_runtime_enable(&pdev->dev);

drivers/scsi/ufs/ufshcd.c

+224 −2

Original line number	Diff line number	Diff line
		@@ -149,6 +149,8 @@
		/* IOCTL opcode for command - ufs set device read only */
		#define UFS_IOCTL_BLKROSET BLKROSET

		#define UFSHCD_PM_QOS_UNVOTE_TIMEOUT_US (10000) /* microseconds */

		#define ufshcd_toggle_vreg(_dev, _vreg, _on) \
		({ \
		int _ret; \
		@@ -1405,15 +1407,193 @@ static void ufshcd_exit_hibern8_on_idle(struct ufs_hba *hba)
		device_remove_file(hba->dev, &hba->hibern8_on_idle.enable_attr);
		}

		#ifdef CONFIG_SMP

		/* Host lock is assumed to be held by caller */
		static int __ufshcd_pm_qos_hold(struct ufs_hba *hba, bool async)
		{
		int ret = 0;
		unsigned long flags;

		if (!hba->pm_qos.cpu_dma_latency_us)
		goto out;

		hba->pm_qos.active_reqs++;
		if (hba->pm_qos.is_suspended)
		goto out;
		start:
		switch (hba->pm_qos.state) {
		case PM_QOS_VOTED:
		/* nothing to do */
		break;
		case PM_QOS_REQ_UNVOTE:
		/*
		* Fall-through - unvoting is either running or completed,
		* so need to perform voting.
		*/
		case PM_QOS_UNVOTED:
		scsi_block_requests(hba->host);
		hba->pm_qos.state = PM_QOS_REQ_VOTE;
		schedule_work(&hba->pm_qos.vote_work);
		/* fall-through */
		case PM_QOS_REQ_VOTE:
		if (async) {
		hba->pm_qos.active_reqs--;
		ret = -EAGAIN;
		break;
		}
		spin_unlock_irqrestore(hba->host->host_lock, flags);
		flush_work(&hba->pm_qos.vote_work);
		spin_lock_irqsave(hba->host->host_lock, flags);
		goto start;
		default:
		dev_err(hba->dev, "%s: PM QoS invalid state %d\n", __func__,
		hba->pm_qos.state);
		ret = -EINVAL;
		break;
		}
		out:
		return ret;
		}

		static int ufshcd_pm_qos_hold(struct ufs_hba *hba, bool async)
		{
		unsigned long flags;
		int ret;

		spin_lock_irqsave(hba->host->host_lock, flags);
		ret = __ufshcd_pm_qos_hold(hba, async);
		spin_unlock_irqrestore(hba->host->host_lock, flags);

		return ret;
		}

		/* Host lock is assumed to be held by caller */
		static void __ufshcd_pm_qos_release(struct ufs_hba *hba)
		{
		if (!hba->pm_qos.cpu_dma_latency_us)
		return;

		if (--hba->pm_qos.active_reqs)
		return;

		hba->pm_qos.state = PM_QOS_REQ_UNVOTE;
		schedule_work(&hba->pm_qos.unvote_work);
		}

		static void ufshcd_pm_qos_release(struct ufs_hba *hba)
		{
		unsigned long flags;

		spin_lock_irqsave(hba->host->host_lock, flags);
		__ufshcd_pm_qos_release(hba);
		spin_unlock_irqrestore(hba->host->host_lock, flags);
		}

		static void ufshcd_pm_qos_vote_work(struct work_struct *work)
		{
		struct ufshcd_pm_qos *ufs_pm_qos =
		container_of(work, struct ufshcd_pm_qos, vote_work);
		struct ufs_hba *hba = container_of(ufs_pm_qos, struct ufs_hba, pm_qos);
		unsigned long flags;

		/*
		* un-voting work might be running when a new request arrives
		* and causes voting work to schedule. To prevent race condition
		* make sure the un-voting is finished.
		*/
		cancel_work_sync(&hba->pm_qos.unvote_work);

		pm_qos_update_request(&hba->pm_qos.req,
		hba->pm_qos.cpu_dma_latency_us);

		spin_lock_irqsave(hba->host->host_lock, flags);
		hba->pm_qos.state = PM_QOS_VOTED;
		spin_unlock_irqrestore(hba->host->host_lock, flags);

		scsi_unblock_requests(hba->host);
		}

		static void ufshcd_pm_qos_unvote_work(struct work_struct *work)
		{
		struct ufshcd_pm_qos *ufs_pm_qos =
		container_of(work, struct ufshcd_pm_qos, unvote_work);
		struct ufs_hba *hba = container_of(ufs_pm_qos, struct ufs_hba, pm_qos);
		unsigned long flags;

		/*
		* Check if new requests were submitted in the meantime and do not
		* unvote if so.
		*/
		spin_lock_irqsave(hba->host->host_lock, flags);
		if (hba->pm_qos.active_reqs) {
		spin_unlock_irqrestore(hba->host->host_lock, flags);
		return;
		}
		spin_unlock_irqrestore(hba->host->host_lock, flags);

		/*
		* When PM QoS voting is suspended (clocks scaled down or PM suspend
		* taking place) we can un-vote immediately. Otherwise, un-voting is
		* best done a bit later to accommodate for a burst of new upcoming
		* requests.
		*/
		if (hba->pm_qos.is_suspended)
		pm_qos_update_request(&hba->pm_qos.req, PM_QOS_DEFAULT_VALUE);
		else
		pm_qos_update_request_timeout(&hba->pm_qos.req,
		PM_QOS_DEFAULT_VALUE, UFSHCD_PM_QOS_UNVOTE_TIMEOUT_US);

		spin_lock_irqsave(hba->host->host_lock, flags);
		hba->pm_qos.state = PM_QOS_UNVOTED;
		spin_unlock_irqrestore(hba->host->host_lock, flags);
		}

		static int ufshcd_pm_qos_init(struct ufs_hba *hba)
		{
		/*
		* PM_QOS_REQ_AFFINE_CORES request type is used for targets that have
		* little cluster and will update/apply the vote to all the cores in
		* the little cluster.
		*/
		if (hba->pm_qos.req.type == PM_QOS_REQ_AFFINE_CORES)
		hba->pm_qos.req.cpus_affine.bits[0] = 0x0F;
		else if (hba->pm_qos.req.type == PM_QOS_REQ_AFFINE_IRQ)
		hba->pm_qos.req.irq = hba->irq;

		if (hba->pm_qos.cpu_dma_latency_us)
		pm_qos_add_request(&hba->pm_qos.req,
		PM_QOS_CPU_DMA_LATENCY, hba->pm_qos.cpu_dma_latency_us);
		else
		pm_qos_add_request(&hba->pm_qos.req,
		PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE);
		hba->pm_qos.state = PM_QOS_VOTED;
		hba->pm_qos.active_reqs = 0;
		hba->pm_qos.is_suspended = false;
		INIT_WORK(&hba->pm_qos.vote_work, ufshcd_pm_qos_vote_work);
		INIT_WORK(&hba->pm_qos.unvote_work, ufshcd_pm_qos_unvote_work);

		return 0;
		}

		static void ufshcd_pm_qos_remove(struct ufs_hba *hba)
		{
		pm_qos_remove_request(&hba->pm_qos.req);
		}

		#endif /* CONFIG_SMP */

		static void ufshcd_hold_all(struct ufs_hba *hba)
		{
		ufshcd_hold(hba, false);
		ufshcd_pm_qos_hold(hba, false);
		ufshcd_hibern8_hold(hba, false);
		}

		static void ufshcd_release_all(struct ufs_hba *hba)
		{
		ufshcd_hibern8_release(hba);
		ufshcd_pm_qos_release(hba);
		ufshcd_release(hba);
		}

		@@ -1443,6 +1623,7 @@ static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba)
		scaling->is_busy_started = false;
		}
		}

		/**
		* ufshcd_send_command - Send SCSI or device management commands
		* @hba: per adapter instance
		@@ -2013,10 +2194,19 @@ static int ufshcd_queuecommand(struct Scsi_Host host, struct scsi_cmnd cmd)
		}
		WARN_ON(hba->clk_gating.state != CLKS_ON);

		err = ufshcd_pm_qos_hold(hba, true);
		if (err) {
		err = SCSI_MLQUEUE_HOST_BUSY;
		clear_bit_unlock(tag, &hba->lrb_in_use);
		ufshcd_release(hba);
		goto out;
		}

		err = ufshcd_hibern8_hold(hba, true);
		if (err) {
		clear_bit_unlock(tag, &hba->lrb_in_use);
		err = SCSI_MLQUEUE_HOST_BUSY;
		ufshcd_pm_qos_release(hba);
		ufshcd_release(hba);
		goto out;
		}
		@@ -4275,6 +4465,7 @@ static void ufshcd_transfer_req_compl(struct ufs_hba *hba)
		/* Do not touch lrbp after scsi done */
		cmd->scsi_done(cmd);
		__ufshcd_release(hba);
		__ufshcd_pm_qos_release(hba);
		__ufshcd_hibern8_release(hba);
		} else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE) {
		if (hba->dev_cmd.complete)
		@@ -6506,6 +6697,7 @@ static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
		enum ufs_pm_level pm_lvl;
		enum ufs_dev_pwr_mode req_dev_pwr_mode;
		enum uic_link_state req_link_state;
		unsigned long flags;

		hba->pm_op_in_progress = 1;
		if (!ufshcd_is_shutdown_pm(pm_op)) {
		@@ -6528,6 +6720,13 @@ static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
		hba->clk_gating.is_suspended = true;
		hba->hibern8_on_idle.is_suspended = true;

		/* While entering PM suspend release the PM QoS vote and suspend it */
		if (hba->pm_qos.state == PM_QOS_VOTED) {
		pm_qos_update_request(&hba->pm_qos.req, PM_QOS_DEFAULT_VALUE);
		hba->pm_qos.state = PM_QOS_UNVOTED;
		}
		hba->pm_qos.is_suspended = true;

		if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE &&
		req_link_state == UIC_LINK_ACTIVE_STATE) {
		goto disable_clks;
		@@ -6632,6 +6831,9 @@ set_dev_active:
		enable_gating:
		hba->hibern8_on_idle.is_suspended = false;
		hba->clk_gating.is_suspended = false;
		spin_lock_irqsave(hba->host->host_lock, flags);
		hba->pm_qos.is_suspended = false;
		spin_unlock_irqrestore(hba->host->host_lock, flags);
		ufshcd_release_all(hba);
		out:
		hba->pm_op_in_progress = 0;
		@@ -6656,6 +6858,7 @@ static int ufshcd_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
		{
		int ret;
		enum uic_link_state old_link_state;
		unsigned long flags;

		hba->pm_op_in_progress = 1;
		old_link_state = hba->uic_link_state;
		@@ -6720,6 +6923,10 @@ static int ufshcd_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
		hba->clk_gating.is_suspended = false;
		hba->hibern8_on_idle.is_suspended = false;

		spin_lock_irqsave(hba->host->host_lock, flags);
		hba->pm_qos.is_suspended = false;
		spin_unlock_irqrestore(hba->host->host_lock, flags);

		if (hba->clk_scaling.is_allowed)
		ufshcd_resume_clkscaling(hba);

		@@ -7151,6 +7358,7 @@ static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up)
		struct list_head *head = &hba->clk_list_head;
		ktime_t start = ktime_get();
		bool clk_state_changed = false;
		unsigned long flags;

		if (!head \|\| list_empty(head))
		goto out;
		@@ -7203,6 +7411,11 @@ static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up)
		clki->name, clk_get_rate(clki->clk));
		}

		/* Suspend PM QoS voting when clocks are scaled down and vise versa */
		spin_lock_irqsave(hba->host->host_lock, flags);
		hba->pm_qos.is_suspended = !scale_up;
		spin_unlock_irqrestore(hba->host->host_lock, flags);

		ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);

		out:
		@@ -7401,6 +7614,13 @@ int ufshcd_init(struct ufs_hba hba, void __iomem mmio_base, unsigned int irq)
		goto out_disable;
		}

		/* Configure PM_QOS latency */
		err = ufshcd_pm_qos_init(hba);
		if (err) {
		dev_err(hba->dev, "ufshcd_pm_qos_init failed, err=%d\n", err);
		goto exit_gating;
		}

		/* Configure LRB */
		ufshcd_host_memory_configure(hba);

		@@ -7452,7 +7672,7 @@ int ufshcd_init(struct ufs_hba hba, void __iomem mmio_base, unsigned int irq)
		err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba);
		if (err) {
		dev_err(hba->dev, "request irq failed\n");
		goto exit_gating;
		goto pm_qos_remove;
		} else {
		hba->is_irq_enabled = true;
		}
		@@ -7460,7 +7680,7 @@ int ufshcd_init(struct ufs_hba hba, void __iomem mmio_base, unsigned int irq)
		err = scsi_add_host(host, hba->dev);
		if (err) {
		dev_err(hba->dev, "scsi_add_host failed\n");
		goto exit_gating;
		goto pm_qos_remove;
		}

		/* Host controller enable */
		@@ -7517,6 +7737,8 @@ int ufshcd_init(struct ufs_hba hba, void __iomem mmio_base, unsigned int irq)

		out_remove_scsi_host:
		scsi_remove_host(hba->host);
		pm_qos_remove:
		ufshcd_pm_qos_remove(hba);
		exit_gating:
		ufshcd_exit_clk_gating(hba);
		out_disable:

include/linux/scsi/ufs/ufshcd.h

+71 −0

Original line number	Diff line number	Diff line
		@@ -54,6 +54,7 @@
		#include <linux/clk.h>
		#include <linux/completion.h>
		#include <linux/regulator/consumer.h>
		#include <linux/pm_qos.h>

		#include <asm/irq.h>
		#include <asm/byteorder.h>
		@@ -448,6 +449,38 @@ enum ts_types {
		};
		#endif

		/* PM QoS voting state */
		enum ufshcd_pm_qos_state {
		PM_QOS_UNVOTED,
		PM_QOS_VOTED,
		PM_QOS_REQ_VOTE,
		PM_QOS_REQ_UNVOTE,
		};

		/* Default latency for PM QOS */
		#define UFS_DEFAULT_CPU_DMA_LATENCY_US 200 /* microseconds */

		/**
		* struct ufshcd_pm_qos - data related to PM QoS voting logic
		* @vote_work: work object for voting procedure
		* @unvote_work: work object for un-voting procedure
		* @req: request object for PM QoS
		* @cpu_dma_latency_us: requested latency value used for voting in microseconds
		* @state: voting state machine current state
		* @active_reqs: number of active requests requiring PM QoS voting
		* @is_suspended: flag specifying whether voting logic is suspended.
		* When set, voting will not occur for pending requests.
		*/
		struct ufshcd_pm_qos {
		struct work_struct vote_work;
		struct work_struct unvote_work;
		struct pm_qos_request req;
		u32 cpu_dma_latency_us;
		enum ufshcd_pm_qos_state state;
		int active_reqs;
		bool is_suspended;
		};

		/**
		* struct ufs_hba - per adapter private structure
		* @mmio_base: UFSHCI base register address
		@@ -637,6 +670,9 @@ struct ufs_hba {

		bool wlun_dev_clr_ua;

		/* PM Quality-of-Service (QoS) data */
		struct ufshcd_pm_qos pm_qos;

		struct ufs_pa_layer_attr pwr_info;
		struct ufs_pwr_mode_info max_pwr_info;

		@@ -834,6 +870,41 @@ static const struct ufs_hba_variant_ops ufs_hba_qcom_vops = {
		};
		#endif

		#ifndef CONFIG_SMP
		static inline int ufshcd_pm_qos_init(struct ufs_hba *hba)
		{
		return 0;
		}

		static inline int ufshcd_pm_qos_hold(struct ufs_hba *hba, bool async)
		{
		return 0;
		}

		static inline int __ufshcd_pm_qos_hold(struct ufs_hba *hba, bool async)
		{
		return 0;
		}

		static inline int ufshcd_pm_qos_release(struct ufs_hba *hba)
		{
		return 0;
		}

		static inline int __ufshcd_pm_qos_release(struct ufs_hba *hba)
		{
		return 0;
		}

		static inline void ufshcd_pm_qos_remove(struct ufs_hba *hba)
		{
		}

		static inline void ufshcd_parse_pm_qos(struct ufs_hba *hba)
		{
		}
		#endif /* CONFIG_SMP */

		/* Expose Query-Request API */
		int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
		enum flag_idn idn, bool *flag_res);