Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e34ad72e authored by Gilad Broner's avatar Gilad Broner
Browse files

mmc: sdhci-msm: add PM QoS voting 



Add PM QoS voting mechanism to sdhci-msm driver for command queueing.
Two types of voting schemes are supported:
1) Vote for HW IRQ
2) Vote for a cpu group according to the request's designated cpu
Using PM QoS voting should benefit performance.

Change-Id: I8a20653eeb6348d5b442c846708d92c8fb64a8e9
Signed-off-by: default avatarGilad Broner <gbroner@codeaurora.org>
parent ee630a97

drivers/mmc/card/queue.c

+3 −0
Original line number Diff line number Diff line
@@ -342,6 +342,9 @@ int mmc_init_queue(struct mmc_queue *mq, struct mmc_card *card, 
			blk_cleanup_queue(mq->queue);

		} else {

			sema_init(&mq->thread_sem, 1);

			/* hook for pm qos cmdq init */

			if (card->host->cmdq_ops->init)

				card->host->cmdq_ops->init(card->host);

			mq->queue->queuedata = mq;

			card->host->cmdq_ctx.q = mq->queue;

			mq->thread = kthread_run(mmc_cmdq_thread, mq,

drivers/mmc/host/cmdq_hci.c

+47 −0
Original line number Diff line number Diff line
@@ -24,8 +24,12 @@ 
#include <linux/mmc/host.h>

#include <linux/mmc/card.h>

#include <linux/pm_runtime.h>

#include <linux/mmc/sdhci.h>

#include <linux/workqueue.h>



#include "cmdq_hci.h"

#include "sdhci.h"

#include "sdhci-msm.h"



#define DCMD_SLOT 31

#define NUM_SLOTS 32
@@ -575,6 +579,21 @@ static void cmdq_prep_dcmd_desc(struct mmc_host *mmc, 


}



static void cmdq_pm_qos_vote(struct sdhci_host *host, struct mmc_request *mrq)

{

	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);

	struct sdhci_msm_host *msm_host = pltfm_host->priv;



	sdhci_msm_pm_qos_cpu_vote(host,

		msm_host->pdata->pm_qos_data.cmdq_latency, mrq->req->cpu);

}



static void cmdq_pm_qos_unvote(struct sdhci_host *host, struct mmc_request *mrq)

{

	/* use async as we're inside an atomic context (soft-irq) */

	sdhci_msm_pm_qos_cpu_unvote(host, mrq->req->cpu, true);

}



static int cmdq_request(struct mmc_host *mmc, struct mmc_request *mrq)

{

	int err = 0;
@@ -582,6 +601,7 @@ static int cmdq_request(struct mmc_host *mmc, struct mmc_request *mrq) 
	u64 *task_desc = NULL;

	u32 tag = mrq->cmdq_req->tag;

	struct cmdq_host *cq_host = (struct cmdq_host *)mmc_cmdq_private(mmc);

	struct sdhci_host *host = mmc_priv(mmc);



	if (!cq_host->enabled) {

		pr_err("%s: CMDQ host not enabled yet !!!\n",
@@ -628,6 +648,9 @@ static int cmdq_request(struct mmc_host *mmc, struct mmc_request *mrq) 
	if (cq_host->ops->set_tranfer_params)

		cq_host->ops->set_tranfer_params(mmc);



	/* PM QoS */

	sdhci_msm_pm_qos_irq_vote(host);

	cmdq_pm_qos_vote(host, mrq);

ring_doorbell:

	/* Ensure the task descriptor list is flushed before ringing doorbell */

	wmb();
@@ -781,6 +804,7 @@ static void cmdq_post_req(struct mmc_host *host, struct mmc_request *mrq, 
			  int err)

{

	struct mmc_data *data = mrq->data;

	struct sdhci_host *sdhci_host = mmc_priv(host);



	if (data) {

		data->error = err;
@@ -791,6 +815,10 @@ static void cmdq_post_req(struct mmc_host *host, struct mmc_request *mrq, 
			data->bytes_xfered = 0;

		else

			data->bytes_xfered = blk_rq_bytes(mrq->req);



		/* we're in atomic context (soft-irq) so unvote async. */

		sdhci_msm_pm_qos_irq_unvote(sdhci_host, true);

		cmdq_pm_qos_unvote(sdhci_host, mrq);

	}

}
@@ -802,7 +830,26 @@ static void cmdq_dumpstate(struct mmc_host *mmc) 
	cmdq_runtime_pm_put(cq_host);

}



static int cmdq_late_init(struct mmc_host *mmc)

{

	struct sdhci_host *host = mmc_priv(mmc);

	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);

	struct sdhci_msm_host *msm_host = pltfm_host->priv;



	/*

	 * TODO: This should basically move to something like "sdhci-cmdq-msm"

	 * for msm specific implementation.

	 */

	sdhci_msm_pm_qos_irq_init(host);



	if (msm_host->pdata->pm_qos_data.cmdq_valid)

		sdhci_msm_pm_qos_cpu_init(host,

			msm_host->pdata->pm_qos_data.cmdq_latency);

	return 0;

}



static const struct mmc_cmdq_host_ops cmdq_host_ops = {

	.init = cmdq_late_init,

	.enable = cmdq_enable,

	.disable = cmdq_disable,

	.request = cmdq_request,

drivers/mmc/host/sdhci-msm.c

+208 −0
Original line number Diff line number Diff line
@@ -3141,6 +3141,214 @@ void sdhci_msm_reset_workaround(struct sdhci_host *host, u32 enable) 
	}

}



static void sdhci_msm_pm_qos_irq_unvote_work(struct work_struct *work)

{

	struct sdhci_msm_pm_qos_irq *pm_qos_irq =

		container_of(work, struct sdhci_msm_pm_qos_irq, unvote_work);



	if (atomic_read(&pm_qos_irq->counter))

		return;



	pm_qos_irq->latency = PM_QOS_DEFAULT_VALUE;

	pm_qos_update_request(&pm_qos_irq->req, pm_qos_irq->latency);

}



void sdhci_msm_pm_qos_irq_vote(struct sdhci_host *host)

{

	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);

	struct sdhci_msm_host *msm_host = pltfm_host->priv;

	struct sdhci_msm_pm_qos_latency *latency =

		&msm_host->pdata->pm_qos_data.irq_latency;

	int counter;



	if (!msm_host->pm_qos_irq.enabled)

		return;



	counter = atomic_inc_return(&msm_host->pm_qos_irq.counter);

	/* Make sure to update the voting in case power policy has changed */

	if (msm_host->pm_qos_irq.latency == latency->latency[host->power_policy]

		&& counter > 1)

		return;



	cancel_work_sync(&msm_host->pm_qos_irq.unvote_work);

	msm_host->pm_qos_irq.latency = latency->latency[host->power_policy];

	pm_qos_update_request(&msm_host->pm_qos_irq.req,

				msm_host->pm_qos_irq.latency);

}



void sdhci_msm_pm_qos_irq_unvote(struct sdhci_host *host, bool async)

{

	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);

	struct sdhci_msm_host *msm_host = pltfm_host->priv;

	int counter;



	if (!msm_host->pm_qos_irq.enabled)

		return;



	counter = atomic_dec_return(&msm_host->pm_qos_irq.counter);

	if (counter < 0) {

		pr_err("%s: counter=%d\n", __func__, counter);

		BUG();

	}

	if (counter)

		return;



	if (async) {

		schedule_work(&msm_host->pm_qos_irq.unvote_work);

		return;

	}



	msm_host->pm_qos_irq.latency = PM_QOS_DEFAULT_VALUE;

	pm_qos_update_request(&msm_host->pm_qos_irq.req,

			msm_host->pm_qos_irq.latency);

}



void sdhci_msm_pm_qos_irq_init(struct sdhci_host *host)

{

	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);

	struct sdhci_msm_host *msm_host = pltfm_host->priv;

	struct sdhci_msm_pm_qos_latency *irq_latency;



	if (!msm_host->pdata->pm_qos_data.irq_valid)

		return;



	/* Initialize only once as this gets called per partition */

	if (msm_host->pm_qos_irq.enabled)

		return;



	atomic_set(&msm_host->pm_qos_irq.counter, 0);

	msm_host->pm_qos_irq.req.type =

			msm_host->pdata->pm_qos_data.irq_req_type;

	if (msm_host->pm_qos_irq.req.type == PM_QOS_REQ_AFFINE_IRQ)

		msm_host->pm_qos_irq.req.irq = host->irq;

	else

		cpumask_copy(&msm_host->pm_qos_irq.req.cpus_affine,

			cpumask_of(msm_host->pdata->pm_qos_data.irq_cpu));



	INIT_WORK(&msm_host->pm_qos_irq.unvote_work,

		sdhci_msm_pm_qos_irq_unvote_work);

	/* For initialization phase, set the performance latency */

	irq_latency = &msm_host->pdata->pm_qos_data.irq_latency;

	msm_host->pm_qos_irq.latency =

		irq_latency->latency[SDHCI_PERFORMANCE_MODE];

	pm_qos_add_request(&msm_host->pm_qos_irq.req, PM_QOS_CPU_DMA_LATENCY,

			msm_host->pm_qos_irq.latency);

	msm_host->pm_qos_irq.enabled = true;

}



static int sdhci_msm_get_cpu_group(struct sdhci_msm_host *msm_host, int cpu)

{

	int i;

	struct sdhci_msm_cpu_group_map *map =

			&msm_host->pdata->pm_qos_data.cpu_group_map;



	if (cpu < 0)

		goto not_found;



	for (i = 0; i < map->nr_groups; i++)

		if (cpumask_test_cpu(cpu, &map->mask[i]))

			return i;



not_found:

	return -EINVAL;

}



void sdhci_msm_pm_qos_cpu_vote(struct sdhci_host *host,

		struct sdhci_msm_pm_qos_latency *latency, int cpu)

{

	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);

	struct sdhci_msm_host *msm_host = pltfm_host->priv;

	int group = sdhci_msm_get_cpu_group(msm_host, cpu);

	struct sdhci_msm_pm_qos_group *pm_qos_group;

	int counter;



	if (!msm_host->pm_qos_group_enable || group < 0)

		return;



	pm_qos_group = &msm_host->pm_qos[group];

	counter = atomic_inc_return(&pm_qos_group->counter);



	/* Make sure to update the voting in case power policy has changed */

	if (pm_qos_group->latency == latency->latency[host->power_policy]

		&& counter > 1)

		return;



	cancel_work_sync(&pm_qos_group->unvote_work);



	pm_qos_group->latency = latency->latency[host->power_policy];

	pm_qos_update_request(&pm_qos_group->req, pm_qos_group->latency);

}



static void sdhci_msm_pm_qos_cpu_unvote_work(struct work_struct *work)

{

	struct sdhci_msm_pm_qos_group *group =

		container_of(work, struct sdhci_msm_pm_qos_group, unvote_work);



	if (atomic_read(&group->counter))

		return;



	group->latency = PM_QOS_DEFAULT_VALUE;

	pm_qos_update_request(&group->req, group->latency);

}



void sdhci_msm_pm_qos_cpu_unvote(struct sdhci_host *host, int cpu, bool async)

{

	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);

	struct sdhci_msm_host *msm_host = pltfm_host->priv;

	int group = sdhci_msm_get_cpu_group(msm_host, cpu);



	if (!msm_host->pm_qos_group_enable || group < 0 ||

		atomic_dec_return(&msm_host->pm_qos[group].counter))

		return;



	if (async) {

		schedule_work(&msm_host->pm_qos[group].unvote_work);

		return;

	}



	msm_host->pm_qos[group].latency = PM_QOS_DEFAULT_VALUE;

	pm_qos_update_request(&msm_host->pm_qos[group].req,

				msm_host->pm_qos[group].latency);

}



void sdhci_msm_pm_qos_cpu_init(struct sdhci_host *host,

		struct sdhci_msm_pm_qos_latency *latency)

{

	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);

	struct sdhci_msm_host *msm_host = pltfm_host->priv;

	int nr_groups = msm_host->pdata->pm_qos_data.cpu_group_map.nr_groups;

	struct sdhci_msm_pm_qos_group *group;

	int i;



	if (msm_host->pm_qos_group_enable)

		return;



	msm_host->pm_qos = kcalloc(nr_groups, sizeof(*msm_host->pm_qos),

			GFP_KERNEL);

	if (!msm_host->pm_qos)

		return;



	for (i = 0; i < nr_groups; i++) {

		group = &msm_host->pm_qos[i];

		INIT_WORK(&group->unvote_work,

			sdhci_msm_pm_qos_cpu_unvote_work);

		atomic_set(&group->counter, 0);

		group->req.type = PM_QOS_REQ_AFFINE_CORES;

		cpumask_copy(&group->req.cpus_affine,

			&msm_host->pdata->pm_qos_data.cpu_group_map.mask[i]);

		/* For initialization phase, set the performance mode latency */

		group->latency = latency[i].latency[SDHCI_PERFORMANCE_MODE];

		pm_qos_add_request(&group->req, PM_QOS_CPU_DMA_LATENCY,

			group->latency);

		pr_info("%s (): voted for group #%d (mask=0x%lx) latency=%d (0x%p)\n",

			__func__, i,

			group->req.cpus_affine.bits[0],

			group->latency,

			&latency[i].latency[SDHCI_PERFORMANCE_MODE]);

	}

	msm_host->pm_qos_group_enable = true;

}



static struct sdhci_ops sdhci_msm_ops = {

	.crypto_engine_cfg = sdhci_msm_ice_cfg,

	.crypto_engine_reset = sdhci_msm_ice_reset,

drivers/mmc/host/sdhci-msm.h

+35 −0
Original line number Diff line number Diff line
@@ -105,6 +105,26 @@ struct sdhci_msm_pm_qos_data { 
	bool legacy_valid;

};



/*

 * PM QoS for group voting management - each cpu group defined is associated

 * with 1 instance of this structure.

 */

struct sdhci_msm_pm_qos_group {

	struct pm_qos_request req;

	struct work_struct unvote_work;

	atomic_t counter;

	s32 latency;

};



/* PM QoS HW IRQ voting */

struct sdhci_msm_pm_qos_irq {

	struct pm_qos_request req;

	struct work_struct unvote_work;

	atomic_t counter;

	s32 latency;

	bool enabled;

};



struct sdhci_msm_pltfm_data {

	/* Supported UHS-I Modes */

	u32 caps;
@@ -182,8 +202,23 @@ struct sdhci_msm_host { 
	u32 caps_0;

	struct sdhci_msm_ice_data ice;

	u32 ice_clk_rate;

	struct sdhci_msm_pm_qos_group *pm_qos;

	int pm_qos_prev_group;

	bool pm_qos_group_enable;

	struct sdhci_msm_pm_qos_irq pm_qos_irq;

};



extern char *saved_command_line;



void sdhci_msm_pm_qos_irq_init(struct sdhci_host *host);

void sdhci_msm_pm_qos_irq_vote(struct sdhci_host *host);

void sdhci_msm_pm_qos_irq_unvote(struct sdhci_host *host, bool async);



void sdhci_msm_pm_qos_cpu_init(struct sdhci_host *host,

		struct sdhci_msm_pm_qos_latency *latency);

void sdhci_msm_pm_qos_cpu_vote(struct sdhci_host *host,

		struct sdhci_msm_pm_qos_latency *latency, int cpu);

void sdhci_msm_pm_qos_cpu_unvote(struct sdhci_host *host, int cpu, bool async);





#endif /* __SDHCI_MSM_H__ */

include/linux/mmc/host.h

+1 −0
Original line number Diff line number Diff line
@@ -90,6 +90,7 @@ enum mmc_load { 
};



struct mmc_cmdq_host_ops {

	int (*init)(struct mmc_host *host);

	int (*enable)(struct mmc_host *host);

	void (*disable)(struct mmc_host *host, bool soft);

	int (*request)(struct mmc_host *host, struct mmc_request *mrq);