Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit d5792fe6 authored by Chintan Pandya's avatar Chintan Pandya
Browse files

iommu: msm: Add support for dynamic domain switch 



Dynamic domains are used by clients like KGSL for
very fast switching of page tables for different
processes during per-process page table use-cases.

Add support for attach/detach of dynamic domains
to MSM IOMMU driver.

Change-Id: I6c9118f2be2fc883db00be31255f1e162a377128
Signed-off-by: default avatarChintan Pandya <cpandya@codeaurora.org>
parent 90c540cd

Documentation/devicetree/bindings/iommu/dynamic_domain.txt

0 → 100644
+80 −0
Original line number Diff line number Diff line 
This document describes the concept of Dynamic domains, how to use dynamic

domains and their implementation.



Dynamic Domains:

================



Some clients like graphics has many processes and hence

many domains (one for each process). While switching

between processes, context bank is updated by client

driver to point to new domain (or say TTBR0 is updated

to point to new page table). During this time, client

driver also invalidates the TLB to discard old page

table entries from TLB. This is costly operation in

time.



SMMU provides ASID tagging support with each page table.

If we tag different ASID with each domain, we can omit

TLB invalidation of out going domain. To make this

happen, client can attach one base domain and multiple

_dynamic_ domains. Dynamic domain would represent

each translation context with unique ASID on same

context bank.



How to use dynamic domains:

===========================



Following is example code of how to use dynamic domains.



void fn()

{

    struct iommu_domain *base_domain, *dyn_domain1, *dyn_domain2;



    base_domain = iommu_domain_alloc(bus);

    dyn_domain1 = iommu_domain_alloc(bus);

    dyn_domain2 = iommu_domain_alloc(bus);



    iommu_attach_device(base_domain, dev);



    iommu_domain_set_attr(dyn_domain1, DOMAIN_ATTR_DYNAMIC, &dynamic);

    iommu_domain_set_attr(dyn_domain2, DOMAIN_ATTR_DYNAMIC, &dynamic);

    iommu_attach_device(dyn_domain1, dev);

    iommu_attach_device(dyn_domain2, dev);



    while (keep_going) {

            iommu_map(dyn_domain1, ...);

            iommu_map(dyn_domain2, ...);

            use_domain(dyn_domain1, job1);

            use_domain(dyn_domain2, job2);

            iommu_unmap(dyn_domain1, ...);

            iommu_unmap(dyn_domain2, ...);

    }



    iommu_detach_device(dyn_domain2, dev);

    iommu_detach_device(dyn_domain1, dev);

    iommu_detach_device(base_domain, dev);

}



Some constraints while using dynamic domains:

============================================



* Before detaching the base_domain, we must detach all the dynamic

  domains. This is because, dynamic domain detach will invalidate

  the TLB. At the time of TLB invalidate, we really want the CB

  programming intact. And detach of base_domain will clear the CB

  programming which can cause some issues in TLB invalidation of

  dynamic domains.



* At the time of fault, the domain used by fault handler is the

  base_domain. So, any debug info which comes from attached domain

  would be from base_domain where as at the time of fault, it may

  be possible that some dynamic_domain is attached. So, don't trust

  debug info comes out from domain.



* Dynamic domains doesn't store its CB device identity. IOMMU drivers

  assumes that only one domain can be attached to one CB at any given

  time. We still attach multiple dynamic domains on top of one

  base_domain to CB. In the background, client driver uses these dynamic

  domains to switch the per-process page tables about which IOMMU driver

  can never be aware. Hence, dynamic domains never really knows its CB

  device.

drivers/iommu/msm_iommu-v1.c

+202 −10
Original line number Diff line number Diff line
@@ -43,9 +43,11 @@ 
#if defined(CONFIG_IOMMU_LPAE) || defined(CONFIG_IOMMU_AARCH64)

/* bitmap of the page sizes currently supported */

#define MSM_IOMMU_PGSIZES	(SZ_4K | SZ_64K | SZ_2M | SZ_32M | SZ_1G)

#define IS_CB_FORMAT_LONG	1

#else

/* bitmap of the page sizes currently supported */

#define MSM_IOMMU_PGSIZES	(SZ_4K | SZ_64K | SZ_1M | SZ_16M)

#define IS_CB_FORMAT_LONG	0

#endif



#define IOMMU_USEC_STEP		10
@@ -54,6 +56,9 @@ 
/* commands for SCM_SVC_SMMU_PROGRAM */

#define SMMU_CHANGE_PAGETABLE_FORMAT    0X01



/* Max ASID width is 8-bit */

#define MAX_ASID	0xff



/*

 * msm_iommu_spin_lock protects anything that can race with map

 * and unmap. msm_iommu_lock for everything else.
@@ -352,6 +357,11 @@ void iommu_resume(const struct msm_iommu_drvdata *iommu_drvdata) 
	}

}



static inline bool is_domain_dynamic(struct msm_iommu_priv *priv)

{

	return (priv->attributes & (1 << DOMAIN_ATTR_DYNAMIC));

}



static void __sync_tlb(struct msm_iommu_drvdata *iommu_drvdata, int ctx,

		struct msm_iommu_priv *priv)

{
@@ -373,11 +383,27 @@ static void __sync_tlb(struct msm_iommu_drvdata *iommu_drvdata, int ctx, 
static int __flush_iotlb(struct iommu_domain *domain)

{

	struct msm_iommu_priv *priv = domain->priv;

	struct msm_iommu_priv *base_priv;

	struct msm_iommu_drvdata *iommu_drvdata;

	struct msm_iommu_ctx_drvdata *ctx_drvdata;

	int ret = 0;



	list_for_each_entry(ctx_drvdata, &priv->list_attached, attached_elm) {

	/*

	 * Context banks are properly attached to base domain and not dynamic

	 * domains. So, we must get the base domain and CBs attached to it

	 * for TLB invalidation.

	 */

	if (is_domain_dynamic(priv)) {

		if (!priv->base)

			return 0;



		base_priv = priv->base->priv;

	} else {

		base_priv = priv;

	}



	list_for_each_entry(ctx_drvdata, &base_priv->list_attached,

			attached_elm) {

		BUG_ON(!ctx_drvdata->pdev || !ctx_drvdata->pdev->dev.parent);



		iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent);
@@ -388,7 +414,7 @@ static int __flush_iotlb(struct iommu_domain *domain) 
			goto fail;



		SET_TLBIASID(iommu_drvdata->cb_base, ctx_drvdata->num,

			     ctx_drvdata->asid);

			     priv->asid);

		__sync_tlb(iommu_drvdata, ctx_drvdata->num, priv);

		__disable_clocks(iommu_drvdata);

	}
@@ -400,11 +426,22 @@ fail: 
static int __flush_iotlb_va(struct iommu_domain *domain, unsigned long va)

{

	struct msm_iommu_priv *priv = domain->priv;

	struct msm_iommu_priv *base_priv;

	struct msm_iommu_drvdata *iommu_drvdata;

	struct msm_iommu_ctx_drvdata *ctx_drvdata;

	int ret = 0;



	list_for_each_entry(ctx_drvdata, &priv->list_attached, attached_elm) {

	if (is_domain_dynamic(priv)) {

		if (!priv->base)

			return 0;



		base_priv = priv->base->priv;

	} else {

		base_priv = priv;

	}



	list_for_each_entry(ctx_drvdata, &base_priv->list_attached,

			attached_elm) {

		BUG_ON(!ctx_drvdata->pdev || !ctx_drvdata->pdev->dev.parent);



		iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent);
@@ -415,7 +452,7 @@ static int __flush_iotlb_va(struct iommu_domain *domain, unsigned long va) 
			goto fail;



		SET_TLBIVA(iommu_drvdata->cb_base, ctx_drvdata->num,

				ctx_drvdata->asid | (va & CB_TLBIVA_VA));

				priv->asid | (va & CB_TLBIVA_VA));

		__sync_tlb(iommu_drvdata, ctx_drvdata->num, priv);

		__disable_clocks(iommu_drvdata);

	}
@@ -617,6 +654,12 @@ static void __set_cb_format(struct msm_iommu_drvdata *iommu_drvdata, 
{

}



static u64 get_full_ttbr0(struct msm_iommu_priv *priv)

{

	return (virt_to_phys(priv->pt.fl_table) |

			(priv->asid << CB_TTBR0_ASID_SHIFT));

}



static void msm_iommu_set_ASID(void __iomem *base, unsigned int ctx_num,

			       unsigned int asid)

{
@@ -694,6 +737,12 @@ static void __set_cb_format(struct msm_iommu_drvdata *iommu_drvdata, 
	}

}



static u64 get_full_ttbr0(struct msm_iommu_priv *priv)

{

	return (virt_to_phys(priv->pt.fl_table) |

			(priv->asid << CB_TTBR0_ASID_SHIFT));

}



static void msm_iommu_set_ASID(void __iomem *base, unsigned int ctx_num,

			       unsigned int asid)

{
@@ -742,6 +791,11 @@ static void __set_cb_format(struct msm_iommu_drvdata *iommu_drvdata, 
{

}



static u64 get_full_ttbr0(struct msm_iommu_priv *priv)

{

	return virt_to_phys(priv->pt.fl_table);

}



static void msm_iommu_set_ASID(void __iomem *base, unsigned int ctx_num,

			       unsigned int asid)

{
@@ -756,6 +810,13 @@ static void msm_iommu_assign_ASID(const struct msm_iommu_drvdata *iommu_drvdata, 
	void __iomem *cb_base = iommu_drvdata->cb_base;



	curr_ctx->asid = curr_ctx->num;



	/*

	 * Domain also keeps the ASID info separately. This is because with

	 * dynamic domain, each domain will have different ASID but their

	 * attached CB is the same

	 */

	priv->asid = curr_ctx->num;

	msm_iommu_set_ASID(cb_base, curr_ctx->num, curr_ctx->asid);

}
@@ -925,6 +986,34 @@ static void msm_iommu_domain_destroy(struct iommu_domain *domain) 
	mutex_unlock(&msm_iommu_lock);

}



static int msm_iommu_dynamic_attach(struct iommu_domain *domain,

				struct msm_iommu_drvdata *iommu_drvdata,

				struct msm_iommu_ctx_drvdata *ctx_drvdata)

{

	int ret;

	struct msm_iommu_priv *priv;



	priv = domain->priv;



	/* Check if the domain is already attached or not */

	if (priv->asid < MAX_ASID)

		return -EBUSY;



	ret = idr_alloc_cyclic(&iommu_drvdata->asid_idr, priv,

			iommu_drvdata->ncb + 2, MAX_ASID + 1, GFP_KERNEL);



	if (ret < 0)

		return -ENOSPC;



	priv->asid = ret;

	priv->base = ctx_drvdata->attached_domain;



	/* Once the CB is dynamic, it is always dynamic */

	ctx_drvdata->dynamic = true;



	return 0;

}



static int msm_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)

{

	struct msm_iommu_priv *priv;
@@ -951,6 +1040,13 @@ static int msm_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) 
		goto unlock;

	}



	if (is_domain_dynamic(priv)) {

		ret = msm_iommu_dynamic_attach(domain,

				iommu_drvdata, ctx_drvdata);

		mutex_unlock(&msm_iommu_lock);

		return ret;

	}



	++ctx_drvdata->attach_count;



	if (ctx_drvdata->attach_count > 1)
@@ -1039,6 +1135,33 @@ unlock: 
	return ret;

}



static void msm_iommu_dynamic_detach(struct iommu_domain *domain,

				struct msm_iommu_drvdata *iommu_drvdata,

				struct msm_iommu_ctx_drvdata *ctx_drvdata)

{

	int ret;

	struct msm_iommu_priv *priv;



	priv = domain->priv;

	if (ctx_drvdata->attach_count > 0) {

		ret = __enable_clocks(iommu_drvdata);

		if (ret)

			return;



		SET_TLBIASID(iommu_drvdata->cb_base, ctx_drvdata->num,

			     priv->asid);

		__sync_tlb(iommu_drvdata, ctx_drvdata->num, priv);

		__disable_clocks(iommu_drvdata);

	}



	BUG_ON(priv->asid == -1);



	idr_remove(&iommu_drvdata->asid_idr, priv->asid);



	priv->asid = (-1);

	priv->base = NULL;

}



static void msm_iommu_detach_dev(struct iommu_domain *domain,

				 struct device *dev)

{
@@ -1064,6 +1187,13 @@ static void msm_iommu_detach_dev(struct iommu_domain *domain, 
	if (!iommu_drvdata || !ctx_drvdata || !ctx_drvdata->attached_domain)

		goto unlock;



	if (is_domain_dynamic(priv)) {

		msm_iommu_dynamic_detach(domain,

				iommu_drvdata, ctx_drvdata);

		mutex_unlock(&msm_iommu_lock);

		return;

	}



	--ctx_drvdata->attach_count;

	BUG_ON(ctx_drvdata->attach_count < 0);
@@ -1235,11 +1365,9 @@ static phys_addr_t msm_iommu_iova_to_phys(struct iommu_domain *domain, 
	phys_addr_t ret;

	unsigned long flags;



	mutex_lock(&msm_iommu_lock);

	spin_lock_irqsave(&msm_iommu_spin_lock, flags);

	ret = msm_iommu_iova_to_phys_soft(domain, va);

	spin_unlock_irqrestore(&msm_iommu_spin_lock, flags);

	mutex_unlock(&msm_iommu_lock);

	return ret;

}
@@ -1262,11 +1390,15 @@ static phys_addr_t msm_iommu_iova_to_phys_hard(struct iommu_domain *domain, 
	if (list_empty(&priv->list_attached))

		goto fail;





	spin_lock_irqsave(&msm_iommu_spin_lock, flags);

	ctx_drvdata = list_entry(priv->list_attached.next,

				 struct msm_iommu_ctx_drvdata, attached_elm);

	spin_unlock_irqrestore(&msm_iommu_spin_lock, flags);



	if (is_domain_dynamic(priv) || ctx_drvdata->dynamic)

		goto fail;



	iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent);



	if (iommu_drvdata->model == MMU_500) {
@@ -1667,10 +1799,39 @@ static void __do_get_redirect(struct iommu_domain *domain, void *data) 
static int msm_iommu_domain_set_attr(struct iommu_domain *domain,

				enum iommu_attr attr, void *data)

{

	struct msm_iommu_priv *priv = domain->priv;

	struct msm_iommu_ctx_drvdata *ctx_drvdata = NULL;

	int dynamic;



	if (!list_empty(&priv->list_attached)) {

		ctx_drvdata = list_first_entry(&priv->list_attached,

			struct msm_iommu_ctx_drvdata, attached_elm);

	}



	switch (attr) {

	case DOMAIN_ATTR_COHERENT_HTW_DISABLE:

		__do_set_redirect(domain, data);

		break;

	case DOMAIN_ATTR_PROCID:

		priv->procid = *((u32 *)data);

		break;

	case DOMAIN_ATTR_DYNAMIC:

		dynamic = *((int *)data);



		if (ctx_drvdata)

			return -EBUSY;



		if (dynamic)

			priv->attributes |= 1 << DOMAIN_ATTR_DYNAMIC;

		else

			priv->attributes &= ~(1 << DOMAIN_ATTR_DYNAMIC);

		break;

	case DOMAIN_ATTR_CONTEXT_BANK:

		/*

		 * We don't need to do anything here because CB allocation

		 * is not dynamic in this driver.

		 */

		break;

	default:

		return -EINVAL;

	}
@@ -1681,7 +1842,13 @@ static int msm_iommu_domain_get_attr(struct iommu_domain *domain, 
				enum iommu_attr attr, void *data)

{

	struct msm_iommu_priv *priv = domain->priv;

	struct msm_iommu_ctx_drvdata *ctx_drvdata;

	struct msm_iommu_ctx_drvdata *ctx_drvdata = NULL;

	u64 ttbr0;

	u32 ctxidr;



	if (!list_empty(&priv->list_attached))

		ctx_drvdata = list_first_entry(&priv->list_attached,

			struct msm_iommu_ctx_drvdata, attached_elm);



	switch (attr) {

	case DOMAIN_ATTR_COHERENT_HTW_DISABLE:
@@ -1691,13 +1858,38 @@ static int msm_iommu_domain_get_attr(struct iommu_domain *domain, 
		*((phys_addr_t *)data) = virt_to_phys(priv->pt.fl_table);

		break;

	case DOMAIN_ATTR_CONTEXT_BANK:

		if (list_empty(&priv->list_attached))

		if (!ctx_drvdata)

			return -ENODEV;



		ctx_drvdata = list_first_entry(&priv->list_attached,

			struct msm_iommu_ctx_drvdata, attached_elm);

		*((unsigned int *) data) = ctx_drvdata->num;

		break;

	case DOMAIN_ATTR_TTBR0:

		if (!ctx_drvdata)

			return -ENODEV;



		ttbr0 = get_full_ttbr0(priv);



		*((u64 *)data) = ttbr0;

		break;

	case DOMAIN_ATTR_CONTEXTIDR:

		if (!ctx_drvdata)

			return -ENODEV;



		if (IS_CB_FORMAT_LONG)

			ctxidr = priv->procid;

		else

			ctxidr = (priv->asid & CB_CONTEXTIDR_ASID_MASK) |

				(priv->procid << CB_CONTEXTIDR_PROCID_SHIFT);



		*((u32 *)data) = ctxidr;

		break;

	case DOMAIN_ATTR_PROCID:

		*((u32 *)data) = priv->procid;

		break;

	case DOMAIN_ATTR_DYNAMIC:

		*((int *)data) = !!(priv->attributes

					& (1 << DOMAIN_ATTR_DYNAMIC));

		break;

	default:

		return -EINVAL;

	}

drivers/iommu/msm_iommu_dev-v1.c

+3 −0
Original line number Diff line number Diff line
@@ -483,6 +483,8 @@ static int msm_iommu_probe(struct platform_device *pdev) 
					global_client_irq, ret);

	}



	idr_init(&drvdata->asid_idr);



	ret = of_platform_populate(pdev->dev.of_node, msm_iommu_ctx_match_table,

				   NULL, &pdev->dev);

fail:
@@ -506,6 +508,7 @@ static int msm_iommu_remove(struct platform_device *pdev) 


	drv = platform_get_drvdata(pdev);

	if (drv) {

		idr_destroy(&drv->asid_idr);

		__put_bus_vote_client(drv);

		clk_unprepare(drv->clk);

		clk_unprepare(drv->pclk);

drivers/iommu/msm_iommu_priv.h

+8 −0
Original line number Diff line number Diff line
@@ -55,11 +55,19 @@ struct msm_iommu_pt { 
 * pt: Page table attribute structure

 * list_attached: List of devices (contexts) attached to this domain.

 * client_name: Name of the domain client.

 * procid: Procid used by the clients

 * asid: Unique asid assigned to each domain

 * attributes: Attributes associated with domains, like DYNAMIC attributes

 * base: If the domain is dynamic in nature, it must point to its base domain

 */

struct msm_iommu_priv {

	struct msm_iommu_pt pt;

	struct list_head list_attached;

	const char *client_name;

	u32 procid;

	u32 asid;

	u32 attributes;

	struct iommu_domain *base;

};



#endif

include/linux/qcom_iommu.h

+4 −0
Original line number Diff line number Diff line
@@ -18,6 +18,7 @@ 
#include <linux/list.h>

#include <linux/regulator/consumer.h>

#include <linux/platform_device.h>

#include <linux/idr.h>

#include <soc/qcom/socinfo.h>



extern pgprot_t     pgprot_kernel;
@@ -137,6 +138,7 @@ struct msm_iommu_drvdata { 
	int needs_rem_spinlock;

	int powered_on;

	unsigned int model;

	struct idr asid_idr;

};



/**
@@ -184,6 +186,7 @@ void iommu_resume(const struct msm_iommu_drvdata *iommu_drvdata); 
 * @asid		ASID used with this context.

 * @attach_count	Number of time this context has been attached.

 * @report_error_on_fault - true if error is returned back to master

 * @dynamic		true if any dynamic domain is ever attached to this CB

 *

 * A msm_iommu_ctx_drvdata holds the driver data for a single context bank

 * within each IOMMU hardware instance
@@ -203,6 +206,7 @@ struct msm_iommu_ctx_drvdata { 
	unsigned int n_sid_mask;

	bool report_error_on_fault;

	unsigned int prefetch_depth;

	bool dynamic;

};



enum dump_reg {