iommu/arm-smmu: Abstract GR0 accesses (00320ce6) · Commits · e / devices / android_kernel_fairphone_FP5

drivers/iommu/arm-smmu.c

+58 −48

Original line number	Original line	Diff line number	Diff line
	@@ -69,19 +69,6 @@
	/* Maximum number of context banks per SMMU */		/* Maximum number of context banks per SMMU */
	#define ARM_SMMU_MAX_CBS 128		#define ARM_SMMU_MAX_CBS 128

	/* SMMU global address space */
	#define ARM_SMMU_GR0(smmu) ((smmu)->base)

	/*
	* SMMU global address space with conditional offset to access secure
	* aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
	* nsGFSYNR0: 0x450)
	*/
	#define ARM_SMMU_GR0_NS(smmu) \
	((smmu)->base + \
	((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS) \
	? 0x400 : 0))

	#define MSI_IOVA_BASE 0x8000000		#define MSI_IOVA_BASE 0x8000000
	#define MSI_IOVA_LENGTH 0x100000		#define MSI_IOVA_LENGTH 0x100000

	@@ -246,6 +233,21 @@ struct arm_smmu_domain {
	struct iommu_domain domain;		struct iommu_domain domain;
	};		};

			static int arm_smmu_gr0_ns(int offset)
			{
			switch(offset) {
			case ARM_SMMU_GR0_sCR0:
			case ARM_SMMU_GR0_sACR:
			case ARM_SMMU_GR0_sGFSR:
			case ARM_SMMU_GR0_sGFSYNR0:
			case ARM_SMMU_GR0_sGFSYNR1:
			case ARM_SMMU_GR0_sGFSYNR2:
			return offset + 0x400;
			default:
			return offset;
			}
			}

	static void __iomem arm_smmu_page(struct arm_smmu_device smmu, int n)		static void __iomem arm_smmu_page(struct arm_smmu_device smmu, int n)
	{		{
	return smmu->base + (n << smmu->pgshift);		return smmu->base + (n << smmu->pgshift);
	@@ -253,12 +255,18 @@ static void __iomem arm_smmu_page(struct arm_smmu_device smmu, int n)

	static u32 arm_smmu_readl(struct arm_smmu_device *smmu, int page, int offset)		static u32 arm_smmu_readl(struct arm_smmu_device *smmu, int page, int offset)
	{		{
			if ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS) && page == 0)
			offset = arm_smmu_gr0_ns(offset);

	return readl_relaxed(arm_smmu_page(smmu, page) + offset);		return readl_relaxed(arm_smmu_page(smmu, page) + offset);
	}		}

	static void arm_smmu_writel(struct arm_smmu_device *smmu, int page, int offset,		static void arm_smmu_writel(struct arm_smmu_device *smmu, int page, int offset,
	u32 val)		u32 val)
	{		{
			if ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS) && page == 0)
			offset = arm_smmu_gr0_ns(offset);

	writel_relaxed(val, arm_smmu_page(smmu, page) + offset);		writel_relaxed(val, arm_smmu_page(smmu, page) + offset);
	}		}

	@@ -273,9 +281,15 @@ static void arm_smmu_writeq(struct arm_smmu_device *smmu, int page, int offset,
	writeq_relaxed(val, arm_smmu_page(smmu, page) + offset);		writeq_relaxed(val, arm_smmu_page(smmu, page) + offset);
	}		}

			#define ARM_SMMU_GR0 0
	#define ARM_SMMU_GR1 1		#define ARM_SMMU_GR1 1
	#define ARM_SMMU_CB(s, n) ((s)->numpage + (n))		#define ARM_SMMU_CB(s, n) ((s)->numpage + (n))

			#define arm_smmu_gr0_read(s, o) \
			arm_smmu_readl((s), ARM_SMMU_GR0, (o))
			#define arm_smmu_gr0_write(s, o, v) \
			arm_smmu_writel((s), ARM_SMMU_GR0, (o), (v))

	#define arm_smmu_gr1_read(s, o) \		#define arm_smmu_gr1_read(s, o) \
	arm_smmu_readl((s), ARM_SMMU_GR1, (o))		arm_smmu_readl((s), ARM_SMMU_GR1, (o))
	#define arm_smmu_gr1_write(s, o, v) \		#define arm_smmu_gr1_write(s, o, v) \
	@@ -470,7 +484,7 @@ static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
	unsigned long flags;		unsigned long flags;

	spin_lock_irqsave(&smmu->global_sync_lock, flags);		spin_lock_irqsave(&smmu->global_sync_lock, flags);
	__arm_smmu_tlb_sync(smmu, 0, ARM_SMMU_GR0_sTLBGSYNC,		__arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC,
	ARM_SMMU_GR0_sTLBGSTATUS);		ARM_SMMU_GR0_sTLBGSTATUS);
	spin_unlock_irqrestore(&smmu->global_sync_lock, flags);		spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
	}		}
	@@ -511,10 +525,10 @@ static void arm_smmu_tlb_inv_context_s2(void *cookie)
	{		{
	struct arm_smmu_domain *smmu_domain = cookie;		struct arm_smmu_domain *smmu_domain = cookie;
	struct arm_smmu_device *smmu = smmu_domain->smmu;		struct arm_smmu_device *smmu = smmu_domain->smmu;
	void __iomem *base = ARM_SMMU_GR0(smmu);

	/* NOTE: see above */		/* See above */
	writel(smmu_domain->cfg.vmid, base + ARM_SMMU_GR0_TLBIVMID);		wmb();
			arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
	arm_smmu_tlb_sync_global(smmu);		arm_smmu_tlb_sync_global(smmu);
	}		}

	@@ -579,12 +593,12 @@ static void arm_smmu_tlb_inv_vmid_nosync(unsigned long iova, size_t size,
	size_t granule, bool leaf, void *cookie)		size_t granule, bool leaf, void *cookie)
	{		{
	struct arm_smmu_domain *smmu_domain = cookie;		struct arm_smmu_domain *smmu_domain = cookie;
	void __iomem *base = ARM_SMMU_GR0(smmu_domain->smmu);		struct arm_smmu_device *smmu = smmu_domain->smmu;

	if (smmu_domain->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)		if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
	wmb();		wmb();

	writel_relaxed(smmu_domain->cfg.vmid, base + ARM_SMMU_GR0_TLBIVMID);		arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
	}		}

	static const struct iommu_gather_ops arm_smmu_s1_tlb_ops = {		static const struct iommu_gather_ops arm_smmu_s1_tlb_ops = {
	@@ -634,12 +648,11 @@ static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
	{		{
	u32 gfsr, gfsynr0, gfsynr1, gfsynr2;		u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
	struct arm_smmu_device *smmu = dev;		struct arm_smmu_device *smmu = dev;
	void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);

	gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);		gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
	gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);		gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0);
	gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);		gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1);
	gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);		gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2);

	if (!gfsr)		if (!gfsr)
	return IRQ_NONE;		return IRQ_NONE;
	@@ -650,7 +663,7 @@ static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
	"\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",		"\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
	gfsr, gfsynr0, gfsynr1, gfsynr2);		gfsr, gfsynr0, gfsynr1, gfsynr2);

	writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);		arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr);
	return IRQ_HANDLED;		return IRQ_HANDLED;
	}		}

	@@ -1056,7 +1069,7 @@ static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)

	if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)		if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
	reg \|= SMR_VALID;		reg \|= SMR_VALID;
	writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx));		arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
	}		}

	static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)		static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
	@@ -1069,7 +1082,7 @@ static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
	if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&		if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
	smmu->smrs[idx].valid)		smmu->smrs[idx].valid)
	reg \|= S2CR_EXIDVALID;		reg \|= S2CR_EXIDVALID;
	writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_S2CR(idx));		arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
	}		}

	static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)		static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
	@@ -1085,7 +1098,6 @@ static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
	*/		*/
	static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)		static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
	{		{
	void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
	u32 smr;		u32 smr;

	if (!smmu->smrs)		if (!smmu->smrs)
	@@ -1097,13 +1109,13 @@ static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
	* masters later if they try to claim IDs outside these masks.		* masters later if they try to claim IDs outside these masks.
	*/		*/
	smr = FIELD_PREP(SMR_ID, smmu->streamid_mask);		smr = FIELD_PREP(SMR_ID, smmu->streamid_mask);
	writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));		arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(0), smr);
	smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));		smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(0));
	smmu->streamid_mask = FIELD_GET(SMR_ID, smr);		smmu->streamid_mask = FIELD_GET(SMR_ID, smr);

	smr = FIELD_PREP(SMR_MASK, smmu->streamid_mask);		smr = FIELD_PREP(SMR_MASK, smmu->streamid_mask);
	writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));		arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(0), smr);
	smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));		smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(0));
	smmu->smr_mask_mask = FIELD_GET(SMR_MASK, smr);		smmu->smr_mask_mask = FIELD_GET(SMR_MASK, smr);
	}		}

	@@ -1736,13 +1748,12 @@ static struct iommu_ops arm_smmu_ops = {

	static void arm_smmu_device_reset(struct arm_smmu_device *smmu)		static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
	{		{
	void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
	int i;		int i;
	u32 reg, major;		u32 reg, major;

	/* clear global FSR */		/* clear global FSR */
	reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);		reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
	writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);		arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg);

	/*		/*
	* Reset stream mapping groups: Initial values mark all SMRn as		* Reset stream mapping groups: Initial values mark all SMRn as
	@@ -1757,9 +1768,9 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
	* clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK		* clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
	* bit is only present in MMU-500r2 onwards.		* bit is only present in MMU-500r2 onwards.
	*/		*/
	reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);		reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID7);
	major = FIELD_GET(ID7_MAJOR, reg);		major = FIELD_GET(ID7_MAJOR, reg);
	reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR);		reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sACR);
	if (major >= 2)		if (major >= 2)
	reg &= ~ARM_MMU500_ACR_CACHE_LOCK;		reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
	/*		/*
	@@ -1767,7 +1778,7 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
	* TLB entries for reduced latency.		* TLB entries for reduced latency.
	*/		*/
	reg \|= ARM_MMU500_ACR_SMTNMB_TLBEN \| ARM_MMU500_ACR_S2CRB_TLBEN;		reg \|= ARM_MMU500_ACR_SMTNMB_TLBEN \| ARM_MMU500_ACR_S2CRB_TLBEN;
	writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR);		arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sACR, reg);
	}		}

	/* Make sure all context banks are disabled and clear CB_FSR */		/* Make sure all context banks are disabled and clear CB_FSR */
	@@ -1786,10 +1797,10 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
	}		}

	/* Invalidate the TLB, just in case */		/* Invalidate the TLB, just in case */
	writel_relaxed(QCOM_DUMMY_VAL, gr0_base + ARM_SMMU_GR0_TLBIALLH);		arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL);
	writel_relaxed(QCOM_DUMMY_VAL, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);		arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL);

	reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);		reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);

	/* Enable fault reporting */		/* Enable fault reporting */
	reg \|= (sCR0_GFRE \| sCR0_GFIE \| sCR0_GCFGFRE \| sCR0_GCFGFIE);		reg \|= (sCR0_GFRE \| sCR0_GFIE \| sCR0_GCFGFRE \| sCR0_GCFGFIE);
	@@ -1818,7 +1829,7 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu)

	/* Push the button */		/* Push the button */
	arm_smmu_tlb_sync_global(smmu);		arm_smmu_tlb_sync_global(smmu);
	writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);		arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
	}		}

	static int arm_smmu_id_size_to_bits(int size)		static int arm_smmu_id_size_to_bits(int size)
	@@ -1843,7 +1854,6 @@ static int arm_smmu_id_size_to_bits(int size)
	static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)		static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
	{		{
	unsigned int size;		unsigned int size;
	void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
	u32 id;		u32 id;
	bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;		bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
	int i;		int i;
	@@ -1853,7 +1863,7 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
	smmu->version == ARM_SMMU_V2 ? 2 : 1);		smmu->version == ARM_SMMU_V2 ? 2 : 1);

	/* ID0 */		/* ID0 */
	id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);		id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);

	/* Restrict available stages based on module parameter */		/* Restrict available stages based on module parameter */
	if (force_stage == 1)		if (force_stage == 1)
	@@ -1947,7 +1957,7 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
	}		}

	/* ID1 */		/* ID1 */
	id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);		id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
	smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;		smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;

	/* Check for size mismatch of SMMU address space from mapped region */		/* Check for size mismatch of SMMU address space from mapped region */
	@@ -1985,7 +1995,7 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
	return -ENOMEM;		return -ENOMEM;

	/* ID2 */		/* ID2 */
	id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);		id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
	size = arm_smmu_id_size_to_bits(FIELD_GET(ID2_IAS, id));		size = arm_smmu_id_size_to_bits(FIELD_GET(ID2_IAS, id));
	smmu->ipa_size = size;		smmu->ipa_size = size;

	@@ -2372,7 +2382,7 @@ static void arm_smmu_device_shutdown(struct platform_device *pdev)

	arm_smmu_rpm_get(smmu);		arm_smmu_rpm_get(smmu);
	/* Turn the thing off */		/* Turn the thing off */
	writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);		arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, sCR0_CLIENTPD);
	arm_smmu_rpm_put(smmu);		arm_smmu_rpm_put(smmu);

	if (pm_runtime_enabled(smmu->dev))		if (pm_runtime_enabled(smmu->dev))