Merge "iommu/arm-smmu: replicate faulty transaction" (b38c88d9) · Commits · e / devices / android_kernel_oneplus_sm7250

drivers/iommu/arm-smmu-regs.h

+3 −0

Original line number	Diff line number	Diff line
		@@ -258,6 +258,9 @@ enum arm_smmu_s2cr_privcfg {
		FSR_EF \| FSR_PF \| FSR_TF \| FSR_IGN)

		#define FSYNR0_WNR (1 << 4)
		#define FSYNR0_PNU (1 << 5)
		#define FSYNR0_IND (1 << 6)
		#define FSYNR0_NSATTR (1 << 8)

		#define IMPL_DEF1_MICRO_MMU_CTRL 0
		#define MICRO_MMU_CTRL_LOCAL_HALT_REQ (1 << 2)

drivers/iommu/arm-smmu.c

+86 −35

Original line number	Diff line number	Diff line
		@@ -457,7 +457,7 @@ static struct arm_smmu_option_prop arm_smmu_options[] = {
		static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
		dma_addr_t iova);
		static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
		dma_addr_t iova);
		dma_addr_t iova, unsigned long trans_flags);
		static void arm_smmu_destroy_domain_context(struct iommu_domain *domain);

		static int arm_smmu_prepare_pgtable(void addr, void cookie);
		@@ -708,7 +708,7 @@ struct arm_smmu_arch_ops {
		int (init)(struct arm_smmu_device smmu);
		void (device_reset)(struct arm_smmu_device smmu);
		phys_addr_t (iova_to_phys_hard)(struct iommu_domain domain,
		dma_addr_t iova);
		dma_addr_t iova, unsigned long trans_flags);
		void (init_context_bank)(struct arm_smmu_domain smmu_domain,
		struct device *dev);
		int (device_group)(struct device dev, struct iommu_group *group);
		@@ -1599,18 +1599,18 @@ static void print_ctx_regs(struct arm_smmu_device *smmu, struct arm_smmu_cfg
		dev_err(smmu->dev,
		"FSR = 0x%08x [%s%s%s%s%s%s%s%s%s%s]\n",
		fsr,
		(fsr & 0x02) ? (fsynr0 & 0x10 ?
		(fsr & FSR_TF) ? (fsynr0 & FSYNR0_WNR ?
		"TF W " : "TF R ") : "",
		(fsr & 0x04) ? "AFF " : "",
		(fsr & 0x08) ? (fsynr0 & 0x10 ?
		(fsr & FSR_AFF) ? "AFF " : "",
		(fsr & FSR_PF) ? (fsynr0 & FSYNR0_WNR ?
		"PF W " : "PF R ") : "",
		(fsr & 0x10) ? "EF " : "",
		(fsr & 0x20) ? "TLBMCF " : "",
		(fsr & 0x40) ? "TLBLKF " : "",
		(fsr & 0x80) ? "MHF " : "",
		(fsr & 0x100) ? "UUT " : "",
		(fsr & 0x40000000) ? "SS " : "",
		(fsr & 0x80000000) ? "MULTI " : "");
		(fsr & FSR_EF) ? "EF " : "",
		(fsr & FSR_TLBMCF) ? "TLBMCF " : "",
		(fsr & FSR_TLBLKF) ? "TLBLKF " : "",
		(fsr & FSR_ASF) ? "ASF " : "",
		(fsr & FSR_UUT) ? "UUT " : "",
		(fsr & FSR_SS) ? "SS " : "",
		(fsr & FSR_MULTI) ? "MULTI " : "");

		if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
		dev_err(smmu->dev, "TTBR0 = 0x%pK\n",
		@@ -1641,24 +1641,56 @@ static void print_ctx_regs(struct arm_smmu_device *smmu, struct arm_smmu_cfg
		}

		static phys_addr_t arm_smmu_verify_fault(struct iommu_domain *domain,
		dma_addr_t iova, u32 fsr)
		dma_addr_t iova, u32 fsr, u32 fsynr0)
		{
		struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
		struct arm_smmu_device *smmu = smmu_domain->smmu;
		phys_addr_t phys;
		phys_addr_t phys_post_tlbiall;
		phys_addr_t phys_hard_priv = 0;
		phys_addr_t phys_stimu, phys_stimu_post_tlbiall;
		unsigned long flags = 0;

		/* Get the transaction type */
		if (fsynr0 & FSYNR0_WNR)
		flags \|= IOMMU_TRANS_WRITE;
		if (fsynr0 & FSYNR0_PNU)
		flags \|= IOMMU_TRANS_PRIV;
		if (fsynr0 & FSYNR0_IND)
		flags \|= IOMMU_TRANS_INST;

		/* Now replicate the faulty transaction */
		phys_stimu = arm_smmu_iova_to_phys_hard(domain, iova, flags);

		/*
		* If the replicated transaction fails, it could be due to legitimate
		* unmapped access (translation fault) or stale TLB with insufficient
		* privileges (permission fault). Try ATOS operation with full access
		* privileges to rule out stale entry with insufficient privileges case.
		*/
		if (!phys_stimu)
		phys_hard_priv = arm_smmu_iova_to_phys_hard(domain, iova,
		IOMMU_TRANS_DEFAULT \|
		IOMMU_TRANS_PRIV);

		phys = arm_smmu_iova_to_phys_hard(domain, iova);
		/* Now replicate the faulty transaction post tlbiall */
		smmu_domain->pgtbl_cfg.tlb->tlb_flush_all(smmu_domain);
		phys_post_tlbiall = arm_smmu_iova_to_phys_hard(domain, iova);
		phys_stimu_post_tlbiall = arm_smmu_iova_to_phys_hard(domain, iova,
		flags);

		if (!phys_stimu && phys_hard_priv) {
		dev_err(smmu->dev,
		"ATOS results differed across access privileges...\n"
		"Before: %pa After: %pa\n",
		&phys_stimu, &phys_hard_priv);
		}

		if (phys != phys_post_tlbiall) {
		if (phys_stimu != phys_stimu_post_tlbiall) {
		dev_err(smmu->dev,
		"ATOS results differed across TLBIALL...\n"
		"Before: %pa After: %pa\n", &phys, &phys_post_tlbiall);
		"Before: %pa After: %pa\n", &phys_stimu,
		&phys_stimu_post_tlbiall);
		}

		return (phys == 0 ? phys_post_tlbiall : phys);
		return (phys_stimu == 0 ? phys_stimu_post_tlbiall : phys_stimu);
		}

		static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
		@@ -1731,7 +1763,8 @@ static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
		phys_addr_t phys_atos;

		print_ctx_regs(smmu, cfg, fsr);
		phys_atos = arm_smmu_verify_fault(domain, iova, fsr);
		phys_atos = arm_smmu_verify_fault(domain, iova, fsr,
		fsynr0);
		dev_err(smmu->dev,
		"Unhandled context fault: iova=0x%08lx, cb=%d, fsr=0x%x, fsynr0=0x%x, fsynr1=0x%x\n",
		iova, cfg->cbndx, fsr, fsynr0, fsynr1);
		@@ -3426,7 +3459,7 @@ static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
		* original iova_to_phys() op.
		*/
		static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
		dma_addr_t iova)
		dma_addr_t iova, unsigned long trans_flags)
		{
		phys_addr_t ret = 0;
		unsigned long flags;
		@@ -3442,7 +3475,7 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
		if (smmu_domain->smmu->arch_ops &&
		smmu_domain->smmu->arch_ops->iova_to_phys_hard) {
		ret = smmu_domain->smmu->arch_ops->iova_to_phys_hard(
		domain, iova);
		domain, iova, trans_flags);
		goto out;
		}

		@@ -4333,7 +4366,7 @@ static void qsmmuv2_device_reset(struct arm_smmu_device *smmu)
		}

		static phys_addr_t qsmmuv2_iova_to_phys_hard(struct iommu_domain *domain,
		dma_addr_t iova)
		dma_addr_t iova, unsigned long trans_flags)
		{
		struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
		struct arm_smmu_device *smmu = smmu_domain->smmu;
		@@ -5459,8 +5492,14 @@ module_exit(arm_smmu_exit);

		#define DEBUG_TXN_TRIGG_REG 0x18
		#define DEBUG_TXN_AXPROT_SHIFT 6
		#define DEBUG_TXN_AXPROT_PRIV (0x1 << DEBUG_TXN_AXPROT_SHIFT)
		#define DEBUG_TXN_AXPROT_UNPRIV (0x0 << DEBUG_TXN_AXPROT_SHIFT)
		#define DEBUG_TXN_AXPROT_NSEC (0x2 << DEBUG_TXN_AXPROT_SHIFT)
		#define DEBUG_TXN_AXPROT_SEC (0x0 << DEBUG_TXN_AXPROT_SHIFT)
		#define DEBUG_TXN_AXPROT_INST (0x4 << DEBUG_TXN_AXPROT_SHIFT)
		#define DEBUG_TXN_AXPROT_DATA (0x0 << DEBUG_TXN_AXPROT_SHIFT)
		#define DEBUG_TXN_AXCACHE_SHIFT 2
		#define DEBUG_TRX_WRITE (0x1 << 1)
		#define DEBUG_TXN_WRITE (0x1 << 1)
		#define DEBUG_TXN_READ (0x0 << 1)
		#define DEBUG_TXN_TRIGGER 0x1

		@@ -5644,7 +5683,8 @@ static void qsmmuv500_ecats_unlock(struct arm_smmu_domain *smmu_domain,
		* Zero means failure.
		*/
		static phys_addr_t qsmmuv500_iova_to_phys(
		struct iommu_domain *domain, dma_addr_t iova, u32 sid)
		struct iommu_domain *domain, dma_addr_t iova,
		u32 sid, unsigned long trans_flags)
		{
		struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
		struct arm_smmu_device *smmu = smmu_domain->smmu;
		@@ -5722,13 +5762,24 @@ static phys_addr_t qsmmuv500_iova_to_phys(
		DEBUG_AXUSER_CDMID_SHIFT;
		writeq_relaxed(val, tbu->base + DEBUG_AXUSER_REG);

		/*
		* Write-back Read and Write-Allocate
		* Priviledged, nonsecure, data transaction
		* Read operation.
		*/
		/* Write-back Read and Write-Allocate */
		val = 0xF << DEBUG_TXN_AXCACHE_SHIFT;
		val \|= 0x3 << DEBUG_TXN_AXPROT_SHIFT;

		/* Non-secure Access */
		val \|= DEBUG_TXN_AXPROT_NSEC;

		/* Write or Read Access */
		if (flags & IOMMU_TRANS_WRITE)
		val \|= DEBUG_TXN_WRITE;

		/* Priviledged or Unpriviledged Access */
		if (flags & IOMMU_TRANS_PRIV)
		val \|= DEBUG_TXN_AXPROT_PRIV;

		/* Data or Instruction Access */
		if (flags & IOMMU_TRANS_INST)
		val \|= DEBUG_TXN_AXPROT_INST;

		val \|= DEBUG_TXN_TRIGGER;
		writeq_relaxed(val, tbu->base + DEBUG_TXN_TRIGG_REG);

		@@ -5808,7 +5859,8 @@ static phys_addr_t qsmmuv500_iova_to_phys(
		}

		static phys_addr_t qsmmuv500_iova_to_phys_hard(
		struct iommu_domain *domain, dma_addr_t iova)
		struct iommu_domain *domain, dma_addr_t iova,
		unsigned long trans_flags)
		{
		u16 sid;
		struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
		@@ -5818,7 +5870,6 @@ static phys_addr_t qsmmuv500_iova_to_phys_hard(
		void __iomem *gr1_base;
		u32 frsynra;


		/* Check to see if the domain is associated with the test
		* device. If the domain belongs to the test device, then
		* pick the SID from fwspec.
		@@ -5837,7 +5888,7 @@ static phys_addr_t qsmmuv500_iova_to_phys_hard(
		frsynra &= CBFRSYNRA_SID_MASK;
		sid = frsynra;
		}
		return qsmmuv500_iova_to_phys(domain, iova, sid);
		return qsmmuv500_iova_to_phys(domain, iova, sid, trans_flags);
		}

		static void qsmmuv500_release_group_iommudata(void *data)

drivers/iommu/iommu-debug.c

+11 −6

Original line number	Diff line number	Diff line
		@@ -884,7 +884,8 @@ static int __rand_va_sweep(struct device dev, struct seq_file s,
		static int __check_mapping(struct device dev, struct iommu_domain domain,
		dma_addr_t iova, phys_addr_t expected)
		{
		phys_addr_t res = iommu_iova_to_phys_hard(domain, iova);
		phys_addr_t res = iommu_iova_to_phys_hard(domain, iova,
		IOMMU_TRANS_DEFAULT);
		phys_addr_t res2 = iommu_iova_to_phys(domain, iova);

		WARN(res != res2, "hard/soft iova_to_phys fns don't agree...");
		@@ -1109,7 +1110,8 @@ static int __functional_dma_api_basic_test(struct device *dev,
		memset(data, 0xa5, size);
		iova = dma_map_single(dev, data, size, DMA_TO_DEVICE);
		pa = iommu_iova_to_phys(domain, iova);
		pa2 = iommu_iova_to_phys_hard(domain, iova);
		pa2 = iommu_iova_to_phys_hard(domain, iova,
		IOMMU_TRANS_DEFAULT);
		if (pa != pa2) {
		dev_err_ratelimited(dev,
		"iova_to_phys doesn't match iova_to_phys_hard: %pa != %pa\n",
		@@ -1182,7 +1184,8 @@ static int __functional_dma_api_map_sg_test(struct device *dev,
		for_each_sg(table.sgl, sg, count, i) {
		iova = sg_dma_address(sg);
		pa = iommu_iova_to_phys(domain, iova);
		pa2 = iommu_iova_to_phys_hard(domain, iova);
		pa2 = iommu_iova_to_phys_hard(domain, iova,
		IOMMU_TRANS_DEFAULT);
		if (pa != pa2) {
		dev_err_ratelimited(dev,
		"iova_to_phys doesn't match iova_to_phys_hard: %pa != %pa\n",
		@@ -1193,7 +1196,8 @@ static int __functional_dma_api_map_sg_test(struct device *dev,
		/* check mappings at end of buffer */
		iova += sg_dma_len(sg) - 1;
		pa = iommu_iova_to_phys(domain, iova);
		pa2 = iommu_iova_to_phys_hard(domain, iova);
		pa2 = iommu_iova_to_phys_hard(domain, iova,
		IOMMU_TRANS_DEFAULT);
		if (pa != pa2) {
		dev_err_ratelimited(dev,
		"iova_to_phys doesn't match iova_to_phys_hard: %pa != %pa\n",
		@@ -1533,7 +1537,8 @@ static ssize_t iommu_debug_atos_read(struct file file, char __user ubuf,

		memset(buf, 0, 100);

		phys = iommu_iova_to_phys_hard(ddev->domain, ddev->iova);
		phys = iommu_iova_to_phys_hard(ddev->domain, ddev->iova,
		IOMMU_TRANS_DEFAULT);
		if (!phys) {
		strlcpy(buf, "FAIL\n", 100);
		phys = iommu_iova_to_phys(ddev->domain, ddev->iova);
		@@ -1576,7 +1581,7 @@ static ssize_t iommu_debug_dma_atos_read(struct file file, char __user ubuf,
		memset(buf, 0, sizeof(buf));

		phys = iommu_iova_to_phys_hard(ddev->domain,
		ddev->iova);
		ddev->iova, IOMMU_TRANS_DEFAULT);
		if (!phys)
		strlcpy(buf, "FAIL\n", sizeof(buf));
		else

drivers/iommu/iommu.c

+2 −2

Original line number	Diff line number	Diff line
		@@ -1509,12 +1509,12 @@ phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
		EXPORT_SYMBOL_GPL(iommu_iova_to_phys);

		phys_addr_t iommu_iova_to_phys_hard(struct iommu_domain *domain,
		dma_addr_t iova)
		dma_addr_t iova, unsigned long trans_flags)
		{
		if (unlikely(domain->ops->iova_to_phys_hard == NULL))
		return 0;

		return domain->ops->iova_to_phys_hard(domain, iova);
		return domain->ops->iova_to_phys_hard(domain, iova, trans_flags);
		}

		uint64_t iommu_iova_to_pte(struct iommu_domain *domain,

include/linux/iommu.h

+12 −4

Original line number	Diff line number	Diff line
		@@ -71,6 +71,15 @@ struct iommu_domain_geometry {
		bool force_aperture; /* DMA only allowed in mappable range? */
		};

		/* iommu transaction flags */
		#define IOMMU_TRANS_WRITE BIT(0) /* 1 Write, 0 Read */
		#define IOMMU_TRANS_PRIV BIT(1) /* 1 Privileged, 0 Unprivileged */
		#define IOMMU_TRANS_INST BIT(2) /* 1 Instruction fetch, 0 Data access */
		#define IOMMU_TRANS_SEC BIT(3) /* 1 Secure, 0 Non-secure access*/

		/* Non secure unprivileged Data read operation */
		#define IOMMU_TRANS_DEFAULT (0U)

		struct iommu_pgtbl_info {
		void *ops;
		};
		@@ -258,7 +267,7 @@ struct iommu_ops {
		void (iotlb_sync)(struct iommu_domain domain);
		phys_addr_t (iova_to_phys)(struct iommu_domain domain, dma_addr_t iova);
		phys_addr_t (iova_to_phys_hard)(struct iommu_domain domain,
		dma_addr_t iova);
		dma_addr_t iova, unsigned long trans_flags);
		int (add_device)(struct device dev);
		void (remove_device)(struct device dev);
		struct iommu_group (device_group)(struct device *dev);
		@@ -370,7 +379,7 @@ extern size_t default_iommu_map_sg(struct iommu_domain *domain, unsigned long io
		struct scatterlist *sg,unsigned int nents, int prot);
		extern phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova);
		extern phys_addr_t iommu_iova_to_phys_hard(struct iommu_domain *domain,
		dma_addr_t iova);
		dma_addr_t iova, unsigned long trans_flags);
		extern bool iommu_is_iova_coherent(struct iommu_domain *domain,
		dma_addr_t iova);
		extern void iommu_set_fault_handler(struct iommu_domain *domain,
		@@ -606,7 +615,7 @@ static inline phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_ad
		}

		static inline phys_addr_t iommu_iova_to_phys_hard(struct iommu_domain *domain,
		dma_addr_t iova)
		dma_addr_t iova, unsigned long trans_flags)
		{
		return 0;
		}
		@@ -626,7 +635,6 @@ static inline void iommu_get_resv_regions(struct device *dev,
		struct list_head *list)
		{
		}

		static inline void iommu_put_resv_regions(struct device *dev,
		struct list_head *list)
		{