Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit efa31e1d authored by Sudarshan Rajagopalan's avatar Sudarshan Rajagopalan
Browse files

iommu/arm-smmu: replicate faulty transaction 



With hardware having the capability, replicate the faulty transaction
using ATOS to better reflect the fault transaction type with Read/Write,
Privileged/Unprivileged and Data/Instruction access. This is done by
extending the iova_to_phys_hard to account for the transaction type.

Change-Id: Ifb3150ad78f159f04da9ba02b49851e9e119e172
Signed-off-by: default avatarSudarshan Rajagopalan <sudaraja@codeaurora.org>
parent dd22fab7

drivers/iommu/arm-smmu-regs.h

+3 −0
Original line number Diff line number Diff line
@@ -258,6 +258,9 @@ enum arm_smmu_s2cr_privcfg { 
					 FSR_EF | FSR_PF | FSR_TF | FSR_IGN)



#define FSYNR0_WNR			(1 << 4)

#define FSYNR0_PNU			(1 << 5)

#define FSYNR0_IND			(1 << 6)

#define FSYNR0_NSATTR			(1 << 8)



#define IMPL_DEF1_MICRO_MMU_CTRL	0

#define MICRO_MMU_CTRL_LOCAL_HALT_REQ	(1 << 2)

drivers/iommu/arm-smmu.c

+86 −35
Original line number Diff line number Diff line
@@ -457,7 +457,7 @@ static struct arm_smmu_option_prop arm_smmu_options[] = { 
static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,

					dma_addr_t iova);

static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,

					      dma_addr_t iova);

				    dma_addr_t iova, unsigned long trans_flags);

static void arm_smmu_destroy_domain_context(struct iommu_domain *domain);



static int arm_smmu_prepare_pgtable(void *addr, void *cookie);
@@ -708,7 +708,7 @@ struct arm_smmu_arch_ops { 
	int (*init)(struct arm_smmu_device *smmu);

	void (*device_reset)(struct arm_smmu_device *smmu);

	phys_addr_t (*iova_to_phys_hard)(struct iommu_domain *domain,

					 dma_addr_t iova);

			dma_addr_t iova, unsigned long trans_flags);

	void (*init_context_bank)(struct arm_smmu_domain *smmu_domain,

					struct device *dev);

	int (*device_group)(struct device *dev, struct iommu_group *group);
@@ -1599,18 +1599,18 @@ static void print_ctx_regs(struct arm_smmu_device *smmu, struct arm_smmu_cfg 
	dev_err(smmu->dev,

		"FSR    = 0x%08x [%s%s%s%s%s%s%s%s%s%s]\n",

		fsr,

		(fsr & 0x02) ?  (fsynr0 & 0x10 ?

		(fsr & FSR_TF) ?  (fsynr0 & FSYNR0_WNR ?

				 "TF W " : "TF R ") : "",

		(fsr & 0x04) ? "AFF " : "",

		(fsr & 0x08) ? (fsynr0 & 0x10 ?

		(fsr & FSR_AFF) ? "AFF " : "",

		(fsr & FSR_PF) ? (fsynr0 & FSYNR0_WNR ?

				"PF W " : "PF R ") : "",

		(fsr & 0x10) ? "EF " : "",

		(fsr & 0x20) ? "TLBMCF " : "",

		(fsr & 0x40) ? "TLBLKF " : "",

		(fsr & 0x80) ? "MHF " : "",

		(fsr & 0x100) ? "UUT " : "",

		(fsr & 0x40000000) ? "SS " : "",

		(fsr & 0x80000000) ? "MULTI " : "");

		(fsr & FSR_EF) ? "EF " : "",

		(fsr & FSR_TLBMCF) ? "TLBMCF " : "",

		(fsr & FSR_TLBLKF) ? "TLBLKF " : "",

		(fsr & FSR_ASF) ? "ASF " : "",

		(fsr & FSR_UUT) ? "UUT " : "",

		(fsr & FSR_SS) ? "SS " : "",

		(fsr & FSR_MULTI) ? "MULTI " : "");



	if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {

		dev_err(smmu->dev, "TTBR0  = 0x%pK\n",
@@ -1641,24 +1641,56 @@ static void print_ctx_regs(struct arm_smmu_device *smmu, struct arm_smmu_cfg 
}



static phys_addr_t arm_smmu_verify_fault(struct iommu_domain *domain,

					 dma_addr_t iova, u32 fsr)

					 dma_addr_t iova, u32 fsr, u32 fsynr0)

{

	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);

	struct arm_smmu_device *smmu = smmu_domain->smmu;

	phys_addr_t phys;

	phys_addr_t phys_post_tlbiall;

	phys_addr_t phys_hard_priv = 0;

	phys_addr_t phys_stimu, phys_stimu_post_tlbiall;

	unsigned long flags = 0;



	/* Get the transaction type */

	if (fsynr0 & FSYNR0_WNR)

		flags |= IOMMU_TRANS_WRITE;

	if (fsynr0 & FSYNR0_PNU)

		flags |= IOMMU_TRANS_PRIV;

	if (fsynr0 & FSYNR0_IND)

		flags |= IOMMU_TRANS_INST;



	/* Now replicate the faulty transaction */

	phys_stimu = arm_smmu_iova_to_phys_hard(domain, iova, flags);



	/*

	 * If the replicated transaction fails, it could be due to legitimate

	 * unmapped access (translation fault) or stale TLB with insufficient

	 * privileges (permission fault). Try ATOS operation with full access

	 * privileges to rule out stale entry with insufficient privileges case.

	 */

	if (!phys_stimu)

		phys_hard_priv = arm_smmu_iova_to_phys_hard(domain, iova,

						       IOMMU_TRANS_DEFAULT |

						       IOMMU_TRANS_PRIV);



	phys = arm_smmu_iova_to_phys_hard(domain, iova);

	/* Now replicate the faulty transaction post tlbiall */

	smmu_domain->pgtbl_cfg.tlb->tlb_flush_all(smmu_domain);

	phys_post_tlbiall = arm_smmu_iova_to_phys_hard(domain, iova);

	phys_stimu_post_tlbiall = arm_smmu_iova_to_phys_hard(domain, iova,

							     flags);



	if (!phys_stimu && phys_hard_priv) {

		dev_err(smmu->dev,

			"ATOS results differed across access privileges...\n"

			"Before: %pa After: %pa\n",

			&phys_stimu, &phys_hard_priv);

	}



	if (phys != phys_post_tlbiall) {

	if (phys_stimu != phys_stimu_post_tlbiall) {

		dev_err(smmu->dev,

			"ATOS results differed across TLBIALL...\n"

			"Before: %pa After: %pa\n", &phys, &phys_post_tlbiall);

			"Before: %pa After: %pa\n", &phys_stimu,

						&phys_stimu_post_tlbiall);

	}



	return (phys == 0 ? phys_post_tlbiall : phys);

	return (phys_stimu == 0 ? phys_stimu_post_tlbiall : phys_stimu);

}



static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
@@ -1731,7 +1763,8 @@ static irqreturn_t arm_smmu_context_fault(int irq, void *dev) 
			phys_addr_t phys_atos;



			print_ctx_regs(smmu, cfg, fsr);

			phys_atos = arm_smmu_verify_fault(domain, iova, fsr);

			phys_atos = arm_smmu_verify_fault(domain, iova, fsr,

							  fsynr0);

			dev_err(smmu->dev,

				"Unhandled context fault: iova=0x%08lx, cb=%d, fsr=0x%x, fsynr0=0x%x, fsynr1=0x%x\n",

				iova, cfg->cbndx, fsr, fsynr0, fsynr1);
@@ -3424,7 +3457,7 @@ static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain, 
 * original iova_to_phys() op.

 */

static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,

					dma_addr_t iova)

				    dma_addr_t iova, unsigned long trans_flags)

{

	phys_addr_t ret = 0;

	unsigned long flags;
@@ -3440,7 +3473,7 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, 
	if (smmu_domain->smmu->arch_ops &&

	    smmu_domain->smmu->arch_ops->iova_to_phys_hard) {

		ret = smmu_domain->smmu->arch_ops->iova_to_phys_hard(

						domain, iova);

						domain, iova, trans_flags);

		goto out;

	}
@@ -4325,7 +4358,7 @@ static void qsmmuv2_device_reset(struct arm_smmu_device *smmu) 
}



static phys_addr_t qsmmuv2_iova_to_phys_hard(struct iommu_domain *domain,

				dma_addr_t iova)

				dma_addr_t iova, unsigned long trans_flags)

{

	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);

	struct arm_smmu_device *smmu = smmu_domain->smmu;
@@ -5451,8 +5484,14 @@ module_exit(arm_smmu_exit); 


#define DEBUG_TXN_TRIGG_REG		0x18

#define DEBUG_TXN_AXPROT_SHIFT		6

#define DEBUG_TXN_AXPROT_PRIV	(0x1 << DEBUG_TXN_AXPROT_SHIFT)

#define DEBUG_TXN_AXPROT_UNPRIV	(0x0 << DEBUG_TXN_AXPROT_SHIFT)

#define DEBUG_TXN_AXPROT_NSEC	(0x2 << DEBUG_TXN_AXPROT_SHIFT)

#define DEBUG_TXN_AXPROT_SEC	(0x0 << DEBUG_TXN_AXPROT_SHIFT)

#define DEBUG_TXN_AXPROT_INST	(0x4 << DEBUG_TXN_AXPROT_SHIFT)

#define DEBUG_TXN_AXPROT_DATA	(0x0 << DEBUG_TXN_AXPROT_SHIFT)

#define DEBUG_TXN_AXCACHE_SHIFT		2

#define DEBUG_TRX_WRITE			(0x1 << 1)

#define DEBUG_TXN_WRITE			(0x1 << 1)

#define DEBUG_TXN_READ			(0x0 << 1)

#define DEBUG_TXN_TRIGGER		0x1
@@ -5636,7 +5675,8 @@ static void qsmmuv500_ecats_unlock(struct arm_smmu_domain *smmu_domain, 
 * Zero means failure.

 */

static phys_addr_t qsmmuv500_iova_to_phys(

		struct iommu_domain *domain, dma_addr_t iova, u32 sid)

		struct iommu_domain *domain, dma_addr_t iova,

		u32 sid, unsigned long trans_flags)

{

	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);

	struct arm_smmu_device *smmu = smmu_domain->smmu;
@@ -5714,13 +5754,24 @@ static phys_addr_t qsmmuv500_iova_to_phys( 
		DEBUG_AXUSER_CDMID_SHIFT;

	writeq_relaxed(val, tbu->base + DEBUG_AXUSER_REG);



	/*

	 * Write-back Read and Write-Allocate

	 * Priviledged, nonsecure, data transaction

	 * Read operation.

	 */

	/* Write-back Read and Write-Allocate */

	val = 0xF << DEBUG_TXN_AXCACHE_SHIFT;

	val |= 0x3 << DEBUG_TXN_AXPROT_SHIFT;



	/* Non-secure Access */

	val |= DEBUG_TXN_AXPROT_NSEC;



	/* Write or Read Access */

	if (flags & IOMMU_TRANS_WRITE)

		val |= DEBUG_TXN_WRITE;



	/* Priviledged or Unpriviledged Access */

	if (flags & IOMMU_TRANS_PRIV)

		val |= DEBUG_TXN_AXPROT_PRIV;



	/* Data or Instruction Access */

	if (flags & IOMMU_TRANS_INST)

		val |= DEBUG_TXN_AXPROT_INST;



	val |= DEBUG_TXN_TRIGGER;

	writeq_relaxed(val, tbu->base + DEBUG_TXN_TRIGG_REG);
@@ -5800,7 +5851,8 @@ static phys_addr_t qsmmuv500_iova_to_phys( 
}



static phys_addr_t qsmmuv500_iova_to_phys_hard(

		struct iommu_domain *domain, dma_addr_t iova)

		struct iommu_domain *domain, dma_addr_t iova,

		unsigned long trans_flags)

{

	u16 sid;

	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
@@ -5810,7 +5862,6 @@ static phys_addr_t qsmmuv500_iova_to_phys_hard( 
	void __iomem *gr1_base;

	u32 frsynra;





	/* Check to see if the domain is associated with the test

	 * device. If the domain belongs to the test device, then

	 * pick the SID from fwspec.
@@ -5829,7 +5880,7 @@ static phys_addr_t qsmmuv500_iova_to_phys_hard( 
		frsynra &= CBFRSYNRA_SID_MASK;

		sid      = frsynra;

	}

	return qsmmuv500_iova_to_phys(domain, iova, sid);

	return qsmmuv500_iova_to_phys(domain, iova, sid, trans_flags);

}



static void qsmmuv500_release_group_iommudata(void *data)

drivers/iommu/iommu-debug.c

+11 −6
Original line number Diff line number Diff line
@@ -884,7 +884,8 @@ static int __rand_va_sweep(struct device *dev, struct seq_file *s, 
static int __check_mapping(struct device *dev, struct iommu_domain *domain,

			   dma_addr_t iova, phys_addr_t expected)

{

	phys_addr_t res = iommu_iova_to_phys_hard(domain, iova);

	phys_addr_t res = iommu_iova_to_phys_hard(domain, iova,

						  IOMMU_TRANS_DEFAULT);

	phys_addr_t res2 = iommu_iova_to_phys(domain, iova);



	WARN(res != res2, "hard/soft iova_to_phys fns don't agree...");
@@ -1109,7 +1110,8 @@ static int __functional_dma_api_basic_test(struct device *dev, 
		memset(data, 0xa5, size);

		iova = dma_map_single(dev, data, size, DMA_TO_DEVICE);

		pa = iommu_iova_to_phys(domain, iova);

		pa2 = iommu_iova_to_phys_hard(domain, iova);

		pa2 = iommu_iova_to_phys_hard(domain, iova,

					      IOMMU_TRANS_DEFAULT);

		if (pa != pa2) {

			dev_err_ratelimited(dev,

				"iova_to_phys doesn't match iova_to_phys_hard: %pa != %pa\n",
@@ -1182,7 +1184,8 @@ static int __functional_dma_api_map_sg_test(struct device *dev, 
		for_each_sg(table.sgl, sg, count, i) {

			iova = sg_dma_address(sg);

			pa = iommu_iova_to_phys(domain, iova);

			pa2 = iommu_iova_to_phys_hard(domain, iova);

			pa2 = iommu_iova_to_phys_hard(domain, iova,

						      IOMMU_TRANS_DEFAULT);

			if (pa != pa2) {

				dev_err_ratelimited(dev,

					"iova_to_phys doesn't match iova_to_phys_hard: %pa != %pa\n",
@@ -1193,7 +1196,8 @@ static int __functional_dma_api_map_sg_test(struct device *dev, 
			/* check mappings at end of buffer */

			iova += sg_dma_len(sg) - 1;

			pa = iommu_iova_to_phys(domain, iova);

			pa2 = iommu_iova_to_phys_hard(domain, iova);

			pa2 = iommu_iova_to_phys_hard(domain, iova,

						      IOMMU_TRANS_DEFAULT);

			if (pa != pa2) {

				dev_err_ratelimited(dev,

					"iova_to_phys doesn't match iova_to_phys_hard: %pa != %pa\n",
@@ -1533,7 +1537,8 @@ static ssize_t iommu_debug_atos_read(struct file *file, char __user *ubuf, 


	memset(buf, 0, 100);



	phys = iommu_iova_to_phys_hard(ddev->domain, ddev->iova);

	phys = iommu_iova_to_phys_hard(ddev->domain, ddev->iova,

				       IOMMU_TRANS_DEFAULT);

	if (!phys) {

		strlcpy(buf, "FAIL\n", 100);

		phys = iommu_iova_to_phys(ddev->domain, ddev->iova);
@@ -1576,7 +1581,7 @@ static ssize_t iommu_debug_dma_atos_read(struct file *file, char __user *ubuf, 
	memset(buf, 0, sizeof(buf));



	phys = iommu_iova_to_phys_hard(ddev->domain,

			ddev->iova);

			ddev->iova, IOMMU_TRANS_DEFAULT);

	if (!phys)

		strlcpy(buf, "FAIL\n", sizeof(buf));

	else

drivers/iommu/iommu.c

+2 −2
Original line number Diff line number Diff line
@@ -1509,12 +1509,12 @@ phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) 
EXPORT_SYMBOL_GPL(iommu_iova_to_phys);



phys_addr_t iommu_iova_to_phys_hard(struct iommu_domain *domain,

				    dma_addr_t iova)

				    dma_addr_t iova, unsigned long trans_flags)

{

	if (unlikely(domain->ops->iova_to_phys_hard == NULL))

		return 0;



	return domain->ops->iova_to_phys_hard(domain, iova);

	return domain->ops->iova_to_phys_hard(domain, iova, trans_flags);

}



uint64_t iommu_iova_to_pte(struct iommu_domain *domain,

include/linux/iommu.h

+12 −4
Original line number Diff line number Diff line
@@ -71,6 +71,15 @@ struct iommu_domain_geometry { 
	bool force_aperture;       /* DMA only allowed in mappable range? */

};



/* iommu transaction flags */

#define IOMMU_TRANS_WRITE	BIT(0)	/* 1 Write, 0 Read */

#define IOMMU_TRANS_PRIV	BIT(1)	/* 1 Privileged, 0 Unprivileged */

#define IOMMU_TRANS_INST	BIT(2)	/* 1 Instruction fetch, 0 Data access */

#define IOMMU_TRANS_SEC	BIT(3)	/* 1 Secure, 0 Non-secure access*/



/* Non secure unprivileged Data read operation */

#define IOMMU_TRANS_DEFAULT	(0U)



struct iommu_pgtbl_info {

	void *ops;

};
@@ -258,7 +267,7 @@ struct iommu_ops { 
	void (*iotlb_sync)(struct iommu_domain *domain);

	phys_addr_t (*iova_to_phys)(struct iommu_domain *domain, dma_addr_t iova);

	phys_addr_t (*iova_to_phys_hard)(struct iommu_domain *domain,

					 dma_addr_t iova);

				 dma_addr_t iova, unsigned long trans_flags);

	int (*add_device)(struct device *dev);

	void (*remove_device)(struct device *dev);

	struct iommu_group *(*device_group)(struct device *dev);
@@ -370,7 +379,7 @@ extern size_t default_iommu_map_sg(struct iommu_domain *domain, unsigned long io 
			   struct scatterlist *sg,unsigned int nents, int prot);

extern phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova);

extern phys_addr_t iommu_iova_to_phys_hard(struct iommu_domain *domain,

					   dma_addr_t iova);

				   dma_addr_t iova, unsigned long trans_flags);

extern bool iommu_is_iova_coherent(struct iommu_domain *domain,

				dma_addr_t iova);

extern void iommu_set_fault_handler(struct iommu_domain *domain,
@@ -606,7 +615,7 @@ static inline phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_ad 
}



static inline phys_addr_t iommu_iova_to_phys_hard(struct iommu_domain *domain,

						  dma_addr_t iova)

				dma_addr_t iova, unsigned long trans_flags)

{

	return 0;

}
@@ -626,7 +635,6 @@ static inline void iommu_get_resv_regions(struct device *dev, 
					struct list_head *list)

{

}



static inline void iommu_put_resv_regions(struct device *dev,

					struct list_head *list)

{