Merge "iommu/arm-smmu: enhance tlb sync timeout handler" (2178bbdb) · Commits · e / devices / android_kernel_oneplus_sm7250

drivers/iommu/arm-smmu-regs.h

+13 −0

Original line number	Diff line number	Diff line
		@@ -192,8 +192,21 @@ enum arm_smmu_s2cr_privcfg {
		#define ARM_SMMU_CB_ATS1PR 0x800
		#define ARM_SMMU_CB_ATSR 0x8f0
		#define ARM_SMMU_STATS_SYNC_INV_TBU_ACK 0x25dc
		#define TBU_SYNC_ACK_MASK 0x1ff
		#define TBU_SYNC_ACK_SHIFT 17
		#define TBU_SYNC_REQ_MASK 0x1
		#define TBU_SYNC_REQ_SHIFT 16
		#define TBU_INV_ACK_MASK 0x1ff
		#define TBU_INV_ACK_SHIFT 1
		#define TBU_INV_REQ_MASK 0x1
		#define TBU_INV_REQ_SHIFT 0
		#define ARM_SMMU_TBU_PWR_STATUS 0x2204
		#define ARM_SMMU_MMU2QSS_AND_SAFE_WAIT_CNTR 0x2670
		#define TCU_SYNC_IN_PRGSS_MASK 0x1
		#define TCU_SYNC_IN_PRGSS_SHIFT 20
		#define TCU_INV_IN_PRGSS_MASK 0x1
		#define TCU_INV_IN_PRGSS_SHIFT 16
		#define TBUID_SHIFT 10

		#define SCTLR_MEM_ATTR_SHIFT 16
		#define SCTLR_SHCFG_SHIFT 22

drivers/iommu/arm-smmu.c

+90 −19

Original line number	Diff line number	Diff line
		@@ -1183,19 +1183,103 @@ static void arm_smmu_testbus_dump(struct arm_smmu_device *smmu, u16 sid)
		data->tcu_base,
		tbu_testbus_sel,
		data->testbus_version);

		arm_smmu_debug_dump_tcu_testbus(smmu->dev, ARM_SMMU_GR0(smmu),
		else
		arm_smmu_debug_dump_tcu_testbus(smmu->dev,
		ARM_SMMU_GR0(smmu),
		data->tcu_base,
		tcu_testbus_sel);
		spin_unlock(&testbus_lock);
		}
		}

		static void __arm_smmu_tlb_sync_timeout(struct arm_smmu_device *smmu)
		{
		u32 sync_inv_ack, tbu_pwr_status, sync_inv_progress;
		u32 tbu_inv_pending = 0, tbu_sync_pending = 0;
		u32 tbu_inv_acked = 0, tbu_sync_acked = 0;
		u32 tcu_inv_pending = 0, tcu_sync_pending = 0;
		u32 tbu_ids = 0;
		phys_addr_t base_phys = smmu->phys_addr;


		static DEFINE_RATELIMIT_STATE(_rs,
		DEFAULT_RATELIMIT_INTERVAL,
		DEFAULT_RATELIMIT_BURST);

		sync_inv_ack = scm_io_read(base_phys + ARM_SMMU_STATS_SYNC_INV_TBU_ACK);
		tbu_pwr_status = scm_io_read(base_phys + ARM_SMMU_TBU_PWR_STATUS);
		sync_inv_progress = scm_io_read(base_phys +
		ARM_SMMU_MMU2QSS_AND_SAFE_WAIT_CNTR);

		if (sync_inv_ack) {
		tbu_inv_pending = (sync_inv_ack >> TBU_INV_REQ_SHIFT) &
		TBU_INV_REQ_MASK;
		tbu_inv_acked = (sync_inv_ack >> TBU_INV_ACK_SHIFT) &
		TBU_INV_ACK_MASK;
		tbu_sync_pending = (sync_inv_ack >> TBU_SYNC_REQ_SHIFT) &
		TBU_SYNC_REQ_MASK;
		tbu_sync_acked = (sync_inv_ack >> TBU_SYNC_ACK_SHIFT) &
		TBU_SYNC_ACK_MASK;
		}

		if (tbu_pwr_status) {
		if (tbu_sync_pending)
		tbu_ids = tbu_pwr_status & ~tbu_sync_acked;
		else if (tbu_inv_pending)
		tbu_ids = tbu_pwr_status & ~tbu_inv_acked;
		}

		tcu_inv_pending = (sync_inv_progress >> TCU_INV_IN_PRGSS_SHIFT) &
		TCU_INV_IN_PRGSS_MASK;
		tcu_sync_pending = (sync_inv_progress >> TCU_SYNC_IN_PRGSS_SHIFT) &
		TCU_SYNC_IN_PRGSS_MASK;

		if (__ratelimit(&_rs)) {
		unsigned long tbu_id, tbus_t = tbu_ids;

		dev_err(smmu->dev,
		"TLB sync timed out -- SMMU may be deadlocked\n"
		"TBU ACK 0x%x TBU PWR 0x%x TCU sync_inv 0x%x\n",
		sync_inv_ack, tbu_pwr_status, sync_inv_progress);
		dev_err(smmu->dev,
		"TCU invalidation %s, TCU sync %s\n",
		tcu_inv_pending?"pending":"completed",
		tcu_sync_pending?"pending":"completed");

		dev_err(smmu->dev, "TBU PWR status 0x%x\n", tbu_pwr_status);

		while (tbus_t) {
		struct qsmmuv500_tbu_device *tbu;

		tbu_id = __ffs(tbus_t);
		tbus_t = tbus_t & ~(1 << tbu_id);
		tbu = qsmmuv500_find_tbu(smmu,
		(u16)(tbu_id << TBUID_SHIFT));
		if (tbu) {
		dev_err(smmu->dev,
		"TBU %s ack pending for TBU %s, %s\n",
		tbu_sync_pending?"sync" : "inv",
		dev_name(tbu->dev),
		tbu_sync_pending ?
		"check pending transactions on TBU"
		: "check for TBU power status");
		arm_smmu_testbus_dump(smmu,
		(u16)(tbu_id << TBUID_SHIFT));
		}
		}

		/dump TCU testbus/
		arm_smmu_testbus_dump(smmu, U16_MAX);
		}

		BUG_ON(IS_ENABLED(CONFIG_IOMMU_TLBSYNC_DEBUG));
		}

		/* Wait for any pending TLB invalidations to complete */
		static int __arm_smmu_tlb_sync(struct arm_smmu_device *smmu,
		void __iomem sync, void __iomem status)
		{
		unsigned int spin_cnt, delay;
		u32 sync_inv_ack, tbu_pwr_status, sync_inv_progress;

		writel_relaxed(QCOM_DUMMY_VAL, sync);
		for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
		@@ -1206,16 +1290,8 @@ static int __arm_smmu_tlb_sync(struct arm_smmu_device *smmu,
		}
		udelay(delay);
		}
		sync_inv_ack = scm_io_read((unsigned long)(smmu->phys_addr +
		ARM_SMMU_STATS_SYNC_INV_TBU_ACK));
		tbu_pwr_status = scm_io_read((unsigned long)(smmu->phys_addr +
		ARM_SMMU_TBU_PWR_STATUS));
		sync_inv_progress = scm_io_read((unsigned long)(smmu->phys_addr +
		ARM_SMMU_MMU2QSS_AND_SAFE_WAIT_CNTR));
		trace_tlbsync_timeout(smmu->dev, 0);
		dev_err_ratelimited(smmu->dev,
		"TLB sync timed out -- SMMU may be deadlocked ack 0x%x pwr 0x%x sync and invalidation progress 0x%x\n",
		sync_inv_ack, tbu_pwr_status, sync_inv_progress);
		__arm_smmu_tlb_sync_timeout(smmu);
		return -EINVAL;
		}

		@@ -1229,8 +1305,6 @@ static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
		base + ARM_SMMU_GR0_sTLBGSTATUS)) {
		dev_err_ratelimited(smmu->dev,
		"TLB global sync failed!\n");
		arm_smmu_testbus_dump(smmu, U16_MAX);
		BUG_ON(IS_ENABLED(CONFIG_IOMMU_TLBSYNC_DEBUG));
		}
		spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
		}
		@@ -1239,7 +1313,6 @@ static void arm_smmu_tlb_sync_context(void *cookie)
		{
		struct arm_smmu_domain *smmu_domain = cookie;
		struct arm_smmu_device *smmu = smmu_domain->smmu;
		struct iommu_fwspec *fwspec = smmu_domain->dev->iommu_fwspec;
		void __iomem *base = ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx);
		unsigned long flags;

		@@ -1250,8 +1323,6 @@ static void arm_smmu_tlb_sync_context(void *cookie)
		"TLB sync on cb%d failed for device %s\n",
		smmu_domain->cfg.cbndx,
		dev_name(smmu_domain->dev));
		arm_smmu_testbus_dump(smmu, (u16)fwspec->ids[0]);
		BUG_ON(IS_ENABLED(CONFIG_IOMMU_TLBSYNC_DEBUG));
		}
		spin_unlock_irqrestore(&smmu_domain->sync_lock, flags);
		}