Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 8c1202be authored by Patrick Daly's avatar Patrick Daly
Browse files

iommu: arm-smmu: Support ECATS during context fault 



The TBU will complete the halt sequence only if there are no pending
transactions. This includes transactions stalled by an ongoing context
fault. When halting the TBU, disable stall on fault and fault
interrupt reporting. Clear any pending faults via terminate register.

Additionally, add an error message for maximum address range.

Change-Id: If1a6059594f8ec98c91ea0be59d8c7c6bf2f55a3
Signed-off-by: default avatarPatrick Daly <pdaly@codeaurora.org>
parent 727fcc69

drivers/iommu/arm-smmu.c

+70 −33
Original line number Diff line number Diff line
@@ -4341,7 +4341,7 @@ IOMMU_OF_DECLARE(cavium_smmuv2, "cavium,smmu-v2", arm_smmu_of_init); 
#define DEBUG_PAR_PA_SHIFT		12

#define DEBUG_PAR_FAULT_VAL		0x1



#define TBU_DBG_TIMEOUT_US		30000

#define TBU_DBG_TIMEOUT_US		100



#define QSMMUV500_ACTLR_DEEP_PREFETCH_MASK	0x3

#define QSMMUV500_ACTLR_DEEP_PREFETCH_SHIFT	0x8
@@ -4509,11 +4509,12 @@ static struct iommu_gather_ops qsmmuv500_errata1_smmu_gather_ops = { 
	.free_pages_exact = arm_smmu_free_pages_exact,

};



static int qsmmuv500_tbu_halt(struct qsmmuv500_tbu_device *tbu)

static int qsmmuv500_tbu_halt(struct qsmmuv500_tbu_device *tbu,

				struct arm_smmu_domain *smmu_domain)

{

	unsigned long flags;

	u32 val;

	void __iomem *base;

	u32 halt, fsr, sctlr_orig, sctlr, status;

	void __iomem *base, *cb_base;



	spin_lock_irqsave(&tbu->halt_lock, flags);

	if (tbu->halt_count) {
@@ -4522,19 +4523,49 @@ static int qsmmuv500_tbu_halt(struct qsmmuv500_tbu_device *tbu) 
		return 0;

	}



	cb_base = ARM_SMMU_CB_BASE(smmu_domain->smmu) +

		ARM_SMMU_CB(smmu_domain->smmu, smmu_domain->cfg.cbndx);

	base = tbu->base;

	val = readl_relaxed(base + DEBUG_SID_HALT_REG);

	val |= DEBUG_SID_HALT_VAL;

	writel_relaxed(val, base + DEBUG_SID_HALT_REG);

	halt = readl_relaxed(base + DEBUG_SID_HALT_REG);

	halt |= DEBUG_SID_HALT_VAL;

	writel_relaxed(halt, base + DEBUG_SID_HALT_REG);



	if (readl_poll_timeout_atomic(base + DEBUG_SR_HALT_ACK_REG,

					val, (val & DEBUG_SR_HALT_ACK_VAL),

					0, TBU_DBG_TIMEOUT_US)) {

	if (!readl_poll_timeout_atomic(base + DEBUG_SR_HALT_ACK_REG, status,

					(status & DEBUG_SR_HALT_ACK_VAL),

					0, TBU_DBG_TIMEOUT_US))

		goto out;



	fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);

	if (!(fsr & FSR_FAULT)) {

		dev_err(tbu->dev, "Couldn't halt TBU!\n");

		spin_unlock_irqrestore(&tbu->halt_lock, flags);

		return -ETIMEDOUT;

	}



	/*

	 * We are in a fault; Our request to halt the bus will not complete

	 * until transactions in front of us (such as the fault itself) have

	 * completed. Disable iommu faults and terminate any existing

	 * transactions.

	 */

	sctlr_orig = readl_relaxed(cb_base + ARM_SMMU_CB_SCTLR);

	sctlr = sctlr_orig & ~(SCTLR_CFCFG | SCTLR_CFIE);

	writel_relaxed(sctlr, cb_base + ARM_SMMU_CB_SCTLR);



	writel_relaxed(fsr, cb_base + ARM_SMMU_CB_FSR);

	writel_relaxed(RESUME_TERMINATE, cb_base + ARM_SMMU_CB_RESUME);



	if (readl_poll_timeout_atomic(base + DEBUG_SR_HALT_ACK_REG, status,

					(status & DEBUG_SR_HALT_ACK_VAL),

					0, TBU_DBG_TIMEOUT_US)) {

		dev_err(tbu->dev, "Couldn't halt TBU from fault context!\n");

		writel_relaxed(sctlr_orig, cb_base + ARM_SMMU_CB_SCTLR);

		spin_unlock_irqrestore(&tbu->halt_lock, flags);

		return -ETIMEDOUT;

	}



	writel_relaxed(sctlr_orig, cb_base + ARM_SMMU_CB_SCTLR);

out:

	tbu->halt_count = 1;

	spin_unlock_irqrestore(&tbu->halt_lock, flags);

	return 0;
@@ -4635,6 +4666,14 @@ static phys_addr_t qsmmuv500_iova_to_phys( 
	void __iomem *cb_base;

	u32 sctlr_orig, sctlr;

	int needs_redo = 0;

	ktime_t timeout;



	/* only 36 bit iova is supported */

	if (iova >= (1ULL << 36)) {

		dev_err_ratelimited(smmu->dev, "ECATS: address too large: %pad\n",

					&iova);

		return 0;

	}



	cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);

	tbu = qsmmuv500_find_tbu(smmu, sid);
@@ -4645,35 +4684,23 @@ static phys_addr_t qsmmuv500_iova_to_phys( 
	if (ret)

		return 0;



	/*

	 * Disable client transactions & wait for existing operations to

	 * complete.

	 */

	ret = qsmmuv500_tbu_halt(tbu);

	ret = qsmmuv500_tbu_halt(tbu, smmu_domain);

	if (ret)

		goto out_power_off;



	/* Only one concurrent atos operation */

	ret = qsmmuv500_ecats_lock(smmu_domain, tbu, &flags);

	if (ret)

		goto out_resume;



	/*

	 * We can be called from an interrupt handler with FSR already set

	 * so terminate the faulting transaction prior to starting ecats.

	 * No new racing faults can occur since we in the halted state.

	 * ECATS can trigger the fault interrupt, so disable it temporarily

	 * and check for an interrupt manually.

	 */

	fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);

	if (fsr & FSR_FAULT) {

		writel_relaxed(fsr, cb_base + ARM_SMMU_CB_FSR);

		writel_relaxed(RESUME_TERMINATE, cb_base + ARM_SMMU_CB_RESUME);

	}

	sctlr_orig = readl_relaxed(cb_base + ARM_SMMU_CB_SCTLR);

	sctlr = sctlr_orig & ~(SCTLR_CFCFG | SCTLR_CFIE);

	writel_relaxed(sctlr, cb_base + ARM_SMMU_CB_SCTLR);



	/* Only one concurrent atos operation */

	ret = qsmmuv500_ecats_lock(smmu_domain, tbu, &flags);

	if (ret)

		goto out_resume;



redo:

	/* Set address and stream-id */

	val = readq_relaxed(tbu->base + DEBUG_SID_HALT_REG);
@@ -4692,16 +4719,26 @@ static phys_addr_t qsmmuv500_iova_to_phys( 
	writeq_relaxed(val, tbu->base + DEBUG_TXN_TRIGG_REG);



	ret = 0;

	if (readl_poll_timeout_atomic(tbu->base + DEBUG_SR_HALT_ACK_REG,

				val, !(val & DEBUG_SR_ECATS_RUNNING_VAL),

				0, TBU_DBG_TIMEOUT_US)) {

	//based on readx_poll_timeout_atomic

	timeout = ktime_add_us(ktime_get(), TBU_DBG_TIMEOUT_US);

	for (;;) {

		val = readl_relaxed(tbu->base + DEBUG_SR_HALT_ACK_REG);

		if (!(val & DEBUG_SR_ECATS_RUNNING_VAL))

			break;

		val = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);

		if (val & FSR_FAULT)

			break;

		if (ktime_compare(ktime_get(), timeout) > 0) {

			dev_err(tbu->dev, "ECATS translation timed out!\n");

			ret = -ETIMEDOUT;

			break;

		}

	}



	fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);

	if (fsr & FSR_FAULT) {

		dev_err(tbu->dev, "ECATS generated a fault interrupt! FSR = %llx\n",

			val);

			fsr);

		ret = -EINVAL;



		writel_relaxed(val, cb_base + ARM_SMMU_CB_FSR);