Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 37294737 authored by Isaac J. Manjarres's avatar Isaac J. Manjarres
Browse files

iommu/arm-smmu: Prepare to migrate QSMMUV500 implementation details 



All of the implementation details related to the QSMMUV500 model
of the SMMU-V2 architecture currently reside in arm-smmu.c.
This clashes with the current ARM SMMU SW architecture, as
arm-smmu.c is meant to house architecture specific details,
and arm-smmu-[$VENDOR].c is meant to house vendor specific
implementation details about those architectures.

To prepare to move to comply with this architecture, migrate
the QSMMUV500 related logic to arm-smmu-qcom.c, as well as
export any shared data structures. This patch should not
introduce any functional changes.

Change-Id: I711e41e38cd35b6727faf5a399eefcf5db453612
Signed-off-by: default avatarIsaac J. Manjarres <isaacm@codeaurora.org>
parent 816c0efa

drivers/iommu/arm-smmu-impl.c

+0 −1
Original line number Diff line number Diff line
@@ -105,7 +105,6 @@ static struct arm_smmu_device *cavium_smmu_impl_init(struct arm_smmu_device *smm 


#define ARM_MMU500_ACTLR_CPRE		(1 << 1)



#define ARM_MMU500_ACR_CACHE_LOCK	(1 << 26)

#define ARM_MMU500_ACR_S2CRB_TLBEN	(1 << 10)

#define ARM_MMU500_ACR_SMTNMB_TLBEN	(1 << 8)

drivers/iommu/arm-smmu-qcom.c

+830 −1
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0-only

/*

 * Copyright (c) 2019, The Linux Foundation. All rights reserved.

 * Copyright (c) 2019-2020, The Linux Foundation. All rights reserved.

 */



#include <linux/bitfield.h>

#include <linux/io.h>

#include <linux/iopoll.h>

#include <linux/of.h>

#include <linux/of_platform.h>

#include <linux/platform_device.h>

#include <linux/qcom_scm.h>

#include <linux/slab.h>

#include <linux/workqueue.h>



#include "arm-smmu.h"
@@ -69,6 +76,828 @@ static const struct arm_smmu_impl qcom_smmu_impl = { 
	.reset = qcom_sdm845_smmu500_reset,

};



#define TCU_HW_VERSION_HLOS1		(0x18)



#define DEBUG_SID_HALT_REG		0x0

#define DEBUG_SID_HALT_REQ		BIT(16)

#define DEBUG_SID_HALT_SID		GENMASK(9, 0)



#define DEBUG_VA_ADDR_REG		0x8



#define DEBUG_TXN_TRIGG_REG		0x18

#define DEBUG_TXN_AXPROT		GENMASK(8, 6)

#define DEBUG_TXN_AXCACHE		GENMASK(5, 2)

#define DEBUG_TXN_WRITE			BIT(1)

#define DEBUG_TXN_AXPROT_PRIV		0x1

#define DEBUG_TXN_AXPROT_UNPRIV		0x0

#define DEBUG_TXN_AXPROT_NSEC		0x2

#define DEBUG_TXN_AXPROT_SEC		0x0

#define DEBUG_TXN_AXPROT_INST		0x4

#define DEBUG_TXN_AXPROT_DATA		0x0

#define DEBUG_TXN_READ			(0x0 << 1)

#define DEBUG_TXN_TRIGGER		BIT(0)



#define DEBUG_SR_HALT_ACK_REG		0x20

#define DEBUG_SR_HALT_ACK_VAL		(0x1 << 1)

#define DEBUG_SR_ECATS_RUNNING_VAL	(0x1 << 0)



#define DEBUG_PAR_REG			0x28

#define DEBUG_PAR_PA			GENMASK_ULL(47, 12)

#define DEBUG_PAR_FAULT_VAL		BIT(0)



#define DEBUG_AXUSER_REG		0x30

#define DEBUG_AXUSER_CDMID		GENMASK_ULL(43, 36)

#define DEBUG_AXUSER_CDMID_VAL          255



#define TBU_DBG_TIMEOUT_US		100



#define TNX_TCR_CNTL			0x130

#define TNX_TCR_CNTL_TBU_OT_CAPTURE_EN	BIT(18)

#define TNX_TCR_CNTL_ALWAYS_CAPTURE	BIT(15)

#define TNX_TCR_CNTL_MATCH_MASK_UPD	BIT(7)

#define TNX_TCR_CNTL_MATCH_MASK_VALID	BIT(6)



#define CAPTURE1_SNAPSHOT_1		0x138



#define TNX_TCR_CNTL_2			0x178

#define TNX_TCR_CNTL_2_CAP1_VALID	BIT(0)



struct actlr_setting {

	struct arm_smmu_smr smr;

	u32 actlr;

};



struct qsmmuv500_archdata {

	struct list_head		tbus;

	void __iomem			*tcu_base;

	u32				version;

	struct actlr_setting		*actlrs;

	u32				actlr_tbl_size;

	struct work_struct		outstanding_tnx_work;

	struct arm_smmu_device		*smmu;

};

#define get_qsmmuv500_archdata(smmu)				\

	((struct qsmmuv500_archdata *)(smmu->archdata))



struct qsmmuv500_group_iommudata {

	bool has_actlr;

	u32 actlr;

};

#define to_qsmmuv500_group_iommudata(group)				\

	((struct qsmmuv500_group_iommudata *)				\

		(iommu_group_get_iommudata(group)))



static struct qsmmuv500_tbu_device *qsmmuv500_find_tbu(

	struct arm_smmu_device *smmu, u32 sid);



/*

 * Provides mutually exclusive access to the registers used by the

 * outstanding transaction snapshot feature and the transaction

 * snapshot capture feature.

 */

static DEFINE_MUTEX(capture_reg_lock);



static void qsmmuv500_log_outstanding_transactions(struct work_struct *work)

{

	struct qsmmuv500_tbu_device *tbu = NULL;

	u64 outstanding_tnxs;

	u64 tcr_cntl_val, res;

	struct qsmmuv500_archdata *data = container_of(work,

						struct qsmmuv500_archdata,

						outstanding_tnx_work);

	struct arm_smmu_device *smmu = data->smmu;

	void __iomem *base;



	if (!mutex_trylock(&capture_reg_lock)) {

		dev_warn_ratelimited(smmu->dev,

			"Tnx snapshot regs in use, not dumping OT tnxs.\n");

		goto bug;

	}



	if (arm_smmu_power_on(smmu->pwr)) {

		dev_err_ratelimited(smmu->dev,

				    "%s: Failed to power on SMMU.\n",

				    __func__);

		goto unlock;

	}



	list_for_each_entry(tbu, &data->tbus, list) {

		if (arm_smmu_power_on(tbu->pwr)) {

			dev_err_ratelimited(tbu->dev,

					    "%s: Failed to power on TBU.\n",

					    __func__);

			continue;

		}

		base = tbu->base;



		tcr_cntl_val = readq_relaxed(base + TNX_TCR_CNTL);



		/* Write 1 into MATCH_MASK_UPD of TNX_TCR_CNTL */

		writeq_relaxed(tcr_cntl_val | TNX_TCR_CNTL_MATCH_MASK_UPD,

			       base + TNX_TCR_CNTL);



		/*

		 * Simultaneously write 0 into MATCH_MASK_UPD, 0 into

		 * ALWAYS_CAPTURE, 0 into MATCH_MASK_VALID, and 1 into

		 * TBU_OT_CAPTURE_EN of TNX_TCR_CNTL

		 */

		tcr_cntl_val &= ~(TNX_TCR_CNTL_MATCH_MASK_UPD |

				  TNX_TCR_CNTL_ALWAYS_CAPTURE |

				  TNX_TCR_CNTL_MATCH_MASK_VALID);

		writeq_relaxed(tcr_cntl_val | TNX_TCR_CNTL_TBU_OT_CAPTURE_EN,

			       base + TNX_TCR_CNTL);



		/* Poll for CAPTURE1_VALID to become 1 on TNX_TCR_CNTL_2 */

		if (readq_poll_timeout_atomic(base + TNX_TCR_CNTL_2, res,

					      res & TNX_TCR_CNTL_2_CAP1_VALID,

					      0, TBU_DBG_TIMEOUT_US)) {

			dev_err_ratelimited(tbu->dev,

					    "Timeout on TNX snapshot poll\n");

			goto poll_timeout;

		}



		/* Read Register CAPTURE1_SNAPSHOT_1 */

		outstanding_tnxs = readq_relaxed(base + CAPTURE1_SNAPSHOT_1);

		dev_err_ratelimited(tbu->dev,

				    "Outstanding Transaction Bitmap: 0x%llx\n",

				    outstanding_tnxs);

poll_timeout:

		/* Write TBU_OT_CAPTURE_EN to 0 of TNX_TCR_CNTL */

		writeq_relaxed(tcr_cntl_val & ~TNX_TCR_CNTL_TBU_OT_CAPTURE_EN,

			       tbu->base + TNX_TCR_CNTL);



		arm_smmu_power_off(smmu, tbu->pwr);

	}



	arm_smmu_power_off(smmu, smmu->pwr);

unlock:

	mutex_unlock(&capture_reg_lock);

bug:

	BUG_ON(IS_ENABLED(CONFIG_IOMMU_TLBSYNC_DEBUG));

}



static void qsmmuv500_tlb_sync_timeout(struct arm_smmu_device *smmu)

{

	u32 sync_inv_ack, tbu_pwr_status, sync_inv_progress;

	u32 tbu_inv_pending = 0, tbu_sync_pending = 0;

	u32 tbu_inv_acked = 0, tbu_sync_acked = 0;

	u32 tcu_inv_pending = 0, tcu_sync_pending = 0;

	u32 tbu_ids = 0;

	struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu);

	int ret;



	static DEFINE_RATELIMIT_STATE(_rs,

				      DEFAULT_RATELIMIT_INTERVAL,

				      DEFAULT_RATELIMIT_BURST);



	dev_err_ratelimited(smmu->dev,

			    "TLB sync timed out -- SMMU may be deadlocked\n");



	sync_inv_ack = arm_smmu_readl(smmu,

				      ARM_SMMU_IMPL_DEF0,

				      ARM_SMMU_STATS_SYNC_INV_TBU_ACK);

	ret = qcom_scm_io_readl((unsigned long)(smmu->phys_addr +

				ARM_SMMU_TBU_PWR_STATUS), &tbu_pwr_status);

	if (ret) {

		dev_err_ratelimited(smmu->dev,

				    "SCM read of TBU power status fails: %d\n",

				    ret);

		goto out;

	}



	ret = qcom_scm_io_readl((unsigned long)(smmu->phys_addr +

				ARM_SMMU_MMU2QSS_AND_SAFE_WAIT_CNTR),

				&sync_inv_progress);

	if (ret) {

		dev_err_ratelimited(smmu->dev,

				    "SCM read of TBU sync/inv prog fails: %d\n",

				    ret);

		goto out;

	}



	if (sync_inv_ack) {

		tbu_inv_pending = FIELD_GET(TBU_INV_REQ, sync_inv_ack);

		tbu_inv_acked = FIELD_GET(TBU_INV_ACK, sync_inv_ack);

		tbu_sync_pending = FIELD_GET(TBU_SYNC_REQ, sync_inv_ack);

		tbu_sync_acked = FIELD_GET(TBU_SYNC_ACK, sync_inv_ack);

	}



	if (tbu_pwr_status) {

		if (tbu_sync_pending)

			tbu_ids = tbu_pwr_status & ~tbu_sync_acked;

		else if (tbu_inv_pending)

			tbu_ids = tbu_pwr_status & ~tbu_inv_acked;

	}



	tcu_inv_pending = FIELD_GET(TCU_INV_IN_PRGSS, sync_inv_progress);

	tcu_sync_pending = FIELD_GET(TCU_SYNC_IN_PRGSS, sync_inv_progress);



	if (__ratelimit(&_rs)) {

		unsigned long tbu_id;



		dev_err(smmu->dev,

			"TBU ACK 0x%x TBU PWR 0x%x TCU sync_inv 0x%x\n",

			sync_inv_ack, tbu_pwr_status, sync_inv_progress);

		dev_err(smmu->dev,

			"TCU invalidation %s, TCU sync %s\n",

			tcu_inv_pending?"pending":"completed",

			tcu_sync_pending?"pending":"completed");



		for_each_set_bit(tbu_id, (unsigned long *) &tbu_ids,

				 sizeof(tbu_ids) * BITS_PER_BYTE) {



			struct qsmmuv500_tbu_device *tbu;



			tbu = qsmmuv500_find_tbu(smmu,

						 (u16)(tbu_id << TBUID_SHIFT));

			if (tbu) {

				dev_err(smmu->dev,

					"TBU %s ack pending for TBU %s, %s\n",

					tbu_sync_pending?"sync" : "inv",

					dev_name(tbu->dev),

					tbu_sync_pending ?

					"check pending transactions on TBU"

					: "check for TBU power status");

			}

		}

	}



	if (tcu_sync_pending) {

		schedule_work(&data->outstanding_tnx_work);

		return;

	}

out:

	BUG_ON(IS_ENABLED(CONFIG_IOMMU_TLBSYNC_DEBUG));

}



static void qsmmuv500_device_remove(struct arm_smmu_device *smmu)

{

	struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu);



	cancel_work_sync(&data->outstanding_tnx_work);

}



static bool arm_smmu_fwspec_match_smr(struct iommu_fwspec *fwspec,

				      struct arm_smmu_smr *smr)

{

	struct arm_smmu_smr *smr2;

	struct arm_smmu_device *smmu = fwspec_smmu(fwspec);

	int i, idx;



	for_each_cfg_sme(fwspec, i, idx) {

		smr2 = &smmu->smrs[idx];

		/* Continue if table entry does not match */

		if ((smr->id ^ smr2->id) & ~(smr->mask | smr2->mask))

			continue;

		return true;

	}

	return false;

}



static int qsmmuv500_tbu_halt(struct qsmmuv500_tbu_device *tbu,

				struct arm_smmu_domain *smmu_domain)

{

	unsigned long flags;

	struct arm_smmu_device *smmu = smmu_domain->smmu;

	int idx = smmu_domain->cfg.cbndx;

	u32 halt, fsr, status;

	void __iomem *tbu_base;



	if (of_property_read_bool(tbu->dev->of_node,

						"qcom,opt-out-tbu-halting")) {

		dev_notice(tbu->dev, "TBU opted-out for halting!\n");

		return -EBUSY;

	}



	spin_lock_irqsave(&tbu->halt_lock, flags);

	if (tbu->halt_count) {

		tbu->halt_count++;

		spin_unlock_irqrestore(&tbu->halt_lock, flags);

		return 0;

	}



	tbu_base = tbu->base;

	halt = readl_relaxed(tbu_base + DEBUG_SID_HALT_REG);

	halt |= DEBUG_SID_HALT_REQ;

	writel_relaxed(halt, tbu_base + DEBUG_SID_HALT_REG);



	fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);

	if ((fsr & FSR_FAULT) && (fsr & FSR_SS)) {

		u32 sctlr_orig, sctlr;

		/*

		 * We are in a fault; Our request to halt the bus will not

		 * complete until transactions in front of us (such as the fault

		 * itself) have completed. Disable iommu faults and terminate

		 * any existing transactions.

		 */

		sctlr_orig = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_SCTLR);

		sctlr = sctlr_orig & ~(SCTLR_CFCFG | SCTLR_CFIE);

		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr);



		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr);

		/*

		 * Barrier required to ensure that the FSR is cleared

		 * before resuming SMMU operation

		 */

		wmb();

		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME,

				  RESUME_TERMINATE);



		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr_orig);

	}



	if (readl_poll_timeout_atomic(tbu_base + DEBUG_SR_HALT_ACK_REG, status,

					(status & DEBUG_SR_HALT_ACK_VAL),

					0, TBU_DBG_TIMEOUT_US)) {

		dev_err(tbu->dev, "Couldn't halt TBU!\n");



		halt = readl_relaxed(tbu_base + DEBUG_SID_HALT_REG);

		halt &= ~DEBUG_SID_HALT_REQ;

		writel_relaxed(halt, tbu_base + DEBUG_SID_HALT_REG);



		spin_unlock_irqrestore(&tbu->halt_lock, flags);

		return -ETIMEDOUT;

	}



	tbu->halt_count = 1;

	spin_unlock_irqrestore(&tbu->halt_lock, flags);

	return 0;

}



static void qsmmuv500_tbu_resume(struct qsmmuv500_tbu_device *tbu)

{

	unsigned long flags;

	u32 val;

	void __iomem *base;



	spin_lock_irqsave(&tbu->halt_lock, flags);

	if (!tbu->halt_count) {

		WARN(1, "%s: bad tbu->halt_count", dev_name(tbu->dev));

		spin_unlock_irqrestore(&tbu->halt_lock, flags);

		return;



	} else if (tbu->halt_count > 1) {

		tbu->halt_count--;

		spin_unlock_irqrestore(&tbu->halt_lock, flags);

		return;

	}



	base = tbu->base;

	val = readl_relaxed(base + DEBUG_SID_HALT_REG);

	val &= ~DEBUG_SID_HALT_REQ;

	writel_relaxed(val, base + DEBUG_SID_HALT_REG);



	tbu->halt_count = 0;

	spin_unlock_irqrestore(&tbu->halt_lock, flags);

}



static struct qsmmuv500_tbu_device *qsmmuv500_find_tbu(

	struct arm_smmu_device *smmu, u32 sid)

{

	struct qsmmuv500_tbu_device *tbu = NULL;

	struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu);



	list_for_each_entry(tbu, &data->tbus, list) {

		if (tbu->sid_start <= sid &&

		    sid < tbu->sid_start + tbu->num_sids)

			return tbu;

	}

	return NULL;

}



static int qsmmuv500_ecats_lock(struct arm_smmu_domain *smmu_domain,

				struct qsmmuv500_tbu_device *tbu,

				unsigned long *flags)

	__acquires(&smmu->atos_lock)

{

	struct arm_smmu_device *smmu = tbu->smmu;

	struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu);

	u32 val;



	spin_lock_irqsave(&smmu->atos_lock, *flags);

	/* The status register is not accessible on version 1.0 */

	if (data->version == 0x01000000)

		return 0;



	if (readl_poll_timeout_atomic(tbu->status_reg,

					val, (val == 0x1), 0,

					TBU_DBG_TIMEOUT_US)) {

		dev_err(tbu->dev, "ECATS hw busy!\n");

		spin_unlock_irqrestore(&smmu->atos_lock, *flags);

		return  -ETIMEDOUT;

	}



	return 0;

}



static void qsmmuv500_ecats_unlock(struct arm_smmu_domain *smmu_domain,

					struct qsmmuv500_tbu_device *tbu,

					unsigned long *flags)

	__releases(&smmu->atos_lock)

{

	struct arm_smmu_device *smmu = tbu->smmu;

	struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu);



	/* The status register is not accessible on version 1.0 */

	if (data->version != 0x01000000)

		writel_relaxed(0, tbu->status_reg);

	spin_unlock_irqrestore(&smmu->atos_lock, *flags);

}



/*

 * Zero means failure.

 */

static phys_addr_t qsmmuv500_iova_to_phys(

		struct arm_smmu_domain *smmu_domain, dma_addr_t iova, u32 sid,

		unsigned long trans_flags)

{

	struct arm_smmu_device *smmu = smmu_domain->smmu;

	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;

	struct qsmmuv500_tbu_device *tbu;

	int ret;

	phys_addr_t phys = 0;

	u64 val, fsr;

	unsigned long flags;

	int idx = cfg->cbndx;

	u32 sctlr_orig, sctlr;

	int needs_redo = 0;

	ktime_t timeout;



	/* only 36 bit iova is supported */

	if (iova >= (1ULL << 36)) {

		dev_err_ratelimited(smmu->dev, "ECATS: address too large: %pad\n",

					&iova);

		return 0;

	}



	tbu = qsmmuv500_find_tbu(smmu, sid);

	if (!tbu)

		return 0;



	ret = arm_smmu_power_on(tbu->pwr);

	if (ret)

		return 0;



	ret = qsmmuv500_tbu_halt(tbu, smmu_domain);

	if (ret)

		goto out_power_off;



	/*

	 * ECATS can trigger the fault interrupt, so disable it temporarily

	 * and check for an interrupt manually.

	 */

	sctlr_orig = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_SCTLR);

	sctlr = sctlr_orig & ~(SCTLR_CFCFG | SCTLR_CFIE);

	arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr);



	fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);

	if (fsr & FSR_FAULT) {

		/* Clear pending interrupts */

		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr);

		/*

		 * Barrier required to ensure that the FSR is cleared

		 * before resuming SMMU operation.

		 */

		wmb();



		/*

		 * TBU halt takes care of resuming any stalled transcation.

		 * Kept it here for completeness sake.

		 */

		if (fsr & FSR_SS)

			arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME,

					  RESUME_TERMINATE);

	}



	/* Only one concurrent atos operation */

	ret = qsmmuv500_ecats_lock(smmu_domain, tbu, &flags);

	if (ret)

		goto out_resume;



redo:

	/* Set address and stream-id */

	val = readq_relaxed(tbu->base + DEBUG_SID_HALT_REG);

	val &= ~DEBUG_SID_HALT_SID;

	val |= FIELD_PREP(DEBUG_SID_HALT_SID, sid);

	writeq_relaxed(val, tbu->base + DEBUG_SID_HALT_REG);

	writeq_relaxed(iova, tbu->base + DEBUG_VA_ADDR_REG);

	val = FIELD_PREP(DEBUG_AXUSER_CDMID, DEBUG_AXUSER_CDMID_VAL);

	writeq_relaxed(val, tbu->base + DEBUG_AXUSER_REG);



	/* Write-back Read and Write-Allocate */

	val = FIELD_PREP(DEBUG_TXN_AXCACHE, 0xF);



	/* Non-secure Access */

	val |= FIELD_PREP(DEBUG_TXN_AXPROT, DEBUG_TXN_AXPROT_NSEC);



	/* Write or Read Access */

	if (flags & IOMMU_TRANS_WRITE)

		val |= DEBUG_TXN_WRITE;



	/* Priviledged or Unpriviledged Access */

	if (flags & IOMMU_TRANS_PRIV)

		val |= FIELD_PREP(DEBUG_TXN_AXPROT, DEBUG_TXN_AXPROT_PRIV);



	/* Data or Instruction Access */

	if (flags & IOMMU_TRANS_INST)

		val |= FIELD_PREP(DEBUG_TXN_AXPROT, DEBUG_TXN_AXPROT_INST);



	val |= DEBUG_TXN_TRIGGER;

	writeq_relaxed(val, tbu->base + DEBUG_TXN_TRIGG_REG);



	ret = 0;

	timeout = ktime_add_us(ktime_get(), TBU_DBG_TIMEOUT_US);

	for (;;) {

		val = readl_relaxed(tbu->base + DEBUG_SR_HALT_ACK_REG);

		if (!(val & DEBUG_SR_ECATS_RUNNING_VAL))

			break;

		val = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);

		if (val & FSR_FAULT)

			break;

		if (ktime_compare(ktime_get(), timeout) > 0) {

			dev_err_ratelimited(tbu->dev, "ECATS translation timed out!\n");

			ret = -ETIMEDOUT;

			break;

		}

	}



	val = readq_relaxed(tbu->base + DEBUG_PAR_REG);

	fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);

	if (val & DEBUG_PAR_FAULT_VAL) {

		dev_err(tbu->dev, "ECATS generated a fault interrupt! FSR = %llx, SID=0x%x\n",

			fsr, sid);



		dev_err(tbu->dev, "ECATS translation failed! PAR = %llx\n",

			val);

		/* Clear pending interrupts */

		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr);

		/*

		 * Barrier required to ensure that the FSR is cleared

		 * before resuming SMMU operation.

		 */

		wmb();



		if (fsr & FSR_SS)

			arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME,

					  RESUME_TERMINATE);



		ret = -EINVAL;

	}



	phys = FIELD_GET(DEBUG_PAR_PA, val);

	if (ret < 0)

		phys = 0;



	/* Reset hardware */

	writeq_relaxed(0, tbu->base + DEBUG_TXN_TRIGG_REG);

	writeq_relaxed(0, tbu->base + DEBUG_VA_ADDR_REG);

	val = readl_relaxed(tbu->base + DEBUG_SID_HALT_REG);

	val &= ~DEBUG_SID_HALT_SID;

	writel_relaxed(val, tbu->base + DEBUG_SID_HALT_REG);



	/*

	 * After a failed translation, the next successful translation will

	 * incorrectly be reported as a failure.

	 */

	if (!phys && needs_redo++ < 2)

		goto redo;



	arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr_orig);

	qsmmuv500_ecats_unlock(smmu_domain, tbu, &flags);



out_resume:

	qsmmuv500_tbu_resume(tbu);



out_power_off:

	/* Read to complete prior write transcations */

	val = readl_relaxed(tbu->base + DEBUG_SR_HALT_ACK_REG);



	/* Wait for read to complete before off */

	rmb();



	arm_smmu_power_off(tbu->smmu, tbu->pwr);



	return phys;

}



static phys_addr_t qsmmuv500_iova_to_phys_hard(

					struct arm_smmu_domain *smmu_domain,

					dma_addr_t iova,

					unsigned long trans_flags)

{

	u16 sid;

	struct msm_iommu_domain *msm_domain = &smmu_domain->domain;

	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;

	struct arm_smmu_device *smmu = smmu_domain->smmu;

	struct iommu_fwspec *fwspec;

	u32 frsynra;



	/* Check to see if the domain is associated with the test

	 * device. If the domain belongs to the test device, then

	 * pick the SID from fwspec.

	 */

	if (msm_domain->is_debug_domain) {

		fwspec = dev_iommu_fwspec_get(smmu_domain->dev);

		sid    = (u16)fwspec->ids[0];

	} else {



		/* If the domain belongs to an actual device, read

		 * SID from the corresponding frsynra register

		 */

		frsynra = arm_smmu_gr1_read(smmu,

					    ARM_SMMU_GR1_CBFRSYNRA(cfg->cbndx));

		frsynra &= CBFRSYNRA_SID_MASK;

		sid      = frsynra;

	}

	return qsmmuv500_iova_to_phys(smmu_domain, iova, sid, trans_flags);

}



static void qsmmuv500_release_group_iommudata(void *data)

{

	kfree(data);

}



/* If a device has a valid actlr, it must match */

static int qsmmuv500_device_group(struct device *dev,

				struct iommu_group *group)

{

	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);

	struct arm_smmu_device *smmu = fwspec_smmu(fwspec);

	struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu);

	struct qsmmuv500_group_iommudata *iommudata;

	u32 actlr, i;

	struct arm_smmu_smr *smr;



	iommudata = to_qsmmuv500_group_iommudata(group);

	if (!iommudata) {

		iommudata = kzalloc(sizeof(*iommudata), GFP_KERNEL);

		if (!iommudata)

			return -ENOMEM;



		iommu_group_set_iommudata(group, iommudata,

				qsmmuv500_release_group_iommudata);

	}



	for (i = 0; i < data->actlr_tbl_size; i++) {

		smr = &data->actlrs[i].smr;

		actlr = data->actlrs[i].actlr;



		if (!arm_smmu_fwspec_match_smr(fwspec, smr))

			continue;



		if (!iommudata->has_actlr) {

			iommudata->actlr = actlr;

			iommudata->has_actlr = true;

		} else if (iommudata->actlr != actlr) {

			return -EINVAL;

		}

	}



	return 0;

}



static void qsmmuv500_init_cb(struct arm_smmu_domain *smmu_domain,

				struct device *dev)

{

	struct arm_smmu_device *smmu = smmu_domain->smmu;

	struct qsmmuv500_group_iommudata *iommudata =

		to_qsmmuv500_group_iommudata(dev->iommu_group);

	int idx = smmu_domain->cfg.cbndx;

	const struct iommu_flush_ops *tlb;



	if (!iommudata->has_actlr)

		return;



	tlb = smmu_domain->pgtbl_info.pgtbl_cfg.tlb;



	arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ACTLR, iommudata->actlr);



	/*

	 * Flush the context bank after modifying ACTLR to ensure there

	 * are no cache entries with stale state

	 */

	tlb->tlb_flush_all(smmu_domain);

}



static int qsmmuv500_tbu_register(struct device *dev, void *cookie)

{

	struct arm_smmu_device *smmu = cookie;

	struct qsmmuv500_tbu_device *tbu;

	struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu);



	if (!dev->driver) {

		dev_err(dev, "TBU failed probe, QSMMUV500 cannot continue!\n");

		return -EINVAL;

	}



	tbu = dev_get_drvdata(dev);



	INIT_LIST_HEAD(&tbu->list);

	tbu->smmu = smmu;

	list_add(&tbu->list, &data->tbus);

	return 0;

}



static int qsmmuv500_read_actlr_tbl(struct arm_smmu_device *smmu)

{

	int len, i;

	struct device *dev = smmu->dev;

	struct qsmmuv500_archdata *data = get_qsmmuv500_archdata(smmu);

	struct actlr_setting *actlrs;

	const __be32 *cell;



	cell = of_get_property(dev->of_node, "qcom,actlr", NULL);

	if (!cell)

		return 0;



	len = of_property_count_elems_of_size(dev->of_node, "qcom,actlr",

						sizeof(u32) * 3);

	if (len < 0)

		return 0;



	actlrs = devm_kzalloc(dev, sizeof(*actlrs) * len, GFP_KERNEL);

	if (!actlrs)

		return -ENOMEM;



	for (i = 0; i < len; i++) {

		actlrs[i].smr.id = of_read_number(cell++, 1);

		actlrs[i].smr.mask = of_read_number(cell++, 1);

		actlrs[i].actlr = of_read_number(cell++, 1);

	}



	data->actlrs = actlrs;

	data->actlr_tbl_size = len;

	return 0;

}



static int qsmmuv500_arch_init(struct arm_smmu_device *smmu)

{

	struct resource *res;

	struct device *dev = smmu->dev;

	struct qsmmuv500_archdata *data;

	struct platform_device *pdev;

	int ret;

	u32 val;



	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);

	if (!data)

		return -ENOMEM;



	INIT_LIST_HEAD(&data->tbus);



	pdev = container_of(dev, struct platform_device, dev);

	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "tcu-base");

	if (!res) {

		dev_err(dev, "Unable to get the tcu-base\n");

		return -EINVAL;

	}

	data->tcu_base = devm_ioremap(dev, res->start, resource_size(res));

	if (IS_ERR(data->tcu_base))

		return PTR_ERR(data->tcu_base);



	data->version = readl_relaxed(data->tcu_base + TCU_HW_VERSION_HLOS1);

	smmu->archdata = data;

	data->smmu = smmu;



	ret = qsmmuv500_read_actlr_tbl(smmu);

	if (ret)

		return ret;



	val = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sACR);

	val &= ~ARM_MMU500_ACR_CACHE_LOCK;

	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sACR, val);

	val = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sACR);

	/*

	 * Modifiying the nonsecure copy of the sACR register is only

	 * allowed if permission is given in the secure sACR register.

	 * Attempt to detect if we were able to update the value.

	 */

	WARN_ON(val & ARM_MMU500_ACR_CACHE_LOCK);



	ret = of_platform_populate(dev->of_node, NULL, NULL, dev);

	if (ret)

		return ret;



	INIT_WORK(&data->outstanding_tnx_work,

		  qsmmuv500_log_outstanding_transactions);



	/* Attempt to register child devices */

	ret = device_for_each_child(dev, smmu, qsmmuv500_tbu_register);

	if (ret)

		return -EPROBE_DEFER;



	return 0;

}



struct arm_smmu_arch_ops qsmmuv500_arch_ops = {

	.init = qsmmuv500_arch_init,

	.iova_to_phys_hard = qsmmuv500_iova_to_phys_hard,

	.init_context_bank = qsmmuv500_init_cb,

	.device_group = qsmmuv500_device_group,

	.tlb_sync_timeout = qsmmuv500_tlb_sync_timeout,

	.device_remove = qsmmuv500_device_remove,

};



struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu)

{

	struct qcom_smmu *qsmmu;