Merge "iommu/io-pgtable-arm: Don't leave dangling table entries" (f7d9a03f) · Commits · e / devices / android_kernel_xiaomi_markw

drivers/iommu/arm-smmu.c

+220 −2

Original line number	Diff line number	Diff line
		@@ -200,6 +200,7 @@
		#define ARM_SMMU_CB_PAR_LO 0x50
		#define ARM_SMMU_CB_PAR_HI 0x54
		#define ARM_SMMU_CB_FSR 0x58
		#define ARM_SMMU_CB_FSRRESTORE 0x5c
		#define ARM_SMMU_CB_FAR_LO 0x60
		#define ARM_SMMU_CB_FAR_HI 0x64
		#define ARM_SMMU_CB_FSYNR0 0x68
		@@ -290,6 +291,13 @@

		#define FSYNR0_WNR (1 << 4)

		#define ARM_SMMU_IMPL_DEF0(smmu) \
		((smmu)->base + (2 * (1 << (smmu)->pgshift)))
		#define ARM_SMMU_IMPL_DEF1(smmu) \
		((smmu)->base + (6 * (1 << (smmu)->pgshift)))
		#define IMPL_DEF1_MICRO_MMU_CTRL 0
		#define MICRO_MMU_CTRL_LOCAL_HALT_REQ (1 << 2)

		static int force_stage;
		module_param_named(force_stage, force_stage, int, S_IRUGO \| S_IWUSR);
		MODULE_PARM_DESC(force_stage,
		@@ -391,7 +399,7 @@ struct arm_smmu_device {
		struct arm_smmu_impl_def_reg *impl_def_attach_registers;
		unsigned int num_impl_def_attach_registers;

		struct mutex atos_lock;
		spinlock_t atos_lock;
		unsigned int clock_refs_count;
		spinlock_t clock_refs_lock;
		};
		@@ -951,9 +959,66 @@ static struct iommu_gather_ops arm_smmu_gather_ops = {
		.unprepare_pgtable = arm_smmu_unprepare_pgtable,
		};

		static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
		dma_addr_t iova);
		static phys_addr_t arm_smmu_iova_to_phys_hard_no_halt(
		struct iommu_domain *domain, dma_addr_t iova);
		static int arm_smmu_wait_for_halt(struct arm_smmu_device *smmu);
		static int arm_smmu_halt_nowait(struct arm_smmu_device *smmu);
		static void arm_smmu_resume(struct arm_smmu_device *smmu);
		static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
		dma_addr_t iova);

		static phys_addr_t arm_smmu_verify_fault(struct iommu_domain *domain,
		dma_addr_t iova, u32 fsr)
		{
		struct arm_smmu_domain *smmu_domain = domain->priv;
		struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
		struct arm_smmu_device *smmu;
		void __iomem *cb_base;
		u64 sctlr, sctlr_orig;
		phys_addr_t phys;

		smmu = smmu_domain->smmu;
		cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);

		arm_smmu_halt_nowait(smmu);

		writel_relaxed(RESUME_TERMINATE, cb_base + ARM_SMMU_CB_RESUME);

		arm_smmu_wait_for_halt(smmu);

		/* clear FSR to allow ATOS to log any faults */
		writel_relaxed(fsr, cb_base + ARM_SMMU_CB_FSR);

		/* disable stall mode momentarily */
		sctlr_orig = readl_relaxed(cb_base + ARM_SMMU_CB_SCTLR);
		sctlr = sctlr_orig & ~SCTLR_CFCFG;
		writel_relaxed(sctlr, cb_base + ARM_SMMU_CB_SCTLR);

		phys = arm_smmu_iova_to_phys_hard_no_halt(domain, iova);

		if (!phys) {
		dev_err(smmu->dev,
		"ATOS failed. Will issue a TLBIALL and try again...\n");
		arm_smmu_tlb_inv_context(smmu_domain);
		phys = arm_smmu_iova_to_phys_hard_no_halt(domain, iova);
		if (phys)
		dev_err(smmu->dev,
		"ATOS succeeded this time. Maybe we missed a TLB invalidation while messing with page tables earlier??\n");
		else
		dev_err(smmu->dev,
		"ATOS still failed. If the page tables look good (check the software table walk) then hardware might be misbehaving.\n");
		}

		/* restore SCTLR */
		writel_relaxed(sctlr_orig, cb_base + ARM_SMMU_CB_SCTLR);

		arm_smmu_resume(smmu);

		return phys;
		}

		static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
		{
		int flags, ret, tmp;
		@@ -1024,6 +1089,9 @@ static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
		ret = IRQ_HANDLED;
		resume = ctx_hang_errata ? RESUME_TERMINATE : RESUME_RETRY;
		} else {
		phys_addr_t phys_atos = arm_smmu_verify_fault(domain, iova,
		fsr);

		dev_err(smmu->dev,
		"Unhandled context fault: iova=0x%08lx, fsr=0x%x, fsynr=0x%x, cb=%d\n",
		iova, fsr, fsynr, cfg->cbndx);
		@@ -1040,6 +1108,8 @@ static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
		(fsr & 0x80000000) ? "MULTI " : "");
		dev_err(smmu->dev,
		"soft iova-to-phys=%pa\n", &phys_soft);
		dev_err(smmu->dev,
		"hard iova-to-phys (ATOS)=%pa\n", &phys_atos);
		dev_err(smmu->dev, "SID=0x%x\n", frsynra & 0x1FF);
		ret = IRQ_NONE;
		resume = RESUME_TERMINATE;
		@@ -1112,6 +1182,29 @@ static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
		return IRQ_HANDLED;
		}

		static void arm_smmu_trigger_fault(struct iommu_domain *domain)
		{
		struct arm_smmu_domain *smmu_domain = domain->priv;
		struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
		struct arm_smmu_device *smmu;
		void __iomem *cb_base;

		if (!smmu_domain->smmu) {
		pr_err("Can't trigger faults on non-attached domains\n");
		return;
		}

		smmu = smmu_domain->smmu;

		cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
		arm_smmu_enable_clocks(smmu);
		/* trigger a translation fault */
		dev_err(smmu->dev, "Triggering translation fault on cb %d\n",
		cfg->cbndx);
		writel_relaxed(FSR_TF, cb_base + ARM_SMMU_CB_FSRRESTORE);
		arm_smmu_disable_clocks(smmu);
		}

		static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
		struct io_pgtable_cfg *pgtbl_cfg)
		{
		@@ -1384,6 +1477,8 @@ static int arm_smmu_domain_init(struct iommu_domain *domain)
		return -ENOMEM;

		smmu_domain->secure_vmid = VMID_INVAL;
		/* disable coherent htw by default */
		smmu_domain->attributes = (1 << DOMAIN_ATTR_COHERENT_HTW_DISABLE);
		INIT_LIST_HEAD(&smmu_domain->pte_info_list);
		smmu_domain->cfg.cbndx = INVALID_CBNDX;
		smmu_domain->cfg.irptndx = INVALID_IRPTNDX;
		@@ -1960,6 +2055,127 @@ static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
		return ret;
		}

		static int arm_smmu_wait_for_halt(struct arm_smmu_device *smmu)
		{
		void __iomem *impl_def1_base = ARM_SMMU_IMPL_DEF1(smmu);
		u32 tmp;

		if (readl_poll_timeout_atomic(impl_def1_base + IMPL_DEF1_MICRO_MMU_CTRL,
		tmp, (tmp & MICRO_MMU_CTRL_IDLE),
		0, 30000)) {
		dev_err(smmu->dev, "Couldn't halt SMMU!\n");
		return -EBUSY;
		}

		return 0;
		}

		static int __arm_smmu_halt(struct arm_smmu_device *smmu, bool wait)
		{
		void __iomem *impl_def1_base = ARM_SMMU_IMPL_DEF1(smmu);

		writel_relaxed(MICRO_MMU_CTRL_LOCAL_HALT_REQ,
		impl_def1_base + IMPL_DEF1_MICRO_MMU_CTRL);

		return wait ? arm_smmu_wait_for_halt(smmu) : 0;
		}

		static int arm_smmu_halt(struct arm_smmu_device *smmu)
		{
		return __arm_smmu_halt(smmu, true);
		}

		static int arm_smmu_halt_nowait(struct arm_smmu_device *smmu)
		{
		return __arm_smmu_halt(smmu, false);
		}

		static void arm_smmu_resume(struct arm_smmu_device *smmu)
		{
		void __iomem *impl_def1_base = ARM_SMMU_IMPL_DEF1(smmu);
		u32 reg;

		reg = readl_relaxed(impl_def1_base + IMPL_DEF1_MICRO_MMU_CTRL);
		reg &= ~MICRO_MMU_CTRL_LOCAL_HALT_REQ;
		writel_relaxed(reg, impl_def1_base + IMPL_DEF1_MICRO_MMU_CTRL);
		}

		static phys_addr_t __arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
		dma_addr_t iova, bool do_halt)
		{
		struct arm_smmu_domain *smmu_domain = domain->priv;
		struct arm_smmu_device *smmu = smmu_domain->smmu;
		struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
		struct device *dev = smmu->dev;
		void __iomem *cb_base;
		u32 tmp;
		u64 phys;
		unsigned long flags;

		arm_smmu_enable_clocks(smmu);

		cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);

		spin_lock_irqsave(&smmu->atos_lock, flags);

		if (do_halt && arm_smmu_halt(smmu))
		goto err_unlock;

		if (smmu->version == 1)
		writel_relaxed(iova & ~0xfff, cb_base + ARM_SMMU_CB_ATS1PR_LO);
		else
		writeq_relaxed(iova & ~0xfffULL,
		cb_base + ARM_SMMU_CB_ATS1PR_LO);

		if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
		!(tmp & ATSR_ACTIVE), 5, 50)) {
		dev_err(dev, "iova to phys timed out\n");
		goto err_resume;
		}

		phys = readl_relaxed(cb_base + ARM_SMMU_CB_PAR_LO);
		phys \|= ((u64) readl_relaxed(cb_base + ARM_SMMU_CB_PAR_HI)) << 32;

		if (do_halt)
		arm_smmu_resume(smmu);
		spin_unlock_irqrestore(&smmu->atos_lock, flags);

		if (phys & CB_PAR_F) {
		dev_err(dev, "translation fault on %s!\n", dev_name(dev));
		dev_err(dev, "PAR = 0x%llx\n", phys);
		phys = 0;
		} else {
		phys = (phys & (PHYS_MASK & ~0xfffULL)) \| (iova & 0xfff);
		}

		arm_smmu_disable_clocks(smmu);
		return phys;

		err_resume:
		if (do_halt)
		arm_smmu_resume(smmu);
		err_unlock:
		spin_unlock_irqrestore(&smmu->atos_lock, flags);
		arm_smmu_disable_clocks(smmu);
		phys = arm_smmu_iova_to_phys(domain, iova);
		dev_err(dev,
		"iova to phys failed 0x%pa. software table walk result=%pa.\n",
		&iova, &phys);
		return 0;
		}

		static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
		dma_addr_t iova)
		{
		return __arm_smmu_iova_to_phys_hard(domain, iova, true);
		}

		static phys_addr_t arm_smmu_iova_to_phys_hard_no_halt(
		struct iommu_domain *domain, dma_addr_t iova)
		{
		return __arm_smmu_iova_to_phys_hard(domain, iova, false);
		}

		static bool arm_smmu_capable(enum iommu_cap cap)
		{
		switch (cap) {
		@@ -2253,12 +2469,14 @@ static struct iommu_ops arm_smmu_ops = {
		.unmap = arm_smmu_unmap,
		.map_sg = arm_smmu_map_sg,
		.iova_to_phys = arm_smmu_iova_to_phys,
		.iova_to_phys_hard = arm_smmu_iova_to_phys_hard,
		.add_device = arm_smmu_add_device,
		.remove_device = arm_smmu_remove_device,
		.domain_get_attr = arm_smmu_domain_get_attr,
		.domain_set_attr = arm_smmu_domain_set_attr,
		.pgsize_bitmap = -1UL, /* Restricted during device attach */
		.dma_supported = arm_smmu_dma_supported,
		.trigger_fault = arm_smmu_trigger_fault,
		};

		static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
		@@ -2624,7 +2842,7 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
		}
		smmu->dev = dev;
		mutex_init(&smmu->attach_lock);
		mutex_init(&smmu->atos_lock);
		spin_lock_init(&smmu->atos_lock);
		spin_lock_init(&smmu->clock_refs_lock);

		of_id = of_match_node(arm_smmu_of_match, dev->of_node);

drivers/iommu/io-pgtable-arm.c

+111 −18

Original line number	Diff line number	Diff line
		@@ -164,7 +164,7 @@

		/* IOPTE accessors */
		#define iopte_deref(pte, d) \
		(__va((pte) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1) \
		(__va(iopte_val(pte) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1) \
		& ~((1ULL << (d)->pg_shift) - 1)))

		#define iopte_type(pte,l) \
		@@ -196,12 +196,82 @@ struct arm_lpae_io_pgtable {

		typedef u64 arm_lpae_iopte;

		/*
		* We'll use some ignored bits in table entries to keep track of the number
		* of page mappings beneath the table. The maximum number of entries
		* beneath any table mapping in armv8 is 8192 (which is possible at the
		* 2nd- and 3rd-level when using a 64K granule size). The bits at our
		* disposal are:
		*
		* 4k granule: [58..52], [11..2]
		* 64k granule: [58..52], [15..2]
		*
		* [58..52], [11..2] is enough bits for tracking table mappings at any
		* level for any granule, so we'll use those.
		*/
		#define BOTTOM_IGNORED_MASK 0x3ff
		#define BOTTOM_IGNORED_SHIFT 2
		#define BOTTOM_IGNORED_NUM_BITS 10
		#define TOP_IGNORED_MASK 0x7fULL
		#define TOP_IGNORED_SHIFT 52
		#define IOPTE_RESERVED_MASK ((BOTTOM_IGNORED_MASK << BOTTOM_IGNORED_SHIFT) \| \
		(TOP_IGNORED_MASK << TOP_IGNORED_SHIFT))

		static arm_lpae_iopte iopte_val(arm_lpae_iopte table_pte)
		{
		return table_pte & ~IOPTE_RESERVED_MASK;
		}

		static arm_lpae_iopte _iopte_bottom_ignored_val(arm_lpae_iopte table_pte)
		{
		return (table_pte & (BOTTOM_IGNORED_MASK << BOTTOM_IGNORED_SHIFT))
		>> BOTTOM_IGNORED_SHIFT;
		}

		static arm_lpae_iopte _iopte_top_ignored_val(arm_lpae_iopte table_pte)
		{
		return (table_pte & (TOP_IGNORED_MASK << TOP_IGNORED_SHIFT))
		>> TOP_IGNORED_SHIFT;
		}

		static int iopte_tblcnt(arm_lpae_iopte table_pte)
		{
		return (_iopte_bottom_ignored_val(table_pte) \|
		(_iopte_top_ignored_val(table_pte) << BOTTOM_IGNORED_NUM_BITS));
		}

		static void iopte_tblcnt_set(arm_lpae_iopte *table_pte, int val)
		{
		arm_lpae_iopte pte = iopte_val(*table_pte);

		pte \|= ((val & BOTTOM_IGNORED_MASK) << BOTTOM_IGNORED_SHIFT) \|
		(((val & (TOP_IGNORED_MASK << BOTTOM_IGNORED_NUM_BITS))
		>> BOTTOM_IGNORED_NUM_BITS) << TOP_IGNORED_SHIFT);
		*table_pte = pte;
		}

		static void iopte_tblcnt_sub(arm_lpae_iopte *table_ptep, int cnt)
		{
		arm_lpae_iopte current_cnt = iopte_tblcnt(*table_ptep);

		current_cnt -= cnt;
		iopte_tblcnt_set(table_ptep, current_cnt);
		}

		static void iopte_tblcnt_add(arm_lpae_iopte *table_ptep, int cnt)
		{
		arm_lpae_iopte current_cnt = iopte_tblcnt(*table_ptep);

		current_cnt += cnt;
		iopte_tblcnt_set(table_ptep, current_cnt);
		}

		static bool suppress_map_failures;

		static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data,
		unsigned long iova, phys_addr_t paddr,
		arm_lpae_iopte prot, int lvl,
		arm_lpae_iopte *ptep)
		arm_lpae_iopte ptep, arm_lpae_iopte prev_ptep)
		{
		arm_lpae_iopte pte = prot;

		@@ -224,12 +294,17 @@ static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data,

		*ptep = pte;
		data->iop.cfg.tlb->flush_pgtable(ptep, sizeof(*ptep), data->iop.cookie);

		if (prev_ptep)
		iopte_tblcnt_add(prev_ptep, 1);

		return 0;
		}

		static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
		phys_addr_t paddr, size_t size, arm_lpae_iopte prot,
		int lvl, arm_lpae_iopte *ptep)
		int lvl, arm_lpae_iopte *ptep,
		arm_lpae_iopte *prev_ptep)
		{
		arm_lpae_iopte *cptep, pte;
		void *cookie = data->iop.cookie;
		@@ -240,7 +315,8 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,

		/* If we can install a leaf entry at this level, then do so */
		if (size == block_size && (size & data->iop.cfg.pgsize_bitmap))
		return arm_lpae_init_pte(data, iova, paddr, prot, lvl, ptep);
		return arm_lpae_init_pte(data, iova, paddr, prot, lvl, ptep,
		prev_ptep);

		/* We can't allocate tables at the final level */
		if (WARN_ON(lvl >= ARM_LPAE_MAX_LEVELS - 1))
		@@ -249,7 +325,7 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
		/* Grab a pointer to the next level */
		pte = *ptep;
		if (!pte) {
		cptep = alloc_pages_exact(1UL << data->pg_shift,
		cptep = io_pgtable_alloc_pages_exact(1UL << data->pg_shift,
		GFP_ATOMIC \| __GFP_ZERO);
		if (!cptep)
		return -ENOMEM;
		@@ -267,7 +343,8 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
		}

		/* Rinse, repeat */
		return __arm_lpae_map(data, iova, paddr, size, prot, lvl + 1, cptep);
		return __arm_lpae_map(data, iova, paddr, size, prot, lvl + 1, cptep,
		ptep);
		}

		static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data,
		@@ -323,7 +400,7 @@ static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova,
		return 0;

		prot = arm_lpae_prot_to_pte(data, iommu_prot);
		return __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep);
		return __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep, NULL);
		}

		static int arm_lpae_map_sg(struct io_pgtable_ops *ops, unsigned long iova,
		@@ -365,7 +442,7 @@ static int arm_lpae_map_sg(struct io_pgtable_ops *ops, unsigned long iova,
		size_t pgsize = iommu_pgsize(
		data->iop.cfg.pgsize_bitmap, iova \| phys, size);
		ret = __arm_lpae_map(data, iova, phys, pgsize, prot,
		lvl, ptep);
		lvl, ptep, NULL);
		if (ret)
		goto out_err;

		@@ -414,7 +491,7 @@ static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,
		}

		data->iop.cfg.tlb->unprepare_pgtable(data->iop.cookie, start);
		free_pages_exact(start, table_size);
		io_pgtable_free_pages_exact(start, table_size);
		}

		static void arm_lpae_free_pgtable(struct io_pgtable *iop)
		@@ -428,7 +505,8 @@ static void arm_lpae_free_pgtable(struct io_pgtable *iop)
		static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data,
		unsigned long iova, size_t size,
		arm_lpae_iopte prot, int lvl,
		arm_lpae_iopte *ptep, size_t blk_size)
		arm_lpae_iopte *ptep,
		arm_lpae_iopte *prev_ptep, size_t blk_size)
		{
		unsigned long blk_start, blk_end;
		phys_addr_t blk_paddr;
		@@ -450,7 +528,7 @@ static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data,
		/* __arm_lpae_map expects a pointer to the start of the table */
		tablep = &table - ARM_LPAE_LVL_IDX(blk_start, lvl, data);
		if (__arm_lpae_map(data, blk_start, blk_paddr, size, prot, lvl,
		tablep) < 0) {
		tablep, prev_ptep) < 0) {
		if (table) {
		/* Free the table we allocated */
		tablep = iopte_deref(table, data);
		@@ -467,7 +545,7 @@ static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data,

		static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
		unsigned long iova, size_t size, int lvl,
		arm_lpae_iopte *ptep)
		arm_lpae_iopte ptep, arm_lpae_iopte prev_ptep)
		{
		arm_lpae_iopte pte;
		const struct iommu_gather_ops *tlb = data->iop.cfg.tlb;
		@@ -495,6 +573,7 @@ static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
		return size;
		} else if ((lvl == ARM_LPAE_MAX_LEVELS - 2) && !iopte_leaf(pte, lvl)) {
		arm_lpae_iopte *table = iopte_deref(pte, data);
		arm_lpae_iopte *table_base = table;
		int tl_offset = ARM_LPAE_LVL_IDX(iova, lvl + 1, data);
		int entry_size = (1 << data->pg_shift);
		int max_entries = ARM_LPAE_BLOCK_SIZE(lvl, data) / entry_size;
		@@ -514,6 +593,15 @@ static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
		memset(table, 0, table_len);
		tlb->flush_pgtable(table, table_len, cookie);

		iopte_tblcnt_sub(ptep, entries);
		if (!iopte_tblcnt(*ptep)) {
		/* no valid mappings left under this table. free it. */
		*ptep = 0;
		tlb->flush_pgtable(ptep, sizeof(*ptep), cookie);
		io_pgtable_free_pages_exact(
		table_base, max_entries * sizeof(*table_base));
		}

		return entries * entry_size;
		} else if (iopte_leaf(pte, lvl)) {
		/*
		@@ -522,12 +610,14 @@ static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
		*/
		return arm_lpae_split_blk_unmap(data, iova, size,
		iopte_prot(pte), lvl, ptep,
		prev_ptep,
		blk_size);
		}

		/* Keep on walkin' */
		prev_ptep = ptep;
		ptep = iopte_deref(pte, data);
		return __arm_lpae_unmap(data, iova, size, lvl + 1, ptep);
		return __arm_lpae_unmap(data, iova, size, lvl + 1, ptep, prev_ptep);
		}

		static size_t arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova,
		@@ -546,7 +636,8 @@ static size_t arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova,
		size_to_unmap = remaining < SZ_2M
		? remaining
		: iommu_pgsize(data->iop.cfg.pgsize_bitmap, iova, size);
		ret = __arm_lpae_unmap(data, iova, size_to_unmap, lvl, ptep);
		ret = __arm_lpae_unmap(data, iova, size_to_unmap, lvl, ptep,
		NULL);
		if (ret == 0)
		break;
		unmapped += ret;
		@@ -733,7 +824,8 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg cfg, void cookie)
		cfg->arm_lpae_s1_cfg.mair[1] = 0;

		/* Looking good; allocate a pgd */
		data->pgd = alloc_pages_exact(data->pgd_size, GFP_KERNEL \| __GFP_ZERO);
		data->pgd = io_pgtable_alloc_pages_exact(data->pgd_size,
		GFP_KERNEL \| __GFP_ZERO);
		if (!data->pgd)
		goto out_free_data;

		@@ -821,7 +913,8 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg cfg, void cookie)
		cfg->arm_lpae_s2_cfg.vtcr = reg;

		/* Allocate pgd pages */
		data->pgd = alloc_pages_exact(data->pgd_size, GFP_KERNEL \| __GFP_ZERO);
		data->pgd = io_pgtable_alloc_pages_exact(data->pgd_size,
		GFP_KERNEL \| __GFP_ZERO);
		if (!data->pgd)
		goto out_free_data;

drivers/iommu/io-pgtable.c

+48 −0

Original line number	Diff line number	Diff line
		@@ -18,9 +18,14 @@
		* Author: Will Deacon <will.deacon@arm.com>
		*/

		#define pr_fmt(fmt) "io-pgtable: " fmt

		#include <linux/bug.h>
		#include <linux/kernel.h>
		#include <linux/types.h>
		#include <linux/iommu.h>
		#include <linux/debugfs.h>
		#include <linux/atomic.h>

		#include "io-pgtable.h"

		@@ -40,6 +45,8 @@ io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] =
		#endif
		};

		static struct dentry *io_pgtable_top;

		struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt,
		struct io_pgtable_cfg *cfg,
		void *cookie)
		@@ -80,3 +87,44 @@ void free_io_pgtable_ops(struct io_pgtable_ops *ops)
		iop->cfg.tlb->tlb_flush_all(iop->cookie);
		io_pgtable_init_table[iop->fmt]->free(iop);
		}

		static atomic_t pages_allocated;

		void *io_pgtable_alloc_pages_exact(size_t size, gfp_t gfp_mask)
		{
		void *ret = alloc_pages_exact(size, gfp_mask);

		if (likely(ret))
		atomic_add(1 << get_order(size), &pages_allocated);
		return ret;
		}

		void io_pgtable_free_pages_exact(void *virt, size_t size)
		{
		free_pages_exact(virt, size);
		atomic_sub(1 << get_order(size), &pages_allocated);
		}

		static int io_pgtable_init(void)
		{
		io_pgtable_top = debugfs_create_dir("io-pgtable", iommu_debugfs_top);

		if (!io_pgtable_top)
		return -ENODEV;

		if (!debugfs_create_atomic_t("pages", 0600,
		io_pgtable_top, &pages_allocated)) {
		debugfs_remove_recursive(io_pgtable_top);
		return -ENODEV;
		}

		return 0;
		}

		static void io_pgtable_exit(void)
		{
		debugfs_remove_recursive(io_pgtable_top);
		}

		module_init(io_pgtable_init);
		module_exit(io_pgtable_exit);

drivers/iommu/io-pgtable.h

+20 −0

Original line number	Diff line number	Diff line
		@@ -148,4 +148,24 @@ struct io_pgtable_init_fns {
		void (free)(struct io_pgtable iop);
		};

		/**
		* io_pgtable_alloc_pages_exact - allocate an exact number physically-contiguous pages.
		* @size: the number of bytes to allocate
		* @gfp_mask: GFP flags for the allocation
		*
		* Like alloc_pages_exact(), but with some additional accounting for debug
		* purposes.
		*/
		void *io_pgtable_alloc_pages_exact(size_t size, gfp_t gfp_mask);

		/**
		* io_pgtable_free_pages_exact - release memory allocated via io_pgtable_alloc_pages_exact()
		* @virt: the value returned by alloc_pages_exact.
		* @size: size of allocation, same value as passed to alloc_pages_exact().
		*
		* Like free_pages_exact(), but with some additional accounting for debug
		* purposes.
		*/
		void io_pgtable_free_pages_exact(void *virt, size_t size);

		#endif /* __IO_PGTABLE_H */

drivers/iommu/iommu-debug.c

+484 −44

File changed.

Preview size limit exceeded, changes collapsed.