Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit ad5f4df4 authored by qctecmdr's avatar qctecmdr Committed by Gerrit - the friendly Code Review server
Browse files

Merge "iommu: arm-smmu: iommu_unmap_fast support"

parents a88ca368 49d6dec0

drivers/iommu/arm-smmu-v3.c

+10 −0
Original line number Diff line number Diff line
@@ -1759,6 +1759,14 @@ arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size) 
	return ops->unmap(ops, iova, size);

}



static void arm_smmu_iotlb_sync(struct iommu_domain *domain)

{

	struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;



	if (smmu)

		__arm_smmu_tlb_sync(smmu);

}



static phys_addr_t

arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)

{
@@ -1979,6 +1987,8 @@ static struct iommu_ops arm_smmu_ops = { 
	.map			= arm_smmu_map,

	.unmap			= arm_smmu_unmap,

	.map_sg			= default_iommu_map_sg,

	.flush_iotlb_all	= arm_smmu_iotlb_sync,

	.iotlb_sync		= arm_smmu_iotlb_sync,

	.iova_to_phys		= arm_smmu_iova_to_phys,

	.add_device		= arm_smmu_add_device,

	.remove_device		= arm_smmu_remove_device,

drivers/iommu/arm-smmu.c

+47 −34
Original line number Diff line number Diff line
@@ -360,7 +360,7 @@ struct arm_smmu_domain { 
	struct arm_smmu_device		*smmu;

	struct device			*dev;

	struct io_pgtable_ops		*pgtbl_ops;



	const struct iommu_gather_ops	*tlb_ops;

	struct arm_smmu_cfg		cfg;

	enum arm_smmu_domain_stage	stage;

	struct mutex			init_mutex; /* Protects smmu pointer */
@@ -1138,12 +1138,31 @@ static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu) 
	spin_unlock_irqrestore(&smmu->global_sync_lock, flags);

}



static void arm_smmu_tlb_inv_context_s1(void *cookie);



static void arm_smmu_tlb_sync_context(void *cookie)

{

	struct arm_smmu_domain *smmu_domain = cookie;

	struct arm_smmu_device *smmu = smmu_domain->smmu;

	struct device *dev = smmu_domain->dev;

	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;

	void __iomem *base = ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx);

	unsigned long flags;

	size_t ret;

	bool use_tlbiall = smmu->options & ARM_SMMU_OPT_NO_ASID_RETENTION;

	ktime_t cur = ktime_get();



	ret = arm_smmu_domain_power_on(&smmu_domain->domain,

				       smmu_domain->smmu);

	if (ret)

		return;



	trace_tlbi_start(dev, 0);



	if (!use_tlbiall)

		writel_relaxed(cfg->asid, base + ARM_SMMU_CB_S1_TLBIASID);

	else

		writel_relaxed(0, base + ARM_SMMU_CB_S1_TLBIALL);



	spin_lock_irqsave(&smmu_domain->sync_lock, flags);

	if (__arm_smmu_tlb_sync(smmu, base + ARM_SMMU_CB_TLBSYNC,
@@ -1153,6 +1172,10 @@ static void arm_smmu_tlb_sync_context(void *cookie) 
				    smmu_domain->cfg.cbndx,

				    dev_name(smmu_domain->dev));

	spin_unlock_irqrestore(&smmu_domain->sync_lock, flags);



	trace_tlbi_end(dev, ktime_us_delta(ktime_get(), cur));



	arm_smmu_domain_power_off(&smmu_domain->domain, smmu_domain->smmu);

}



static void arm_smmu_tlb_sync_vmid(void *cookie)
@@ -1164,23 +1187,7 @@ static void arm_smmu_tlb_sync_vmid(void *cookie) 


static void arm_smmu_tlb_inv_context_s1(void *cookie)

{

	struct arm_smmu_domain *smmu_domain = cookie;

	struct device *dev = smmu_domain->dev;

	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;

	struct arm_smmu_device *smmu = smmu_domain->smmu;

	void __iomem *base = ARM_SMMU_CB(smmu_domain->smmu, cfg->cbndx);

	bool use_tlbiall = smmu->options & ARM_SMMU_OPT_NO_ASID_RETENTION;

	ktime_t cur = ktime_get();



	trace_tlbi_start(dev, 0);



	if (!use_tlbiall)

		writel_relaxed(cfg->asid, base + ARM_SMMU_CB_S1_TLBIASID);

	else

		writel_relaxed(0, base + ARM_SMMU_CB_S1_TLBIALL);



	arm_smmu_tlb_sync_context(cookie);

	trace_tlbi_end(dev, ktime_us_delta(ktime_get(), cur));

	return;

}



static void arm_smmu_tlb_inv_context_s2(void *cookie)
@@ -1471,6 +1478,7 @@ static phys_addr_t arm_smmu_verify_fault(struct iommu_domain *domain, 


	phys = arm_smmu_iova_to_phys_hard(domain, iova);

	smmu_domain->pgtbl_cfg.tlb->tlb_flush_all(smmu_domain);

	smmu_domain->pgtbl_cfg.tlb->tlb_sync(smmu_domain);

	phys_post_tlbiall = arm_smmu_iova_to_phys_hard(domain, iova);



	if (phys != phys_post_tlbiall) {
@@ -1886,7 +1894,6 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, 
	enum io_pgtable_fmt fmt;

	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);

	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;

	const struct iommu_gather_ops *tlb_ops;

	bool is_fast = smmu_domain->attributes & (1 << DOMAIN_ATTR_FAST);

	unsigned long quirks = 0;

	bool dynamic;
@@ -1978,7 +1985,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, 
			ias = min(ias, 32UL);

			oas = min(oas, 32UL);

		}

		tlb_ops = &arm_smmu_s1_tlb_ops;

		smmu_domain->tlb_ops = &arm_smmu_s1_tlb_ops;

		break;

	case ARM_SMMU_DOMAIN_NESTED:

		/*
@@ -1998,9 +2005,9 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, 
			oas = min(oas, 40UL);

		}

		if (smmu->version == ARM_SMMU_V2)

			tlb_ops = &arm_smmu_s2_tlb_ops_v2;

			smmu_domain->tlb_ops = &arm_smmu_s2_tlb_ops_v2;

		else

			tlb_ops = &arm_smmu_s2_tlb_ops_v1;

			smmu_domain->tlb_ops = &arm_smmu_s2_tlb_ops_v1;

		break;

	default:

		ret = -EINVAL;
@@ -2022,7 +2029,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, 
		quirks |= IO_PGTABLE_QUIRK_QSMMUV500_NON_SHAREABLE;



	if (arm_smmu_is_slave_side_secure(smmu_domain))

		tlb_ops = &msm_smmu_gather_ops;

		smmu_domain->tlb_ops = &msm_smmu_gather_ops;



	ret = arm_smmu_alloc_cb(domain, smmu, dev);

	if (ret < 0)
@@ -2038,7 +2045,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, 
				.sec_id = smmu->sec_id,

				.cbndx = cfg->cbndx,

			},

			.tlb		= tlb_ops,

			.tlb		= smmu_domain->tlb_ops,

			.iommu_dev      = smmu->dev,

		};

		fmt = ARM_MSM_SECURE;
@@ -2048,7 +2055,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, 
			.pgsize_bitmap	= smmu->pgsize_bitmap,

			.ias		= ias,

			.oas		= oas,

			.tlb		= tlb_ops,

			.tlb		= smmu_domain->tlb_ops,

			.iommu_dev	= smmu->dev,

		};

	}
@@ -2503,6 +2510,7 @@ static void arm_smmu_domain_remove_master(struct arm_smmu_domain *smmu_domain, 


	/* Ensure there are no stale mappings for this context bank */

	tlb->tlb_flush_all(smmu_domain);

	tlb->tlb_sync(smmu_domain);

}



static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
@@ -2846,17 +2854,12 @@ static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, 
	if (arm_smmu_is_slave_side_secure(smmu_domain))

		return msm_secure_smmu_unmap(domain, iova, size);



	ret = arm_smmu_domain_power_on(domain, smmu_domain->smmu);

	if (ret)

		return ret;



	arm_smmu_secure_domain_lock(smmu_domain);



	spin_lock_irqsave(&smmu_domain->cb_lock, flags);

	ret = ops->unmap(ops, iova, size);

	spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);



	arm_smmu_domain_power_off(domain, smmu_domain->smmu);

	/*

	 * While splitting up block mappings, we might allocate page table

	 * memory during unmap, so the vmids needs to be assigned to the
@@ -3002,6 +3005,14 @@ static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain, 
	return ret;

}



static void arm_smmu_iotlb_sync(struct iommu_domain *domain)

{

	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);



	if (smmu_domain->tlb_ops)

		smmu_domain->tlb_ops->tlb_sync(smmu_domain);

}



/*

 * This function can sleep, and cannot be called from atomic context. Will

 * power on register block if required. This restriction does not apply to the
@@ -3811,6 +3822,8 @@ static struct iommu_ops arm_smmu_ops = { 
	.map			= arm_smmu_map,

	.unmap			= arm_smmu_unmap,

	.map_sg			= arm_smmu_map_sg,

	.flush_iotlb_all	= arm_smmu_iotlb_sync,

	.iotlb_sync		= arm_smmu_iotlb_sync,

	.iova_to_phys		= arm_smmu_iova_to_phys,

	.iova_to_phys_hard	= arm_smmu_iova_to_phys_hard,

	.add_device		= arm_smmu_add_device,

drivers/iommu/dma-mapping-fast.c

+1 −0
Original line number Diff line number Diff line
@@ -201,6 +201,7 @@ static dma_addr_t __fast_smmu_alloc_iova(struct dma_fast_smmu_mapping *mapping, 
		bool skip_sync = (attrs & DMA_ATTR_SKIP_CPU_SYNC);



		iommu_tlbiall(mapping->domain);

		iommu_tlb_sync(mapping->domain);

		mapping->have_stale_tlbs = false;

		av8l_fast_clear_stale_ptes(mapping->pgtbl_pmds, skip_sync);

	}

drivers/iommu/io-pgtable.c

+1 −0
Original line number Diff line number Diff line
@@ -89,6 +89,7 @@ void free_io_pgtable_ops(struct io_pgtable_ops *ops) 


	iop = container_of(ops, struct io_pgtable, ops);

	io_pgtable_tlb_flush_all(iop);

	io_pgtable_tlb_sync(iop);

	io_pgtable_init_table[iop->fmt]->free(iop);

}

drivers/iommu/ipmmu-vmsa.c

+10 −0
Original line number Diff line number Diff line
@@ -621,6 +621,14 @@ static size_t ipmmu_unmap(struct iommu_domain *io_domain, unsigned long iova, 
	return domain->iop->unmap(domain->iop, iova, size);

}



static void ipmmu_iotlb_sync(struct iommu_domain *io_domain)

{

	struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);



	if (domain->mmu)

		ipmmu_tlb_flush_all(domain);

}



static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain,

				      dma_addr_t iova)

{
@@ -878,6 +886,8 @@ static const struct iommu_ops ipmmu_ops = { 
	.detach_dev = ipmmu_detach_device,

	.map = ipmmu_map,

	.unmap = ipmmu_unmap,

	.flush_iotlb_all = ipmmu_iotlb_sync,

	.iotlb_sync = ipmmu_iotlb_sync,

	.map_sg = default_iommu_map_sg,

	.iova_to_phys = ipmmu_iova_to_phys,

	.add_device = ipmmu_add_device_dma,