Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e9e8346e authored by Linux Build Service Account's avatar Linux Build Service Account Committed by Gerrit - the friendly Code Review server
Browse files

Merge "iommu: arm-smmu: Support ECATS during context fault"

parents 5323339b 8c1202be

drivers/iommu/arm-smmu.c

+70 −33
Original line number Diff line number Diff line
@@ -4341,7 +4341,7 @@ IOMMU_OF_DECLARE(cavium_smmuv2, "cavium,smmu-v2", arm_smmu_of_init); 
#define DEBUG_PAR_PA_SHIFT		12

#define DEBUG_PAR_FAULT_VAL		0x1



#define TBU_DBG_TIMEOUT_US		30000

#define TBU_DBG_TIMEOUT_US		100



#define QSMMUV500_ACTLR_DEEP_PREFETCH_MASK	0x3

#define QSMMUV500_ACTLR_DEEP_PREFETCH_SHIFT	0x8
@@ -4509,11 +4509,12 @@ static struct iommu_gather_ops qsmmuv500_errata1_smmu_gather_ops = { 
	.free_pages_exact = arm_smmu_free_pages_exact,

};



static int qsmmuv500_tbu_halt(struct qsmmuv500_tbu_device *tbu)

static int qsmmuv500_tbu_halt(struct qsmmuv500_tbu_device *tbu,

				struct arm_smmu_domain *smmu_domain)

{

	unsigned long flags;

	u32 val;

	void __iomem *base;

	u32 halt, fsr, sctlr_orig, sctlr, status;

	void __iomem *base, *cb_base;



	spin_lock_irqsave(&tbu->halt_lock, flags);

	if (tbu->halt_count) {
@@ -4522,19 +4523,49 @@ static int qsmmuv500_tbu_halt(struct qsmmuv500_tbu_device *tbu) 
		return 0;

	}



	cb_base = ARM_SMMU_CB_BASE(smmu_domain->smmu) +

		ARM_SMMU_CB(smmu_domain->smmu, smmu_domain->cfg.cbndx);

	base = tbu->base;

	val = readl_relaxed(base + DEBUG_SID_HALT_REG);

	val |= DEBUG_SID_HALT_VAL;

	writel_relaxed(val, base + DEBUG_SID_HALT_REG);

	halt = readl_relaxed(base + DEBUG_SID_HALT_REG);

	halt |= DEBUG_SID_HALT_VAL;

	writel_relaxed(halt, base + DEBUG_SID_HALT_REG);



	if (readl_poll_timeout_atomic(base + DEBUG_SR_HALT_ACK_REG,

					val, (val & DEBUG_SR_HALT_ACK_VAL),

					0, TBU_DBG_TIMEOUT_US)) {

	if (!readl_poll_timeout_atomic(base + DEBUG_SR_HALT_ACK_REG, status,

					(status & DEBUG_SR_HALT_ACK_VAL),

					0, TBU_DBG_TIMEOUT_US))

		goto out;



	fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);

	if (!(fsr & FSR_FAULT)) {

		dev_err(tbu->dev, "Couldn't halt TBU!\n");

		spin_unlock_irqrestore(&tbu->halt_lock, flags);

		return -ETIMEDOUT;

	}



	/*

	 * We are in a fault; Our request to halt the bus will not complete

	 * until transactions in front of us (such as the fault itself) have

	 * completed. Disable iommu faults and terminate any existing

	 * transactions.

	 */

	sctlr_orig = readl_relaxed(cb_base + ARM_SMMU_CB_SCTLR);

	sctlr = sctlr_orig & ~(SCTLR_CFCFG | SCTLR_CFIE);

	writel_relaxed(sctlr, cb_base + ARM_SMMU_CB_SCTLR);



	writel_relaxed(fsr, cb_base + ARM_SMMU_CB_FSR);

	writel_relaxed(RESUME_TERMINATE, cb_base + ARM_SMMU_CB_RESUME);



	if (readl_poll_timeout_atomic(base + DEBUG_SR_HALT_ACK_REG, status,

					(status & DEBUG_SR_HALT_ACK_VAL),

					0, TBU_DBG_TIMEOUT_US)) {

		dev_err(tbu->dev, "Couldn't halt TBU from fault context!\n");

		writel_relaxed(sctlr_orig, cb_base + ARM_SMMU_CB_SCTLR);

		spin_unlock_irqrestore(&tbu->halt_lock, flags);

		return -ETIMEDOUT;

	}



	writel_relaxed(sctlr_orig, cb_base + ARM_SMMU_CB_SCTLR);

out:

	tbu->halt_count = 1;

	spin_unlock_irqrestore(&tbu->halt_lock, flags);

	return 0;
@@ -4635,6 +4666,14 @@ static phys_addr_t qsmmuv500_iova_to_phys( 
	void __iomem *cb_base;

	u32 sctlr_orig, sctlr;

	int needs_redo = 0;

	ktime_t timeout;



	/* only 36 bit iova is supported */

	if (iova >= (1ULL << 36)) {

		dev_err_ratelimited(smmu->dev, "ECATS: address too large: %pad\n",

					&iova);

		return 0;

	}



	cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);

	tbu = qsmmuv500_find_tbu(smmu, sid);
@@ -4645,35 +4684,23 @@ static phys_addr_t qsmmuv500_iova_to_phys( 
	if (ret)

		return 0;



	/*

	 * Disable client transactions & wait for existing operations to

	 * complete.

	 */

	ret = qsmmuv500_tbu_halt(tbu);

	ret = qsmmuv500_tbu_halt(tbu, smmu_domain);

	if (ret)

		goto out_power_off;



	/* Only one concurrent atos operation */

	ret = qsmmuv500_ecats_lock(smmu_domain, tbu, &flags);

	if (ret)

		goto out_resume;



	/*

	 * We can be called from an interrupt handler with FSR already set

	 * so terminate the faulting transaction prior to starting ecats.

	 * No new racing faults can occur since we in the halted state.

	 * ECATS can trigger the fault interrupt, so disable it temporarily

	 * and check for an interrupt manually.

	 */

	fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);

	if (fsr & FSR_FAULT) {

		writel_relaxed(fsr, cb_base + ARM_SMMU_CB_FSR);

		writel_relaxed(RESUME_TERMINATE, cb_base + ARM_SMMU_CB_RESUME);

	}

	sctlr_orig = readl_relaxed(cb_base + ARM_SMMU_CB_SCTLR);

	sctlr = sctlr_orig & ~(SCTLR_CFCFG | SCTLR_CFIE);

	writel_relaxed(sctlr, cb_base + ARM_SMMU_CB_SCTLR);



	/* Only one concurrent atos operation */

	ret = qsmmuv500_ecats_lock(smmu_domain, tbu, &flags);

	if (ret)

		goto out_resume;



redo:

	/* Set address and stream-id */

	val = readq_relaxed(tbu->base + DEBUG_SID_HALT_REG);
@@ -4692,16 +4719,26 @@ static phys_addr_t qsmmuv500_iova_to_phys( 
	writeq_relaxed(val, tbu->base + DEBUG_TXN_TRIGG_REG);



	ret = 0;

	if (readl_poll_timeout_atomic(tbu->base + DEBUG_SR_HALT_ACK_REG,

				val, !(val & DEBUG_SR_ECATS_RUNNING_VAL),

				0, TBU_DBG_TIMEOUT_US)) {

	//based on readx_poll_timeout_atomic

	timeout = ktime_add_us(ktime_get(), TBU_DBG_TIMEOUT_US);

	for (;;) {

		val = readl_relaxed(tbu->base + DEBUG_SR_HALT_ACK_REG);

		if (!(val & DEBUG_SR_ECATS_RUNNING_VAL))

			break;

		val = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);

		if (val & FSR_FAULT)

			break;

		if (ktime_compare(ktime_get(), timeout) > 0) {

			dev_err(tbu->dev, "ECATS translation timed out!\n");

			ret = -ETIMEDOUT;

			break;

		}

	}



	fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);

	if (fsr & FSR_FAULT) {

		dev_err(tbu->dev, "ECATS generated a fault interrupt! FSR = %llx\n",

			val);

			fsr);

		ret = -EINVAL;



		writel_relaxed(val, cb_base + ARM_SMMU_CB_FSR);

drivers/iommu/iommu-debug.c

+4 −1
Original line number Diff line number Diff line
@@ -165,6 +165,7 @@ static void *test_virt_addr; 
struct iommu_debug_device {

	struct device *dev;

	struct iommu_domain *domain;

	struct dma_iommu_mapping *mapping;

	u64 iova;

	u64 phys;

	size_t len;
@@ -1251,6 +1252,8 @@ static ssize_t __iommu_debug_dma_attach_write(struct file *file, 


		if (arm_iommu_attach_device(dev, dma_mapping))

			goto out_release_mapping;



		ddev->mapping = dma_mapping;

		pr_err("Attached\n");

	} else {

		if (!dev->archdata.mapping) {
@@ -1264,7 +1267,7 @@ static ssize_t __iommu_debug_dma_attach_write(struct file *file, 
			goto out;

		}

		arm_iommu_detach_device(dev);

		arm_iommu_release_mapping(dev->archdata.mapping);

		arm_iommu_release_mapping(ddev->mapping);

		pr_err("Detached\n");

	}

	retval = count;

drivers/staging/android/ion/ion_cma_heap.c

+16 −6
Original line number Diff line number Diff line
@@ -57,6 +57,7 @@ static int ion_cma_get_sgtable(struct device *dev, struct sg_table *sgt, 
		return ret;



	sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);

	sg_dma_address(sgt->sgl) = sg_phys(sgt->sgl);

	return 0;

}
@@ -97,9 +98,9 @@ static int ion_cma_allocate(struct ion_heap *heap, struct ion_buffer *buffer, 
							&info->handle,

							GFP_KERNEL);

	else

		info->cpu_addr = dma_alloc_nonconsistent(dev, len,

							 &info->handle,

							 GFP_KERNEL);

		info->cpu_addr = dma_alloc_attrs(dev, len, &info->handle,

						GFP_KERNEL,

						DMA_ATTR_FORCE_COHERENT);



	if (!info->cpu_addr) {

		dev_err(dev, "Fail to allocate buffer\n");
@@ -115,6 +116,11 @@ static int ion_cma_allocate(struct ion_heap *heap, struct ion_buffer *buffer, 
	ion_cma_get_sgtable(dev,

			    info->table, info->cpu_addr, info->handle, len);



	/* Ensure memory is dma-ready - refer to ion_buffer_create() */

	if (info->is_cached)

		dma_sync_sg_for_device(dev, info->table->sgl,

				       info->table->nents, DMA_BIDIRECTIONAL);



	/* keep this for memory release */

	buffer->priv_virt = info;

	dev_dbg(dev, "Allocate buffer %pK\n", buffer);
@@ -129,10 +135,13 @@ static void ion_cma_free(struct ion_buffer *buffer) 
{

	struct device *dev = buffer->heap->priv;

	struct ion_cma_buffer_info *info = buffer->priv_virt;

	unsigned long attrs = 0;



	dev_dbg(dev, "Release buffer %pK\n", buffer);

	/* release memory */

	dma_free_coherent(dev, buffer->size, info->cpu_addr, info->handle);

	if (info->is_cached)

		attrs |= DMA_ATTR_FORCE_COHERENT;

	dma_free_attrs(dev, buffer->size, info->cpu_addr, info->handle, attrs);

	sg_free_table(info->table);

	/* release sg table */

	kfree(info->table);
@@ -175,8 +184,9 @@ static int ion_cma_mmap(struct ion_heap *mapper, struct ion_buffer *buffer, 
	struct ion_cma_buffer_info *info = buffer->priv_virt;



	if (info->is_cached)

		return dma_mmap_nonconsistent(dev, vma, info->cpu_addr,

				info->handle, buffer->size);

		return dma_mmap_attrs(dev, vma, info->cpu_addr,

				info->handle, buffer->size,

				DMA_ATTR_FORCE_COHERENT);

	else

		return dma_mmap_writecombine(dev, vma, info->cpu_addr,

				info->handle, buffer->size);

include/linux/gfp.h

+1 −1
Original line number Diff line number Diff line
@@ -189,7 +189,7 @@ struct vm_area_struct; 
#define __GFP_OTHER_NODE ((__force gfp_t)___GFP_OTHER_NODE)



/* Room for N __GFP_FOO bits */

#define __GFP_BITS_SHIFT 26

#define __GFP_BITS_SHIFT 27

#define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1))



/*