Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 2a354cc5 authored by qctecmdr's avatar qctecmdr Committed by Gerrit - the friendly Code Review server
Browse files

Merge "iommu/arm-smmu: add qcom,iommu-geometry DT property"

parents f9b8eceb e98585fe

drivers/iommu/arm-smmu.c

+72 −0
Original line number Diff line number Diff line
@@ -2230,6 +2230,14 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, 


	cfg->cbndx = ret;



	if (!(smmu_domain->attributes & (1 << DOMAIN_ATTR_GEOMETRY))) {

		/* Geometry is not set use the default geometry */

		domain->geometry.aperture_start = 0;

		domain->geometry.aperture_end = (1UL << ias) - 1;

		if (domain->geometry.aperture_end >= SZ_1G * 4ULL)

			domain->geometry.aperture_end = (SZ_1G * 4ULL) - 1;

	}



	if (arm_smmu_is_slave_side_secure(smmu_domain)) {

		smmu_domain->pgtbl_cfg = (struct io_pgtable_cfg) {

			.quirks         = quirks,
@@ -2240,6 +2248,8 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, 
			},

			.tlb		= smmu_domain->tlb_ops,

			.iommu_dev      = smmu->dev,

			.iova_base	= domain->geometry.aperture_start,

			.iova_end	= domain->geometry.aperture_end,

		};

		fmt = ARM_MSM_SECURE;

	} else  {
@@ -2250,6 +2260,8 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, 
			.oas		= oas,

			.tlb		= smmu_domain->tlb_ops,

			.iommu_dev	= smmu->dev,

			.iova_base	= domain->geometry.aperture_start,

			.iova_end	= domain->geometry.aperture_end,

		};

	}
@@ -2927,6 +2939,8 @@ static int arm_smmu_setup_default_domain(struct device *dev, 
	const char *str;

	int attr = 1;

	u32 val;

	const __be32 *ranges;

	int naddr, nsize, len;



	np = arm_iommu_get_of_node(dev);

	if (!np)
@@ -3003,6 +3017,30 @@ static int arm_smmu_setup_default_domain(struct device *dev, 
	if (of_property_read_bool(np, "qcom,iommu-earlymap"))

		__arm_smmu_domain_set_attr(domain,

			DOMAIN_ATTR_EARLY_MAP, &attr);



	ranges = of_get_property(np, "qcom,iommu-geometry", &len);

	if (ranges) {

		struct iommu_domain_geometry geometry;



		len /= sizeof(u32);

		naddr = of_n_addr_cells(np);

		nsize = of_n_size_cells(np);

		if (len < naddr + nsize) {

			dev_err(dev, "Invalid length for qcom,iommu-geometry, expected %d cells\n",

				naddr + nsize);

			return -EINVAL;

		}

		if (naddr == 0 || nsize == 0) {

			dev_err(dev, "Invalid #address-cells %d or #size-cells %d\n",

				naddr, nsize);

			return -EINVAL;

		}



		geometry.aperture_start = of_read_number(ranges, naddr);

		geometry.aperture_end = of_read_number(ranges + naddr, nsize);

		__arm_smmu_domain_set_attr(domain, DOMAIN_ATTR_GEOMETRY,

					   &geometry);

	}

	return 0;

}
@@ -4006,6 +4044,40 @@ static int __arm_smmu_domain_set_attr2(struct iommu_domain *domain, 
		ret = 0;

		break;

	}



	case DOMAIN_ATTR_GEOMETRY: {

		struct iommu_domain_geometry *geometry =

				(struct iommu_domain_geometry *)data;



		if (smmu_domain->smmu != NULL) {

			dev_err(smmu_domain->smmu->dev,

			  "cannot set geometry attribute while attached\n");

			ret = -EBUSY;

			break;

		}



		if (geometry->aperture_start >= SZ_1G * 4ULL ||

		    geometry->aperture_end >= SZ_1G * 4ULL) {

			pr_err("fastmap does not support IOVAs >= 4GB\n");

			ret = -EINVAL;

			break;

		}



		if (geometry->aperture_start

		    < domain->geometry.aperture_start)

			domain->geometry.aperture_start =

				geometry->aperture_start;



		if (geometry->aperture_end

		    > domain->geometry.aperture_end)

			domain->geometry.aperture_end =

				geometry->aperture_end;



		smmu_domain->attributes |= 1 << DOMAIN_ATTR_GEOMETRY;

		ret = 0;

		break;

	}



	default:

		ret = -ENODEV;

	}

drivers/iommu/dma-mapping-fast.c

+10 −3
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0-only

/*

 * Copyright (c) 2016-2019, The Linux Foundation. All rights reserved.

 * Copyright (c) 2016-2017, The Linux Foundation. All rights reserved.

 */



#include <linux/dma-contiguous.h>
@@ -210,7 +210,11 @@ static dma_addr_t __fast_smmu_alloc_iova(struct dma_fast_smmu_mapping *mapping, 


		iommu_tlbiall(mapping->domain);

		mapping->have_stale_tlbs = false;

		av8l_fast_clear_stale_ptes(mapping->pgtbl_ops, skip_sync);

		av8l_fast_clear_stale_ptes(mapping->pgtbl_ops,

				mapping->domain->geometry.aperture_start,

				mapping->base,

				mapping->base + mapping->size - 1,

				skip_sync);

	}



	return (bit << FAST_PAGE_SHIFT) + mapping->base;
@@ -1075,7 +1079,7 @@ static const struct dma_map_ops fast_smmu_dma_ops = { 
 *

 * Creates a mapping structure which holds information about used/unused IO

 * address ranges, which is required to perform mapping with IOMMU aware

 * functions.  The only VA range supported is [0, 4GB).

 * functions. The only VA range supported is [0, 4GB].

 *

 * The client device need to be attached to the mapping with

 * fast_smmu_attach_device function.
@@ -1219,6 +1223,9 @@ int fast_smmu_init_mapping(struct device *dev, 
	fast->dev = dev;

	domain->iova_cookie = fast;



	domain->geometry.aperture_start = mapping->base;

	domain->geometry.aperture_end = mapping->base + size - 1;



	if (iommu_domain_get_attr(domain, DOMAIN_ATTR_PGTBL_INFO,

				  &info)) {

		dev_err(dev, "Couldn't get page table info\n");

drivers/iommu/io-pgtable-fast.c

+80 −40
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0-only

/*

 * Copyright (c) 2016-2019, The Linux Foundation. All rights reserved.

 * Copyright (c) 2016-2017, The Linux Foundation. All rights reserved.

 */



#define pr_fmt(fmt)	"io-pgtable-fast: " fmt
@@ -12,6 +12,7 @@ 
#include <linux/slab.h>

#include <linux/types.h>

#include <linux/io-pgtable-fast.h>

#include <linux/mm.h>

#include <asm/cacheflush.h>

#include <linux/vmalloc.h>
@@ -35,6 +36,10 @@ struct av8l_fast_io_pgtable { 
	av8l_fast_iopte		 *puds[4];

	av8l_fast_iopte		 *pmds;

	struct page		**pages; /* page table memory */

	int			nr_pages;

	dma_addr_t		base;

	dma_addr_t		start;

	dma_addr_t		end;

};



/* Page table bits */
@@ -143,7 +148,7 @@ struct av8l_fast_io_pgtable { 


#define PTE_SH_IDX(pte) (pte & AV8L_FAST_PTE_SH_MASK)



#define iopte_pmd_offset(pmds, iova) (pmds + (iova >> 12))

#define iopte_pmd_offset(pmds, base, iova) (pmds + ((iova - base) >> 12))



#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST_PROVE_TLB
@@ -172,13 +177,15 @@ static void __av8l_check_for_stale_tlb(av8l_fast_iopte *ptep) 
	}

}



void av8l_fast_clear_stale_ptes(struct io_pgtable_ops *ops, bool skip_sync)

void av8l_fast_clear_stale_ptes(struct io_pgtable_ops *ops, u64 base,

		u64 start, u64 end, bool skip_sync)

{

	int i;

	struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops);

	av8l_fast_iopte *pmdp = data->pmds;

	av8l_fast_iopte *pmdp = iopte_pmd_offset(pmds, base, start);



	for (i = 0; i < ((SZ_1G * 4UL) >> AV8L_FAST_PAGE_SHIFT); ++i) {

	for (i = start >> AV8L_FAST_PAGE_SHIFT;

			i <= (end >> AV8L_FAST_PAGE_SHIFT); ++i) {

		if (!(*pmdp & AV8L_FAST_PTE_VALID)) {

			*pmdp = 0;

			if (!skip_sync)
@@ -233,7 +240,7 @@ static int av8l_fast_map(struct io_pgtable_ops *ops, unsigned long iova, 
			 phys_addr_t paddr, size_t size, int prot)

{

	struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops);

	av8l_fast_iopte *ptep = iopte_pmd_offset(data->pmds, iova);

	av8l_fast_iopte *ptep = iopte_pmd_offset(data->pmds, data->base, iova);

	unsigned long i, nptes = size >> AV8L_FAST_PAGE_SHIFT;

	av8l_fast_iopte pte;
@@ -265,7 +272,7 @@ __av8l_fast_unmap(struct io_pgtable_ops *ops, unsigned long iova, 
		? AV8L_FAST_PTE_UNMAPPED_NEED_TLBI

		: 0;



	ptep = iopte_pmd_offset(data->pmds, iova);

	ptep = iopte_pmd_offset(data->pmds, data->base, iova);

	nptes = size >> AV8L_FAST_PAGE_SHIFT;



	memset(ptep, val, sizeof(*ptep) * nptes);
@@ -363,7 +370,7 @@ static bool av8l_fast_iova_coherent(struct io_pgtable_ops *ops, 
					unsigned long iova)

{

	struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops);

	av8l_fast_iopte *ptep = iopte_pmd_offset(data->pmds, iova);

	av8l_fast_iopte *ptep = iopte_pmd_offset(data->pmds, data->base, iova);



	return ((PTE_MAIR_IDX(*ptep) == AV8L_FAST_MAIR_ATTR_IDX_CACHE) &&

		((PTE_SH_IDX(*ptep) == AV8L_FAST_PTE_SH_OS) ||
@@ -397,7 +404,7 @@ av8l_fast_alloc_pgtable_data(struct io_pgtable_cfg *cfg) 
}



/*

 * We need 1 page for the pgd, 4 pages for puds (1GB VA per pud page) and

 * We need max 1 page for the pgd, 4 pages for puds (1GB VA per pud page) and

 * 2048 pages for pmds (each pud page contains 512 table entries, each

 * pointing to a pmd).

 */
@@ -406,12 +413,38 @@ av8l_fast_alloc_pgtable_data(struct io_pgtable_cfg *cfg) 
#define NUM_PMD_PAGES 2048

#define NUM_PGTBL_PAGES (NUM_PGD_PAGES + NUM_PUD_PAGES + NUM_PMD_PAGES)



/* undefine arch specific definitions which depends on page table format */

#undef pud_index

#undef pud_mask

#undef pud_next

#undef pmd_index

#undef pmd_mask

#undef pmd_next



#define pud_index(addr)		(((addr) >> 30) & 0x3)

#define pud_mask(addr)		((addr) & ~((1UL << 30) - 1))

#define pud_next(addr, end)					\

({	unsigned long __boundary = pud_mask(addr + (1UL << 30));\

	(__boundary - 1 < (end) - 1) ? __boundary : (end);	\

})



#define pmd_index(addr)		(((addr) >> 21) & 0x1ff)

#define pmd_mask(addr)		((addr) & ~((1UL << 21) - 1))

#define pmd_next(addr, end)					\

({	unsigned long __boundary = pmd_mask(addr + (1UL << 21));\

	(__boundary - 1 < (end) - 1) ? __boundary : (end);	\

})



static int

av8l_fast_prepopulate_pgtables(struct av8l_fast_io_pgtable *data,

			       struct io_pgtable_cfg *cfg, void *cookie)

{

	int i, j, pg = 0;

	struct page **pages, *page;

	dma_addr_t base = cfg->iova_base;

	dma_addr_t end = cfg->iova_end;

	dma_addr_t pud, pmd;

	int pmd_pg_index;



	pages = kmalloc(sizeof(*pages) * NUM_PGTBL_PAGES, __GFP_NOWARN |

							__GFP_NORETRY);
@@ -429,10 +462,11 @@ av8l_fast_prepopulate_pgtables(struct av8l_fast_io_pgtable *data, 
	data->pgd = page_address(page);



	/*

	 * We need 2048 entries at level 2 to map 4GB of VA space. A page

	 * can hold 512 entries, so we need 4 pages.

	 * We need max 2048 entries at level 2 to map 4GB of VA space. A page

	 * can hold 512 entries, so we need max 4 pages.

	 */

	for (i = 0; i < 4; ++i) {

	for (i = pud_index(base), pud = base; pud < end;

			++i, pud = pud_next(pud, end)) {

		av8l_fast_iopte pte, *ptep;



		page = alloc_page(GFP_KERNEL | __GFP_ZERO);
@@ -447,12 +481,15 @@ av8l_fast_prepopulate_pgtables(struct av8l_fast_io_pgtable *data, 
	dmac_clean_range(data->pgd, data->pgd + 4);



	/*

	 * We have 4 puds, each of which can point to 512 pmds, so we'll

	 * have 2048 pmds, each of which can hold 512 ptes, for a grand

	 * We have max 4 puds, each of which can point to 512 pmds, so we'll

	 * have max 2048 pmds, each of which can hold 512 ptes, for a grand

	 * total of 2048*512=1048576 PTEs.

	 */

	for (i = 0; i < 4; ++i) {

		for (j = 0; j < 512; ++j) {

	pmd_pg_index = pg;

	for (i = pud_index(base), pud = base; pud < end;

			++i, pud = pud_next(pud, end)) {

		for (j = pmd_index(pud), pmd = pud; pmd < pud_next(pud, end);

				++j, pmd = pmd_next(pmd, end)) {

			av8l_fast_iopte pte, *pudp;

			void *addr;
@@ -471,21 +508,21 @@ av8l_fast_prepopulate_pgtables(struct av8l_fast_io_pgtable *data, 
		dmac_clean_range(data->puds[i], data->puds[i] + 512);

	}



	if (WARN_ON(pg != NUM_PGTBL_PAGES))

		goto err_free_pages;



	/*

	 * We map the pmds into a virtually contiguous space so that we

	 * don't have to traverse the first two levels of the page tables

	 * to find the appropriate pud.  Instead, it will be a simple

	 * offset from the virtual base of the pmds.

	 */

	data->pmds = vmap(&pages[NUM_PGD_PAGES + NUM_PUD_PAGES], NUM_PMD_PAGES,

	data->pmds = vmap(&pages[pmd_pg_index], pg - pmd_pg_index,

			  VM_IOREMAP, PAGE_KERNEL);

	if (!data->pmds)

		goto err_free_pages;



	data->pages = pages;

	data->nr_pages = pg;

	data->base = base;

	data->end = end;

	return 0;



err_free_pages:
@@ -591,7 +628,7 @@ static void av8l_fast_free_pgtable(struct io_pgtable *iop) 
	struct av8l_fast_io_pgtable *data = iof_pgtable_to_data(iop);



	vunmap(data->pmds);

	for (i = 0; i < NUM_PGTBL_PAGES; ++i)

	for (i = 0; i < data->nr_pages; ++i)

		__free_page(data->pages[i]);

	kvfree(data->pages);

	kfree(data);
@@ -663,6 +700,7 @@ static int __init av8l_fast_positive_testing(void) 
	struct av8l_fast_io_pgtable *data;

	av8l_fast_iopte *pmds;

	u64 max = SZ_1G * 4ULL - 1;

	u64 base = 0;



	cfg = (struct io_pgtable_cfg) {

		.quirks = 0,
@@ -670,6 +708,8 @@ static int __init av8l_fast_positive_testing(void) 
		.ias = 32,

		.oas = 32,

		.pgsize_bitmap = SZ_4K,

		.iova_base = base,

		.iova_end = max,

	};



	cfg_cookie = &cfg;
@@ -682,81 +722,81 @@ static int __init av8l_fast_positive_testing(void) 
	pmds = data->pmds;



	/* map the entire 4GB VA space with 4K map calls */

	for (iova = 0; iova < max; iova += SZ_4K) {

	for (iova = base; iova < max; iova += SZ_4K) {

		if (WARN_ON(ops->map(ops, iova, iova, SZ_4K, IOMMU_READ))) {

			failed++;

			continue;

		}

	}

	if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, 0, 0,

							  max)))

	if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, base,

					base, max - base)))

		failed++;



	/* unmap it all */

	for (iova = 0; iova < max; iova += SZ_4K) {

	for (iova = base; iova < max; iova += SZ_4K) {

		if (WARN_ON(ops->unmap(ops, iova, SZ_4K) != SZ_4K))

			failed++;

	}



	/* sweep up TLB proving PTEs */

	av8l_fast_clear_stale_ptes(ops, false);

	av8l_fast_clear_stale_ptes(pmds, base, base, max, false);



	/* map the entire 4GB VA space with 8K map calls */

	for (iova = 0; iova < max; iova += SZ_8K) {

	for (iova = base; iova < max; iova += SZ_8K) {

		if (WARN_ON(ops->map(ops, iova, iova, SZ_8K, IOMMU_READ))) {

			failed++;

			continue;

		}

	}



	if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, 0, 0,

							  max)))

	if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, base,

					base, max - base)))

		failed++;



	/* unmap it all with 8K unmap calls */

	for (iova = 0; iova < max; iova += SZ_8K) {

	for (iova = base; iova < max; iova += SZ_8K) {

		if (WARN_ON(ops->unmap(ops, iova, SZ_8K) != SZ_8K))

			failed++;

	}



	/* sweep up TLB proving PTEs */

	av8l_fast_clear_stale_ptes(ops, false);

	av8l_fast_clear_stale_ptes(pmds, base, base, max, false);



	/* map the entire 4GB VA space with 16K map calls */

	for (iova = 0; iova < max; iova += SZ_16K) {

	for (iova = base; iova < max; iova += SZ_16K) {

		if (WARN_ON(ops->map(ops, iova, iova, SZ_16K, IOMMU_READ))) {

			failed++;

			continue;

		}

	}



	if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, 0, 0,

							  max)))

	if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, base,

					base, max - base)))

		failed++;



	/* unmap it all */

	for (iova = 0; iova < max; iova += SZ_16K) {

	for (iova = base; iova < max; iova += SZ_16K) {

		if (WARN_ON(ops->unmap(ops, iova, SZ_16K) != SZ_16K))

			failed++;

	}



	/* sweep up TLB proving PTEs */

	av8l_fast_clear_stale_ptes(ops, false);

	av8l_fast_clear_stale_ptes(pmds, base, base, max, false);



	/* map the entire 4GB VA space with 64K map calls */

	for (iova = 0; iova < max; iova += SZ_64K) {

	for (iova = base; iova < max; iova += SZ_64K) {

		if (WARN_ON(ops->map(ops, iova, iova, SZ_64K, IOMMU_READ))) {

			failed++;

			continue;

		}

	}



	if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, 0, 0,

							  max)))

	if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, base,

					base, max - base)))

		failed++;



	/* unmap it all at once */

	if (WARN_ON(ops->unmap(ops, 0, max) != max))

	if (WARN_ON(ops->unmap(ops, base, max - base) != (max - base)))

		failed++;



	free_io_pgtable_ops(ops);

drivers/iommu/io-pgtable.h

+2 −0
Original line number Diff line number Diff line
@@ -114,6 +114,8 @@ struct io_pgtable_cfg { 
	unsigned int			oas;

	const struct iommu_gather_ops	*tlb;

	struct device			*iommu_dev;

	dma_addr_t			iova_base;

	dma_addr_t			iova_end;



	/* Low-level data specific to the table format */

	union {

drivers/iommu/iommu-debug.c

+8 −0
Original line number Diff line number Diff line
@@ -171,21 +171,25 @@ static void iommu_debug_destroy_phoney_sg_table(struct device *dev, 
struct iommu_debug_attr {

	unsigned long dma_type;

	int vmid;

	struct iommu_domain_geometry geometry;

};



static struct iommu_debug_attr std_attr = {

	.dma_type = 0,

	.vmid = 0,

	.geometry = {0, 0, 0},

};



static struct iommu_debug_attr fastmap_attr = {

	.dma_type = DOMAIN_ATTR_FAST,

	.vmid = 0,

	.geometry = {0, (dma_addr_t)(SZ_1G * 4ULL - 1), 0},

};



static struct iommu_debug_attr secure_attr = {

	.dma_type = 0,

	.vmid = VMID_CP_PIXEL,

	.geometry = {0, 0, 0},

};



static int iommu_debug_set_attrs(struct iommu_debug_device *ddev,
@@ -204,6 +208,10 @@ static int iommu_debug_set_attrs(struct iommu_debug_device *ddev, 
		iommu_domain_set_attr(domain,

			DOMAIN_ATTR_SECURE_VMID, &attrs->vmid);



	if (attrs->geometry.aperture_end || attrs->geometry.aperture_start)

		iommu_domain_set_attr(domain,

			DOMAIN_ATTR_GEOMETRY, &attrs->geometry);



	return 0;

}