Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit dd8e31a4 authored by Mitchel Humpherys's avatar Mitchel Humpherys
Browse files

iommu/io-pgtable: Add fast page table mapper for ARMv8L 



Certain use cases require performance that can't be achieved with the
general-purpose SMMU page table code.  By limiting ourselves to 4K page
mappings (no block mappings) and pre-populating the first and second
levels of the page tables up front, we can eliminate a lot of the work
needed for page table mapping and unmapping.

Add a performance-tuned io-pgtable implementation for ARMv8L page tables
that only supports 4K page mappings.  Any size can be mapped, but only
4K page mappings will be installed in the page tables.

CRs-Fixed: 997751
Change-Id: I5861270709675016988052360d196e0a16a0d103
Signed-off-by: default avatarMitchel Humpherys <mitchelh@codeaurora.org>
parent e05047e2

drivers/iommu/Kconfig

+21 −0
Original line number Diff line number Diff line
@@ -37,6 +37,27 @@ config IOMMU_IO_PGTABLE_LPAE_SELFTEST 


	  If unsure, say N here.



config IOMMU_IO_PGTABLE_FAST

	bool "Fast ARMv7/v8 Long Descriptor Format"

	select IOMMU_IO_PGTABLE

	help

          Enable support for a subset of the ARM long descriptor pagetable

	  format.  This allocator achieves fast performance by

	  pre-allocating and pre-populating page table memory up front.

	  only supports a 32 bit virtual address space.



          This implementation is mainly optimized for use cases where the

          buffers are small (<= 64K) since it only supports 4K page sizes.



config IOMMU_IO_PGTABLE_FAST_SELFTEST

	bool "Fast IO pgtable selftests"

	depends on IOMMU_IO_PGTABLE_FAST

	help

	  Enable self-tests for "fast" page table allocator. This performs

	  a series of page-table consistency checks during boot.



	  If unsure, say N here.



endmenu



config OF_IOMMU

drivers/iommu/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -5,6 +5,7 @@ obj-$(CONFIG_IOMMU_API) += msm_dma_iommu_mapping.o 
obj-$(CONFIG_IOMMU_IO_PGTABLE) += io-pgtable.o

obj-$(CONFIG_IOMMU_IO_PGTABLE_LPAE) += io-pgtable-arm.o

obj-$(CONFIG_MSM_TZ_SMMU) += io-pgtable-msm-secure.o

obj-$(CONFIG_IOMMU_IO_PGTABLE_FAST) += io-pgtable-fast.o

obj-$(CONFIG_OF_IOMMU)	+= of_iommu.o

obj-$(CONFIG_IOMMU_DEBUG) += iommu-debug.o

obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_domains.o msm_iommu_mapping.o

drivers/iommu/io-pgtable-fast.c

0 → 100644
+618 −0
Original line number Diff line number Diff line 
/* Copyright (c) 2016, The Linux Foundation. All rights reserved.

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License version 2 and

 * only version 2 as published by the Free Software Foundation.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 */



#define pr_fmt(fmt)	"io-pgtable-fast: " fmt



#include <linux/iommu.h>

#include <linux/kernel.h>

#include <linux/scatterlist.h>

#include <linux/sizes.h>

#include <linux/slab.h>

#include <linux/types.h>

#include <linux/io-pgtable-fast.h>



#include "io-pgtable.h"



#define AV8L_FAST_MAX_ADDR_BITS		48



/* Struct accessors */

#define iof_pgtable_to_data(x)						\

	container_of((x), struct av8l_fast_io_pgtable, iop)



#define iof_pgtable_ops_to_pgtable(x)					\

	container_of((x), struct io_pgtable, ops)



#define iof_pgtable_ops_to_data(x)					\

	iof_pgtable_to_data(iof_pgtable_ops_to_pgtable(x))



struct av8l_fast_io_pgtable {

	struct io_pgtable	  iop;

	av8l_fast_iopte		 *pgd;

	av8l_fast_iopte		 *puds[4];

	av8l_fast_iopte		 *pmds;

	struct page		**pages; /* page table memory */

};



/* Page table bits */

#define AV8L_FAST_PTE_TYPE_SHIFT	0

#define AV8L_FAST_PTE_TYPE_MASK		0x3



#define AV8L_FAST_PTE_TYPE_BLOCK	1

#define AV8L_FAST_PTE_TYPE_TABLE	3

#define AV8L_FAST_PTE_TYPE_PAGE		3



#define AV8L_FAST_PTE_NSTABLE		(((av8l_fast_iopte)1) << 63)

#define AV8L_FAST_PTE_XN		(((av8l_fast_iopte)3) << 53)

#define AV8L_FAST_PTE_AF		(((av8l_fast_iopte)1) << 10)

#define AV8L_FAST_PTE_SH_NS		(((av8l_fast_iopte)0) << 8)

#define AV8L_FAST_PTE_SH_OS		(((av8l_fast_iopte)2) << 8)

#define AV8L_FAST_PTE_SH_IS		(((av8l_fast_iopte)3) << 8)

#define AV8L_FAST_PTE_NS		(((av8l_fast_iopte)1) << 5)

#define AV8L_FAST_PTE_VALID		(((av8l_fast_iopte)1) << 0)



#define AV8L_FAST_PTE_ATTR_LO_MASK	(((av8l_fast_iopte)0x3ff) << 2)

/* Ignore the contiguous bit for block splitting */

#define AV8L_FAST_PTE_ATTR_HI_MASK	(((av8l_fast_iopte)6) << 52)

#define AV8L_FAST_PTE_ATTR_MASK		(AV8L_FAST_PTE_ATTR_LO_MASK |	\

					 AV8L_FAST_PTE_ATTR_HI_MASK)

#define AV8L_FAST_PTE_ADDR_MASK		((av8l_fast_iopte)0xfffffffff000)





/* Stage-1 PTE */

#define AV8L_FAST_PTE_AP_PRIV_RW	(((av8l_fast_iopte)0) << 6)

#define AV8L_FAST_PTE_AP_RW		(((av8l_fast_iopte)1) << 6)

#define AV8L_FAST_PTE_AP_PRIV_RO	(((av8l_fast_iopte)2) << 6)

#define AV8L_FAST_PTE_AP_RO		(((av8l_fast_iopte)3) << 6)

#define AV8L_FAST_PTE_ATTRINDX_SHIFT	2

#define AV8L_FAST_PTE_nG		(((av8l_fast_iopte)1) << 11)



/* Stage-2 PTE */

#define AV8L_FAST_PTE_HAP_FAULT		(((av8l_fast_iopte)0) << 6)

#define AV8L_FAST_PTE_HAP_READ		(((av8l_fast_iopte)1) << 6)

#define AV8L_FAST_PTE_HAP_WRITE		(((av8l_fast_iopte)2) << 6)

#define AV8L_FAST_PTE_MEMATTR_OIWB	(((av8l_fast_iopte)0xf) << 2)

#define AV8L_FAST_PTE_MEMATTR_NC	(((av8l_fast_iopte)0x5) << 2)

#define AV8L_FAST_PTE_MEMATTR_DEV	(((av8l_fast_iopte)0x1) << 2)



/* Register bits */

#define ARM_32_LPAE_TCR_EAE		(1 << 31)

#define ARM_64_LPAE_S2_TCR_RES1		(1 << 31)



#define AV8L_FAST_TCR_TG0_4K		(0 << 14)

#define AV8L_FAST_TCR_TG0_64K		(1 << 14)

#define AV8L_FAST_TCR_TG0_16K		(2 << 14)



#define AV8L_FAST_TCR_SH0_SHIFT		12

#define AV8L_FAST_TCR_SH0_MASK		0x3

#define AV8L_FAST_TCR_SH_NS		0

#define AV8L_FAST_TCR_SH_OS		2

#define AV8L_FAST_TCR_SH_IS		3



#define AV8L_FAST_TCR_ORGN0_SHIFT	10

#define AV8L_FAST_TCR_IRGN0_SHIFT	8

#define AV8L_FAST_TCR_RGN_MASK		0x3

#define AV8L_FAST_TCR_RGN_NC		0

#define AV8L_FAST_TCR_RGN_WBWA		1

#define AV8L_FAST_TCR_RGN_WT		2

#define AV8L_FAST_TCR_RGN_WB		3



#define AV8L_FAST_TCR_SL0_SHIFT		6

#define AV8L_FAST_TCR_SL0_MASK		0x3



#define AV8L_FAST_TCR_T0SZ_SHIFT	0

#define AV8L_FAST_TCR_SZ_MASK		0xf



#define AV8L_FAST_TCR_PS_SHIFT		16

#define AV8L_FAST_TCR_PS_MASK		0x7



#define AV8L_FAST_TCR_IPS_SHIFT		32

#define AV8L_FAST_TCR_IPS_MASK		0x7



#define AV8L_FAST_TCR_PS_32_BIT		0x0ULL

#define AV8L_FAST_TCR_PS_36_BIT		0x1ULL

#define AV8L_FAST_TCR_PS_40_BIT		0x2ULL

#define AV8L_FAST_TCR_PS_42_BIT		0x3ULL

#define AV8L_FAST_TCR_PS_44_BIT		0x4ULL

#define AV8L_FAST_TCR_PS_48_BIT		0x5ULL



#define AV8L_FAST_TCR_EPD1_SHIFT	23

#define AV8L_FAST_TCR_EPD1_FAULT	1



#define AV8L_FAST_MAIR_ATTR_SHIFT(n)	((n) << 3)

#define AV8L_FAST_MAIR_ATTR_MASK	0xff

#define AV8L_FAST_MAIR_ATTR_DEVICE	0x04

#define AV8L_FAST_MAIR_ATTR_NC		0x44

#define AV8L_FAST_MAIR_ATTR_WBRWA	0xff

#define AV8L_FAST_MAIR_ATTR_IDX_NC	0

#define AV8L_FAST_MAIR_ATTR_IDX_CACHE	1

#define AV8L_FAST_MAIR_ATTR_IDX_DEV	2



#define AV8L_FAST_PAGE_SHIFT		12





/* caller must take care of cache maintenance on *ptep */

int av8l_fast_map_public(av8l_fast_iopte *ptep, phys_addr_t paddr, size_t size,

			 int prot)

{

	int i, nptes = size >> AV8L_FAST_PAGE_SHIFT;

	av8l_fast_iopte pte = AV8L_FAST_PTE_XN

		| AV8L_FAST_PTE_TYPE_PAGE

		| AV8L_FAST_PTE_AF

		| AV8L_FAST_PTE_nG

		| AV8L_FAST_PTE_SH_IS;



	if (prot & IOMMU_DEVICE)

		pte |= (AV8L_FAST_MAIR_ATTR_IDX_DEV

			<< AV8L_FAST_PTE_ATTRINDX_SHIFT);

	else if (prot & IOMMU_CACHE)

		pte |= (AV8L_FAST_MAIR_ATTR_IDX_CACHE

			<< AV8L_FAST_PTE_ATTRINDX_SHIFT);



	if (!(prot & IOMMU_WRITE))

		pte |= AV8L_FAST_PTE_AP_RO;

	else

		pte |= AV8L_FAST_PTE_AP_RW;



	paddr &= AV8L_FAST_PTE_ADDR_MASK;

	for (i = 0; i < nptes; i++, paddr += SZ_4K)

		*(ptep + i) = pte | paddr;



	return 0;

}



static int av8l_fast_map(struct io_pgtable_ops *ops, unsigned long iova,

			 phys_addr_t paddr, size_t size, int prot)

{

	struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops);

	av8l_fast_iopte *ptep = iopte_pmd_offset(data->pmds, iova);

	unsigned long nptes = size >> AV8L_FAST_PAGE_SHIFT;



	av8l_fast_map_public(ptep, paddr, size, prot);

	data->iop.cfg.tlb->flush_pgtable(

		ptep, sizeof(*ptep) * nptes,

		data->iop.cookie);



	return 0;

}



/* caller must take care of cache maintenance on *ptep */

void av8l_fast_unmap_public(av8l_fast_iopte *ptep, size_t size)

{

	unsigned long nptes = size >> AV8L_FAST_PAGE_SHIFT;



	memset(ptep, 0, sizeof(*ptep) * nptes);

}



/* upper layer must take care of TLB invalidation */

static size_t av8l_fast_unmap(struct io_pgtable_ops *ops, unsigned long iova,

			      size_t size)

{

	struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops);

	av8l_fast_iopte *ptep = iopte_pmd_offset(data->pmds, iova);

	unsigned long nptes = size >> AV8L_FAST_PAGE_SHIFT;



	av8l_fast_unmap_public(ptep, size);



	data->iop.cfg.tlb->flush_pgtable(

		ptep, sizeof(*ptep) * nptes,

		data->iop.cookie);



	return size;

}



static phys_addr_t av8l_fast_iova_to_phys(struct io_pgtable_ops *ops,

					  unsigned long iova)

{

	struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops);

	av8l_fast_iopte pte, *pgdp, *pudp, *pmdp;

	phys_addr_t phys;

	const unsigned long pts = AV8L_FAST_PTE_TYPE_SHIFT;

	const unsigned long ptm = AV8L_FAST_PTE_TYPE_MASK;

	const unsigned long ptt = AV8L_FAST_PTE_TYPE_TABLE;

	const unsigned long ptp = AV8L_FAST_PTE_TYPE_PAGE;

	const av8l_fast_iopte am = AV8L_FAST_PTE_ADDR_MASK;



	/* TODO: clean up some of these magic numbers... */



	pgdp = (av8l_fast_iopte *)

		(((unsigned long)data->pgd) | ((iova & 0x7fc0000000) >> 27));

	pte = *pgdp;

	if (((pte >> pts) & ptm) != ptt)

		return 0;

	pudp = phys_to_virt((pte & am) | ((iova & 0x3fe00000) >> 18));



	pte = *pudp;

	if (((pte >> pts) & ptm) != ptt)

		return 0;

	pmdp = phys_to_virt((pte & am) | ((iova & 0x1ff000) >> 9));



	pte = *pmdp;

	if (((pte >> pts) & ptm) != ptp)

		return 0;

	phys = pte & am;



	return phys | (iova & 0xfff);

}



static int av8l_fast_map_sg(struct io_pgtable_ops *ops, unsigned long iova,

			    struct scatterlist *sg, unsigned int nents,

			    int prot, size_t *size)

{

	return -ENODEV;

}



static struct av8l_fast_io_pgtable *

av8l_fast_alloc_pgtable_data(struct io_pgtable_cfg *cfg)

{

	struct av8l_fast_io_pgtable *data;



	data = kmalloc(sizeof(*data), GFP_KERNEL);

	if (!data)

		return NULL;



	data->iop.ops = (struct io_pgtable_ops) {

		.map		= av8l_fast_map,

		.map_sg		= av8l_fast_map_sg,

		.unmap		= av8l_fast_unmap,

		.iova_to_phys	= av8l_fast_iova_to_phys,

	};



	return data;

}



/*

 * We need 1 page for the pgd, 4 pages for puds (1GB VA per pud page) and

 * 2048 pages for pmds (each pud page contains 512 table entries, each

 * pointing to a pmd).

 */

#define NUM_PGD_PAGES 1

#define NUM_PUD_PAGES 4

#define NUM_PMD_PAGES 2048

#define NUM_PGTBL_PAGES (NUM_PGD_PAGES + NUM_PUD_PAGES + NUM_PMD_PAGES)



static int

av8l_fast_prepopulate_pgtables(struct av8l_fast_io_pgtable *data,

			       struct io_pgtable_cfg *cfg, void *cookie)

{

	int i, j, pg = 0;

	struct page **pages, *page;



	pages = kmalloc(sizeof(*pages) * NUM_PGTBL_PAGES, GFP_KERNEL);

	if (!pages)

		return -ENOMEM;



	page = alloc_page(GFP_KERNEL | __GFP_ZERO);

	if (!page)

		goto err_free_pages_arr;

	pages[pg++] = page;

	data->pgd = page_address(page);



	/*

	 * We need 2048 entries at level 2 to map 4GB of VA space. A page

	 * can hold 512 entries, so we need 4 pages.

	 */

	for (i = 0; i < 4; ++i) {

		av8l_fast_iopte pte, *ptep;



		page = alloc_page(GFP_KERNEL | __GFP_ZERO);

		if (!page)

			goto err_free_pages;

		pages[pg++] = page;

		data->puds[i] = page_address(page);

		pte = page_to_phys(page) | AV8L_FAST_PTE_TYPE_TABLE;

		ptep = ((av8l_fast_iopte *)data->pgd) + i;

		*ptep = pte;

	}

	cfg->tlb->flush_pgtable(data->pgd, sizeof(av8l_fast_iopte) * 4, cookie);



	/*

	 * We have 4 puds, each of which can point to 512 pmds, so we'll

	 * have 2048 pmds, each of which can hold 512 ptes, for a grand

	 * total of 2048*512=1048576 PTEs.

	 */

	for (i = 0; i < 4; ++i) {

		for (j = 0; j < 512; ++j) {

			av8l_fast_iopte pte, *pudp;



			page = alloc_page(GFP_KERNEL | __GFP_ZERO);

			if (!page)

				goto err_free_pages;

			pages[pg++] = page;

			pte = page_to_phys(page) | AV8L_FAST_PTE_TYPE_TABLE;

			pudp = data->puds[i] + j;

			*pudp = pte;

		}

		cfg->tlb->flush_pgtable(data->puds[i], SZ_4K, cookie);

	}



	BUG_ON(pg != NUM_PGTBL_PAGES);



	/*

	 * We map the pmds into a virtually contiguous space so that we

	 * don't have to traverse the first two levels of the page tables

	 * to find the appropriate pud.  Instead, it will be a simple

	 * offset from the virtual base of the pmds.

	 */

	data->pmds = vmap(&pages[NUM_PGD_PAGES + NUM_PUD_PAGES], NUM_PMD_PAGES,

			  VM_IOREMAP, PAGE_KERNEL);

	if (!data->pmds)

		goto err_free_pages;



	data->pages = pages;

	return 0;



err_free_pages:

	for (i = 0; i < pg; ++i)

		__free_page(pages[i]);

err_free_pages_arr:

	kfree(pages);

	return -ENOMEM;

}



static struct io_pgtable *

av8l_fast_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie)

{

	u64 reg;

	struct av8l_fast_io_pgtable *data =

		av8l_fast_alloc_pgtable_data(cfg);



	/* restrict according to the fast map requirements */

	cfg->ias = 32;

	cfg->pgsize_bitmap = SZ_4K;



	/* TCR */

	reg = (AV8L_FAST_TCR_SH_IS << AV8L_FAST_TCR_SH0_SHIFT) |

	      (AV8L_FAST_TCR_RGN_NC << AV8L_FAST_TCR_IRGN0_SHIFT) |

	      (AV8L_FAST_TCR_RGN_NC << AV8L_FAST_TCR_ORGN0_SHIFT);



	reg |= AV8L_FAST_TCR_TG0_4K;



	switch (cfg->oas) {

	case 32:

		reg |= (AV8L_FAST_TCR_PS_32_BIT << AV8L_FAST_TCR_IPS_SHIFT);

		break;

	case 36:

		reg |= (AV8L_FAST_TCR_PS_36_BIT << AV8L_FAST_TCR_IPS_SHIFT);

		break;

	case 40:

		reg |= (AV8L_FAST_TCR_PS_40_BIT << AV8L_FAST_TCR_IPS_SHIFT);

		break;

	case 42:

		reg |= (AV8L_FAST_TCR_PS_42_BIT << AV8L_FAST_TCR_IPS_SHIFT);

		break;

	case 44:

		reg |= (AV8L_FAST_TCR_PS_44_BIT << AV8L_FAST_TCR_IPS_SHIFT);

		break;

	case 48:

		reg |= (AV8L_FAST_TCR_PS_48_BIT << AV8L_FAST_TCR_IPS_SHIFT);

		break;

	default:

		goto out_free_data;

	}



	reg |= (64ULL - cfg->ias) << AV8L_FAST_TCR_T0SZ_SHIFT;

	reg |= AV8L_FAST_TCR_EPD1_FAULT << AV8L_FAST_TCR_EPD1_SHIFT;

	cfg->av8l_fast_cfg.tcr = reg;



	/* MAIRs */

	reg = (AV8L_FAST_MAIR_ATTR_NC

	       << AV8L_FAST_MAIR_ATTR_SHIFT(AV8L_FAST_MAIR_ATTR_IDX_NC)) |

	      (AV8L_FAST_MAIR_ATTR_WBRWA

	       << AV8L_FAST_MAIR_ATTR_SHIFT(AV8L_FAST_MAIR_ATTR_IDX_CACHE)) |

	      (AV8L_FAST_MAIR_ATTR_DEVICE

	       << AV8L_FAST_MAIR_ATTR_SHIFT(AV8L_FAST_MAIR_ATTR_IDX_DEV));



	cfg->av8l_fast_cfg.mair[0] = reg;

	cfg->av8l_fast_cfg.mair[1] = 0;



	/* Allocate all page table memory! */

	if (av8l_fast_prepopulate_pgtables(data, cfg, cookie))

		goto out_free_data;



	cfg->av8l_fast_cfg.pmds = data->pmds;



	/* TTBRs */

	cfg->av8l_fast_cfg.ttbr[0] = virt_to_phys(data->pgd);

	cfg->av8l_fast_cfg.ttbr[1] = 0;

	return &data->iop;



out_free_data:

	kfree(data);

	return NULL;

}



static void av8l_fast_free_pgtable(struct io_pgtable *iop)

{

	int i;

	struct av8l_fast_io_pgtable *data = iof_pgtable_to_data(iop);



	vunmap(data->pmds);

	for (i = 0; i < NUM_PGTBL_PAGES; ++i)

		__free_page(data->pages[i]);

	kfree(data->pages);

	kfree(data);

}



struct io_pgtable_init_fns io_pgtable_av8l_fast_init_fns = {

	.alloc	= av8l_fast_alloc_pgtable,

	.free	= av8l_fast_free_pgtable,

};





#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST_SELFTEST



#include <linux/dma-contiguous.h>



static struct io_pgtable_cfg *cfg_cookie;



static void dummy_tlb_flush_all(void *cookie)

{

	WARN_ON(cookie != cfg_cookie);

}



static void dummy_tlb_add_flush(unsigned long iova, size_t size, bool leaf,

				void *cookie)

{

	WARN_ON(cookie != cfg_cookie);

	WARN_ON(!(size & cfg_cookie->pgsize_bitmap));

}



static void dummy_tlb_sync(void *cookie)

{

	WARN_ON(cookie != cfg_cookie);

}



static void dummy_flush_pgtable(void *ptr, size_t size, void *cookie)

{

	WARN_ON(cookie != cfg_cookie);

}



static struct iommu_gather_ops dummy_tlb_ops __initdata = {

	.tlb_flush_all	= dummy_tlb_flush_all,

	.tlb_add_flush	= dummy_tlb_add_flush,

	.tlb_sync	= dummy_tlb_sync,

	.flush_pgtable	= dummy_flush_pgtable,

};



/*

 * Returns true if the iova range is successfully mapped to the contiguous

 * phys range in ops.

 */

static bool av8l_fast_range_has_specific_mapping(struct io_pgtable_ops *ops,

						 const unsigned long iova_start,

						 const phys_addr_t phys_start,

						 const size_t size)

{

	unsigned long iova = iova_start;

	phys_addr_t phys = phys_start;



	while (iova < (iova_start + size)) {

		/* + 42 just to make sure offsetting is working */

		if (ops->iova_to_phys(ops, iova + 42) != (phys + 42))

			return false;

		iova += SZ_4K;

		phys += SZ_4K;

	}

	return true;

}



static int __init av8l_fast_positive_testing(void)

{

	int failed = 0;

	unsigned long iova;

	struct io_pgtable_ops *ops;

	struct io_pgtable_cfg cfg;

	struct av8l_fast_io_pgtable *data;

	av8l_fast_iopte *pmds;



	cfg = (struct io_pgtable_cfg) {

		.tlb = &dummy_tlb_ops,

		.ias = 32,

		.oas = 32,

		.pgsize_bitmap = SZ_4K,

	};



	cfg_cookie = &cfg;

	ops = alloc_io_pgtable_ops(ARM_V8L_FAST, &cfg, &cfg);



	if (WARN_ON(!ops))

		return 1;



	data = iof_pgtable_ops_to_data(ops);

	pmds = data->pmds;



	/* map the entire 4GB VA space with 4K map calls */

	for (iova = 0; iova < SZ_1G * 4UL; iova += SZ_4K) {

		if (WARN_ON(ops->map(ops, iova, iova, SZ_4K, IOMMU_READ))) {

			failed++;

			continue;

		}

	}



	if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, 0, 0,

							  SZ_1G * 4UL)))

		failed++;



	/* unmap it all */

	for (iova = 0; iova < SZ_1G * 4UL; iova += SZ_4K) {

		if (WARN_ON(ops->unmap(ops, iova, SZ_4K) != SZ_4K))

			failed++;

	}



	/* map the entire 4GB VA space with 8K map calls */

	for (iova = 0; iova < SZ_1G * 4UL; iova += SZ_8K) {

		if (WARN_ON(ops->map(ops, iova, iova, SZ_8K, IOMMU_READ))) {

			failed++;

			continue;

		}

	}



	if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, 0, 0,

							  SZ_1G * 4UL)))

		failed++;



	/* unmap it all with 8K unmap calls */

	for (iova = 0; iova < SZ_1G * 4UL; iova += SZ_8K) {

		if (WARN_ON(ops->unmap(ops, iova, SZ_8K) != SZ_8K))

			failed++;

	}



	/* map the entire 4GB VA space with 16K map calls */

	for (iova = 0; iova < SZ_1G * 4UL; iova += SZ_16K) {

		if (WARN_ON(ops->map(ops, iova, iova, SZ_16K, IOMMU_READ))) {

			failed++;

			continue;

		}

	}



	if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, 0, 0,

							  SZ_1G * 4UL)))

		failed++;



	/* unmap it all */

	for (iova = 0; iova < SZ_1G * 4UL; iova += SZ_16K) {

		if (WARN_ON(ops->unmap(ops, iova, SZ_16K) != SZ_16K))

			failed++;

	}



	/* map the entire 4GB VA space with 64K map calls */

	for (iova = 0; iova < SZ_1G * 4UL; iova += SZ_64K) {

		if (WARN_ON(ops->map(ops, iova, iova, SZ_64K, IOMMU_READ))) {

			failed++;

			continue;

		}

	}



	if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, 0, 0,

							  SZ_1G * 4UL)))

		failed++;



	/* unmap it all at once */

	if (WARN_ON(ops->unmap(ops, 0, SZ_1G * 4UL) != SZ_1G * 4UL))

		failed++;



	free_io_pgtable_ops(ops);

	return failed;

}



static int __init av8l_fast_do_selftests(void)

{

	int failed = 0;



	failed += av8l_fast_positive_testing();



	pr_err("selftest: completed with %d failures\n", failed);



	return 0;

}

subsys_initcall(av8l_fast_do_selftests);

#endif

drivers/iommu/io-pgtable.c

+4 −0
Original line number Diff line number Diff line
@@ -34,6 +34,7 @@ extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns; 
extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns;

extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns;

extern struct io_pgtable_init_fns io_pgtable_arm_msm_secure_init_fns;

extern struct io_pgtable_init_fns io_pgtable_av8l_fast_init_fns;



static const struct io_pgtable_init_fns *

io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] =
@@ -47,6 +48,9 @@ io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = 
#ifdef CONFIG_MSM_TZ_SMMU

	[ARM_MSM_SECURE] = &io_pgtable_arm_msm_secure_init_fns,

#endif

#ifdef CONFIG_IOMMU_IO_PGTABLE_FAST

	[ARM_V8L_FAST] = &io_pgtable_av8l_fast_init_fns,

#endif

};



static struct dentry *io_pgtable_top;

drivers/iommu/io-pgtable.h

+8 −0
Original line number Diff line number Diff line
@@ -13,6 +13,7 @@ enum io_pgtable_fmt { 
	ARM_64_LPAE_S1,

	ARM_64_LPAE_S2,

	ARM_MSM_SECURE,

	ARM_V8L_FAST,

	IO_PGTABLE_NUM_FMTS,

};
@@ -77,6 +78,13 @@ struct io_pgtable_cfg { 
			enum tz_smmu_device_id sec_id;

			int cbndx;

		} arm_msm_secure_cfg;



		struct {

			u64	ttbr[2];

			u64	tcr;

			u64	mair[2];

			void	*pmds;

		} av8l_fast_cfg;

	};

};