ANDROID: GKI: iommu: Snapshot of vendor changes (e24979f0) · Commits · e / devices / android_kernel_oneplus_sm7250

arch/arm64/mm/dma-mapping.c

+77 −91

Original line number	Diff line number	Diff line
		@@ -37,6 +37,7 @@
		#include <asm/cacheflush.h>
		#include <asm/tlbflush.h>
		#include <asm/dma-iommu.h>
		#include <linux/of_address.h>
		#include <linux/dma-mapping-fast.h>

		static int swiotlb __ro_after_init;
		@@ -51,6 +52,18 @@ static pgprot_t __get_dma_pgprot(unsigned long attrs, pgprot_t prot,
		return prot;
		}

		static bool is_dma_coherent(struct device *dev, unsigned long attrs)
		{

		if (attrs & DMA_ATTR_FORCE_COHERENT)
		return true;
		else if (attrs & DMA_ATTR_FORCE_NON_COHERENT)
		return false;
		else if (is_device_dma_coherent(dev))
		return true;
		else
		return false;
		}
		static struct gen_pool *atomic_pool __ro_after_init;

		#define NO_KERNEL_MAPPING_DUMMY 0x2222
		@@ -144,7 +157,7 @@ static void __dma_alloc(struct device dev, size_t size,
		{
		struct page *page;
		void ptr, coherent_ptr;
		bool coherent = is_device_dma_coherent(dev);
		bool coherent = is_dma_coherent(dev, attrs);
		pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL, false);

		size = PAGE_ALIGN(size);
		@@ -160,6 +173,7 @@ static void __dma_alloc(struct device dev, size_t size,
		}

		ptr = swiotlb_alloc(dev, size, dma_handle, flags, attrs);

		if (!ptr)
		goto no_mem;

		@@ -227,7 +241,7 @@ static dma_addr_t __swiotlb_map_page(struct device dev, struct page page,
		dma_addr_t dev_addr;

		dev_addr = swiotlb_map_page(dev, page, offset, size, dir, attrs);
		if (!is_device_dma_coherent(dev) &&
		if (!is_dma_coherent(dev, attrs) &&
		(attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
		__dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);

		@@ -239,7 +253,7 @@ static void __swiotlb_unmap_page(struct device *dev, dma_addr_t dev_addr,
		size_t size, enum dma_data_direction dir,
		unsigned long attrs)
		{
		if (!is_device_dma_coherent(dev) &&
		if (!is_dma_coherent(dev, attrs) &&
		(attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
		__dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
		swiotlb_unmap_page(dev, dev_addr, size, dir, attrs);
		@@ -253,7 +267,7 @@ static int __swiotlb_map_sg_attrs(struct device dev, struct scatterlist sgl,
		int i, ret;

		ret = swiotlb_map_sg_attrs(dev, sgl, nelems, dir, attrs);
		if (!is_device_dma_coherent(dev) &&
		if (!is_dma_coherent(dev, attrs) &&
		(attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
		for_each_sg(sgl, sg, ret, i)
		__dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
		@@ -270,7 +284,7 @@ static void __swiotlb_unmap_sg_attrs(struct device *dev,
		struct scatterlist *sg;
		int i;

		if (!is_device_dma_coherent(dev) &&
		if (!is_dma_coherent(dev, attrs) &&
		(attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
		for_each_sg(sgl, sg, nelems, i)
		__dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
		@@ -347,11 +361,11 @@ static int __swiotlb_mmap(struct device *dev,
		void *cpu_addr, dma_addr_t dma_addr, size_t size,
		unsigned long attrs)
		{
		int ret;
		int ret = -ENXIO;
		unsigned long pfn = dma_to_phys(dev, dma_addr) >> PAGE_SHIFT;

		vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot,
		is_device_dma_coherent(dev));
		is_dma_coherent(dev, attrs));

		if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
		return ret;
		@@ -531,6 +545,7 @@ static void __dummy_alloc(struct device dev, size_t size,
		dma_addr_t *dma_handle, gfp_t flags,
		unsigned long attrs)
		{
		WARN(1, "dma alloc failure, device may be missing a call to arch_setup_dma_ops");
		return NULL;
		}

		@@ -645,7 +660,7 @@ static void __iommu_alloc_attrs(struct device dev, size_t size,
		dma_addr_t *handle, gfp_t gfp,
		unsigned long attrs)
		{
		bool coherent = is_device_dma_coherent(dev);
		bool coherent = is_dma_coherent(dev, attrs);
		int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs);
		size_t iosize = size;
		void *addr;
		@@ -659,6 +674,7 @@ static void __iommu_alloc_attrs(struct device dev, size_t size,
		* Some drivers rely on this, and we probably don't want the
		* possibility of stale kernel data being read by devices anyway.
		*/
		if (!(attrs & DMA_ATTR_SKIP_ZEROING))
		gfp \|= __GFP_ZERO;

		if (!gfpflags_allow_blocking(gfp)) {
		@@ -751,7 +767,6 @@ static void __iommu_free_attrs(struct device dev, size_t size, void cpu_addr,
		__free_from_pool(cpu_addr, size);
		} else if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) {
		struct page *page = vmalloc_to_page(cpu_addr);

		iommu_dma_unmap_page(dev, handle, iosize, 0, attrs);
		dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT);
		dma_common_free_remap(cpu_addr, size, VM_USERMAP, false);
		@@ -774,32 +789,31 @@ static int __iommu_mmap_attrs(struct device dev, struct vm_area_struct vma,
		{
		struct vm_struct *area;
		int ret;
		unsigned long pfn = 0;

		vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot,
		is_device_dma_coherent(dev));
		is_dma_coherent(dev, attrs));

		if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
		return ret;

		if (!is_vmalloc_addr(cpu_addr)) {
		unsigned long pfn = page_to_pfn(virt_to_page(cpu_addr));
		return __swiotlb_mmap_pfn(vma, pfn, size);
		}
		area = find_vm_area(cpu_addr);

		if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) {
		if (area && area->pages)
		return iommu_dma_mmap(area->pages, size, vma);
		else if (!is_vmalloc_addr(cpu_addr))
		pfn = page_to_pfn(virt_to_page(cpu_addr));
		else if (is_vmalloc_addr(cpu_addr))
		/*
		* DMA_ATTR_FORCE_CONTIGUOUS allocations are always remapped,
		* hence in the vmalloc space.
		* DMA_ATTR_FORCE_CONTIGUOUS and atomic pool allocations are
		* always remapped, hence in the vmalloc space.
		*/
		unsigned long pfn = vmalloc_to_pfn(cpu_addr);
		pfn = vmalloc_to_pfn(cpu_addr);

		if (pfn)
		return __swiotlb_mmap_pfn(vma, pfn, size);
		}

		area = find_vm_area(cpu_addr);
		if (WARN_ON(!area \|\| !area->pages))
		return -ENXIO;

		return iommu_dma_mmap(area->pages, size, vma);
		}

		static int __iommu_get_sgtable(struct device dev, struct sg_table sgt,
		@@ -807,27 +821,24 @@ static int __iommu_get_sgtable(struct device dev, struct sg_table sgt,
		size_t size, unsigned long attrs)
		{
		unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
		struct page *page = NULL;
		struct vm_struct *area = find_vm_area(cpu_addr);

		if (!is_vmalloc_addr(cpu_addr)) {
		struct page *page = virt_to_page(cpu_addr);
		return __swiotlb_get_sgtable_page(sgt, page, size);
		}

		if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) {
		if (area && area->pages)
		return sg_alloc_table_from_pages(sgt, area->pages, count, 0,
		size, GFP_KERNEL);
		else if (!is_vmalloc_addr(cpu_addr))
		page = virt_to_page(cpu_addr);
		else if (is_vmalloc_addr(cpu_addr))
		/*
		* DMA_ATTR_FORCE_CONTIGUOUS allocations are always remapped,
		* hence in the vmalloc space.
		* DMA_ATTR_FORCE_CONTIGUOUS and atomic pool allocations
		* are always remapped, hence in the vmalloc space.
		*/
		struct page *page = vmalloc_to_page(cpu_addr);
		return __swiotlb_get_sgtable_page(sgt, page, size);
		}
		page = vmalloc_to_page(cpu_addr);

		if (WARN_ON(!area \|\| !area->pages))
		if (page)
		return __swiotlb_get_sgtable_page(sgt, page, size);
		return -ENXIO;

		return sg_alloc_table_from_pages(sgt, area->pages, count, 0, size,
		GFP_KERNEL);
		}

		static void __iommu_sync_single_for_cpu(struct device *dev,
		@@ -835,11 +846,12 @@ static void __iommu_sync_single_for_cpu(struct device *dev,
		enum dma_data_direction dir)
		{
		phys_addr_t phys;
		struct iommu_domain *domain = iommu_get_domain_for_dev(dev);

		if (is_device_dma_coherent(dev))
		if (!domain \|\| iommu_is_iova_coherent(domain, dev_addr))
		return;

		phys = iommu_iova_to_phys(iommu_get_domain_for_dev(dev), dev_addr);
		phys = iommu_iova_to_phys(domain, dev_addr);
		__dma_unmap_area(phys_to_virt(phys), size, dir);
		}

		@@ -848,11 +860,12 @@ static void __iommu_sync_single_for_device(struct device *dev,
		enum dma_data_direction dir)
		{
		phys_addr_t phys;
		struct iommu_domain *domain = iommu_get_domain_for_dev(dev);

		if (is_device_dma_coherent(dev))
		if (!domain \|\| iommu_is_iova_coherent(domain, dev_addr))
		return;

		phys = iommu_iova_to_phys(iommu_get_domain_for_dev(dev), dev_addr);
		phys = iommu_iova_to_phys(domain, dev_addr);
		__dma_map_area(phys_to_virt(phys), size, dir);
		}

		@@ -861,7 +874,7 @@ static dma_addr_t __iommu_map_page(struct device dev, struct page page,
		enum dma_data_direction dir,
		unsigned long attrs)
		{
		bool coherent = is_device_dma_coherent(dev);
		bool coherent = is_dma_coherent(dev, attrs);
		int prot = dma_info_to_prot(dir, coherent, attrs);
		dma_addr_t dev_addr = iommu_dma_map_page(dev, page, offset, size, prot);

		@@ -887,9 +900,11 @@ static void __iommu_sync_sg_for_cpu(struct device *dev,
		enum dma_data_direction dir)
		{
		struct scatterlist *sg;
		dma_addr_t iova = sg_dma_address(sgl);
		struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
		int i;

		if (is_device_dma_coherent(dev))
		if (!domain \|\| iommu_is_iova_coherent(domain, iova))
		return;

		for_each_sg(sgl, sg, nelems, i)
		@@ -901,9 +916,11 @@ static void __iommu_sync_sg_for_device(struct device *dev,
		enum dma_data_direction dir)
		{
		struct scatterlist *sg;
		dma_addr_t iova = sg_dma_address(sgl);
		struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
		int i;

		if (is_device_dma_coherent(dev))
		if (!domain \|\| iommu_is_iova_coherent(domain, iova))
		return;

		for_each_sg(sgl, sg, nelems, i)
		@@ -914,13 +931,18 @@ static int __iommu_map_sg_attrs(struct device dev, struct scatterlist sgl,
		int nelems, enum dma_data_direction dir,
		unsigned long attrs)
		{
		bool coherent = is_device_dma_coherent(dev);
		bool coherent = is_dma_coherent(dev, attrs);
		int ret;

		ret = iommu_dma_map_sg(dev, sgl, nelems,
		dma_info_to_prot(dir, coherent, attrs));
		if (!ret)
		return ret;

		if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
		__iommu_sync_sg_for_device(dev, sgl, nelems, dir);

		return iommu_dma_map_sg(dev, sgl, nelems,
		dma_info_to_prot(dir, coherent, attrs));
		return ret;
		}

		static void __iommu_unmap_sg_attrs(struct device *dev,
		@@ -958,50 +980,14 @@ static int __init __iommu_dma_init(void)
		}
		arch_initcall(__iommu_dma_init);

		static void __iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
		const struct iommu_ops *ops)
		{
		struct iommu_domain *domain;

		if (!ops)
		return;

		/*
		* The IOMMU core code allocates the default DMA domain, which the
		* underlying IOMMU driver needs to support via the dma-iommu layer.
		*/
		domain = iommu_get_domain_for_dev(dev);

		if (!domain)
		goto out_err;

		if (domain->type == IOMMU_DOMAIN_DMA) {
		if (iommu_dma_init_domain(domain, dma_base, size, dev))
		goto out_err;

		dev->dma_ops = &iommu_dma_ops;
		}

		return;

		out_err:
		pr_warn("Failed to set up IOMMU for device %s; retaining platform DMA ops\n",
		dev_name(dev));
		}

		void arch_teardown_dma_ops(struct device *dev)
		{
		dev->dma_ops = NULL;
		}

		#else

		static void __iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
		const struct iommu_ops *iommu)
		{ }

		#endif /* CONFIG_IOMMU_DMA */

		static void arm_iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size);

		void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
		const struct iommu_ops *iommu, bool coherent)
		{
		@@ -1013,7 +999,7 @@ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
		}

		dev->archdata.dma_coherent = coherent;
		__iommu_setup_dma_ops(dev, dma_base, size, iommu);
		arm_iommu_setup_dma_ops(dev, dma_base, size);

		#ifdef CONFIG_XEN
		if (xen_initial_domain()) {

drivers/iommu/dma-iommu.c

+98 −17

Original line number	Diff line number	Diff line
		@@ -307,6 +307,7 @@ int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base,
		return 0;
		}

		iovad->end_pfn = end_pfn;
		init_iova_domain(iovad, 1UL << order, base_pfn);
		if (!dev)
		return 0;
		@@ -315,6 +316,48 @@ int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base,
		}
		EXPORT_SYMBOL(iommu_dma_init_domain);

		/*
		* Should be called prior to using dma-apis
		*/
		int iommu_dma_reserve_iova(struct device *dev, dma_addr_t base,
		u64 size)
		{
		struct iommu_domain *domain;
		struct iova_domain *iovad;
		unsigned long pfn_lo, pfn_hi;

		domain = iommu_get_domain_for_dev(dev);
		if (!domain \|\| !domain->iova_cookie)
		return -EINVAL;

		iovad = &((struct iommu_dma_cookie *)domain->iova_cookie)->iovad;

		/* iova will be freed automatically by put_iova_domain() */
		pfn_lo = iova_pfn(iovad, base);
		pfn_hi = iova_pfn(iovad, base + size - 1);
		if (!reserve_iova(iovad, pfn_lo, pfn_hi))
		return -EINVAL;

		return 0;
		}

		/*
		* Should be called prior to using dma-apis.
		*/
		int iommu_dma_enable_best_fit_algo(struct device *dev)
		{
		struct iommu_domain *domain;
		struct iova_domain *iovad;

		domain = iommu_get_domain_for_dev(dev);
		if (!domain \|\| !domain->iova_cookie)
		return -EINVAL;

		iovad = &((struct iommu_dma_cookie *)domain->iova_cookie)->iovad;
		iovad->best_fit = true;
		return 0;
		}

		/**
		* dma_info_to_prot - Translate DMA API directions and attributes to IOMMU API
		* page flags.
		@@ -332,6 +375,15 @@ int dma_info_to_prot(enum dma_data_direction dir, bool coherent,
		if (attrs & DMA_ATTR_PRIVILEGED)
		prot \|= IOMMU_PRIV;

		if (!(attrs & DMA_ATTR_EXEC_MAPPING))
		prot \|= IOMMU_NOEXEC;

		if (attrs & DMA_ATTR_IOMMU_USE_UPSTREAM_HINT)
		prot \|= IOMMU_USE_UPSTREAM_HINT;

		if (attrs & DMA_ATTR_IOMMU_USE_LLC_NWA)
		prot \|= IOMMU_USE_LLC_NWA;

		switch (dir) {
		case DMA_BIDIRECTIONAL:
		return prot \| IOMMU_READ \| IOMMU_WRITE;
		@@ -350,6 +402,7 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,
		struct iommu_dma_cookie *cookie = domain->iova_cookie;
		struct iova_domain *iovad = &cookie->iovad;
		unsigned long shift, iova_len, iova = 0;
		dma_addr_t limit;

		if (cookie->type == IOMMU_DMA_MSI_COOKIE) {
		cookie->msi_iova += size;
		@@ -373,16 +426,27 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,
		if (domain->geometry.force_aperture)
		dma_limit = min(dma_limit, domain->geometry.aperture_end);

		/*
		* Ensure iova is within range specified in iommu_dma_init_domain().
		* This also prevents unnecessary work iterating through the entire
		* rb_tree.
		*/
		limit = min_t(dma_addr_t, DMA_BIT_MASK(32) >> shift,
		iovad->end_pfn);

		/* Try to get PCI devices a SAC address */
		if (dma_limit > DMA_BIT_MASK(32) && dev_is_pci(dev))
		iova = alloc_iova_fast(iovad, iova_len,
		DMA_BIT_MASK(32) >> shift, false);
		iova = alloc_iova_fast(iovad, iova_len, limit, false);

		if (!iova)
		iova = alloc_iova_fast(iovad, iova_len, dma_limit >> shift,
		true);
		if (!iova) {
		limit = min_t(dma_addr_t, dma_limit >> shift,
		iovad->end_pfn);

		iova = alloc_iova_fast(iovad, iova_len, limit, true);
		}

		return (dma_addr_t)iova << shift;

		}

		static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
		@@ -453,8 +517,9 @@ static struct page **__iommu_dma_alloc_pages(unsigned int count,
		unsigned int order = __fls(order_mask);

		order_size = 1U << order;
		page = alloc_pages((order_mask - order_size) ?
		gfp \| __GFP_NORETRY : gfp, order);
		page = alloc_pages(order ?
		(gfp \| __GFP_NORETRY) &
		~__GFP_RECLAIM : gfp, order);
		if (!page)
		continue;
		if (!order)
		@@ -648,7 +713,7 @@ void iommu_dma_unmap_page(struct device *dev, dma_addr_t handle, size_t size,
		* avoid individually crossing any boundaries, so we merely need to check a
		* segment's start address to avoid concatenating across one.
		*/
		static int __finalise_sg(struct device dev, struct scatterlist sg, int nents,
		int iommu_dma_finalise_sg(struct device dev, struct scatterlist sg, int nents,
		dma_addr_t dma_addr)
		{
		struct scatterlist s, cur = sg;
		@@ -701,7 +766,7 @@ static int __finalise_sg(struct device dev, struct scatterlist sg, int nents,
		* If mapping failed, then just restore the original list,
		* but making sure the DMA fields are invalidated.
		*/
		static void __invalidate_sg(struct scatterlist *sg, int nents)
		void iommu_dma_invalidate_sg(struct scatterlist *sg, int nents)
		{
		struct scatterlist *s;
		int i;
		@@ -723,14 +788,10 @@ static void __invalidate_sg(struct scatterlist *sg, int nents)
		* impedance-matching, to be able to hand off a suitably-aligned list,
		* but still preserve the original offsets and sizes for the caller.
		*/
		int iommu_dma_map_sg(struct device dev, struct scatterlist sg,
		int nents, int prot)
		size_t iommu_dma_prepare_map_sg(struct device dev, struct iova_domain iovad,
		struct scatterlist *sg, int nents)
		{
		struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
		struct iommu_dma_cookie *cookie = domain->iova_cookie;
		struct iova_domain *iovad = &cookie->iovad;
		struct scatterlist s, prev = NULL;
		dma_addr_t iova;
		size_t iova_len = 0;
		unsigned long mask = dma_get_seg_boundary(dev);
		int i;
		@@ -774,6 +835,26 @@ int iommu_dma_map_sg(struct device dev, struct scatterlist sg,
		prev = s;
		}

		return iova_len;
		}

		int iommu_dma_map_sg(struct device dev, struct scatterlist sg,
		int nents, int prot)
		{
		struct iommu_domain *domain;
		struct iommu_dma_cookie *cookie;
		struct iova_domain *iovad;
		dma_addr_t iova;
		size_t iova_len;

		domain = iommu_get_domain_for_dev(dev);
		if (!domain)
		return 0;
		cookie = domain->iova_cookie;
		iovad = &cookie->iovad;

		iova_len = iommu_dma_prepare_map_sg(dev, iovad, sg, nents);

		iova = iommu_dma_alloc_iova(domain, iova_len, dma_get_mask(dev), dev);
		if (!iova)
		goto out_restore_sg;
		@@ -785,12 +866,12 @@ int iommu_dma_map_sg(struct device dev, struct scatterlist sg,
		if (iommu_map_sg(domain, iova, sg, nents, prot) < iova_len)
		goto out_free_iova;

		return __finalise_sg(dev, sg, nents, iova);
		return iommu_dma_finalise_sg(dev, sg, nents, iova);

		out_free_iova:
		iommu_dma_free_iova(cookie, iova, iova_len);
		out_restore_sg:
		__invalidate_sg(sg, nents);
		iommu_dma_invalidate_sg(sg, nents);
		return 0;
		}

drivers/iommu/io-pgtable.c

+56 −0

Original line number	Diff line number	Diff line
		@@ -18,10 +18,16 @@
		* Author: Will Deacon <will.deacon@arm.com>
		*/

		#define pr_fmt(fmt) "io-pgtable: " fmt

		#include <linux/bug.h>
		#include <linux/io-pgtable.h>
		#include <linux/kernel.h>
		#include <linux/types.h>
		#include <linux/iommu.h>
		#include <linux/debugfs.h>
		#include <linux/atomic.h>
		#include <linux/module.h>

		static const struct io_pgtable_init_fns *
		io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = {
		@@ -39,6 +45,8 @@ io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = {
		#endif
		};

		static struct dentry *io_pgtable_top;

		struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt,
		struct io_pgtable_cfg *cfg,
		void *cookie)
		@@ -81,3 +89,51 @@ void free_io_pgtable_ops(struct io_pgtable_ops *ops)
		io_pgtable_init_table[iop->fmt]->free(iop);
		}
		EXPORT_SYMBOL_GPL(free_io_pgtable_ops);

		static atomic_t pages_allocated;

		void io_pgtable_alloc_pages_exact(struct io_pgtable_cfg cfg, void *cookie,
		size_t size, gfp_t gfp_mask)
		{
		void *ret;

		if (cfg->tlb->alloc_pages_exact)
		ret = cfg->tlb->alloc_pages_exact(cookie, size, gfp_mask);
		else
		ret = alloc_pages_exact(size, gfp_mask);

		if (likely(ret))
		atomic_add(1 << get_order(size), &pages_allocated);

		return ret;
		}

		void io_pgtable_free_pages_exact(struct io_pgtable_cfg cfg, void cookie,
		void *virt, size_t size)
		{
		if (cfg->tlb->free_pages_exact)
		cfg->tlb->free_pages_exact(cookie, virt, size);
		else
		free_pages_exact(virt, size);

		atomic_sub(1 << get_order(size), &pages_allocated);
		}

		static int __init io_pgtable_init(void)
		{
		static const char io_pgtable_str[] __initconst = "io-pgtable";
		static const char pages_str[] __initconst = "pages";

		io_pgtable_top = debugfs_create_dir(io_pgtable_str, iommu_debugfs_top);
		debugfs_create_atomic_t(pages_str, 0600, io_pgtable_top,
		&pages_allocated);
		return 0;
		}

		static void __exit io_pgtable_exit(void)
		{
		debugfs_remove_recursive(io_pgtable_top);
		}

		module_init(io_pgtable_init);
		module_exit(io_pgtable_exit);

drivers/iommu/iommu.c

+90 −35

Original line number	Diff line number	Diff line
		@@ -1210,7 +1210,6 @@ static int iommu_bus_init(struct bus_type bus, const struct iommu_ops ops)
		if (err)
		goto out_err;


		return 0;

		out_err:
		@@ -1309,6 +1308,8 @@ static struct iommu_domain __iommu_domain_alloc(struct bus_type bus,
		domain->type = type;
		/* Assume all sizes by default; the driver may override this later */
		domain->pgsize_bitmap = bus->iommu_ops->pgsize_bitmap;
		domain->is_debug_domain = false;
		memset(domain->name, 0, IOMMU_DOMAIN_NAME_LEN);

		return domain;
		}
		@@ -1337,8 +1338,14 @@ static int __iommu_attach_device(struct iommu_domain *domain,
		return -ENODEV;

		ret = domain->ops->attach_dev(domain, dev);
		if (!ret)
		if (!ret) {
		trace_attach_device_to_domain(dev);

		if (!strnlen(domain->name, IOMMU_DOMAIN_NAME_LEN)) {
		strlcpy(domain->name, dev_name(dev),
		IOMMU_DOMAIN_NAME_LEN);
		}
		}
		return ret;
		}

		@@ -1445,9 +1452,6 @@ static int __iommu_attach_group(struct iommu_domain *domain,
		{
		int ret;

		if (group->default_domain && group->domain != group->default_domain)
		return -EBUSY;

		ret = __iommu_group_for_each_dev(group, domain,
		iommu_group_do_attach_device);
		if (ret == 0)
		@@ -1477,30 +1481,20 @@ static int iommu_group_do_detach_device(struct device dev, void data)
		return 0;
		}

		/*
		* Although upstream implements detaching the default_domain as a noop,
		* the "SID switch" secure usecase require complete removal of SIDS/SMRS
		* from HLOS iommu registers.
		*/
		static void __iommu_detach_group(struct iommu_domain *domain,
		struct iommu_group *group)
		{
		int ret;

		if (!group->default_domain) {
		__iommu_group_for_each_dev(group, domain,
		iommu_group_do_detach_device);
		group->domain = NULL;
		return;
		}

		if (group->domain == group->default_domain)
		return;

		/* Detach by re-attaching to the default domain */
		ret = __iommu_group_for_each_dev(group, group->default_domain,
		iommu_group_do_attach_device);
		if (ret != 0)
		WARN_ON(1);
		else
		group->domain = group->default_domain;
		}

		void iommu_detach_group(struct iommu_domain domain, struct iommu_group group)
		{
		mutex_lock(&group->mutex);
		@@ -1518,7 +1512,33 @@ phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
		}
		EXPORT_SYMBOL_GPL(iommu_iova_to_phys);

		static size_t iommu_pgsize(struct iommu_domain *domain,
		phys_addr_t iommu_iova_to_phys_hard(struct iommu_domain *domain,
		dma_addr_t iova)
		{
		if (unlikely(domain->ops->iova_to_phys_hard == NULL))
		return 0;

		return domain->ops->iova_to_phys_hard(domain, iova);
		}

		uint64_t iommu_iova_to_pte(struct iommu_domain *domain,
		dma_addr_t iova)
		{
		if (unlikely(domain->ops->iova_to_pte == NULL))
		return 0;

		return domain->ops->iova_to_pte(domain, iova);
		}

		bool iommu_is_iova_coherent(struct iommu_domain *domain, dma_addr_t iova)
		{
		if (unlikely(domain->ops->is_iova_coherent == NULL))
		return 0;

		return domain->ops->is_iova_coherent(domain, iova);
		}

		size_t iommu_pgsize(unsigned long pgsize_bitmap,
		unsigned long addr_merge, size_t size)
		{
		unsigned int pgsize_idx;
		@@ -1538,10 +1558,14 @@ static size_t iommu_pgsize(struct iommu_domain *domain,
		pgsize = (1UL << (pgsize_idx + 1)) - 1;

		/* throw away page sizes not supported by the hardware */
		pgsize &= domain->pgsize_bitmap;
		pgsize &= pgsize_bitmap;

		/* make sure we're still sane */
		BUG_ON(!pgsize);
		if (!pgsize) {
		pr_err("invalid pgsize/addr/size! 0x%lx 0x%lx 0x%zx\n",
		pgsize_bitmap, addr_merge, size);
		BUG();
		}

		/* pick the biggest page */
		pgsize_idx = __fls(pgsize);
		@@ -1583,7 +1607,8 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova,
		pr_debug("map: iova 0x%lx pa %pa size 0x%zx\n", iova, &paddr, size);

		while (size) {
		size_t pgsize = iommu_pgsize(domain, iova \| paddr, size);
		size_t pgsize = iommu_pgsize(domain->pgsize_bitmap,
		iova \| paddr, size);

		pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx\n",
		iova, &paddr, pgsize);
		@@ -1601,7 +1626,7 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova,
		if (ret)
		iommu_unmap(domain, orig_iova, orig_size - size);
		else
		trace_map(orig_iova, orig_paddr, orig_size);
		trace_map(domain, orig_iova, orig_paddr, orig_size, prot);

		return ret;
		}
		@@ -1644,14 +1669,14 @@ static size_t __iommu_unmap(struct iommu_domain *domain,
		* or we hit an area that isn't mapped.
		*/
		while (unmapped < size) {
		size_t pgsize = iommu_pgsize(domain, iova, size - unmapped);
		size_t left = size - unmapped;

		unmapped_page = ops->unmap(domain, iova, pgsize);
		unmapped_page = ops->unmap(domain, iova, left);
		if (!unmapped_page)
		break;

		if (sync && ops->iotlb_range_add)
		ops->iotlb_range_add(domain, iova, pgsize);
		ops->iotlb_range_add(domain, iova, left);

		pr_debug("unmapped: iova 0x%lx size 0x%zx\n",
		iova, unmapped_page);
		@@ -1663,7 +1688,7 @@ static size_t __iommu_unmap(struct iommu_domain *domain,
		if (sync && ops->iotlb_sync)
		ops->iotlb_sync(domain);

		trace_unmap(orig_iova, size, unmapped);
		trace_unmap(domain, orig_iova, size, unmapped);
		return unmapped;
		}

		@@ -1681,7 +1706,19 @@ size_t iommu_unmap_fast(struct iommu_domain *domain,
		}
		EXPORT_SYMBOL_GPL(iommu_unmap_fast);

		size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
		size_t iommu_map_sg(struct iommu_domain *domain,
		unsigned long iova, struct scatterlist *sg,
		unsigned int nents, int prot)
		{
		size_t mapped;

		mapped = domain->ops->map_sg(domain, iova, sg, nents, prot);
		trace_map_sg(domain, iova, mapped, prot);
		return mapped;
		}
		EXPORT_SYMBOL(iommu_map_sg);

		size_t default_iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
		struct scatterlist *sg, unsigned int nents, int prot)
		{
		struct scatterlist *s;
		@@ -1722,7 +1759,7 @@ size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
		return 0;

		}
		EXPORT_SYMBOL_GPL(iommu_map_sg);
		EXPORT_SYMBOL_GPL(default_iommu_map_sg);

		int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr,
		phys_addr_t paddr, u64 size, int prot)
		@@ -1786,6 +1823,9 @@ int report_iommu_fault(struct iommu_domain domain, struct device dev,
		}
		EXPORT_SYMBOL_GPL(report_iommu_fault);

		struct dentry *iommu_debugfs_top;
		EXPORT_SYMBOL_GPL(iommu_debugfs_top);

		static int __init iommu_init(void)
		{
		iommu_group_kset = kset_create_and_add("iommu_groups",
		@@ -1879,6 +1919,21 @@ void iommu_put_resv_regions(struct device dev, struct list_head list)
		ops->put_resv_regions(dev, list);
		}

		/**
		* iommu_trigger_fault() - trigger an IOMMU fault
		* @domain: iommu domain
		*
		* Triggers a fault on the device to which this domain is attached.
		*
		* This function should only be used for debugging purposes, for obvious
		* reasons.
		*/
		void iommu_trigger_fault(struct iommu_domain *domain, unsigned long flags)
		{
		if (domain->ops->trigger_fault)
		domain->ops->trigger_fault(domain, flags);
		}

		struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start,
		size_t length, int prot,
		enum iommu_resv_type type)

drivers/iommu/iova.c

+90 −3

File changed.

Preview size limit exceeded, changes collapsed.