Merge "iommu: dma-mapping: add dma mapper for io-pgtable-fast for 32 bit" into msm-4.9 (63ebee05) · Commits · e / devices / android_kernel_fairphone_FP3

drivers/iommu/dma-mapping-fast.c

+5 −4

Original line number	Diff line number	Diff line
		@@ -17,7 +17,8 @@
		#include <linux/vmalloc.h>
		#include <asm/cacheflush.h>
		#include <asm/dma-iommu.h>

		#include <linux/slab.h>
		#include <linux/vmalloc.h>

		/* some redundant definitions... :( TODO: move to io-pgtable-fast.h */
		#define FAST_PAGE_SHIFT 12
		@@ -633,7 +634,7 @@ static void __fast_smmu_mapped_over_stale(struct dma_fast_smmu_mapping *fast,
		dev_err(fast->dev, "Mapped over stale tlb at %pa\n", &iova);
		dev_err(fast->dev, "bitmap (failure at idx %lu):\n", bitmap_idx);
		dev_err(fast->dev, "ptep: %p pmds: %p diff: %lu\n", ptep,
		fast->pgtbl_pmds, ptep - fast->pgtbl_pmds);
		fast->pgtbl_pmds, bitmap_idx);
		print_hex_dump(KERN_ERR, "bmap: ", DUMP_PREFIX_ADDRESS,
		32, 8, fast->bitmap, fast->bitmap_size, false);
		}
		@@ -683,7 +684,7 @@ static const struct dma_map_ops fast_smmu_dma_ops = {
		* fast_smmu_attach_device function.
		*/
		static struct dma_fast_smmu_mapping *__fast_smmu_create_mapping_sized(
		dma_addr_t base, size_t size)
		dma_addr_t base, u64 size)
		{
		struct dma_fast_smmu_mapping *fast;

		@@ -730,7 +731,7 @@ int fast_smmu_attach_device(struct device *dev,
		int atomic_domain = 1;
		struct iommu_domain *domain = mapping->domain;
		struct iommu_pgtbl_info info;
		size_t size = mapping->bits << PAGE_SHIFT;
		u64 size = (u64)mapping->bits << PAGE_SHIFT;

		if (mapping->base + size > (SZ_1G * 4ULL))
		return -EINVAL;

drivers/iommu/io-pgtable-fast.c

+29 −17

Original line number	Diff line number	Diff line
		@@ -20,6 +20,7 @@
		#include <linux/types.h>
		#include <linux/io-pgtable-fast.h>
		#include <asm/cacheflush.h>
		#include <linux/vmalloc.h>

		#include "io-pgtable.h"

		@@ -268,11 +269,18 @@ static size_t av8l_fast_unmap(struct io_pgtable_ops *ops, unsigned long iova,
		return size;
		}

		#if defined(CONFIG_ARM64)
		#define FAST_PGDNDX(va) (((va) & 0x7fc0000000) >> 27)
		#elif defined(CONFIG_ARM)
		#define FAST_PGDNDX(va) (((va) & 0xc0000000) >> 27)
		#endif

		static phys_addr_t av8l_fast_iova_to_phys(struct io_pgtable_ops *ops,
		unsigned long iova)
		{
		struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops);
		av8l_fast_iopte pte, pgdp, pudp, *pmdp;
		unsigned long pgd;
		phys_addr_t phys;
		const unsigned long pts = AV8L_FAST_PTE_TYPE_SHIFT;
		const unsigned long ptm = AV8L_FAST_PTE_TYPE_MASK;
		@@ -282,8 +290,9 @@ static phys_addr_t av8l_fast_iova_to_phys(struct io_pgtable_ops *ops,

		/* TODO: clean up some of these magic numbers... */

		pgdp = (av8l_fast_iopte *)
		(((unsigned long)data->pgd) \| ((iova & 0x7fc0000000) >> 27));
		pgd = (unsigned long)data->pgd \| FAST_PGDNDX(iova);
		pgdp = (av8l_fast_iopte *)pgd;

		pte = *pgdp;
		if (((pte >> pts) & ptm) != ptt)
		return 0;
		@@ -478,6 +487,9 @@ av8l_fast_alloc_pgtable(struct io_pgtable_cfg cfg, void cookie)

		reg \|= (64ULL - cfg->ias) << AV8L_FAST_TCR_T0SZ_SHIFT;
		reg \|= AV8L_FAST_TCR_EPD1_FAULT << AV8L_FAST_TCR_EPD1_SHIFT;
		#if defined(CONFIG_ARM)
		reg \|= ARM_32_LPAE_TCR_EAE;
		#endif
		cfg->av8l_fast_cfg.tcr = reg;

		/* MAIRs */
		@@ -565,7 +577,7 @@ static bool av8l_fast_range_has_specific_mapping(struct io_pgtable_ops *ops,
		const phys_addr_t phys_start,
		const size_t size)
		{
		unsigned long iova = iova_start;
		u64 iova = iova_start;
		phys_addr_t phys = phys_start;

		while (iova < (iova_start + size)) {
		@@ -581,11 +593,12 @@ static bool av8l_fast_range_has_specific_mapping(struct io_pgtable_ops *ops,
		static int __init av8l_fast_positive_testing(void)
		{
		int failed = 0;
		unsigned long iova;
		u64 iova;
		struct io_pgtable_ops *ops;
		struct io_pgtable_cfg cfg;
		struct av8l_fast_io_pgtable *data;
		av8l_fast_iopte *pmds;
		u64 max = SZ_1G * 4ULL - 1;

		cfg = (struct io_pgtable_cfg) {
		.quirks = 0,
		@@ -605,19 +618,18 @@ static int __init av8l_fast_positive_testing(void)
		pmds = data->pmds;

		/* map the entire 4GB VA space with 4K map calls */
		for (iova = 0; iova < SZ_1G * 4UL; iova += SZ_4K) {
		for (iova = 0; iova < max; iova += SZ_4K) {
		if (WARN_ON(ops->map(ops, iova, iova, SZ_4K, IOMMU_READ))) {
		failed++;
		continue;
		}
		}

		if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, 0, 0,
		SZ_1G * 4UL)))
		max)))
		failed++;

		/* unmap it all */
		for (iova = 0; iova < SZ_1G * 4UL; iova += SZ_4K) {
		for (iova = 0; iova < max; iova += SZ_4K) {
		if (WARN_ON(ops->unmap(ops, iova, SZ_4K) != SZ_4K))
		failed++;
		}
		@@ -626,7 +638,7 @@ static int __init av8l_fast_positive_testing(void)
		av8l_fast_clear_stale_ptes(pmds, false);

		/* map the entire 4GB VA space with 8K map calls */
		for (iova = 0; iova < SZ_1G * 4UL; iova += SZ_8K) {
		for (iova = 0; iova < max; iova += SZ_8K) {
		if (WARN_ON(ops->map(ops, iova, iova, SZ_8K, IOMMU_READ))) {
		failed++;
		continue;
		@@ -634,11 +646,11 @@ static int __init av8l_fast_positive_testing(void)
		}

		if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, 0, 0,
		SZ_1G * 4UL)))
		max)))
		failed++;

		/* unmap it all with 8K unmap calls */
		for (iova = 0; iova < SZ_1G * 4UL; iova += SZ_8K) {
		for (iova = 0; iova < max; iova += SZ_8K) {
		if (WARN_ON(ops->unmap(ops, iova, SZ_8K) != SZ_8K))
		failed++;
		}
		@@ -647,7 +659,7 @@ static int __init av8l_fast_positive_testing(void)
		av8l_fast_clear_stale_ptes(pmds, false);

		/* map the entire 4GB VA space with 16K map calls */
		for (iova = 0; iova < SZ_1G * 4UL; iova += SZ_16K) {
		for (iova = 0; iova < max; iova += SZ_16K) {
		if (WARN_ON(ops->map(ops, iova, iova, SZ_16K, IOMMU_READ))) {
		failed++;
		continue;
		@@ -655,11 +667,11 @@ static int __init av8l_fast_positive_testing(void)
		}

		if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, 0, 0,
		SZ_1G * 4UL)))
		max)))
		failed++;

		/* unmap it all */
		for (iova = 0; iova < SZ_1G * 4UL; iova += SZ_16K) {
		for (iova = 0; iova < max; iova += SZ_16K) {
		if (WARN_ON(ops->unmap(ops, iova, SZ_16K) != SZ_16K))
		failed++;
		}
		@@ -668,7 +680,7 @@ static int __init av8l_fast_positive_testing(void)
		av8l_fast_clear_stale_ptes(pmds, false);

		/* map the entire 4GB VA space with 64K map calls */
		for (iova = 0; iova < SZ_1G * 4UL; iova += SZ_64K) {
		for (iova = 0; iova < max; iova += SZ_64K) {
		if (WARN_ON(ops->map(ops, iova, iova, SZ_64K, IOMMU_READ))) {
		failed++;
		continue;
		@@ -676,11 +688,11 @@ static int __init av8l_fast_positive_testing(void)
		}

		if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, 0, 0,
		SZ_1G * 4UL)))
		max)))
		failed++;

		/* unmap it all at once */
		if (WARN_ON(ops->unmap(ops, 0, SZ_1G * 4UL) != SZ_1G * 4UL))
		if (WARN_ON(ops->unmap(ops, 0, max) != max))
		failed++;

		free_io_pgtable_ops(ops);

drivers/iommu/iommu-debug.c

+22 −18

Original line number	Diff line number	Diff line
		/*
		* Copyright (c) 2015-2016, The Linux Foundation. All rights reserved.
		* Copyright (c) 2015-2017, The Linux Foundation. All rights reserved.
		*
		* This program is free software; you can redistribute it and/or modify
		* it under the terms of the GNU General Public License version 2 and
		@@ -822,7 +822,7 @@ static int iommu_debug_profiling_fast_dma_api_show(struct seq_file *s,
		if (!virt)
		goto out;

		mapping = arm_iommu_create_mapping(&platform_bus_type, 0, SZ_1G * 4UL);
		mapping = arm_iommu_create_mapping(&platform_bus_type, 0, SZ_1G * 4ULL);
		if (!mapping) {
		seq_puts(s, "fast_smmu_create_mapping failed\n");
		goto out_kfree;
		@@ -922,8 +922,8 @@ static const struct file_operations iommu_debug_profiling_fast_dma_api_fops = {
		static int __tlb_stress_sweep(struct device dev, struct seq_file s)
		{
		int i, ret = 0;
		unsigned long iova;
		const unsigned long max = SZ_1G * 4UL;
		u64 iova;
		const u64 max = SZ_1G * 4ULL - 1;
		void *virt;
		phys_addr_t phys;
		dma_addr_t dma_addr;
		@@ -995,8 +995,8 @@ static int __tlb_stress_sweep(struct device dev, struct seq_file s)
		}

		/* we're all full again. unmap everything. */
		for (dma_addr = 0; dma_addr < max; dma_addr += SZ_8K)
		dma_unmap_single(dev, dma_addr, SZ_8K, DMA_TO_DEVICE);
		for (iova = 0; iova < max; iova += SZ_8K)
		dma_unmap_single(dev, (dma_addr_t)iova, SZ_8K, DMA_TO_DEVICE);

		out:
		free_pages((unsigned long)virt, get_order(SZ_8K));
		@@ -1029,7 +1029,7 @@ static int __rand_va_sweep(struct device dev, struct seq_file s,
		const size_t size)
		{
		u64 iova;
		const unsigned long max = SZ_1G * 4UL;
		const u64 max = SZ_1G * 4ULL - 1;
		int i, remapped, unmapped, ret = 0;
		void *virt;
		dma_addr_t dma_addr, dma_addr2;
		@@ -1061,9 +1061,9 @@ static int __rand_va_sweep(struct device dev, struct seq_file s,
		fib_init(&fib);
		for (iova = get_next_fib(&fib) * size;
		iova < max - size;
		iova = get_next_fib(&fib) * size) {
		dma_addr = iova;
		dma_addr2 = max - size - iova;
		iova = (u64)get_next_fib(&fib) * size) {
		dma_addr = (dma_addr_t)(iova);
		dma_addr2 = (dma_addr_t)((max + 1) - size - iova);
		if (dma_addr == dma_addr2) {
		WARN(1,
		"%s test needs update! The random number sequence is folding in on itself and should be changed.\n",
		@@ -1089,8 +1089,8 @@ static int __rand_va_sweep(struct device dev, struct seq_file s,
		ret = -EINVAL;
		}

		for (dma_addr = 0; dma_addr < max; dma_addr += size)
		dma_unmap_single(dev, dma_addr, size, DMA_TO_DEVICE);
		for (iova = 0; iova < max; iova += size)
		dma_unmap_single(dev, (dma_addr_t)iova, size, DMA_TO_DEVICE);

		out:
		free_pages((unsigned long)virt, get_order(size));
		@@ -1118,10 +1118,11 @@ static int __check_mapping(struct device dev, struct iommu_domain domain,
		static int __full_va_sweep(struct device dev, struct seq_file s,
		const size_t size, struct iommu_domain *domain)
		{
		unsigned long iova;
		u64 iova;
		dma_addr_t dma_addr;
		void *virt;
		phys_addr_t phys;
		const u64 max = SZ_1G * 4ULL - 1;
		int ret = 0, i;

		virt = (void *)__get_free_pages(GFP_KERNEL, get_order(size));
		@@ -1136,7 +1137,7 @@ static int __full_va_sweep(struct device dev, struct seq_file s,
		}
		phys = virt_to_phys(virt);

		for (iova = 0, i = 0; iova < SZ_1G * 4UL; iova += size, ++i) {
		for (iova = 0, i = 0; iova < max; iova += size, ++i) {
		unsigned long expected = iova;

		dma_addr = dma_map_single(dev, virt, size, DMA_TO_DEVICE);
		@@ -1184,8 +1185,8 @@ static int __full_va_sweep(struct device dev, struct seq_file s,
		}

		out:
		for (dma_addr = 0; dma_addr < SZ_1G * 4UL; dma_addr += size)
		dma_unmap_single(dev, dma_addr, size, DMA_TO_DEVICE);
		for (iova = 0; iova < max; iova += size)
		dma_unmap_single(dev, (dma_addr_t)iova, size, DMA_TO_DEVICE);

		free_pages((unsigned long)virt, get_order(size));
		return ret;
		@@ -1374,7 +1375,8 @@ static int __apply_to_new_mapping(struct seq_file *s,
		int ret = -EINVAL, fast = 1;
		phys_addr_t pt_phys;

		mapping = arm_iommu_create_mapping(&platform_bus_type, 0, SZ_1G * 4UL);
		mapping = arm_iommu_create_mapping(&platform_bus_type, 0,
		(SZ_1G * 4ULL));
		if (!mapping)
		goto out;

		@@ -1443,7 +1445,9 @@ static int iommu_debug_functional_arm_dma_api_show(struct seq_file *s,
		size_t sizes[] = {SZ_4K, SZ_64K, SZ_2M, SZ_1M * 12, 0};
		int ret = -EINVAL;

		mapping = arm_iommu_create_mapping(&platform_bus_type, 0, SZ_1G * 4UL);
		/* Make the size equal to MAX_ULONG */
		mapping = arm_iommu_create_mapping(&platform_bus_type, 0,
		(SZ_1G * 4ULL - 1));
		if (!mapping)
		goto out;