Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 29f61408 authored by Charan Teja Reddy's avatar Charan Teja Reddy Committed by Patrick Daly
Browse files

iommu: dma-mapping: add dma mapper for io-pgtable-fast for 32 bit 



io-pgtable-fast was implemented to achieve
better performance for IOMMU map/un-map. Add
DMA API support that goes through io-pgtable-fast
for 32 bit targets.

Change-Id: Ib46157cabd1f9b31903837bcc0dcaad87037cdb6
Signed-off-by: default avatarCharan Teja Reddy <charante@codeaurora.org>
[pdaly@codeaurora.org: Included only changes to general iommu files]
Signed-off-by: default avatarPatrick Daly <pdaly@codeaurora.org>
parent 4d4fbba9

drivers/iommu/dma-mapping-fast.c

+5 −4
Original line number Diff line number Diff line
@@ -17,7 +17,8 @@ 
#include <linux/vmalloc.h>

#include <asm/cacheflush.h>

#include <asm/dma-iommu.h>



#include <linux/slab.h>

#include <linux/vmalloc.h>



/* some redundant definitions... :( TODO: move to io-pgtable-fast.h */

#define FAST_PAGE_SHIFT		12
@@ -633,7 +634,7 @@ static void __fast_smmu_mapped_over_stale(struct dma_fast_smmu_mapping *fast, 
	dev_err(fast->dev, "Mapped over stale tlb at %pa\n", &iova);

	dev_err(fast->dev, "bitmap (failure at idx %lu):\n", bitmap_idx);

	dev_err(fast->dev, "ptep: %p pmds: %p diff: %lu\n", ptep,

		fast->pgtbl_pmds, ptep - fast->pgtbl_pmds);

		fast->pgtbl_pmds, bitmap_idx);

	print_hex_dump(KERN_ERR, "bmap: ", DUMP_PREFIX_ADDRESS,

		       32, 8, fast->bitmap, fast->bitmap_size, false);

}
@@ -683,7 +684,7 @@ static const struct dma_map_ops fast_smmu_dma_ops = { 
 * fast_smmu_attach_device function.

 */

static struct dma_fast_smmu_mapping *__fast_smmu_create_mapping_sized(

	dma_addr_t base, size_t size)

	dma_addr_t base, u64 size)

{

	struct dma_fast_smmu_mapping *fast;
@@ -730,7 +731,7 @@ int fast_smmu_attach_device(struct device *dev, 
	int atomic_domain = 1;

	struct iommu_domain *domain = mapping->domain;

	struct iommu_pgtbl_info info;

	size_t size = mapping->bits << PAGE_SHIFT;

	u64 size = (u64)mapping->bits << PAGE_SHIFT;



	if (mapping->base + size > (SZ_1G * 4ULL))

		return -EINVAL;

drivers/iommu/io-pgtable-fast.c

+29 −17
Original line number Diff line number Diff line
@@ -20,6 +20,7 @@ 
#include <linux/types.h>

#include <linux/io-pgtable-fast.h>

#include <asm/cacheflush.h>

#include <linux/vmalloc.h>



#include "io-pgtable.h"
@@ -268,11 +269,18 @@ static size_t av8l_fast_unmap(struct io_pgtable_ops *ops, unsigned long iova, 
	return size;

}



#if defined(CONFIG_ARM64)

#define FAST_PGDNDX(va) (((va) & 0x7fc0000000) >> 27)

#elif defined(CONFIG_ARM)

#define FAST_PGDNDX(va) (((va) & 0xc0000000) >> 27)

#endif



static phys_addr_t av8l_fast_iova_to_phys(struct io_pgtable_ops *ops,

					  unsigned long iova)

{

	struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops);

	av8l_fast_iopte pte, *pgdp, *pudp, *pmdp;

	unsigned long pgd;

	phys_addr_t phys;

	const unsigned long pts = AV8L_FAST_PTE_TYPE_SHIFT;

	const unsigned long ptm = AV8L_FAST_PTE_TYPE_MASK;
@@ -282,8 +290,9 @@ static phys_addr_t av8l_fast_iova_to_phys(struct io_pgtable_ops *ops, 


	/* TODO: clean up some of these magic numbers... */



	pgdp = (av8l_fast_iopte *)

		(((unsigned long)data->pgd) | ((iova & 0x7fc0000000) >> 27));

	pgd = (unsigned long)data->pgd | FAST_PGDNDX(iova);

	pgdp = (av8l_fast_iopte *)pgd;



	pte = *pgdp;

	if (((pte >> pts) & ptm) != ptt)

		return 0;
@@ -478,6 +487,9 @@ av8l_fast_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) 


	reg |= (64ULL - cfg->ias) << AV8L_FAST_TCR_T0SZ_SHIFT;

	reg |= AV8L_FAST_TCR_EPD1_FAULT << AV8L_FAST_TCR_EPD1_SHIFT;

#if defined(CONFIG_ARM)

	reg |= ARM_32_LPAE_TCR_EAE;

#endif

	cfg->av8l_fast_cfg.tcr = reg;



	/* MAIRs */
@@ -565,7 +577,7 @@ static bool av8l_fast_range_has_specific_mapping(struct io_pgtable_ops *ops, 
						 const phys_addr_t phys_start,

						 const size_t size)

{

	unsigned long iova = iova_start;

	u64 iova = iova_start;

	phys_addr_t phys = phys_start;



	while (iova < (iova_start + size)) {
@@ -581,11 +593,12 @@ static bool av8l_fast_range_has_specific_mapping(struct io_pgtable_ops *ops, 
static int __init av8l_fast_positive_testing(void)

{

	int failed = 0;

	unsigned long iova;

	u64 iova;

	struct io_pgtable_ops *ops;

	struct io_pgtable_cfg cfg;

	struct av8l_fast_io_pgtable *data;

	av8l_fast_iopte *pmds;

	u64 max = SZ_1G * 4ULL - 1;



	cfg = (struct io_pgtable_cfg) {

		.quirks = 0,
@@ -605,19 +618,18 @@ static int __init av8l_fast_positive_testing(void) 
	pmds = data->pmds;



	/* map the entire 4GB VA space with 4K map calls */

	for (iova = 0; iova < SZ_1G * 4UL; iova += SZ_4K) {

	for (iova = 0; iova < max; iova += SZ_4K) {

		if (WARN_ON(ops->map(ops, iova, iova, SZ_4K, IOMMU_READ))) {

			failed++;

			continue;

		}

	}



	if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, 0, 0,

							  SZ_1G * 4UL)))

							  max)))

		failed++;



	/* unmap it all */

	for (iova = 0; iova < SZ_1G * 4UL; iova += SZ_4K) {

	for (iova = 0; iova < max; iova += SZ_4K) {

		if (WARN_ON(ops->unmap(ops, iova, SZ_4K) != SZ_4K))

			failed++;

	}
@@ -626,7 +638,7 @@ static int __init av8l_fast_positive_testing(void) 
	av8l_fast_clear_stale_ptes(pmds, false);



	/* map the entire 4GB VA space with 8K map calls */

	for (iova = 0; iova < SZ_1G * 4UL; iova += SZ_8K) {

	for (iova = 0; iova < max; iova += SZ_8K) {

		if (WARN_ON(ops->map(ops, iova, iova, SZ_8K, IOMMU_READ))) {

			failed++;

			continue;
@@ -634,11 +646,11 @@ static int __init av8l_fast_positive_testing(void) 
	}



	if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, 0, 0,

							  SZ_1G * 4UL)))

							  max)))

		failed++;



	/* unmap it all with 8K unmap calls */

	for (iova = 0; iova < SZ_1G * 4UL; iova += SZ_8K) {

	for (iova = 0; iova < max; iova += SZ_8K) {

		if (WARN_ON(ops->unmap(ops, iova, SZ_8K) != SZ_8K))

			failed++;

	}
@@ -647,7 +659,7 @@ static int __init av8l_fast_positive_testing(void) 
	av8l_fast_clear_stale_ptes(pmds, false);



	/* map the entire 4GB VA space with 16K map calls */

	for (iova = 0; iova < SZ_1G * 4UL; iova += SZ_16K) {

	for (iova = 0; iova < max; iova += SZ_16K) {

		if (WARN_ON(ops->map(ops, iova, iova, SZ_16K, IOMMU_READ))) {

			failed++;

			continue;
@@ -655,11 +667,11 @@ static int __init av8l_fast_positive_testing(void) 
	}



	if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, 0, 0,

							  SZ_1G * 4UL)))

							  max)))

		failed++;



	/* unmap it all */

	for (iova = 0; iova < SZ_1G * 4UL; iova += SZ_16K) {

	for (iova = 0; iova < max; iova += SZ_16K) {

		if (WARN_ON(ops->unmap(ops, iova, SZ_16K) != SZ_16K))

			failed++;

	}
@@ -668,7 +680,7 @@ static int __init av8l_fast_positive_testing(void) 
	av8l_fast_clear_stale_ptes(pmds, false);



	/* map the entire 4GB VA space with 64K map calls */

	for (iova = 0; iova < SZ_1G * 4UL; iova += SZ_64K) {

	for (iova = 0; iova < max; iova += SZ_64K) {

		if (WARN_ON(ops->map(ops, iova, iova, SZ_64K, IOMMU_READ))) {

			failed++;

			continue;
@@ -676,11 +688,11 @@ static int __init av8l_fast_positive_testing(void) 
	}



	if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, 0, 0,

							  SZ_1G * 4UL)))

							  max)))

		failed++;



	/* unmap it all at once */

	if (WARN_ON(ops->unmap(ops, 0, SZ_1G * 4UL) != SZ_1G * 4UL))

	if (WARN_ON(ops->unmap(ops, 0, max) != max))

		failed++;



	free_io_pgtable_ops(ops);

drivers/iommu/iommu-debug.c

+22 −18
Original line number Diff line number Diff line 
/*

 * Copyright (c) 2015-2016, The Linux Foundation. All rights reserved.

 * Copyright (c) 2015-2017, The Linux Foundation. All rights reserved.

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License version 2 and
@@ -822,7 +822,7 @@ static int iommu_debug_profiling_fast_dma_api_show(struct seq_file *s, 
	if (!virt)

		goto out;



	mapping = arm_iommu_create_mapping(&platform_bus_type, 0, SZ_1G * 4UL);

	mapping = arm_iommu_create_mapping(&platform_bus_type, 0, SZ_1G * 4ULL);

	if (!mapping) {

		seq_puts(s, "fast_smmu_create_mapping failed\n");

		goto out_kfree;
@@ -922,8 +922,8 @@ static const struct file_operations iommu_debug_profiling_fast_dma_api_fops = { 
static int __tlb_stress_sweep(struct device *dev, struct seq_file *s)

{

	int i, ret = 0;

	unsigned long iova;

	const unsigned long max = SZ_1G * 4UL;

	u64 iova;

	const u64  max = SZ_1G * 4ULL - 1;

	void *virt;

	phys_addr_t phys;

	dma_addr_t dma_addr;
@@ -995,8 +995,8 @@ static int __tlb_stress_sweep(struct device *dev, struct seq_file *s) 
	}



	/* we're all full again. unmap everything. */

	for (dma_addr = 0; dma_addr < max; dma_addr += SZ_8K)

		dma_unmap_single(dev, dma_addr, SZ_8K, DMA_TO_DEVICE);

	for (iova = 0; iova < max; iova += SZ_8K)

		dma_unmap_single(dev, (dma_addr_t)iova, SZ_8K, DMA_TO_DEVICE);



out:

	free_pages((unsigned long)virt, get_order(SZ_8K));
@@ -1029,7 +1029,7 @@ static int __rand_va_sweep(struct device *dev, struct seq_file *s, 
			   const size_t size)

{

	u64 iova;

	const unsigned long max = SZ_1G * 4UL;

	const u64 max = SZ_1G * 4ULL - 1;

	int i, remapped, unmapped, ret = 0;

	void *virt;

	dma_addr_t dma_addr, dma_addr2;
@@ -1061,9 +1061,9 @@ static int __rand_va_sweep(struct device *dev, struct seq_file *s, 
	fib_init(&fib);

	for (iova = get_next_fib(&fib) * size;

	     iova < max - size;

	     iova = get_next_fib(&fib) * size) {

		dma_addr = iova;

		dma_addr2 = max - size - iova;

	     iova = (u64)get_next_fib(&fib) * size) {

		dma_addr = (dma_addr_t)(iova);

		dma_addr2 = (dma_addr_t)((max + 1) - size - iova);

		if (dma_addr == dma_addr2) {

			WARN(1,

			"%s test needs update! The random number sequence is folding in on itself and should be changed.\n",
@@ -1089,8 +1089,8 @@ static int __rand_va_sweep(struct device *dev, struct seq_file *s, 
		ret = -EINVAL;

	}



	for (dma_addr = 0; dma_addr < max; dma_addr += size)

		dma_unmap_single(dev, dma_addr, size, DMA_TO_DEVICE);

	for (iova = 0; iova < max; iova += size)

		dma_unmap_single(dev, (dma_addr_t)iova, size, DMA_TO_DEVICE);



out:

	free_pages((unsigned long)virt, get_order(size));
@@ -1118,10 +1118,11 @@ static int __check_mapping(struct device *dev, struct iommu_domain *domain, 
static int __full_va_sweep(struct device *dev, struct seq_file *s,

			   const size_t size, struct iommu_domain *domain)

{

	unsigned long iova;

	u64 iova;

	dma_addr_t dma_addr;

	void *virt;

	phys_addr_t phys;

	const u64 max = SZ_1G * 4ULL - 1;

	int ret = 0, i;



	virt = (void *)__get_free_pages(GFP_KERNEL, get_order(size));
@@ -1136,7 +1137,7 @@ static int __full_va_sweep(struct device *dev, struct seq_file *s, 
	}

	phys = virt_to_phys(virt);



	for (iova = 0, i = 0; iova < SZ_1G * 4UL; iova += size, ++i) {

	for (iova = 0, i = 0; iova < max; iova += size, ++i) {

		unsigned long expected = iova;



		dma_addr = dma_map_single(dev, virt, size, DMA_TO_DEVICE);
@@ -1184,8 +1185,8 @@ static int __full_va_sweep(struct device *dev, struct seq_file *s, 
	}



out:

	for (dma_addr = 0; dma_addr < SZ_1G * 4UL; dma_addr += size)

		dma_unmap_single(dev, dma_addr, size, DMA_TO_DEVICE);

	for (iova = 0; iova < max; iova += size)

		dma_unmap_single(dev, (dma_addr_t)iova, size, DMA_TO_DEVICE);



	free_pages((unsigned long)virt, get_order(size));

	return ret;
@@ -1374,7 +1375,8 @@ static int __apply_to_new_mapping(struct seq_file *s, 
	int ret = -EINVAL, fast = 1;

	phys_addr_t pt_phys;



	mapping = arm_iommu_create_mapping(&platform_bus_type, 0, SZ_1G * 4UL);

	mapping = arm_iommu_create_mapping(&platform_bus_type, 0,

						(SZ_1G * 4ULL));

	if (!mapping)

		goto out;
@@ -1443,7 +1445,9 @@ static int iommu_debug_functional_arm_dma_api_show(struct seq_file *s, 
	size_t sizes[] = {SZ_4K, SZ_64K, SZ_2M, SZ_1M * 12, 0};

	int ret = -EINVAL;



	mapping = arm_iommu_create_mapping(&platform_bus_type, 0, SZ_1G * 4UL);

	/* Make the size equal to MAX_ULONG */

	mapping = arm_iommu_create_mapping(&platform_bus_type, 0,

						(SZ_1G * 4ULL - 1));

	if (!mapping)

		goto out;