Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 15af6274 authored by Patrick Daly's avatar Patrick Daly
Browse files

dma-mapping-fast: Add support for dma operations with sg_table 



Reuse functionality for reorganizing a sg_table based on
iommu granularity and device dma segment size from dma-iommu.c.

Port dma_sync_sg_for_* from arm64.

Change-Id: I87a7f917a39feff4f201340bbf1b5c2b4c6d9101
Signed-off-by: default avatarPatrick Daly <pdaly@codeaurora.org>
parent 3787dd9c

drivers/iommu/dma-iommu.c

+26 −10
Original line number Diff line number Diff line
@@ -725,7 +725,7 @@ void iommu_dma_unmap_page(struct device *dev, dma_addr_t handle, size_t size, 
 * avoid individually crossing any boundaries, so we merely need to check a

 * segment's start address to avoid concatenating across one.

 */

static int __finalise_sg(struct device *dev, struct scatterlist *sg, int nents,

int iommu_dma_finalise_sg(struct device *dev, struct scatterlist *sg, int nents,

		dma_addr_t dma_addr)

{

	struct scatterlist *s, *cur = sg;
@@ -778,7 +778,7 @@ static int __finalise_sg(struct device *dev, struct scatterlist *sg, int nents, 
 * If mapping failed, then just restore the original list,

 * but making sure the DMA fields are invalidated.

 */

static void __invalidate_sg(struct scatterlist *sg, int nents)

void iommu_dma_invalidate_sg(struct scatterlist *sg, int nents)

{

	struct scatterlist *s;

	int i;
@@ -800,14 +800,10 @@ static void __invalidate_sg(struct scatterlist *sg, int nents) 
 * impedance-matching, to be able to hand off a suitably-aligned list,

 * but still preserve the original offsets and sizes for the caller.

 */

int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,

		int nents, int prot)

size_t iommu_dma_prepare_map_sg(struct device *dev, struct iova_domain *iovad,

				struct scatterlist *sg, int nents)

{

	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);

	struct iommu_dma_cookie *cookie = domain->iova_cookie;

	struct iova_domain *iovad = &cookie->iovad;

	struct scatterlist *s, *prev = NULL;

	dma_addr_t iova;

	size_t iova_len = 0;

	unsigned long mask = dma_get_seg_boundary(dev);

	int i;
@@ -851,6 +847,26 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, 
		prev = s;

	}



	return iova_len;

}



int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,

		int nents, int prot)

{

	struct iommu_domain *domain;

	struct iommu_dma_cookie *cookie;

	struct iova_domain *iovad;

	dma_addr_t iova;

	size_t iova_len;



	domain = iommu_get_domain_for_dev(dev);

	if (!domain)

		return 0;

	cookie = domain->iova_cookie;

	iovad = &cookie->iovad;



	iova_len = iommu_dma_prepare_map_sg(dev, iovad, sg, nents);



	iova = iommu_dma_alloc_iova(domain, iova_len, dma_get_mask(dev), dev);

	if (!iova)

		goto out_restore_sg;
@@ -862,12 +878,12 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, 
	if (iommu_map_sg(domain, iova, sg, nents, prot) < iova_len)

		goto out_free_iova;



	return __finalise_sg(dev, sg, nents, iova);

	return iommu_dma_finalise_sg(dev, sg, nents, iova);



out_free_iova:

	iommu_dma_free_iova(domain, cookie, iova, iova_len);

out_restore_sg:

	__invalidate_sg(sg, nents);

	iommu_dma_invalidate_sg(sg, nents);

	return 0;

}

drivers/iommu/dma-mapping-fast.c

+117 −18
Original line number Diff line number Diff line
@@ -14,6 +14,7 @@ 
#include <linux/vmalloc.h>

#include <linux/pci.h>

#include <linux/dma-iommu.h>

#include <linux/iova.h>

#include <trace/events/iommu.h>

#include "io-pgtable.h"
@@ -412,32 +413,106 @@ static void fast_smmu_sync_single_for_device(struct device *dev, 
	}

}



static void fast_smmu_sync_sg_for_cpu(struct device *dev,

				    struct scatterlist *sgl, int nelems,

				    enum dma_data_direction dir)

{

	struct scatterlist *sg;

	dma_addr_t iova = sg_dma_address(sgl);

	struct dma_fast_smmu_mapping *mapping = dev->archdata.mapping->fast;

	int i;



	if (av8l_fast_iova_coherent_public(mapping->pgtbl_ops, iova))

		return;



	for_each_sg(sgl, sg, nelems, i)

		__dma_unmap_area(sg_virt(sg), sg->length, dir);

}



static void fast_smmu_sync_sg_for_device(struct device *dev,

				       struct scatterlist *sgl, int nelems,

				       enum dma_data_direction dir)

{

	struct scatterlist *sg;

	dma_addr_t iova = sg_dma_address(sgl);

	struct dma_fast_smmu_mapping *mapping = dev->archdata.mapping->fast;

	int i;



	if (av8l_fast_iova_coherent_public(mapping->pgtbl_ops, iova))

		return;



	for_each_sg(sgl, sg, nelems, i)

		__dma_map_area(sg_virt(sg), sg->length, dir);

}



static int fast_smmu_map_sg(struct device *dev, struct scatterlist *sg,

			    int nents, enum dma_data_direction dir,

			    unsigned long attrs)

{

	/* 0 indicates error */

	struct dma_fast_smmu_mapping *mapping = dev->archdata.mapping->fast;

	size_t iova_len;

	bool is_coherent = is_dma_coherent(dev, attrs);

	int prot = dma_info_to_prot(dir, is_coherent, attrs);

	int ret;

	dma_addr_t iova;

	unsigned long flags;

	size_t unused;



	iova_len = iommu_dma_prepare_map_sg(dev, mapping->iovad, sg, nents);



	spin_lock_irqsave(&mapping->lock, flags);

	iova = __fast_smmu_alloc_iova(mapping, attrs, iova_len);

	spin_unlock_irqrestore(&mapping->lock, flags);



	if (unlikely(iova == DMA_ERROR_CODE))

		goto fail;



	av8l_fast_map_sg_public(mapping->pgtbl_ops, iova, sg, nents, prot,

				&unused);



	ret = iommu_dma_finalise_sg(dev, sg, nents, iova);



	if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)

		fast_smmu_sync_sg_for_device(dev, sg, nents, dir);



	return ret;

fail:

	iommu_dma_invalidate_sg(sg, nents);

	return 0;

}



static void fast_smmu_unmap_sg(struct device *dev,

			       struct scatterlist *sg, int nents,

			       struct scatterlist *sg, int nelems,

			       enum dma_data_direction dir,

			       unsigned long attrs)

{

	WARN_ON_ONCE(1);

}

	struct dma_fast_smmu_mapping *mapping = dev->archdata.mapping->fast;

	unsigned long flags;

	dma_addr_t start;

	size_t len;

	struct scatterlist *tmp;

	int i;



static void fast_smmu_sync_sg_for_cpu(struct device *dev,

		struct scatterlist *sg, int nents, enum dma_data_direction dir)

{

	WARN_ON_ONCE(1);

	if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)

		fast_smmu_sync_sg_for_cpu(dev, sg, nelems, dir);



	/*

	 * The scatterlist segments are mapped into a single

	 * contiguous IOVA allocation, so this is incredibly easy.

	 */

	start = sg_dma_address(sg);

	for_each_sg(sg_next(sg), tmp, nelems - 1, i) {

		if (sg_dma_len(tmp) == 0)

			break;

		sg = tmp;

	}

	len = sg_dma_address(sg) + sg_dma_len(sg) - start;



static void fast_smmu_sync_sg_for_device(struct device *dev,

		struct scatterlist *sg, int nents, enum dma_data_direction dir)

{

	WARN_ON_ONCE(1);

	av8l_fast_unmap_public(mapping->pgtbl_ops, start, len);



	spin_lock_irqsave(&mapping->lock, flags);

	__fast_smmu_free_iova(mapping, start, len);

	spin_unlock_irqrestore(&mapping->lock, flags);

}



static void __fast_smmu_free_pages(struct page **pages, int count)
@@ -780,7 +855,16 @@ static struct dma_fast_smmu_mapping *__fast_smmu_create_mapping_sized( 


	spin_lock_init(&fast->lock);



	fast->iovad = kzalloc(sizeof(*fast->iovad), GFP_KERNEL);

	if (!fast->iovad)

		goto err_free_bitmap;

	init_iova_domain(fast->iovad, FAST_PAGE_SIZE,

			base >> FAST_PAGE_SHIFT);



	return fast;



err_free_bitmap:

	kvfree(fast->bitmap);

err2:

	kfree(fast);

err:
@@ -853,6 +937,21 @@ static int fast_smmu_errata_init(struct dma_iommu_mapping *mapping) 
	return 0;

}



static void __fast_smmu_release(struct dma_iommu_mapping *mapping)

{

	struct dma_fast_smmu_mapping *fast = mapping->fast;



	if (fast->iovad) {

		put_iova_domain(fast->iovad);

		kfree(fast->iovad);

	}



	if (fast->bitmap)

		kvfree(fast->bitmap);



	kfree(fast);

}



/**

 * fast_smmu_init_mapping

 * @dev: valid struct device pointer
@@ -878,10 +977,12 @@ int fast_smmu_init_mapping(struct device *dev, 
	mapping->fast = __fast_smmu_create_mapping_sized(mapping->base, size);

	if (IS_ERR(mapping->fast))

		return -ENOMEM;



	mapping->fast->domain = domain;

	mapping->fast->dev = dev;



	if (fast_smmu_errata_init(mapping))

	err = fast_smmu_errata_init(mapping);

	if (err)

		goto release_mapping;



	fast_smmu_reserve_pci_windows(dev, mapping->fast);
@@ -901,8 +1002,7 @@ int fast_smmu_init_mapping(struct device *dev, 
	return 0;



release_mapping:

	kfree(mapping->fast->bitmap);

	kfree(mapping->fast);

	__fast_smmu_release(mapping);

	return err;

}
@@ -917,8 +1017,7 @@ void fast_smmu_release_mapping(struct kref *kref) 
	struct dma_iommu_mapping *mapping =

		container_of(kref, struct dma_iommu_mapping, kref);



	kvfree(mapping->fast->bitmap);

	kfree(mapping->fast);

	__fast_smmu_release(mapping);

	iommu_domain_free(mapping->domain);

	kfree(mapping);

}

drivers/iommu/io-pgtable-fast.c

+22 −7
Original line number Diff line number Diff line
@@ -289,6 +289,28 @@ static size_t av8l_fast_unmap(struct io_pgtable_ops *ops, unsigned long iova, 
	return __av8l_fast_unmap(ops, iova, size, false);

}



static int av8l_fast_map_sg(struct io_pgtable_ops *ops,

			unsigned long iova, struct scatterlist *sgl,

			unsigned int nents, int prot, size_t *size)

{

	struct scatterlist *sg;

	int i;



	for_each_sg(sgl, sg, nents, i) {

		av8l_fast_map(ops, iova, sg_phys(sg), sg->length, prot);

		iova += sg->length;

	}



	return nents;

}



int av8l_fast_map_sg_public(struct io_pgtable_ops *ops,

			    unsigned long iova, struct scatterlist *sgl,

			    unsigned int nents, int prot, size_t *size)

{

	return av8l_fast_map_sg(ops, iova, sgl, nents, prot, size);

}



#if defined(CONFIG_ARM64)

#define FAST_PGDNDX(va) (((va) & 0x7fc0000000) >> 27)

#elif defined(CONFIG_ARM)
@@ -337,13 +359,6 @@ phys_addr_t av8l_fast_iova_to_phys_public(struct io_pgtable_ops *ops, 
	return av8l_fast_iova_to_phys(ops, iova);

}



static int av8l_fast_map_sg(struct io_pgtable_ops *ops, unsigned long iova,

			    struct scatterlist *sg, unsigned int nents,

			    int prot, size_t *size)

{

	return -ENODEV;

}



static bool av8l_fast_iova_coherent(struct io_pgtable_ops *ops,

					unsigned long iova)

{

include/linux/dma-iommu.h

+7 −0
Original line number Diff line number Diff line
@@ -25,6 +25,8 @@ 
#include <linux/iommu.h>

#include <linux/msi.h>



struct iova_domain;



int iommu_dma_init(void);



/* Domain management interface for IOMMU drivers */
@@ -56,6 +58,11 @@ dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, 
		unsigned long offset, size_t size, int prot);

int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,

		int nents, int prot);

size_t iommu_dma_prepare_map_sg(struct device *dev, struct iova_domain *iovad,

				struct scatterlist *sg, int nents);

int iommu_dma_finalise_sg(struct device *dev, struct scatterlist *sg,

		int nents, dma_addr_t dma_addr);

void iommu_dma_invalidate_sg(struct scatterlist *sg, int nents);



/*

 * Arch code with no special attribute handling may use these

include/linux/dma-mapping-fast.h

+3 −0
Original line number Diff line number Diff line
@@ -11,10 +11,13 @@ 


struct dma_iommu_mapping;

struct io_pgtable_ops;

struct iova_domain;



struct dma_fast_smmu_mapping {

	struct device		*dev;

	struct iommu_domain	*domain;

	struct iova_domain	*iovad;



	dma_addr_t	 base;

	size_t		 size;

	size_t		 num_4k_pages;