Merge "arm64: dma-mapping: support per-buffer coherent mappings" into msm-4.9 (60b6e0e3) · Commits · e / devices / android_kernel_fairphone_FP3

arch/arm64/mm/dma-mapping.c

+50 −24

Original line number	Diff line number	Diff line
		@@ -52,6 +52,22 @@ static pgprot_t __get_dma_pgprot(unsigned long attrs, pgprot_t prot,
		return prot;
		}

		static bool is_dma_coherent(struct device *dev, unsigned long attrs)
		{
		bool is_coherent;

		if (attrs & DMA_ATTR_FORCE_COHERENT)
		is_coherent = true;
		else if (attrs & DMA_ATTR_FORCE_NON_COHERENT)
		is_coherent = false;
		else if (is_device_dma_coherent(dev))
		is_coherent = true;
		else
		is_coherent = false;

		return is_coherent;
		}

		static struct gen_pool *atomic_pool;
		#define NO_KERNEL_MAPPING_DUMMY 0x2222
		#define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K
		@@ -207,7 +223,7 @@ static void __dma_alloc(struct device dev, size_t size,
		{
		struct page *page;
		void ptr, coherent_ptr;
		bool coherent = is_device_dma_coherent(dev);
		bool coherent = is_dma_coherent(dev, attrs);

		size = PAGE_ALIGN(size);

		@@ -264,7 +280,7 @@ static void __dma_free(struct device *dev, size_t size,

		size = PAGE_ALIGN(size);

		if (!is_device_dma_coherent(dev)) {
		if (!is_dma_coherent(dev, attrs)) {
		if (__free_from_pool(vaddr, size))
		return;
		if (!(attrs & DMA_ATTR_NO_KERNEL_MAPPING))
		@@ -281,7 +297,7 @@ static dma_addr_t __swiotlb_map_page(struct device dev, struct page page,
		dma_addr_t dev_addr;

		dev_addr = swiotlb_map_page(dev, page, offset, size, dir, attrs);
		if (!is_device_dma_coherent(dev))
		if (!is_dma_coherent(dev, attrs))
		__dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);

		return dev_addr;
		@@ -292,7 +308,7 @@ static void __swiotlb_unmap_page(struct device *dev, dma_addr_t dev_addr,
		size_t size, enum dma_data_direction dir,
		unsigned long attrs)
		{
		if (!is_device_dma_coherent(dev))
		if (!is_dma_coherent(dev, attrs))
		__dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
		swiotlb_unmap_page(dev, dev_addr, size, dir, attrs);
		}
		@@ -305,7 +321,7 @@ static int __swiotlb_map_sg_attrs(struct device dev, struct scatterlist sgl,
		int i, ret;

		ret = swiotlb_map_sg_attrs(dev, sgl, nelems, dir, attrs);
		if (!is_device_dma_coherent(dev))
		if (!is_dma_coherent(dev, attrs))
		for_each_sg(sgl, sg, ret, i)
		__dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
		sg->length, dir);
		@@ -321,7 +337,7 @@ static void __swiotlb_unmap_sg_attrs(struct device *dev,
		struct scatterlist *sg;
		int i;

		if (!is_device_dma_coherent(dev))
		if (!is_dma_coherent(dev, attrs))
		for_each_sg(sgl, sg, nelems, i)
		__dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
		sg->length, dir);
		@@ -387,7 +403,7 @@ static int __swiotlb_mmap(struct device *dev,
		unsigned long off = vma->vm_pgoff;

		vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot,
		is_device_dma_coherent(dev));
		is_dma_coherent(dev, attrs));

		if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
		return ret;
		@@ -687,7 +703,7 @@ static void __iommu_alloc_attrs(struct device dev, size_t size,
		dma_addr_t *handle, gfp_t gfp,
		unsigned long attrs)
		{
		bool coherent = is_device_dma_coherent(dev);
		bool coherent = is_dma_coherent(dev, attrs);
		int ioprot = dma_direction_to_prot(DMA_BIDIRECTIONAL, coherent);
		size_t iosize = size;
		void *addr;
		@@ -784,7 +800,7 @@ static int __iommu_mmap_attrs(struct device dev, struct vm_area_struct vma,
		int ret;

		vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot,
		is_device_dma_coherent(dev));
		is_dma_coherent(dev, attrs));

		if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
		return ret;
		@@ -841,7 +857,7 @@ static dma_addr_t __iommu_map_page(struct device dev, struct page page,
		enum dma_data_direction dir,
		unsigned long attrs)
		{
		bool coherent = is_device_dma_coherent(dev);
		bool coherent = is_dma_coherent(dev, attrs);
		int prot = dma_direction_to_prot(dir, coherent);
		dma_addr_t dev_addr = iommu_dma_map_page(dev, page, offset, size, prot);

		@@ -894,7 +910,7 @@ static int __iommu_map_sg_attrs(struct device dev, struct scatterlist sgl,
		int nelems, enum dma_data_direction dir,
		unsigned long attrs)
		{
		bool coherent = is_device_dma_coherent(dev);
		bool coherent = is_dma_coherent(dev, attrs);

		if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
		__iommu_sync_sg_for_device(dev, sgl, nelems, dir);
		@@ -1209,7 +1225,7 @@ static struct page *__iommu_alloc_buffer(struct device dev, size_t size,
		size_t count = size >> PAGE_SHIFT;
		size_t array_size = count * sizeof(struct page *);
		int i = 0;
		bool is_coherent = is_device_dma_coherent(dev);
		bool is_coherent = is_dma_coherent(dev, attrs);

		if (array_size <= PAGE_SIZE)
		pages = kzalloc(array_size, gfp);
		@@ -1323,7 +1339,7 @@ static dma_addr_t __iommu_create_mapping(struct device *dev,
		return dma_addr;

		prot = __get_iommu_pgprot(attrs, prot,
		is_device_dma_coherent(dev));
		is_dma_coherent(dev, attrs));

		iova = dma_addr;
		for (i = 0; i < count; ) {
		@@ -1403,7 +1419,7 @@ static void __iommu_alloc_atomic(struct device dev, size_t size,
		size_t array_size = count * sizeof(struct page *);
		int i;
		void *addr;
		bool coherent = is_device_dma_coherent(dev);
		bool coherent = is_dma_coherent(dev, attrs);

		if (array_size <= PAGE_SIZE)
		pages = kzalloc(array_size, gfp);
		@@ -1453,7 +1469,7 @@ static void __iommu_free_atomic(struct device dev, void cpu_addr,
		static void arm_iommu_alloc_attrs(struct device dev, size_t size,
		dma_addr_t *handle, gfp_t gfp, unsigned long attrs)
		{
		bool coherent = is_device_dma_coherent(dev);
		bool coherent = is_dma_coherent(dev, attrs);
		pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL, coherent);
		struct page **pages;
		void *addr = NULL;
		@@ -1505,7 +1521,7 @@ static int arm_iommu_mmap_attrs(struct device dev, struct vm_area_struct vma,
		unsigned long uaddr = vma->vm_start;
		unsigned long usize = vma->vm_end - vma->vm_start;
		struct page **pages = __iommu_get_pages(cpu_addr, attrs);
		bool coherent = is_device_dma_coherent(dev);
		bool coherent = is_dma_coherent(dev, attrs);

		vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot,
		coherent);
		@@ -1622,7 +1638,7 @@ int arm_iommu_map_sg(struct device dev, struct scatterlist sg,
		return 0;
		}
		prot = __get_iommu_pgprot(attrs, prot,
		is_device_dma_coherent(dev));
		is_dma_coherent(dev, attrs));

		ret = iommu_map_sg(mapping->domain, iova, sg, nents, prot);
		if (ret != total_length) {
		@@ -1675,8 +1691,11 @@ void arm_iommu_sync_sg_for_cpu(struct device dev, struct scatterlist sg,
		{
		struct scatterlist *s;
		int i;
		struct dma_iommu_mapping *mapping = dev->archdata.mapping;
		dma_addr_t iova = sg_dma_address(sg);
		bool iova_coherent = iommu_is_iova_coherent(mapping->domain, iova);

		if (is_device_dma_coherent(dev))
		if (iova_coherent)
		return;

		for_each_sg(sg, s, nents, i)
		@@ -1696,8 +1715,11 @@ void arm_iommu_sync_sg_for_device(struct device dev, struct scatterlist sg,
		{
		struct scatterlist *s;
		int i;
		struct dma_iommu_mapping *mapping = dev->archdata.mapping;
		dma_addr_t iova = sg_dma_address(sg);
		bool iova_coherent = iommu_is_iova_coherent(mapping->domain, iova);

		if (is_device_dma_coherent(dev))
		if (iova_coherent)
		return;

		for_each_sg(sg, s, nents, i)
		@@ -1729,7 +1751,7 @@ static dma_addr_t arm_coherent_iommu_map_page(struct device *dev,

		prot = __dma_direction_to_prot(dir);
		prot = __get_iommu_pgprot(attrs, prot,
		is_device_dma_coherent(dev));
		is_dma_coherent(dev, attrs));

		ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len,
		prot);
		@@ -1756,7 +1778,7 @@ static dma_addr_t arm_iommu_map_page(struct device dev, struct page page,
		unsigned long offset, size_t size, enum dma_data_direction dir,
		unsigned long attrs)
		{
		if (!is_device_dma_coherent(dev) &&
		if (!is_dma_coherent(dev, attrs) &&
		!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
		__dma_page_cpu_to_dev(page, offset, size, dir);

		@@ -1782,8 +1804,10 @@ static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle,
		mapping->domain, iova));
		int offset = handle & ~PAGE_MASK;
		int len = PAGE_ALIGN(size + offset);
		bool iova_coherent = iommu_is_iova_coherent(mapping->domain,
		handle);

		if (!(is_device_dma_coherent(dev) \|\|
		if (!(iova_coherent \|\|
		(attrs & DMA_ATTR_SKIP_CPU_SYNC)))
		__dma_page_dev_to_cpu(page, offset, size, dir);

		@@ -1799,8 +1823,9 @@ static void arm_iommu_sync_single_for_cpu(struct device *dev,
		struct page *page = phys_to_page(iommu_iova_to_phys(
		mapping->domain, iova));
		unsigned int offset = handle & ~PAGE_MASK;
		bool iova_coherent = iommu_is_iova_coherent(mapping->domain, handle);

		if (!is_device_dma_coherent(dev))
		if (!iova_coherent)
		__dma_page_dev_to_cpu(page, offset, size, dir);
		}

		@@ -1812,8 +1837,9 @@ static void arm_iommu_sync_single_for_device(struct device *dev,
		struct page *page = phys_to_page(iommu_iova_to_phys(
		mapping->domain, iova));
		unsigned int offset = handle & ~PAGE_MASK;
		bool iova_coherent = iommu_is_iova_coherent(mapping->domain, handle);

		if (!is_device_dma_coherent(dev))
		if (!iova_coherent)
		__dma_page_cpu_to_dev(page, offset, size, dir);
		}