Merge "iommu/io-pgtable-arm: Support SMMU coherent page tables" (ec3cc3d9) · Commits · e / devices / android_kernel_sony_msm8998

arch/arm64/mm/dma-mapping.c

+55 −210

Original line number	Original line	Diff line number	Diff line
	@@ -49,6 +49,17 @@ static pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot,
	return prot;		return prot;
	}		}

			static int __get_iommu_pgprot(struct dma_attrs *attrs, int prot,
			bool coherent)
			{
			if (!dma_get_attr(DMA_ATTR_EXEC_MAPPING, attrs))
			prot \|= IOMMU_NOEXEC;
			if (coherent)
			prot \|= IOMMU_CACHE;

			return prot;
			}

	static struct gen_pool *atomic_pool;		static struct gen_pool *atomic_pool;
	#define NO_KERNEL_MAPPING_DUMMY 0x2222		#define NO_KERNEL_MAPPING_DUMMY 0x2222
	#define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K		#define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K
	@@ -1153,7 +1164,7 @@ static int arm_dma_set_mask(struct device *dev, u64 dma_mask)
	/* IOMMU */		/* IOMMU */

	static void __dma_clear_buffer(struct page *page, size_t size,		static void __dma_clear_buffer(struct page *page, size_t size,
	struct dma_attrs *attrs)		struct dma_attrs *attrs, bool is_coherent)
	{		{
	/*		/*
	* Ensure that the allocated pages are zeroed, and that any data		* Ensure that the allocated pages are zeroed, and that any data
	@@ -1162,6 +1173,7 @@ static void __dma_clear_buffer(struct page *page, size_t size,
	void *ptr = page_address(page);		void *ptr = page_address(page);
	if (!dma_get_attr(DMA_ATTR_SKIP_ZEROING, attrs))		if (!dma_get_attr(DMA_ATTR_SKIP_ZEROING, attrs))
	memset(ptr, 0, size);		memset(ptr, 0, size);
			if (!is_coherent)
	dmac_flush_range(ptr, ptr + size);		dmac_flush_range(ptr, ptr + size);
	}		}

	@@ -1212,6 +1224,7 @@ static struct page *__iommu_alloc_buffer(struct device dev, size_t size,
	size_t count = size >> PAGE_SHIFT;		size_t count = size >> PAGE_SHIFT;
	size_t array_size = count * sizeof(struct page *);		size_t array_size = count * sizeof(struct page *);
	int i = 0;		int i = 0;
			bool is_coherent = is_device_dma_coherent(dev);

	if (array_size <= PAGE_SIZE)		if (array_size <= PAGE_SIZE)
	pages = kzalloc(array_size, gfp);		pages = kzalloc(array_size, gfp);
	@@ -1228,7 +1241,7 @@ static struct page *__iommu_alloc_buffer(struct device dev, size_t size,
	if (!page)		if (!page)
	goto error;		goto error;

	__dma_clear_buffer(page, size, attrs);		__dma_clear_buffer(page, size, attrs, is_coherent);

	for (i = 0; i < count; i++)		for (i = 0; i < count; i++)
	pages[i] = page + i;		pages[i] = page + i;
	@@ -1257,7 +1270,8 @@ static struct page *__iommu_alloc_buffer(struct device dev, size_t size,
	pages[i + j] = pages[i] + j;		pages[i + j] = pages[i] + j;
	}		}

	__dma_clear_buffer(pages[i], PAGE_SIZE << order, attrs);		__dma_clear_buffer(pages[i], PAGE_SIZE << order, attrs,
			is_coherent);
	i += 1 << order;		i += 1 << order;
	count -= 1 << order;		count -= 1 << order;
	}		}
	@@ -1322,9 +1336,8 @@ static dma_addr_t __iommu_create_mapping(struct device *dev,
	dma_addr = __alloc_iova(mapping, size);		dma_addr = __alloc_iova(mapping, size);
	if (dma_addr == DMA_ERROR_CODE)		if (dma_addr == DMA_ERROR_CODE)
	return dma_addr;		return dma_addr;
			prot = __get_iommu_pgprot(attrs, prot,
	if (!dma_get_attr(DMA_ATTR_EXEC_MAPPING, attrs))		is_device_dma_coherent(dev));
	prot \|= IOMMU_NOEXEC;

	iova = dma_addr;		iova = dma_addr;
	for (i = 0; i < count; ) {		for (i = 0; i < count; ) {
	@@ -1404,6 +1417,7 @@ static void __iommu_alloc_atomic(struct device dev, size_t size,
	size_t array_size = count * sizeof(struct page *);		size_t array_size = count * sizeof(struct page *);
	int i;		int i;
	void *addr;		void *addr;
			bool coherent = is_device_dma_coherent(dev);

	if (array_size <= PAGE_SIZE)		if (array_size <= PAGE_SIZE)
	pages = kzalloc(array_size, gfp);		pages = kzalloc(array_size, gfp);
	@@ -1413,7 +1427,13 @@ static void __iommu_alloc_atomic(struct device dev, size_t size,
	if (!pages)		if (!pages)
	return NULL;		return NULL;

			if (coherent) {
			page = alloc_pages(gfp, get_order(size));
			addr = page ? page_address(page) : NULL;
			} else {
	addr = __alloc_from_pool(size, &page, gfp);		addr = __alloc_from_pool(size, &page, gfp);
			}

	if (!addr)		if (!addr)
	goto err_free;		goto err_free;

	@@ -1428,6 +1448,9 @@ static void __iommu_alloc_atomic(struct device dev, size_t size,
	return addr;		return addr;

	err_mapping:		err_mapping:
			if (coherent)
			__free_pages(page, get_order(size));
			else
	__free_from_pool(addr, size);		__free_from_pool(addr, size);
	err_free:		err_free:
	kvfree(pages);		kvfree(pages);
	@@ -1444,7 +1467,8 @@ static void __iommu_free_atomic(struct device dev, void cpu_addr,
	static void arm_iommu_alloc_attrs(struct device dev, size_t size,		static void arm_iommu_alloc_attrs(struct device dev, size_t size,
	dma_addr_t handle, gfp_t gfp, struct dma_attrs attrs)		dma_addr_t handle, gfp_t gfp, struct dma_attrs attrs)
	{		{
	pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL, false);		bool coherent = is_device_dma_coherent(dev);
			pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL, coherent);
	struct page **pages;		struct page **pages;
	void *addr = NULL;		void *addr = NULL;

	@@ -1495,8 +1519,10 @@ static int arm_iommu_mmap_attrs(struct device dev, struct vm_area_struct vma,
	unsigned long uaddr = vma->vm_start;		unsigned long uaddr = vma->vm_start;
	unsigned long usize = vma->vm_end - vma->vm_start;		unsigned long usize = vma->vm_end - vma->vm_start;
	struct page **pages = __iommu_get_pages(cpu_addr, attrs);		struct page **pages = __iommu_get_pages(cpu_addr, attrs);
			bool coherent = is_device_dma_coherent(dev);

	vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot, false);		vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot,
			coherent);

	if (!pages)		if (!pages)
	return -ENXIO;		return -ENXIO;
	@@ -1577,121 +1603,6 @@ static int __dma_direction_to_prot(enum dma_data_direction dir)
	return prot;		return prot;
	}		}

	/*
	* Map a part of the scatter-gather list into contiguous io address space
	*/
	static int __map_sg_chunk(struct device dev, struct scatterlist sg,
	size_t size, dma_addr_t *handle,
	enum dma_data_direction dir, struct dma_attrs *attrs,
	bool is_coherent)
	{
	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
	dma_addr_t iova, iova_base;
	int ret = 0;
	unsigned int count;
	struct scatterlist *s;
	int prot;

	size = PAGE_ALIGN(size);
	*handle = DMA_ERROR_CODE;

	iova_base = iova = __alloc_iova(mapping, size);
	if (iova == DMA_ERROR_CODE)
	return -ENOMEM;

	for (count = 0, s = sg; count < (size >> PAGE_SHIFT); s = sg_next(s)) {
	phys_addr_t phys = page_to_phys(sg_page(s));
	unsigned int len = PAGE_ALIGN(s->offset + s->length);

	if (!is_coherent &&
	!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
	__dma_page_cpu_to_dev(sg_page(s), s->offset, s->length,
	dir);

	prot = __dma_direction_to_prot(dir);
	if (!dma_get_attr(DMA_ATTR_EXEC_MAPPING, attrs))
	prot \|= IOMMU_NOEXEC;

	ret = iommu_map(mapping->domain, iova, phys, len, prot);
	if (ret < 0)
	goto fail;
	count += len >> PAGE_SHIFT;
	iova += len;
	}
	*handle = iova_base;

	return 0;
	fail:
	iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE);
	__free_iova(mapping, iova_base, size);
	return ret;
	}

	static int __iommu_map_sg(struct device dev, struct scatterlist sg, int nents,
	enum dma_data_direction dir, struct dma_attrs *attrs,
	bool is_coherent)
	{
	struct scatterlist s = sg, dma = sg, *start = sg;
	int i, count = 0;
	unsigned int offset = s->offset;
	unsigned int size = s->offset + s->length;
	unsigned int max = dma_get_max_seg_size(dev);

	for (i = 1; i < nents; i++) {
	s = sg_next(s);

	s->dma_address = DMA_ERROR_CODE;
	s->dma_length = 0;

	if (s->offset \|\| (size & ~PAGE_MASK)
	\|\| size + s->length > max) {
	if (__map_sg_chunk(dev, start, size, &dma->dma_address,
	dir, attrs, is_coherent) < 0)
	goto bad_mapping;

	dma->dma_address += offset;
	dma->dma_length = size - offset;

	size = offset = s->offset;
	start = s;
	dma = sg_next(dma);
	count += 1;
	}
	size += s->length;
	}
	if (__map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs,
	is_coherent) < 0)
	goto bad_mapping;

	dma->dma_address += offset;
	dma->dma_length = size - offset;

	return count+1;

	bad_mapping:
	for_each_sg(sg, s, count, i)
	__iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s));
	return 0;
	}

	/**
	* arm_coherent_iommu_map_sg - map a set of SG buffers for streaming mode DMA
	* @dev: valid struct device pointer
	* @sg: list of buffers
	* @nents: number of buffers to map
	* @dir: DMA transfer direction
	*
	* Map a set of i/o coherent buffers described by scatterlist in streaming
	* mode for DMA. The scatter gather list elements are merged together (if
	* possible) and tagged with the appropriate dma address and length. They are
	* obtained via sg_dma_{address,length}.
	*/
	int arm_coherent_iommu_map_sg(struct device dev, struct scatterlist sg,
	int nents, enum dma_data_direction dir, struct dma_attrs *attrs)
	{
	return __iommu_map_sg(dev, sg, nents, dir, attrs, true);
	}

	/**		/**
	* arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA		* arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA
	* @dev: valid struct device pointer		* @dev: valid struct device pointer
	@@ -1722,9 +1633,8 @@ int arm_iommu_map_sg(struct device dev, struct scatterlist sg,
	dev_err(dev, "Couldn't allocate iova for sg %p\n", sg);		dev_err(dev, "Couldn't allocate iova for sg %p\n", sg);
	return 0;		return 0;
	}		}
			prot = __get_iommu_pgprot(attrs, prot,
	if (!dma_get_attr(DMA_ATTR_EXEC_MAPPING, attrs))		is_device_dma_coherent(dev));
	prot \|= IOMMU_NOEXEC;

	ret = iommu_map_sg(mapping->domain, iova, sg, nents, prot);		ret = iommu_map_sg(mapping->domain, iova, sg, nents, prot);
	if (ret != total_length) {		if (ret != total_length) {
	@@ -1741,40 +1651,6 @@ int arm_iommu_map_sg(struct device dev, struct scatterlist sg,
	return nents;		return nents;
	}		}

	static void __iommu_unmap_sg(struct device dev, struct scatterlist sg,
	int nents, enum dma_data_direction dir, struct dma_attrs *attrs,
	bool is_coherent)
	{
	struct scatterlist *s;
	int i;

	for_each_sg(sg, s, nents, i) {
	if (sg_dma_len(s))
	__iommu_remove_mapping(dev, sg_dma_address(s),
	sg_dma_len(s));
	if (!is_coherent &&
	!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
	__dma_page_dev_to_cpu(sg_page(s), s->offset,
	s->length, dir);
	}
	}

	/**
	* arm_coherent_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
	* @dev: valid struct device pointer
	* @sg: list of buffers
	* @nents: number of buffers to unmap (same as was passed to dma_map_sg)
	* @dir: DMA transfer direction (same as was passed to dma_map_sg)
	*
	* Unmap a set of streaming mode DMA translations. Again, CPU access
	* rules concerning calls here are the same as for dma_unmap_single().
	*/
	void arm_coherent_iommu_unmap_sg(struct device dev, struct scatterlist sg,
	int nents, enum dma_data_direction dir, struct dma_attrs *attrs)
	{
	__iommu_unmap_sg(dev, sg, nents, dir, attrs, true);
	}

	/**		/**
	* arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg		* arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
	* @dev: valid struct device pointer		* @dev: valid struct device pointer
	@@ -1812,6 +1688,9 @@ void arm_iommu_sync_sg_for_cpu(struct device dev, struct scatterlist sg,
	struct scatterlist *s;		struct scatterlist *s;
	int i;		int i;

			if (is_device_dma_coherent(dev))
			return;

	for_each_sg(sg, s, nents, i)		for_each_sg(sg, s, nents, i)
	__dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir);		__dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir);

	@@ -1830,6 +1709,9 @@ void arm_iommu_sync_sg_for_device(struct device dev, struct scatterlist sg,
	struct scatterlist *s;		struct scatterlist *s;
	int i;		int i;

			if (is_device_dma_coherent(dev))
			return;

	for_each_sg(sg, s, nents, i)		for_each_sg(sg, s, nents, i)
	__dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);		__dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
	}		}
	@@ -1858,8 +1740,8 @@ static dma_addr_t arm_coherent_iommu_map_page(struct device *dev,
	return dma_addr;		return dma_addr;

	prot = __dma_direction_to_prot(dir);		prot = __dma_direction_to_prot(dir);
	if (!dma_get_attr(DMA_ATTR_EXEC_MAPPING, attrs))		prot = __get_iommu_pgprot(attrs, prot,
	prot \|= IOMMU_NOEXEC;		is_device_dma_coherent(dev));

	ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len,		ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len,
	prot);		prot);
	@@ -1886,37 +1768,13 @@ static dma_addr_t arm_iommu_map_page(struct device dev, struct page page,
	unsigned long offset, size_t size, enum dma_data_direction dir,		unsigned long offset, size_t size, enum dma_data_direction dir,
	struct dma_attrs *attrs)		struct dma_attrs *attrs)
	{		{
	if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))		if (!is_device_dma_coherent(dev) &&
			!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
	__dma_page_cpu_to_dev(page, offset, size, dir);		__dma_page_cpu_to_dev(page, offset, size, dir);

	return arm_coherent_iommu_map_page(dev, page, offset, size, dir, attrs);		return arm_coherent_iommu_map_page(dev, page, offset, size, dir, attrs);
	}		}

	/**
	* arm_coherent_iommu_unmap_page
	* @dev: valid struct device pointer
	* @handle: DMA address of buffer
	* @size: size of buffer (same as passed to dma_map_page)
	* @dir: DMA transfer direction (same as passed to dma_map_page)
	*
	* Coherent IOMMU aware version of arm_dma_unmap_page()
	*/
	static void arm_coherent_iommu_unmap_page(struct device *dev, dma_addr_t handle,
	size_t size, enum dma_data_direction dir,
	struct dma_attrs *attrs)
	{
	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
	dma_addr_t iova = handle & PAGE_MASK;
	int offset = handle & ~PAGE_MASK;
	int len = PAGE_ALIGN(size + offset);

	if (!iova)
	return;

	iommu_unmap(mapping->domain, iova, len);
	__free_iova(mapping, iova, len);
	}

	/**		/**
	* arm_iommu_unmap_page		* arm_iommu_unmap_page
	* @dev: valid struct device pointer		* @dev: valid struct device pointer
	@@ -1940,7 +1798,8 @@ static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle,
	if (!iova)		if (!iova)
	return;		return;

	if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))		if (!(is_device_dma_coherent(dev) \|\|
			dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs)))
	__dma_page_dev_to_cpu(page, offset, size, dir);		__dma_page_dev_to_cpu(page, offset, size, dir);

	iommu_unmap(mapping->domain, iova, len);		iommu_unmap(mapping->domain, iova, len);
	@@ -1959,6 +1818,7 @@ static void arm_iommu_sync_single_for_cpu(struct device *dev,
	if (!iova)		if (!iova)
	return;		return;

			if (!is_device_dma_coherent(dev))
	__dma_page_dev_to_cpu(page, offset, size, dir);		__dma_page_dev_to_cpu(page, offset, size, dir);
	}		}

	@@ -1974,6 +1834,7 @@ static void arm_iommu_sync_single_for_device(struct device *dev,
	if (!iova)		if (!iova)
	return;		return;

			if (!is_device_dma_coherent(dev))
	__dma_page_cpu_to_dev(page, offset, size, dir);		__dma_page_cpu_to_dev(page, offset, size, dir);
	}		}

	@@ -2016,22 +1877,6 @@ const struct dma_map_ops iommu_ops = {
	.mapping_error = arm_iommu_mapping_error,		.mapping_error = arm_iommu_mapping_error,
	};		};

	const struct dma_map_ops iommu_coherent_ops = {
	.alloc = arm_iommu_alloc_attrs,
	.free = arm_iommu_free_attrs,
	.mmap = arm_iommu_mmap_attrs,
	.get_sgtable = arm_iommu_get_sgtable,

	.map_page = arm_coherent_iommu_map_page,
	.unmap_page = arm_coherent_iommu_unmap_page,

	.map_sg = arm_coherent_iommu_map_sg,
	.unmap_sg = arm_coherent_iommu_unmap_sg,

	.set_dma_mask = arm_dma_set_mask,
	.dma_supported = arm_iommu_dma_supported,
	};

	/**		/**
	* arm_iommu_create_mapping		* arm_iommu_create_mapping
	* @bus: pointer to the bus holding the client device (for IOMMU calls)		* @bus: pointer to the bus holding the client device (for IOMMU calls)

drivers/iommu/io-pgtable-arm.c

+8 −3

Original line number	Original line	Diff line number	Diff line
	@@ -935,6 +935,11 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg cfg, void cookie)
	return NULL;		return NULL;

	/* TCR */		/* TCR */
			if (cfg->iommu_dev && cfg->iommu_dev->archdata.dma_coherent)
			reg = (ARM_LPAE_TCR_SH_OS << ARM_LPAE_TCR_SH0_SHIFT) \|
			(ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN0_SHIFT) \|
			(ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT);
			else
	reg = (ARM_LPAE_TCR_SH_IS << ARM_LPAE_TCR_SH0_SHIFT) \|		reg = (ARM_LPAE_TCR_SH_IS << ARM_LPAE_TCR_SH0_SHIFT) \|
	(ARM_LPAE_TCR_RGN_NC << ARM_LPAE_TCR_IRGN0_SHIFT) \|		(ARM_LPAE_TCR_RGN_NC << ARM_LPAE_TCR_IRGN0_SHIFT) \|
	(ARM_LPAE_TCR_RGN_NC << ARM_LPAE_TCR_ORGN0_SHIFT);		(ARM_LPAE_TCR_RGN_NC << ARM_LPAE_TCR_ORGN0_SHIFT);