powerpc/iommu: Support "hybrid" iommu/direct DMA ops for coherent_mask < dma_mask (817820b0) · Commits · e / devices / android_kernel_oneplus_sm7250

arch/powerpc/Kconfig

+4 −0

Original line number	Original line	Diff line number	Diff line
	@@ -82,6 +82,9 @@ config GENERIC_HWEIGHT
	bool		bool
	default y		default y

			config ARCH_HAS_DMA_SET_COHERENT_MASK
			bool

	config PPC		config PPC
	bool		bool
	default y		default y
	@@ -155,6 +158,7 @@ config PPC
	select HAVE_PERF_EVENTS_NMI if PPC64		select HAVE_PERF_EVENTS_NMI if PPC64
	select EDAC_SUPPORT		select EDAC_SUPPORT
	select EDAC_ATOMIC_SCRUB		select EDAC_ATOMIC_SCRUB
			select ARCH_HAS_DMA_SET_COHERENT_MASK

	config GENERIC_CSUM		config GENERIC_CSUM
	def_bool CPU_LITTLE_ENDIAN		def_bool CPU_LITTLE_ENDIAN

arch/powerpc/include/asm/dma-mapping.h

+5 −5

Original line number	Original line	Diff line number	Diff line
	@@ -21,10 +21,10 @@
	#define DMA_ERROR_CODE (~(dma_addr_t)0x0)		#define DMA_ERROR_CODE (~(dma_addr_t)0x0)

	/* Some dma direct funcs must be visible for use in other dma_ops */		/* Some dma direct funcs must be visible for use in other dma_ops */
	extern void dma_direct_alloc_coherent(struct device dev, size_t size,		extern void __dma_direct_alloc_coherent(struct device dev, size_t size,
	dma_addr_t *dma_handle, gfp_t flag,		dma_addr_t *dma_handle, gfp_t flag,
	struct dma_attrs *attrs);		struct dma_attrs *attrs);
	extern void dma_direct_free_coherent(struct device *dev, size_t size,		extern void __dma_direct_free_coherent(struct device *dev, size_t size,
	void *vaddr, dma_addr_t dma_handle,		void *vaddr, dma_addr_t dma_handle,
	struct dma_attrs *attrs);		struct dma_attrs *attrs);
	extern int dma_direct_mmap_coherent(struct device *dev,		extern int dma_direct_mmap_coherent(struct device *dev,

arch/powerpc/kernel/dma-iommu.c

+1 −1

Original line number	Original line	Diff line number	Diff line
	@@ -73,7 +73,7 @@ static void dma_iommu_unmap_sg(struct device dev, struct scatterlist sglist,
	}		}

	/* We support DMA to/from any memory page via the iommu */		/* We support DMA to/from any memory page via the iommu */
	static int dma_iommu_dma_supported(struct device *dev, u64 mask)		int dma_iommu_dma_supported(struct device *dev, u64 mask)
	{		{
	struct iommu_table *tbl = get_iommu_table_base(dev);		struct iommu_table *tbl = get_iommu_table_base(dev);

arch/powerpc/kernel/dma-swiotlb.c

+2 −2

Original line number	Original line	Diff line number	Diff line
	@@ -47,8 +47,8 @@ static u64 swiotlb_powerpc_get_required(struct device *dev)
	* for everything else.		* for everything else.
	*/		*/
	struct dma_map_ops swiotlb_dma_ops = {		struct dma_map_ops swiotlb_dma_ops = {
	.alloc = dma_direct_alloc_coherent,		.alloc = __dma_direct_alloc_coherent,
	.free = dma_direct_free_coherent,		.free = __dma_direct_free_coherent,
	.mmap = dma_direct_mmap_coherent,		.mmap = dma_direct_mmap_coherent,
	.map_sg = swiotlb_map_sg_attrs,		.map_sg = swiotlb_map_sg_attrs,
	.unmap_sg = swiotlb_unmap_sg_attrs,		.unmap_sg = swiotlb_unmap_sg_attrs,

arch/powerpc/kernel/dma.c

+93 −18

Original line number	Original line	Diff line number	Diff line
	@@ -16,6 +16,7 @@
	#include <asm/bug.h>		#include <asm/bug.h>
	#include <asm/machdep.h>		#include <asm/machdep.h>
	#include <asm/swiotlb.h>		#include <asm/swiotlb.h>
			#include <asm/iommu.h>

	/*		/*
	* Generic direct DMA implementation		* Generic direct DMA implementation
	@@ -39,7 +40,29 @@ static u64 __maybe_unused get_pfn_limit(struct device *dev)
	return pfn;		return pfn;
	}		}

	void dma_direct_alloc_coherent(struct device dev, size_t size,		static int dma_direct_dma_supported(struct device *dev, u64 mask)
			{
			#ifdef CONFIG_PPC64
			u64 limit = get_dma_offset(dev) + (memblock_end_of_DRAM() - 1);

			/* Limit fits in the mask, we are good */
			if (mask >= limit)
			return 1;

			#ifdef CONFIG_FSL_SOC
			/* Freescale gets another chance via ZONE_DMA/ZONE_DMA32, however
			* that will have to be refined if/when they support iommus
			*/
			return 1;
			#endif
			/* Sorry ... */
			return 0;
			#else
			return 1;
			#endif
			}

			void __dma_direct_alloc_coherent(struct device dev, size_t size,
	dma_addr_t *dma_handle, gfp_t flag,		dma_addr_t *dma_handle, gfp_t flag,
	struct dma_attrs *attrs)		struct dma_attrs *attrs)
	{		{
	@@ -96,7 +119,7 @@ void dma_direct_alloc_coherent(struct device dev, size_t size,
	#endif		#endif
	}		}

	void dma_direct_free_coherent(struct device *dev, size_t size,		void __dma_direct_free_coherent(struct device *dev, size_t size,
	void *vaddr, dma_addr_t dma_handle,		void *vaddr, dma_addr_t dma_handle,
	struct dma_attrs *attrs)		struct dma_attrs *attrs)
	{		{
	@@ -107,6 +130,51 @@ void dma_direct_free_coherent(struct device *dev, size_t size,
	#endif		#endif
	}		}

			static void dma_direct_alloc_coherent(struct device dev, size_t size,
			dma_addr_t *dma_handle, gfp_t flag,
			struct dma_attrs *attrs)
			{
			struct iommu_table *iommu;

			/* The coherent mask may be smaller than the real mask, check if
			* we can really use the direct ops
			*/
			if (dma_direct_dma_supported(dev, dev->coherent_dma_mask))
			return __dma_direct_alloc_coherent(dev, size, dma_handle,
			flag, attrs);

			/* Ok we can't ... do we have an iommu ? If not, fail */
			iommu = get_iommu_table_base(dev);
			if (!iommu)
			return NULL;

			/* Try to use the iommu */
			return iommu_alloc_coherent(dev, iommu, size, dma_handle,
			dev->coherent_dma_mask, flag,
			dev_to_node(dev));
			}

			static void dma_direct_free_coherent(struct device *dev, size_t size,
			void *vaddr, dma_addr_t dma_handle,
			struct dma_attrs *attrs)
			{
			struct iommu_table *iommu;

			/* See comments in dma_direct_alloc_coherent() */
			if (dma_direct_dma_supported(dev, dev->coherent_dma_mask))
			return __dma_direct_free_coherent(dev, size, vaddr, dma_handle,
			attrs);
			/* Maybe we used an iommu ... */
			iommu = get_iommu_table_base(dev);

			/* If we hit that we should have never allocated in the first
			* place so how come we are freeing ?
			*/
			if (WARN_ON(!iommu))
			return;
			iommu_free_coherent(iommu, size, vaddr, dma_handle);
			}

	int dma_direct_mmap_coherent(struct device dev, struct vm_area_struct vma,		int dma_direct_mmap_coherent(struct device dev, struct vm_area_struct vma,
	void *cpu_addr, dma_addr_t handle, size_t size,		void *cpu_addr, dma_addr_t handle, size_t size,
	struct dma_attrs *attrs)		struct dma_attrs *attrs)
	@@ -147,18 +215,6 @@ static void dma_direct_unmap_sg(struct device dev, struct scatterlist sg,
	{		{
	}		}

	static int dma_direct_dma_supported(struct device *dev, u64 mask)
	{
	#ifdef CONFIG_PPC64
	/* Could be improved so platforms can set the limit in case
	* they have limited DMA windows
	*/
	return mask >= get_dma_offset(dev) + (memblock_end_of_DRAM() - 1);
	#else
	return 1;
	#endif
	}

	static u64 dma_direct_get_required_mask(struct device *dev)		static u64 dma_direct_get_required_mask(struct device *dev)
	{		{
	u64 end, mask;		u64 end, mask;
	@@ -230,6 +286,25 @@ struct dma_map_ops dma_direct_ops = {
	};		};
	EXPORT_SYMBOL(dma_direct_ops);		EXPORT_SYMBOL(dma_direct_ops);

			int dma_set_coherent_mask(struct device *dev, u64 mask)
			{
			if (!dma_supported(dev, mask)) {
			/*
			* We need to special case the direct DMA ops which can
			* support a fallback for coherent allocations. There
			* is no dma_op->set_coherent_mask() so we have to do
			* things the hard way:
			*/
			if (get_dma_ops(dev) != &dma_direct_ops \|\|
			get_iommu_table_base(dev) == NULL \|\|
			!dma_iommu_dma_supported(dev, mask))
			return -EIO;
			}
			dev->coherent_dma_mask = mask;
			return 0;
			}
			EXPORT_SYMBOL_GPL(dma_set_coherent_mask);

	#define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)		#define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)

	int __dma_set_mask(struct device *dev, u64 dma_mask)		int __dma_set_mask(struct device *dev, u64 dma_mask)