Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 0176adb0 authored by Christoph Hellwig's avatar Christoph Hellwig
Browse files

swiotlb: refactor coherent buffer allocation 



Factor out a new swiotlb_alloc_buffer helper that allocates DMA coherent
memory from the swiotlb bounce buffer.

This allows to simplify the swiotlb_alloc implemenation that uses
dma_direct_alloc to try to allocate a reachable buffer first.

Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Acked-by: default avatarChristian König <christian.koenig@amd.com>
parent a25381aa

lib/swiotlb.c

+65 −57
Original line number Diff line number Diff line
@@ -709,74 +709,78 @@ void swiotlb_tbl_sync_single(struct device *hwdev, phys_addr_t tlb_addr, 
}

EXPORT_SYMBOL_GPL(swiotlb_tbl_sync_single);



void *

swiotlb_alloc_coherent(struct device *hwdev, size_t size,

		       dma_addr_t *dma_handle, gfp_t flags)

static inline bool dma_coherent_ok(struct device *dev, dma_addr_t addr,

		size_t size)

{

	bool warn = !(flags & __GFP_NOWARN);

	dma_addr_t dev_addr;

	void *ret;

	int order = get_order(size);

	u64 dma_mask = DMA_BIT_MASK(32);



	if (hwdev && hwdev->coherent_dma_mask)

		dma_mask = hwdev->coherent_dma_mask;

	u64 mask = DMA_BIT_MASK(32);



	ret = (void *)__get_free_pages(flags, order);

	if (ret) {

		dev_addr = swiotlb_virt_to_bus(hwdev, ret);

		if (dev_addr + size - 1 > dma_mask) {

			/*

			 * The allocated memory isn't reachable by the device.

			 */

			free_pages((unsigned long) ret, order);

			ret = NULL;

	if (dev && dev->coherent_dma_mask)

		mask = dev->coherent_dma_mask;

	return addr + size - 1 <= mask;

}

	}

	if (!ret) {

		/*

		 * We are either out of memory or the device can't DMA to

		 * GFP_DMA memory; fall back on map_single(), which

		 * will grab memory from the lowest available address range.

		 */

		phys_addr_t paddr = map_single(hwdev, 0, size, DMA_FROM_DEVICE,

					       warn ? 0 : DMA_ATTR_NO_WARN);

		if (paddr == SWIOTLB_MAP_ERROR)

			goto err_warn;



		ret = phys_to_virt(paddr);

		dev_addr = swiotlb_phys_to_dma(hwdev, paddr);

static void *

swiotlb_alloc_buffer(struct device *dev, size_t size, dma_addr_t *dma_handle,

		unsigned long attrs)

{

	phys_addr_t phys_addr;



	if (swiotlb_force == SWIOTLB_NO_FORCE)

		goto out_warn;



	phys_addr = swiotlb_tbl_map_single(dev,

			swiotlb_phys_to_dma(dev, io_tlb_start),

			0, size, DMA_FROM_DEVICE, 0);

	if (phys_addr == SWIOTLB_MAP_ERROR)

		goto out_warn;



	*dma_handle = swiotlb_phys_to_dma(dev, phys_addr);

	if (dma_coherent_ok(dev, *dma_handle, size))

		goto out_unmap;



	memset(phys_to_virt(phys_addr), 0, size);

	return phys_to_virt(phys_addr);



		/* Confirm address can be DMA'd by device */

		if (dev_addr + size - 1 > dma_mask) {

			printk("hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n",

			       (unsigned long long)dma_mask,

			       (unsigned long long)dev_addr);

out_unmap:

	dev_warn(dev, "hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n",

		(unsigned long long)(dev ? dev->coherent_dma_mask : 0),

		(unsigned long long)*dma_handle);



	/*

	 * DMA_TO_DEVICE to avoid memcpy in unmap_single.

			 * The DMA_ATTR_SKIP_CPU_SYNC is optional.

	 * DMA_ATTR_SKIP_CPU_SYNC is optional.

	 */

			swiotlb_tbl_unmap_single(hwdev, paddr,

						 size, DMA_TO_DEVICE,

	swiotlb_tbl_unmap_single(dev, phys_addr, size, DMA_TO_DEVICE,

			DMA_ATTR_SKIP_CPU_SYNC);

			goto err_warn;

out_warn:

	if ((attrs & DMA_ATTR_NO_WARN) && printk_ratelimit()) {

		dev_warn(dev,

			"swiotlb: coherent allocation failed, size=%zu\n",

			size);

		dump_stack();

	}

	return NULL;

}



	*dma_handle = dev_addr;

	memset(ret, 0, size);

void *

swiotlb_alloc_coherent(struct device *hwdev, size_t size,

		       dma_addr_t *dma_handle, gfp_t flags)

{

	int order = get_order(size);

	unsigned long attrs = (flags & __GFP_NOWARN) ? DMA_ATTR_NO_WARN : 0;

	void *ret;



	ret = (void *)__get_free_pages(flags, order);

	if (ret) {

		*dma_handle = swiotlb_virt_to_bus(hwdev, ret);

		if (dma_coherent_ok(hwdev, *dma_handle, size)) {

			memset(ret, 0, size);

			return ret;



err_warn:

	if (warn && printk_ratelimit()) {

		pr_warn("swiotlb: coherent allocation failed for device %s size=%zu\n",

			dev_name(hwdev), size);

		dump_stack();

		}

		free_pages((unsigned long)ret, order);

	}



	return NULL;

	return swiotlb_alloc_buffer(hwdev, size, dma_handle, attrs);

}

EXPORT_SYMBOL(swiotlb_alloc_coherent);
@@ -1103,6 +1107,10 @@ void *swiotlb_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, 
{

	void *vaddr;



	/* temporary workaround: */

	if (gfp & __GFP_NOWARN)

		attrs |= DMA_ATTR_NO_WARN;



	/*

	 * Don't print a warning when the first allocation attempt fails.

	 * swiotlb_alloc_coherent() will print a warning when the DMA memory
@@ -1112,7 +1120,7 @@ void *swiotlb_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, 


	vaddr = dma_direct_alloc(dev, size, dma_handle, gfp, attrs);

	if (!vaddr)

		vaddr = swiotlb_alloc_coherent(dev, size, dma_handle, gfp);

		vaddr = swiotlb_alloc_buffer(dev, size, dma_handle, attrs);

	return vaddr;

}