Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 9b690c3d authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'stable/for-linus-3.8-rc0-tag' of... 

Merge tag 'stable/for-linus-3.8-rc0-tag' of git://git.kernel.org/pub/scm/linux/kernel/git/konrad/swiotlb

Pull swiotlb update from Konrad Rzeszutek Wilk:
 "Feature:
   - Use dma addresses instead of the virt_to_phys and vice versa
     functions.

  Remove the multitude of phys_to_virt/virt_to_phys calls and instead
  operate on the physical addresses instead of virtual in many of the
  internal functions.  This does provide a speed up in interrupt
  handlers that do DMA operations and use SWIOTLB."

* tag 'stable/for-linus-3.8-rc0-tag' of git://git.kernel.org/pub/scm/linux/kernel/git/konrad/swiotlb:
  swiotlb: Do not export swiotlb_bounce since there are no external consumers
  swiotlb: Use physical addresses instead of virtual in swiotlb_tbl_sync_single
  swiotlb: Use physical addresses for swiotlb_tbl_unmap_single
  swiotlb: Return physical addresses when calling swiotlb_tbl_map_single
  swiotlb: Make io_tlb_overflow_buffer a physical address
  swiotlb: Make io_tlb_start a physical address instead of a virtual one
  swiotlb: Make io_tlb_end a physical address instead of a virtual one
parents 36cd5c19 af51a9f1

drivers/xen/swiotlb-xen.c

+12 −13
Original line number Diff line number Diff line
@@ -338,9 +338,8 @@ dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page, 
				enum dma_data_direction dir,

				struct dma_attrs *attrs)

{

	phys_addr_t phys = page_to_phys(page) + offset;

	phys_addr_t map, phys = page_to_phys(page) + offset;

	dma_addr_t dev_addr = xen_phys_to_bus(phys);

	void *map;



	BUG_ON(dir == DMA_NONE);

	/*
@@ -356,10 +355,10 @@ dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page, 
	 * Oh well, have to allocate and map a bounce buffer.

	 */

	map = swiotlb_tbl_map_single(dev, start_dma_addr, phys, size, dir);

	if (!map)

	if (map == SWIOTLB_MAP_ERROR)

		return DMA_ERROR_CODE;



	dev_addr = xen_virt_to_bus(map);

	dev_addr = xen_phys_to_bus(map);



	/*

	 * Ensure that the address returned is DMA'ble
@@ -389,7 +388,7 @@ static void xen_unmap_single(struct device *hwdev, dma_addr_t dev_addr, 


	/* NOTE: We use dev_addr here, not paddr! */

	if (is_xen_swiotlb_buffer(dev_addr)) {

		swiotlb_tbl_unmap_single(hwdev, phys_to_virt(paddr), size, dir);

		swiotlb_tbl_unmap_single(hwdev, paddr, size, dir);

		return;

	}
@@ -434,8 +433,7 @@ xen_swiotlb_sync_single(struct device *hwdev, dma_addr_t dev_addr, 


	/* NOTE: We use dev_addr here, not paddr! */

	if (is_xen_swiotlb_buffer(dev_addr)) {

		swiotlb_tbl_sync_single(hwdev, phys_to_virt(paddr), size, dir,

				       target);

		swiotlb_tbl_sync_single(hwdev, paddr, size, dir, target);

		return;

	}
@@ -494,11 +492,12 @@ xen_swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, 
		if (swiotlb_force ||

		    !dma_capable(hwdev, dev_addr, sg->length) ||

		    range_straddles_page_boundary(paddr, sg->length)) {

			void *map = swiotlb_tbl_map_single(hwdev,

			phys_addr_t map = swiotlb_tbl_map_single(hwdev,

								 start_dma_addr,

								 sg_phys(sg),

							   sg->length, dir);

			if (!map) {

								 sg->length,

								 dir);

			if (map == SWIOTLB_MAP_ERROR) {

				/* Don't panic here, we expect map_sg users

				   to do proper error handling. */

				xen_swiotlb_unmap_sg_attrs(hwdev, sgl, i, dir,
@@ -506,7 +505,7 @@ xen_swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, 
				sgl[0].dma_length = 0;

				return DMA_ERROR_CODE;

			}

			sg->dma_address = xen_virt_to_bus(map);

			sg->dma_address = xen_phys_to_bus(map);

		} else

			sg->dma_address = dev_addr;

		sg->dma_length = sg->length;

include/linux/swiotlb.h

+12 −8
Original line number Diff line number Diff line
@@ -34,21 +34,25 @@ enum dma_sync_target { 
	SYNC_FOR_CPU = 0,

	SYNC_FOR_DEVICE = 1,

};

extern void *swiotlb_tbl_map_single(struct device *hwdev, dma_addr_t tbl_dma_addr,



/* define the last possible byte of physical address space as a mapping error */

#define SWIOTLB_MAP_ERROR (~(phys_addr_t)0x0)



extern phys_addr_t swiotlb_tbl_map_single(struct device *hwdev,

					  dma_addr_t tbl_dma_addr,

					  phys_addr_t phys, size_t size,

					  enum dma_data_direction dir);



extern void swiotlb_tbl_unmap_single(struct device *hwdev, char *dma_addr,

extern void swiotlb_tbl_unmap_single(struct device *hwdev,

				     phys_addr_t tlb_addr,

				     size_t size, enum dma_data_direction dir);



extern void swiotlb_tbl_sync_single(struct device *hwdev, char *dma_addr,

extern void swiotlb_tbl_sync_single(struct device *hwdev,

				    phys_addr_t tlb_addr,

				    size_t size, enum dma_data_direction dir,

				    enum dma_sync_target target);



/* Accessory functions. */

extern void swiotlb_bounce(phys_addr_t phys, char *dma_addr, size_t size,

			   enum dma_data_direction dir);



extern void

*swiotlb_alloc_coherent(struct device *hwdev, size_t size,

			dma_addr_t *dma_handle, gfp_t flags);

lib/swiotlb.c

+139 −130
Original line number Diff line number Diff line
@@ -57,7 +57,7 @@ int swiotlb_force; 
 * swiotlb_tbl_sync_single_*, to see if the memory was in fact allocated by this

 * API.

 */

static char *io_tlb_start, *io_tlb_end;

static phys_addr_t io_tlb_start, io_tlb_end;



/*

 * The number of IO TLB blocks (in groups of 64) between io_tlb_start and
@@ -70,7 +70,7 @@ static unsigned long io_tlb_nslabs; 
 */

static unsigned long io_tlb_overflow = 32*1024;



static void *io_tlb_overflow_buffer;

static phys_addr_t io_tlb_overflow_buffer;



/*

 * This is a free list describing the number of free entries available from
@@ -125,26 +125,37 @@ static dma_addr_t swiotlb_virt_to_bus(struct device *hwdev, 
void swiotlb_print_info(void)

{

	unsigned long bytes = io_tlb_nslabs << IO_TLB_SHIFT;

	phys_addr_t pstart, pend;

	unsigned char *vstart, *vend;



	pstart = virt_to_phys(io_tlb_start);

	pend = virt_to_phys(io_tlb_end);

	vstart = phys_to_virt(io_tlb_start);

	vend = phys_to_virt(io_tlb_end);



	printk(KERN_INFO "software IO TLB [mem %#010llx-%#010llx] (%luMB) mapped at [%p-%p]\n",

	       (unsigned long long)pstart, (unsigned long long)pend - 1,

	       bytes >> 20, io_tlb_start, io_tlb_end - 1);

	       (unsigned long long)io_tlb_start,

	       (unsigned long long)io_tlb_end,

	       bytes >> 20, vstart, vend - 1);

}



void __init swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose)

{

	void *v_overflow_buffer;

	unsigned long i, bytes;



	bytes = nslabs << IO_TLB_SHIFT;



	io_tlb_nslabs = nslabs;

	io_tlb_start = tlb;

	io_tlb_start = __pa(tlb);

	io_tlb_end = io_tlb_start + bytes;



	/*

	 * Get the overflow emergency buffer

	 */

	v_overflow_buffer = alloc_bootmem_low_pages(PAGE_ALIGN(io_tlb_overflow));

	if (!v_overflow_buffer)

		panic("Cannot allocate SWIOTLB overflow buffer!\n");



	io_tlb_overflow_buffer = __pa(v_overflow_buffer);



	/*

	 * Allocate and initialize the free list array.  This array is used

	 * to find contiguous free memory regions of size up to IO_TLB_SEGSIZE
@@ -156,12 +167,6 @@ void __init swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose) 
	io_tlb_index = 0;

	io_tlb_orig_addr = alloc_bootmem_pages(PAGE_ALIGN(io_tlb_nslabs * sizeof(phys_addr_t)));



	/*

	 * Get the overflow emergency buffer

	 */

	io_tlb_overflow_buffer = alloc_bootmem_low_pages(PAGE_ALIGN(io_tlb_overflow));

	if (!io_tlb_overflow_buffer)

		panic("Cannot allocate SWIOTLB overflow buffer!\n");

	if (verbose)

		swiotlb_print_info();

}
@@ -173,6 +178,7 @@ void __init swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose) 
static void __init

swiotlb_init_with_default_size(size_t default_size, int verbose)

{

	unsigned char *vstart;

	unsigned long bytes;



	if (!io_tlb_nslabs) {
@@ -185,11 +191,11 @@ swiotlb_init_with_default_size(size_t default_size, int verbose) 
	/*

	 * Get IO TLB memory from the low pages

	 */

	io_tlb_start = alloc_bootmem_low_pages(PAGE_ALIGN(bytes));

	if (!io_tlb_start)

	vstart = alloc_bootmem_low_pages(PAGE_ALIGN(bytes));

	if (!vstart)

		panic("Cannot allocate SWIOTLB buffer");



	swiotlb_init_with_tbl(io_tlb_start, io_tlb_nslabs, verbose);

	swiotlb_init_with_tbl(vstart, io_tlb_nslabs, verbose);

}



void __init
@@ -207,6 +213,7 @@ int 
swiotlb_late_init_with_default_size(size_t default_size)

{

	unsigned long bytes, req_nslabs = io_tlb_nslabs;

	unsigned char *vstart = NULL;

	unsigned int order;

	int rc = 0;
@@ -223,14 +230,14 @@ swiotlb_late_init_with_default_size(size_t default_size) 
	bytes = io_tlb_nslabs << IO_TLB_SHIFT;



	while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) {

		io_tlb_start = (void *)__get_free_pages(GFP_DMA | __GFP_NOWARN,

		vstart = (void *)__get_free_pages(GFP_DMA | __GFP_NOWARN,

						  order);

		if (io_tlb_start)

		if (vstart)

			break;

		order--;

	}



	if (!io_tlb_start) {

	if (!vstart) {

		io_tlb_nslabs = req_nslabs;

		return -ENOMEM;

	}
@@ -239,9 +246,9 @@ swiotlb_late_init_with_default_size(size_t default_size) 
		       "for software IO TLB\n", (PAGE_SIZE << order) >> 20);

		io_tlb_nslabs = SLABS_PER_PAGE << order;

	}

	rc = swiotlb_late_init_with_tbl(io_tlb_start, io_tlb_nslabs);

	rc = swiotlb_late_init_with_tbl(vstart, io_tlb_nslabs);

	if (rc)

		free_pages((unsigned long)io_tlb_start, order);

		free_pages((unsigned long)vstart, order);

	return rc;

}
@@ -249,14 +256,25 @@ int 
swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs)

{

	unsigned long i, bytes;

	unsigned char *v_overflow_buffer;



	bytes = nslabs << IO_TLB_SHIFT;



	io_tlb_nslabs = nslabs;

	io_tlb_start = tlb;

	io_tlb_start = virt_to_phys(tlb);

	io_tlb_end = io_tlb_start + bytes;



	memset(io_tlb_start, 0, bytes);

	memset(tlb, 0, bytes);



	/*

	 * Get the overflow emergency buffer

	 */

	v_overflow_buffer = (void *)__get_free_pages(GFP_DMA,

						     get_order(io_tlb_overflow));

	if (!v_overflow_buffer)

		goto cleanup2;



	io_tlb_overflow_buffer = virt_to_phys(v_overflow_buffer);



	/*

	 * Allocate and initialize the free list array.  This array is used
@@ -266,7 +284,7 @@ swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs) 
	io_tlb_list = (unsigned int *)__get_free_pages(GFP_KERNEL,

	                              get_order(io_tlb_nslabs * sizeof(int)));

	if (!io_tlb_list)

		goto cleanup2;

		goto cleanup3;



	for (i = 0; i < io_tlb_nslabs; i++)

 		io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE);
@@ -277,18 +295,10 @@ swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs) 
				 get_order(io_tlb_nslabs *

					   sizeof(phys_addr_t)));

	if (!io_tlb_orig_addr)

		goto cleanup3;

		goto cleanup4;



	memset(io_tlb_orig_addr, 0, io_tlb_nslabs * sizeof(phys_addr_t));



	/*

	 * Get the overflow emergency buffer

	 */

	io_tlb_overflow_buffer = (void *)__get_free_pages(GFP_DMA,

	                                          get_order(io_tlb_overflow));

	if (!io_tlb_overflow_buffer)

		goto cleanup4;



	swiotlb_print_info();



	late_alloc = 1;
@@ -296,42 +306,42 @@ swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs) 
	return 0;



cleanup4:

	free_pages((unsigned long)io_tlb_orig_addr,

		   get_order(io_tlb_nslabs * sizeof(phys_addr_t)));

	io_tlb_orig_addr = NULL;

cleanup3:

	free_pages((unsigned long)io_tlb_list, get_order(io_tlb_nslabs *

	                                                 sizeof(int)));

	io_tlb_list = NULL;

cleanup3:

	free_pages((unsigned long)v_overflow_buffer,

		   get_order(io_tlb_overflow));

	io_tlb_overflow_buffer = 0;

cleanup2:

	io_tlb_end = NULL;

	io_tlb_start = NULL;

	io_tlb_end = 0;

	io_tlb_start = 0;

	io_tlb_nslabs = 0;

	return -ENOMEM;

}



void __init swiotlb_free(void)

{

	if (!io_tlb_overflow_buffer)

	if (!io_tlb_orig_addr)

		return;



	if (late_alloc) {

		free_pages((unsigned long)io_tlb_overflow_buffer,

		free_pages((unsigned long)phys_to_virt(io_tlb_overflow_buffer),

			   get_order(io_tlb_overflow));

		free_pages((unsigned long)io_tlb_orig_addr,

			   get_order(io_tlb_nslabs * sizeof(phys_addr_t)));

		free_pages((unsigned long)io_tlb_list, get_order(io_tlb_nslabs *

								 sizeof(int)));

		free_pages((unsigned long)io_tlb_start,

		free_pages((unsigned long)phys_to_virt(io_tlb_start),

			   get_order(io_tlb_nslabs << IO_TLB_SHIFT));

	} else {

		free_bootmem_late(__pa(io_tlb_overflow_buffer),

		free_bootmem_late(io_tlb_overflow_buffer,

				  PAGE_ALIGN(io_tlb_overflow));

		free_bootmem_late(__pa(io_tlb_orig_addr),

				  PAGE_ALIGN(io_tlb_nslabs * sizeof(phys_addr_t)));

		free_bootmem_late(__pa(io_tlb_list),

				  PAGE_ALIGN(io_tlb_nslabs * sizeof(int)));

		free_bootmem_late(__pa(io_tlb_start),

		free_bootmem_late(io_tlb_start,

				  PAGE_ALIGN(io_tlb_nslabs << IO_TLB_SHIFT));

	}

	io_tlb_nslabs = 0;
@@ -339,21 +349,21 @@ void __init swiotlb_free(void) 


static int is_swiotlb_buffer(phys_addr_t paddr)

{

	return paddr >= virt_to_phys(io_tlb_start) &&

		paddr < virt_to_phys(io_tlb_end);

	return paddr >= io_tlb_start && paddr < io_tlb_end;

}



/*

 * Bounce: copy the swiotlb buffer back to the original dma location

 */

void swiotlb_bounce(phys_addr_t phys, char *dma_addr, size_t size,

		    enum dma_data_direction dir)

static void swiotlb_bounce(phys_addr_t orig_addr, phys_addr_t tlb_addr,

			   size_t size, enum dma_data_direction dir)

{

	unsigned long pfn = PFN_DOWN(phys);

	unsigned long pfn = PFN_DOWN(orig_addr);

	unsigned char *vaddr = phys_to_virt(tlb_addr);



	if (PageHighMem(pfn_to_page(pfn))) {

		/* The buffer does not have a mapping.  Map it in and copy */

		unsigned int offset = phys & ~PAGE_MASK;

		unsigned int offset = orig_addr & ~PAGE_MASK;

		char *buffer;

		unsigned int sz = 0;

		unsigned long flags;
@@ -364,32 +374,31 @@ void swiotlb_bounce(phys_addr_t phys, char *dma_addr, size_t size, 
			local_irq_save(flags);

			buffer = kmap_atomic(pfn_to_page(pfn));

			if (dir == DMA_TO_DEVICE)

				memcpy(dma_addr, buffer + offset, sz);

				memcpy(vaddr, buffer + offset, sz);

			else

				memcpy(buffer + offset, dma_addr, sz);

				memcpy(buffer + offset, vaddr, sz);

			kunmap_atomic(buffer);

			local_irq_restore(flags);



			size -= sz;

			pfn++;

			dma_addr += sz;

			vaddr += sz;

			offset = 0;

		}

	} else if (dir == DMA_TO_DEVICE) {

		memcpy(vaddr, phys_to_virt(orig_addr), size);

	} else {

		if (dir == DMA_TO_DEVICE)

			memcpy(dma_addr, phys_to_virt(phys), size);

		else

			memcpy(phys_to_virt(phys), dma_addr, size);

		memcpy(phys_to_virt(orig_addr), vaddr, size);

	}

}

EXPORT_SYMBOL_GPL(swiotlb_bounce);



void *swiotlb_tbl_map_single(struct device *hwdev, dma_addr_t tbl_dma_addr,

			     phys_addr_t phys, size_t size,

phys_addr_t swiotlb_tbl_map_single(struct device *hwdev,

				   dma_addr_t tbl_dma_addr,

				   phys_addr_t orig_addr, size_t size,

				   enum dma_data_direction dir)

{

	unsigned long flags;

	char *dma_addr;

	phys_addr_t tlb_addr;

	unsigned int nslots, stride, index, wrap;

	int i;

	unsigned long mask;
@@ -453,7 +462,7 @@ void *swiotlb_tbl_map_single(struct device *hwdev, dma_addr_t tbl_dma_addr, 
				io_tlb_list[i] = 0;

			for (i = index - 1; (OFFSET(i, IO_TLB_SEGSIZE) != IO_TLB_SEGSIZE - 1) && io_tlb_list[i]; i--)

				io_tlb_list[i] = ++count;

			dma_addr = io_tlb_start + (index << IO_TLB_SHIFT);

			tlb_addr = io_tlb_start + (index << IO_TLB_SHIFT);



			/*

			 * Update the indices to avoid searching in the next
@@ -471,7 +480,7 @@ void *swiotlb_tbl_map_single(struct device *hwdev, dma_addr_t tbl_dma_addr, 


not_found:

	spin_unlock_irqrestore(&io_tlb_lock, flags);

	return NULL;

	return SWIOTLB_MAP_ERROR;

found:

	spin_unlock_irqrestore(&io_tlb_lock, flags);
@@ -481,11 +490,11 @@ void *swiotlb_tbl_map_single(struct device *hwdev, dma_addr_t tbl_dma_addr, 
	 * needed.

	 */

	for (i = 0; i < nslots; i++)

		io_tlb_orig_addr[index+i] = phys + (i << IO_TLB_SHIFT);

		io_tlb_orig_addr[index+i] = orig_addr + (i << IO_TLB_SHIFT);

	if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)

		swiotlb_bounce(phys, dma_addr, size, DMA_TO_DEVICE);

		swiotlb_bounce(orig_addr, tlb_addr, size, DMA_TO_DEVICE);



	return dma_addr;

	return tlb_addr;

}

EXPORT_SYMBOL_GPL(swiotlb_tbl_map_single);
@@ -493,11 +502,10 @@ EXPORT_SYMBOL_GPL(swiotlb_tbl_map_single); 
 * Allocates bounce buffer and returns its kernel virtual address.

 */



static void *

map_single(struct device *hwdev, phys_addr_t phys, size_t size,

phys_addr_t map_single(struct device *hwdev, phys_addr_t phys, size_t size,

		       enum dma_data_direction dir)

{

	dma_addr_t start_dma_addr = swiotlb_virt_to_bus(hwdev, io_tlb_start);

	dma_addr_t start_dma_addr = phys_to_dma(hwdev, io_tlb_start);



	return swiotlb_tbl_map_single(hwdev, start_dma_addr, phys, size, dir);

}
@@ -505,20 +513,19 @@ map_single(struct device *hwdev, phys_addr_t phys, size_t size, 
/*

 * dma_addr is the kernel virtual address of the bounce buffer to unmap.

 */

void

swiotlb_tbl_unmap_single(struct device *hwdev, char *dma_addr, size_t size,

			enum dma_data_direction dir)

void swiotlb_tbl_unmap_single(struct device *hwdev, phys_addr_t tlb_addr,

			      size_t size, enum dma_data_direction dir)

{

	unsigned long flags;

	int i, count, nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;

	int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT;

	phys_addr_t phys = io_tlb_orig_addr[index];

	int index = (tlb_addr - io_tlb_start) >> IO_TLB_SHIFT;

	phys_addr_t orig_addr = io_tlb_orig_addr[index];



	/*

	 * First, sync the memory before unmapping the entry

	 */

	if (phys && ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL)))

		swiotlb_bounce(phys, dma_addr, size, DMA_FROM_DEVICE);

	if (orig_addr && ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL)))

		swiotlb_bounce(orig_addr, tlb_addr, size, DMA_FROM_DEVICE);



	/*

	 * Return the buffer to the free list by setting the corresponding
@@ -547,26 +554,27 @@ swiotlb_tbl_unmap_single(struct device *hwdev, char *dma_addr, size_t size, 
}

EXPORT_SYMBOL_GPL(swiotlb_tbl_unmap_single);



void

swiotlb_tbl_sync_single(struct device *hwdev, char *dma_addr, size_t size,

			enum dma_data_direction dir,

void swiotlb_tbl_sync_single(struct device *hwdev, phys_addr_t tlb_addr,

			     size_t size, enum dma_data_direction dir,

			     enum dma_sync_target target)

{

	int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT;

	phys_addr_t phys = io_tlb_orig_addr[index];

	int index = (tlb_addr - io_tlb_start) >> IO_TLB_SHIFT;

	phys_addr_t orig_addr = io_tlb_orig_addr[index];



	phys += ((unsigned long)dma_addr & ((1 << IO_TLB_SHIFT) - 1));

	orig_addr += (unsigned long)tlb_addr & ((1 << IO_TLB_SHIFT) - 1);



	switch (target) {

	case SYNC_FOR_CPU:

		if (likely(dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL))

			swiotlb_bounce(phys, dma_addr, size, DMA_FROM_DEVICE);

			swiotlb_bounce(orig_addr, tlb_addr,

				       size, DMA_FROM_DEVICE);

		else

			BUG_ON(dir != DMA_TO_DEVICE);

		break;

	case SYNC_FOR_DEVICE:

		if (likely(dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL))

			swiotlb_bounce(phys, dma_addr, size, DMA_TO_DEVICE);

			swiotlb_bounce(orig_addr, tlb_addr,

				       size, DMA_TO_DEVICE);

		else

			BUG_ON(dir != DMA_FROM_DEVICE);

		break;
@@ -589,26 +597,28 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size, 
		dma_mask = hwdev->coherent_dma_mask;



	ret = (void *)__get_free_pages(flags, order);

	if (ret && swiotlb_virt_to_bus(hwdev, ret) + size - 1 > dma_mask) {

	if (ret) {

		dev_addr = swiotlb_virt_to_bus(hwdev, ret);

		if (dev_addr + size - 1 > dma_mask) {

			/*

			 * The allocated memory isn't reachable by the device.

			 */

			free_pages((unsigned long) ret, order);

			ret = NULL;

		}

	}

	if (!ret) {

		/*

		 * We are either out of memory or the device can't DMA to

		 * GFP_DMA memory; fall back on map_single(), which

		 * will grab memory from the lowest available address range.

		 */

		ret = map_single(hwdev, 0, size, DMA_FROM_DEVICE);

		if (!ret)

		phys_addr_t paddr = map_single(hwdev, 0, size, DMA_FROM_DEVICE);

		if (paddr == SWIOTLB_MAP_ERROR)

			return NULL;

	}



	memset(ret, 0, size);

	dev_addr = swiotlb_virt_to_bus(hwdev, ret);

		ret = phys_to_virt(paddr);

		dev_addr = phys_to_dma(hwdev, paddr);



		/* Confirm address can be DMA'd by device */

		if (dev_addr + size - 1 > dma_mask) {
@@ -617,10 +627,15 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size, 
			       (unsigned long long)dev_addr);



			/* DMA_TO_DEVICE to avoid memcpy in unmap_single */

		swiotlb_tbl_unmap_single(hwdev, ret, size, DMA_TO_DEVICE);

			swiotlb_tbl_unmap_single(hwdev, paddr,

						 size, DMA_TO_DEVICE);

			return NULL;

		}

	}



	*dma_handle = dev_addr;

	memset(ret, 0, size);



	return ret;

}

EXPORT_SYMBOL(swiotlb_alloc_coherent);
@@ -636,7 +651,7 @@ swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr, 
		free_pages((unsigned long)vaddr, get_order(size));

	else

		/* DMA_TO_DEVICE to avoid memcpy in swiotlb_tbl_unmap_single */

		swiotlb_tbl_unmap_single(hwdev, vaddr, size, DMA_TO_DEVICE);

		swiotlb_tbl_unmap_single(hwdev, paddr, size, DMA_TO_DEVICE);

}

EXPORT_SYMBOL(swiotlb_free_coherent);
@@ -677,9 +692,8 @@ dma_addr_t swiotlb_map_page(struct device *dev, struct page *page, 
			    enum dma_data_direction dir,

			    struct dma_attrs *attrs)

{

	phys_addr_t phys = page_to_phys(page) + offset;

	phys_addr_t map, phys = page_to_phys(page) + offset;

	dma_addr_t dev_addr = phys_to_dma(dev, phys);

	void *map;



	BUG_ON(dir == DMA_NONE);

	/*
@@ -690,23 +704,19 @@ dma_addr_t swiotlb_map_page(struct device *dev, struct page *page, 
	if (dma_capable(dev, dev_addr, size) && !swiotlb_force)

		return dev_addr;



	/*

	 * Oh well, have to allocate and map a bounce buffer.

	 */

	/* Oh well, have to allocate and map a bounce buffer. */

	map = map_single(dev, phys, size, dir);

	if (!map) {

	if (map == SWIOTLB_MAP_ERROR) {

		swiotlb_full(dev, size, dir, 1);

		map = io_tlb_overflow_buffer;

		return phys_to_dma(dev, io_tlb_overflow_buffer);

	}



	dev_addr = swiotlb_virt_to_bus(dev, map);

	dev_addr = phys_to_dma(dev, map);



	/*

	 * Ensure that the address returned is DMA'ble

	 */

	/* Ensure that the address returned is DMA'ble */

	if (!dma_capable(dev, dev_addr, size)) {

		swiotlb_tbl_unmap_single(dev, map, size, dir);

		dev_addr = swiotlb_virt_to_bus(dev, io_tlb_overflow_buffer);

		return phys_to_dma(dev, io_tlb_overflow_buffer);

	}



	return dev_addr;
@@ -729,7 +739,7 @@ static void unmap_single(struct device *hwdev, dma_addr_t dev_addr, 
	BUG_ON(dir == DMA_NONE);



	if (is_swiotlb_buffer(paddr)) {

		swiotlb_tbl_unmap_single(hwdev, phys_to_virt(paddr), size, dir);

		swiotlb_tbl_unmap_single(hwdev, paddr, size, dir);

		return;

	}
@@ -773,8 +783,7 @@ swiotlb_sync_single(struct device *hwdev, dma_addr_t dev_addr, 
	BUG_ON(dir == DMA_NONE);



	if (is_swiotlb_buffer(paddr)) {

		swiotlb_tbl_sync_single(hwdev, phys_to_virt(paddr), size, dir,

				       target);

		swiotlb_tbl_sync_single(hwdev, paddr, size, dir, target);

		return;

	}
@@ -831,9 +840,9 @@ swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems, 


		if (swiotlb_force ||

		    !dma_capable(hwdev, dev_addr, sg->length)) {

			void *map = map_single(hwdev, sg_phys(sg),

			phys_addr_t map = map_single(hwdev, sg_phys(sg),

						     sg->length, dir);

			if (!map) {

			if (map == SWIOTLB_MAP_ERROR) {

				/* Don't panic here, we expect map_sg users

				   to do proper error handling. */

				swiotlb_full(hwdev, sg->length, dir, 0);
@@ -842,7 +851,7 @@ swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems, 
				sgl[0].dma_length = 0;

				return 0;

			}

			sg->dma_address = swiotlb_virt_to_bus(hwdev, map);

			sg->dma_address = phys_to_dma(hwdev, map);

		} else

			sg->dma_address = dev_addr;

		sg->dma_length = sg->length;
@@ -925,7 +934,7 @@ EXPORT_SYMBOL(swiotlb_sync_sg_for_device); 
int

swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr)

{

	return (dma_addr == swiotlb_virt_to_bus(hwdev, io_tlb_overflow_buffer));

	return (dma_addr == phys_to_dma(hwdev, io_tlb_overflow_buffer));

}

EXPORT_SYMBOL(swiotlb_dma_mapping_error);
@@ -938,6 +947,6 @@ EXPORT_SYMBOL(swiotlb_dma_mapping_error); 
int

swiotlb_dma_supported(struct device *hwdev, u64 mask)

{

	return swiotlb_virt_to_bus(hwdev, io_tlb_end - 1) <= mask;

	return phys_to_dma(hwdev, io_tlb_end - 1) <= mask;

}

EXPORT_SYMBOL(swiotlb_dma_supported);