Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 83cce2b6 authored by Joerg Roedel's avatar Joerg Roedel
Browse files

Merge branches 'amd-iommu/fixes', 'amd-iommu/debug',... 

Merge branches 'amd-iommu/fixes', 'amd-iommu/debug', 'amd-iommu/suspend-resume' and 'amd-iommu/extended-allocator' into amd-iommu/2.6.31

Conflicts:
	arch/x86/kernel/amd_iommu.c
	arch/x86/kernel/amd_iommu_init.c
parents c1eee67b 2e8b5696 736501ee 47bccd6b

Documentation/kernel-parameters.txt

+0 −5
Original line number Diff line number Diff line
@@ -329,11 +329,6 @@ and is between 256 and 4096 characters. It is defined in the file 
				    flushed before they will be reused, which

				    is a lot of faster



	amd_iommu_size= [HW,X86-64]

			Define the size of the aperture for the AMD IOMMU

			driver. Possible values are:

			'32M', '64M' (default), '128M', '256M', '512M', '1G'



	amijoy.map=	[HW,JOY] Amiga joystick support

			Map of devices attached to JOY0DAT and JOY1DAT

			Format: <a>,<b>

arch/x86/Kconfig.debug

+8 −0
Original line number Diff line number Diff line
@@ -159,6 +159,14 @@ config IOMMU_DEBUG 
	  options. See Documentation/x86_64/boot-options.txt for more

	  details.



config IOMMU_STRESS

	bool "Enable IOMMU stress-test mode"

	---help---

	  This option disables various optimizations in IOMMU related

	  code to do real stress testing of the IOMMU code. This option

	  will cause a performance drop and should only be enabled for

	  testing.



config IOMMU_LEAK

	bool "IOMMU leak tracing"

	depends on IOMMU_DEBUG && DMA_API_DEBUG

arch/x86/include/asm/amd_iommu.h

+2 −0
Original line number Diff line number Diff line
@@ -27,6 +27,8 @@ extern int amd_iommu_init(void); 
extern int amd_iommu_init_dma_ops(void);

extern void amd_iommu_detect(void);

extern irqreturn_t amd_iommu_int_handler(int irq, void *data);

extern void amd_iommu_flush_all_domains(void);

extern void amd_iommu_flush_all_devices(void);

#else

static inline int amd_iommu_init(void) { return -ENODEV; }

static inline void amd_iommu_detect(void) { }

arch/x86/include/asm/amd_iommu_types.h

+44 −11
Original line number Diff line number Diff line
@@ -194,6 +194,27 @@ 
#define PD_DMA_OPS_MASK		(1UL << 0) /* domain used for dma_ops */

#define PD_DEFAULT_MASK		(1UL << 1) /* domain is a default dma_ops

					      domain for an IOMMU */

extern bool amd_iommu_dump;

#define DUMP_printk(format, arg...)					\

	do {								\

		if (amd_iommu_dump)						\

			printk(KERN_INFO "AMD IOMMU: " format, ## arg);	\

	} while(0);



/*

 * Make iterating over all IOMMUs easier

 */

#define for_each_iommu(iommu) \

	list_for_each_entry((iommu), &amd_iommu_list, list)

#define for_each_iommu_safe(iommu, next) \

	list_for_each_entry_safe((iommu), (next), &amd_iommu_list, list)



#define APERTURE_RANGE_SHIFT	27	/* 128 MB */

#define APERTURE_RANGE_SIZE	(1ULL << APERTURE_RANGE_SHIFT)

#define APERTURE_RANGE_PAGES	(APERTURE_RANGE_SIZE >> PAGE_SHIFT)

#define APERTURE_MAX_RANGES	32	/* allows 4GB of DMA address space */

#define APERTURE_RANGE_INDEX(a)	((a) >> APERTURE_RANGE_SHIFT)

#define APERTURE_PAGE_INDEX(a)	(((a) >> 21) & 0x3fULL)



/*

 * This structure contains generic data for  IOMMU protection domains
@@ -209,6 +230,26 @@ struct protection_domain { 
	void *priv;		/* private data */

};



/*

 * For dynamic growth the aperture size is split into ranges of 128MB of

 * DMA address space each. This struct represents one such range.

 */

struct aperture_range {



	/* address allocation bitmap */

	unsigned long *bitmap;



	/*

	 * Array of PTE pages for the aperture. In this array we save all the

	 * leaf pages of the domain page table used for the aperture. This way

	 * we don't need to walk the page table to find a specific PTE. We can

	 * just calculate its address in constant time.

	 */

	u64 *pte_pages[64];



	unsigned long offset;

};



/*

 * Data container for a dma_ops specific protection domain

 */
@@ -222,18 +263,10 @@ struct dma_ops_domain { 
	unsigned long aperture_size;



	/* address we start to search for free addresses */

	unsigned long next_bit;



	/* address allocation bitmap */

	unsigned long *bitmap;

	unsigned long next_address;



	/*

	 * Array of PTE pages for the aperture. In this array we save all the

	 * leaf pages of the domain page table used for the aperture. This way

	 * we don't need to walk the page table to find a specific PTE. We can

	 * just calculate its address in constant time.

	 */

	u64 **pte_pages;

	/* address space relevant data */

	struct aperture_range *aperture[APERTURE_MAX_RANGES];



	/* This will be set to true when TLB needs to be flushed */

	bool need_flush;

arch/x86/kernel/amd_iommu.c

+361 −125
Original line number Diff line number Diff line
@@ -55,7 +55,12 @@ struct iommu_cmd { 
static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,

			     struct unity_map_entry *e);

static struct dma_ops_domain *find_protection_domain(u16 devid);



static u64* alloc_pte(struct protection_domain *dom,

		      unsigned long address, u64

		      **pte_page, gfp_t gfp);

static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,

				      unsigned long start_page,

				      unsigned int pages);



#ifndef BUS_NOTIFY_UNBOUND_DRIVER

#define BUS_NOTIFY_UNBOUND_DRIVER 0x0005
@@ -217,7 +222,7 @@ irqreturn_t amd_iommu_int_handler(int irq, void *data) 
{

	struct amd_iommu *iommu;



	list_for_each_entry(iommu, &amd_iommu_list, list)

	for_each_iommu(iommu)

		iommu_poll_events(iommu);



	return IRQ_HANDLED;
@@ -444,7 +449,7 @@ static void iommu_flush_domain(u16 domid) 
	__iommu_build_inv_iommu_pages(&cmd, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,

				      domid, 1, 1);



	list_for_each_entry(iommu, &amd_iommu_list, list) {

	for_each_iommu(iommu) {

		spin_lock_irqsave(&iommu->lock, flags);

		__iommu_queue_command(iommu, &cmd);

		__iommu_completion_wait(iommu);
@@ -453,6 +458,35 @@ static void iommu_flush_domain(u16 domid) 
	}

}



void amd_iommu_flush_all_domains(void)

{

	int i;



	for (i = 1; i < MAX_DOMAIN_ID; ++i) {

		if (!test_bit(i, amd_iommu_pd_alloc_bitmap))

			continue;

		iommu_flush_domain(i);

	}

}



void amd_iommu_flush_all_devices(void)

{

	struct amd_iommu *iommu;

	int i;



	for (i = 0; i <= amd_iommu_last_bdf; ++i) {

		if (amd_iommu_pd_table[i] == NULL)

			continue;



		iommu = amd_iommu_rlookup_table[i];

		if (!iommu)

			continue;



		iommu_queue_inv_dev_entry(iommu, i);

		iommu_completion_wait(iommu);

	}

}



/****************************************************************************

 *

 * The functions below are used the create the page table mappings for
@@ -472,7 +506,7 @@ static int iommu_map_page(struct protection_domain *dom, 
			  unsigned long phys_addr,

			  int prot)

{

	u64 __pte, *pte, *page;

	u64 __pte, *pte;



	bus_addr  = PAGE_ALIGN(bus_addr);

	phys_addr = PAGE_ALIGN(phys_addr);
@@ -481,27 +515,7 @@ static int iommu_map_page(struct protection_domain *dom, 
	if (bus_addr > IOMMU_MAP_SIZE_L3 || !(prot & IOMMU_PROT_MASK))

		return -EINVAL;



	pte = &dom->pt_root[IOMMU_PTE_L2_INDEX(bus_addr)];



	if (!IOMMU_PTE_PRESENT(*pte)) {

		page = (u64 *)get_zeroed_page(GFP_KERNEL);

		if (!page)

			return -ENOMEM;

		*pte = IOMMU_L2_PDE(virt_to_phys(page));

	}



	pte = IOMMU_PTE_PAGE(*pte);

	pte = &pte[IOMMU_PTE_L1_INDEX(bus_addr)];



	if (!IOMMU_PTE_PRESENT(*pte)) {

		page = (u64 *)get_zeroed_page(GFP_KERNEL);

		if (!page)

			return -ENOMEM;

		*pte = IOMMU_L1_PDE(virt_to_phys(page));

	}



	pte = IOMMU_PTE_PAGE(*pte);

	pte = &pte[IOMMU_PTE_L0_INDEX(bus_addr)];

	pte = alloc_pte(dom, bus_addr, NULL, GFP_KERNEL);



	if (IOMMU_PTE_PRESENT(*pte))

		return -EBUSY;
@@ -599,7 +613,8 @@ static int dma_ops_unity_map(struct dma_ops_domain *dma_dom, 
		 * as allocated in the aperture

		 */

		if (addr < dma_dom->aperture_size)

			__set_bit(addr >> PAGE_SHIFT, dma_dom->bitmap);

			__set_bit(addr >> PAGE_SHIFT,

				  dma_dom->aperture[0]->bitmap);

	}



	return 0;
@@ -636,42 +651,191 @@ static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom, 
 ****************************************************************************/



/*

 * The address allocator core function.

 * The address allocator core functions.

 *

 * called with domain->lock held

 */

static unsigned long dma_ops_alloc_addresses(struct device *dev,



/*

 * This function checks if there is a PTE for a given dma address. If

 * there is one, it returns the pointer to it.

 */

static u64* fetch_pte(struct protection_domain *domain,

		      unsigned long address)

{

	u64 *pte;



	pte = &domain->pt_root[IOMMU_PTE_L2_INDEX(address)];



	if (!IOMMU_PTE_PRESENT(*pte))

		return NULL;



	pte = IOMMU_PTE_PAGE(*pte);

	pte = &pte[IOMMU_PTE_L1_INDEX(address)];



	if (!IOMMU_PTE_PRESENT(*pte))

		return NULL;



	pte = IOMMU_PTE_PAGE(*pte);

	pte = &pte[IOMMU_PTE_L0_INDEX(address)];



	return pte;

}



/*

 * This function is used to add a new aperture range to an existing

 * aperture in case of dma_ops domain allocation or address allocation

 * failure.

 */

static int alloc_new_range(struct amd_iommu *iommu,

			   struct dma_ops_domain *dma_dom,

			   bool populate, gfp_t gfp)

{

	int index = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;

	int i;



#ifdef CONFIG_IOMMU_STRESS

	populate = false;

#endif



	if (index >= APERTURE_MAX_RANGES)

		return -ENOMEM;



	dma_dom->aperture[index] = kzalloc(sizeof(struct aperture_range), gfp);

	if (!dma_dom->aperture[index])

		return -ENOMEM;



	dma_dom->aperture[index]->bitmap = (void *)get_zeroed_page(gfp);

	if (!dma_dom->aperture[index]->bitmap)

		goto out_free;



	dma_dom->aperture[index]->offset = dma_dom->aperture_size;



	if (populate) {

		unsigned long address = dma_dom->aperture_size;

		int i, num_ptes = APERTURE_RANGE_PAGES / 512;

		u64 *pte, *pte_page;



		for (i = 0; i < num_ptes; ++i) {

			pte = alloc_pte(&dma_dom->domain, address,

					&pte_page, gfp);

			if (!pte)

				goto out_free;



			dma_dom->aperture[index]->pte_pages[i] = pte_page;



			address += APERTURE_RANGE_SIZE / 64;

		}

	}



	dma_dom->aperture_size += APERTURE_RANGE_SIZE;



	/* Intialize the exclusion range if necessary */

	if (iommu->exclusion_start &&

	    iommu->exclusion_start >= dma_dom->aperture[index]->offset &&

	    iommu->exclusion_start < dma_dom->aperture_size) {

		unsigned long startpage = iommu->exclusion_start >> PAGE_SHIFT;

		int pages = iommu_num_pages(iommu->exclusion_start,

					    iommu->exclusion_length,

					    PAGE_SIZE);

		dma_ops_reserve_addresses(dma_dom, startpage, pages);

	}



	/*

	 * Check for areas already mapped as present in the new aperture

	 * range and mark those pages as reserved in the allocator. Such

	 * mappings may already exist as a result of requested unity

	 * mappings for devices.

	 */

	for (i = dma_dom->aperture[index]->offset;

	     i < dma_dom->aperture_size;

	     i += PAGE_SIZE) {

		u64 *pte = fetch_pte(&dma_dom->domain, i);

		if (!pte || !IOMMU_PTE_PRESENT(*pte))

			continue;



		dma_ops_reserve_addresses(dma_dom, i << PAGE_SHIFT, 1);

	}



	return 0;



out_free:

	free_page((unsigned long)dma_dom->aperture[index]->bitmap);



	kfree(dma_dom->aperture[index]);

	dma_dom->aperture[index] = NULL;



	return -ENOMEM;

}



static unsigned long dma_ops_area_alloc(struct device *dev,

					struct dma_ops_domain *dom,

					unsigned int pages,

					unsigned long align_mask,

					     u64 dma_mask)

					u64 dma_mask,

					unsigned long start)

{

	unsigned long limit;

	unsigned long address;

	unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE;

	int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT;

	int i = start >> APERTURE_RANGE_SHIFT;

	unsigned long boundary_size;

	unsigned long address = -1;

	unsigned long limit;



	next_bit >>= PAGE_SHIFT;



	boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,

			PAGE_SIZE) >> PAGE_SHIFT;

	limit = iommu_device_max_index(dom->aperture_size >> PAGE_SHIFT, 0,



	for (;i < max_index; ++i) {

		unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT;



		if (dom->aperture[i]->offset >= dma_mask)

			break;



		limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset,

					       dma_mask >> PAGE_SHIFT);



	if (dom->next_bit >= limit) {

		dom->next_bit = 0;

		dom->need_flush = true;

		address = iommu_area_alloc(dom->aperture[i]->bitmap,

					   limit, next_bit, pages, 0,

					    boundary_size, align_mask);

		if (address != -1) {

			address = dom->aperture[i]->offset +

				  (address << PAGE_SHIFT);

			dom->next_address = address + (pages << PAGE_SHIFT);

			break;

		}



		next_bit = 0;

	}



	address = iommu_area_alloc(dom->bitmap, limit, dom->next_bit, pages,

				   0 , boundary_size, align_mask);

	return address;

}



static unsigned long dma_ops_alloc_addresses(struct device *dev,

					     struct dma_ops_domain *dom,

					     unsigned int pages,

					     unsigned long align_mask,

					     u64 dma_mask)

{

	unsigned long address;



#ifdef CONFIG_IOMMU_STRESS

	dom->next_address = 0;

	dom->need_flush = true;

#endif



	address = dma_ops_area_alloc(dev, dom, pages, align_mask,

				     dma_mask, dom->next_address);



	if (address == -1) {

		address = iommu_area_alloc(dom->bitmap, limit, 0, pages,

				0, boundary_size, align_mask);

		dom->next_address = 0;

		address = dma_ops_area_alloc(dev, dom, pages, align_mask,

					     dma_mask, 0);

		dom->need_flush = true;

	}



	if (likely(address != -1)) {

		dom->next_bit = address + pages;

		address <<= PAGE_SHIFT;

	} else

	if (unlikely(address == -1))

		address = bad_dma_address;



	WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size);
@@ -688,11 +852,23 @@ static void dma_ops_free_addresses(struct dma_ops_domain *dom, 
				   unsigned long address,

				   unsigned int pages)

{

	address >>= PAGE_SHIFT;

	iommu_area_free(dom->bitmap, address, pages);

	unsigned i = address >> APERTURE_RANGE_SHIFT;

	struct aperture_range *range = dom->aperture[i];



	BUG_ON(i >= APERTURE_MAX_RANGES || range == NULL);



	if (address >= dom->next_bit)

#ifdef CONFIG_IOMMU_STRESS

	if (i < 4)

		return;

#endif



	if (address >= dom->next_address)

		dom->need_flush = true;



	address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT;



	iommu_area_free(range->bitmap, address, pages);



}



/****************************************************************************
@@ -740,12 +916,16 @@ static void dma_ops_reserve_addresses(struct dma_ops_domain *dom, 
				      unsigned long start_page,

				      unsigned int pages)

{

	unsigned int last_page = dom->aperture_size >> PAGE_SHIFT;

	unsigned int i, last_page = dom->aperture_size >> PAGE_SHIFT;



	if (start_page + pages > last_page)

		pages = last_page - start_page;



	iommu_area_reserve(dom->bitmap, start_page, pages);

	for (i = start_page; i < start_page + pages; ++i) {

		int index = i / APERTURE_RANGE_PAGES;

		int page  = i % APERTURE_RANGE_PAGES;

		__set_bit(page, dom->aperture[index]->bitmap);

	}

}



static void free_pagetable(struct protection_domain *domain)
@@ -784,14 +964,19 @@ static void free_pagetable(struct protection_domain *domain) 
 */

static void dma_ops_domain_free(struct dma_ops_domain *dom)

{

	int i;



	if (!dom)

		return;



	free_pagetable(&dom->domain);



	kfree(dom->pte_pages);



	kfree(dom->bitmap);

	for (i = 0; i < APERTURE_MAX_RANGES; ++i) {

		if (!dom->aperture[i])

			continue;

		free_page((unsigned long)dom->aperture[i]->bitmap);

		kfree(dom->aperture[i]);

	}



	kfree(dom);

}
@@ -801,19 +986,9 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom) 
 * It also intializes the page table and the address allocator data

 * structures required for the dma_ops interface

 */

static struct dma_ops_domain *dma_ops_domain_alloc(struct amd_iommu *iommu,

						   unsigned order)

static struct dma_ops_domain *dma_ops_domain_alloc(struct amd_iommu *iommu)

{

	struct dma_ops_domain *dma_dom;

	unsigned i, num_pte_pages;

	u64 *l2_pde;

	u64 address;



	/*

	 * Currently the DMA aperture must be between 32 MB and 1GB in size

	 */

	if ((order < 25) || (order > 30))

		return NULL;



	dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);

	if (!dma_dom)
@@ -830,55 +1005,20 @@ static struct dma_ops_domain *dma_ops_domain_alloc(struct amd_iommu *iommu, 
	dma_dom->domain.priv = dma_dom;

	if (!dma_dom->domain.pt_root)

		goto free_dma_dom;

	dma_dom->aperture_size = (1ULL << order);

	dma_dom->bitmap = kzalloc(dma_dom->aperture_size / (PAGE_SIZE * 8),

				  GFP_KERNEL);

	if (!dma_dom->bitmap)

		goto free_dma_dom;

	/*

	 * mark the first page as allocated so we never return 0 as

	 * a valid dma-address. So we can use 0 as error value

	 */

	dma_dom->bitmap[0] = 1;

	dma_dom->next_bit = 0;



	dma_dom->need_flush = false;

	dma_dom->target_dev = 0xffff;



	/* Intialize the exclusion range if necessary */

	if (iommu->exclusion_start &&

	    iommu->exclusion_start < dma_dom->aperture_size) {

		unsigned long startpage = iommu->exclusion_start >> PAGE_SHIFT;

		int pages = iommu_num_pages(iommu->exclusion_start,

					    iommu->exclusion_length,

					    PAGE_SIZE);

		dma_ops_reserve_addresses(dma_dom, startpage, pages);

	}

	if (alloc_new_range(iommu, dma_dom, true, GFP_KERNEL))

		goto free_dma_dom;



	/*

	 * At the last step, build the page tables so we don't need to

	 * allocate page table pages in the dma_ops mapping/unmapping

	 * path.

	 * mark the first page as allocated so we never return 0 as

	 * a valid dma-address. So we can use 0 as error value

	 */

	num_pte_pages = dma_dom->aperture_size / (PAGE_SIZE * 512);

	dma_dom->pte_pages = kzalloc(num_pte_pages * sizeof(void *),

			GFP_KERNEL);

	if (!dma_dom->pte_pages)

		goto free_dma_dom;

	dma_dom->aperture[0]->bitmap[0] = 1;

	dma_dom->next_address = 0;



	l2_pde = (u64 *)get_zeroed_page(GFP_KERNEL);

	if (l2_pde == NULL)

		goto free_dma_dom;



	dma_dom->domain.pt_root[0] = IOMMU_L2_PDE(virt_to_phys(l2_pde));



	for (i = 0; i < num_pte_pages; ++i) {

		dma_dom->pte_pages[i] = (u64 *)get_zeroed_page(GFP_KERNEL);

		if (!dma_dom->pte_pages[i])

			goto free_dma_dom;

		address = virt_to_phys(dma_dom->pte_pages[i]);

		l2_pde[i] = IOMMU_L1_PDE(address);

	}



	return dma_dom;
@@ -987,7 +1127,6 @@ static int device_change_notifier(struct notifier_block *nb, 
	struct protection_domain *domain;

	struct dma_ops_domain *dma_domain;

	struct amd_iommu *iommu;

	int order = amd_iommu_aperture_order;

	unsigned long flags;



	if (devid > amd_iommu_last_bdf)
@@ -1013,8 +1152,9 @@ static int device_change_notifier(struct notifier_block *nb, 
		if (!dma_domain)

			dma_domain = iommu->default_dom;

		attach_device(iommu, &dma_domain->domain, devid);

		printk(KERN_INFO "AMD IOMMU: Using protection domain %d for "

		       "device %s\n", dma_domain->domain.id, dev_name(dev));

		DUMP_printk(KERN_INFO "AMD IOMMU: Using protection domain "

			    "%d for device %s\n",

			    dma_domain->domain.id, dev_name(dev));

		break;

	case BUS_NOTIFY_UNBOUND_DRIVER:

		if (!domain)
@@ -1026,7 +1166,7 @@ static int device_change_notifier(struct notifier_block *nb, 
		dma_domain = find_protection_domain(devid);

		if (dma_domain)

			goto out;

		dma_domain = dma_ops_domain_alloc(iommu, order);

		dma_domain = dma_ops_domain_alloc(iommu);

		if (!dma_domain)

			goto out;

		dma_domain->target_dev = devid;
@@ -1137,8 +1277,9 @@ static int get_device_resources(struct device *dev, 
			dma_dom = (*iommu)->default_dom;

		*domain = &dma_dom->domain;

		attach_device(*iommu, *domain, *bdf);

		printk(KERN_INFO "AMD IOMMU: Using protection domain %d for "

				"device %s\n", (*domain)->id, dev_name(dev));

		DUMP_printk(KERN_INFO "AMD IOMMU: Using protection domain "

				"%d for device %s\n",

				(*domain)->id, dev_name(dev));

	}



	if (domain_for_device(_bdf) == NULL)
@@ -1147,6 +1288,66 @@ static int get_device_resources(struct device *dev, 
	return 1;

}



/*

 * If the pte_page is not yet allocated this function is called

 */

static u64* alloc_pte(struct protection_domain *dom,

		      unsigned long address, u64 **pte_page, gfp_t gfp)

{

	u64 *pte, *page;



	pte = &dom->pt_root[IOMMU_PTE_L2_INDEX(address)];



	if (!IOMMU_PTE_PRESENT(*pte)) {

		page = (u64 *)get_zeroed_page(gfp);

		if (!page)

			return NULL;

		*pte = IOMMU_L2_PDE(virt_to_phys(page));

	}



	pte = IOMMU_PTE_PAGE(*pte);

	pte = &pte[IOMMU_PTE_L1_INDEX(address)];



	if (!IOMMU_PTE_PRESENT(*pte)) {

		page = (u64 *)get_zeroed_page(gfp);

		if (!page)

			return NULL;

		*pte = IOMMU_L1_PDE(virt_to_phys(page));

	}



	pte = IOMMU_PTE_PAGE(*pte);



	if (pte_page)

		*pte_page = pte;



	pte = &pte[IOMMU_PTE_L0_INDEX(address)];



	return pte;

}



/*

 * This function fetches the PTE for a given address in the aperture

 */

static u64* dma_ops_get_pte(struct dma_ops_domain *dom,

			    unsigned long address)

{

	struct aperture_range *aperture;

	u64 *pte, *pte_page;



	aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];

	if (!aperture)

		return NULL;



	pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];

	if (!pte) {

		pte = alloc_pte(&dom->domain, address, &pte_page, GFP_ATOMIC);

		aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page;

	} else

		pte += IOMMU_PTE_L0_INDEX(address);



	return pte;

}



/*

 * This is the generic map function. It maps one 4kb page at paddr to

 * the given address in the DMA address space for the domain.
@@ -1163,8 +1364,9 @@ static dma_addr_t dma_ops_domain_map(struct amd_iommu *iommu, 


	paddr &= PAGE_MASK;



	pte  = dom->pte_pages[IOMMU_PTE_L1_INDEX(address)];

	pte += IOMMU_PTE_L0_INDEX(address);

	pte  = dma_ops_get_pte(dom, address);

	if (!pte)

		return bad_dma_address;



	__pte = paddr | IOMMU_PTE_P | IOMMU_PTE_FC;
@@ -1189,14 +1391,20 @@ static void dma_ops_domain_unmap(struct amd_iommu *iommu, 
				 struct dma_ops_domain *dom,

				 unsigned long address)

{

	struct aperture_range *aperture;

	u64 *pte;



	if (address >= dom->aperture_size)

		return;



	WARN_ON(address & ~PAGE_MASK || address >= dom->aperture_size);

	aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];

	if (!aperture)

		return;



	pte  = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];

	if (!pte)

		return;



	pte  = dom->pte_pages[IOMMU_PTE_L1_INDEX(address)];

	pte += IOMMU_PTE_L0_INDEX(address);



	WARN_ON(!*pte);
@@ -1220,7 +1428,7 @@ static dma_addr_t __map_single(struct device *dev, 
			       u64 dma_mask)

{

	dma_addr_t offset = paddr & ~PAGE_MASK;

	dma_addr_t address, start;

	dma_addr_t address, start, ret;

	unsigned int pages;

	unsigned long align_mask = 0;

	int i;
@@ -1236,14 +1444,33 @@ static dma_addr_t __map_single(struct device *dev, 
	if (align)

		align_mask = (1UL << get_order(size)) - 1;



retry:

	address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,

					  dma_mask);

	if (unlikely(address == bad_dma_address))

	if (unlikely(address == bad_dma_address)) {

		/*

		 * setting next_address here will let the address

		 * allocator only scan the new allocated range in the

		 * first run. This is a small optimization.

		 */

		dma_dom->next_address = dma_dom->aperture_size;



		if (alloc_new_range(iommu, dma_dom, false, GFP_ATOMIC))

			goto out;



		/*

		 * aperture was sucessfully enlarged by 128 MB, try

		 * allocation again

		 */

		goto retry;

	}



	start = address;

	for (i = 0; i < pages; ++i) {

		dma_ops_domain_map(iommu, dma_dom, start, paddr, dir);

		ret = dma_ops_domain_map(iommu, dma_dom, start, paddr, dir);

		if (ret == bad_dma_address)

			goto out_unmap;



		paddr += PAGE_SIZE;

		start += PAGE_SIZE;

	}
@@ -1259,6 +1486,17 @@ static dma_addr_t __map_single(struct device *dev, 


out:

	return address;



out_unmap:



	for (--i; i >= 0; --i) {

		start -= PAGE_SIZE;

		dma_ops_domain_unmap(iommu, dma_dom, start);

	}



	dma_ops_free_addresses(dma_dom, address, pages);



	return bad_dma_address;

}



/*
@@ -1629,7 +1867,6 @@ static void prealloc_protection_domains(void) 
	struct pci_dev *dev = NULL;

	struct dma_ops_domain *dma_dom;

	struct amd_iommu *iommu;

	int order = amd_iommu_aperture_order;

	u16 devid;



	while ((dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, dev)) != NULL) {
@@ -1642,7 +1879,7 @@ static void prealloc_protection_domains(void) 
		iommu = amd_iommu_rlookup_table[devid];

		if (!iommu)

			continue;

		dma_dom = dma_ops_domain_alloc(iommu, order);

		dma_dom = dma_ops_domain_alloc(iommu);

		if (!dma_dom)

			continue;

		init_unity_mappings_for_device(dma_dom, devid);
@@ -1668,7 +1905,6 @@ static struct dma_map_ops amd_iommu_dma_ops = { 
int __init amd_iommu_init_dma_ops(void)

{

	struct amd_iommu *iommu;

	int order = amd_iommu_aperture_order;

	int ret;



	/*
@@ -1676,8 +1912,8 @@ int __init amd_iommu_init_dma_ops(void) 
	 * found in the system. Devices not assigned to any other

	 * protection domain will be assigned to the default one.

	 */

	list_for_each_entry(iommu, &amd_iommu_list, list) {

		iommu->default_dom = dma_ops_domain_alloc(iommu, order);

	for_each_iommu(iommu) {

		iommu->default_dom = dma_ops_domain_alloc(iommu);

		if (iommu->default_dom == NULL)

			return -ENOMEM;

		iommu->default_dom->domain.flags |= PD_DEFAULT_MASK;
@@ -1714,7 +1950,7 @@ int __init amd_iommu_init_dma_ops(void) 


free_domains:



	list_for_each_entry(iommu, &amd_iommu_list, list) {

	for_each_iommu(iommu) {

		if (iommu->default_dom)

			dma_ops_domain_free(iommu->default_dom);

	}