Merge branches 'core/iommu', 'x86/amd-iommu' and 'x86/iommu' into x86-v28-for-linus-phase3-B (725c2581) · Commits · e / devices / android_kernel_xiaomi_nabu

Documentation/kernel-parameters.txt

+5 −0

Original line number	Original line	Diff line number	Diff line
	@@ -284,6 +284,11 @@ and is between 256 and 4096 characters. It is defined in the file
	isolate - enable device isolation (each device, as far		isolate - enable device isolation (each device, as far
	as possible, will get its own protection		as possible, will get its own protection
	domain)		domain)
			fullflush - enable flushing of IO/TLB entries when
			they are unmapped. Otherwise they are
			flushed before they will be reused, which
			is a lot of faster

	amd_iommu_size= [HW,X86-64]		amd_iommu_size= [HW,X86-64]
	Define the size of the aperture for the AMD IOMMU		Define the size of the aperture for the AMD IOMMU
	driver. Possible values are:		driver. Possible values are:

MAINTAINERS

+1 −0

Original line number	Original line	Diff line number	Diff line
	@@ -387,6 +387,7 @@ AMD IOMMU (AMD-VI)
	P: Joerg Roedel		P: Joerg Roedel
	M: joerg.roedel@amd.com		M: joerg.roedel@amd.com
	L: iommu@lists.linux-foundation.org		L: iommu@lists.linux-foundation.org
			T: git://git.kernel.org/pub/scm/linux/kernel/git/joro/linux-2.6-iommu.git
	S: Supported		S: Supported

	AMS (Apple Motion Sensor) DRIVER		AMS (Apple Motion Sensor) DRIVER

arch/ia64/include/asm/dma-mapping.h

+3 −1

Original line number	Original line	Diff line number	Diff line
	@@ -8,7 +8,9 @@
	#include <asm/machvec.h>		#include <asm/machvec.h>
	#include <linux/scatterlist.h>		#include <linux/scatterlist.h>

	#define dma_alloc_coherent platform_dma_alloc_coherent		#define dma_alloc_coherent(dev, size, handle, gfp) \
			platform_dma_alloc_coherent(dev, size, handle, (gfp) \| GFP_DMA)

	/* coherent mem. is cheap */		/* coherent mem. is cheap */
	static inline void *		static inline void *
	dma_alloc_noncoherent(struct device dev, size_t size, dma_addr_t dma_handle,		dma_alloc_noncoherent(struct device dev, size_t size, dma_addr_t dma_handle,

arch/x86/Kconfig

+1 −0

Original line number	Original line	Diff line number	Diff line
	@@ -554,6 +554,7 @@ config CALGARY_IOMMU_ENABLED_BY_DEFAULT
	config AMD_IOMMU		config AMD_IOMMU
	bool "AMD IOMMU support"		bool "AMD IOMMU support"
	select SWIOTLB		select SWIOTLB
			select PCI_MSI
	depends on X86_64 && PCI && ACPI		depends on X86_64 && PCI && ACPI
	help		help
	With this option you can enable support for AMD IOMMU hardware in		With this option you can enable support for AMD IOMMU hardware in

arch/x86/kernel/amd_iommu.c

+267 −61

Original line number	Original line	Diff line number	Diff line
	@@ -33,6 +33,10 @@

	static DEFINE_RWLOCK(amd_iommu_devtable_lock);		static DEFINE_RWLOCK(amd_iommu_devtable_lock);

			/* A list of preallocated protection domains */
			static LIST_HEAD(iommu_pd_list);
			static DEFINE_SPINLOCK(iommu_pd_list_lock);

	/*		/*
	* general struct to manage commands send to an IOMMU		* general struct to manage commands send to an IOMMU
	*/		*/
	@@ -49,6 +53,102 @@ static int iommu_has_npcache(struct amd_iommu *iommu)
	return iommu->cap & IOMMU_CAP_NPCACHE;		return iommu->cap & IOMMU_CAP_NPCACHE;
	}		}

			/****************************************************************************
			*
			* Interrupt handling functions
			*
			****************************************************************************/

			static void iommu_print_event(void *__evt)
			{
			u32 *event = __evt;
			int type = (event[1] >> EVENT_TYPE_SHIFT) & EVENT_TYPE_MASK;
			int devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
			int domid = (event[1] >> EVENT_DOMID_SHIFT) & EVENT_DOMID_MASK;
			int flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
			u64 address = (u64)(((u64)event[3]) << 32) \| event[2];

			printk(KERN_ERR "AMD IOMMU: Event logged [");

			switch (type) {
			case EVENT_TYPE_ILL_DEV:
			printk("ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x "
			"address=0x%016llx flags=0x%04x]\n",
			PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
			address, flags);
			break;
			case EVENT_TYPE_IO_FAULT:
			printk("IO_PAGE_FAULT device=%02x:%02x.%x "
			"domain=0x%04x address=0x%016llx flags=0x%04x]\n",
			PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
			domid, address, flags);
			break;
			case EVENT_TYPE_DEV_TAB_ERR:
			printk("DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
			"address=0x%016llx flags=0x%04x]\n",
			PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
			address, flags);
			break;
			case EVENT_TYPE_PAGE_TAB_ERR:
			printk("PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
			"domain=0x%04x address=0x%016llx flags=0x%04x]\n",
			PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
			domid, address, flags);
			break;
			case EVENT_TYPE_ILL_CMD:
			printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address);
			break;
			case EVENT_TYPE_CMD_HARD_ERR:
			printk("COMMAND_HARDWARE_ERROR address=0x%016llx "
			"flags=0x%04x]\n", address, flags);
			break;
			case EVENT_TYPE_IOTLB_INV_TO:
			printk("IOTLB_INV_TIMEOUT device=%02x:%02x.%x "
			"address=0x%016llx]\n",
			PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
			address);
			break;
			case EVENT_TYPE_INV_DEV_REQ:
			printk("INVALID_DEVICE_REQUEST device=%02x:%02x.%x "
			"address=0x%016llx flags=0x%04x]\n",
			PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
			address, flags);
			break;
			default:
			printk(KERN_ERR "UNKNOWN type=0x%02x]\n", type);
			}
			}

			static void iommu_poll_events(struct amd_iommu *iommu)
			{
			u32 head, tail;
			unsigned long flags;

			spin_lock_irqsave(&iommu->lock, flags);

			head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
			tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);

			while (head != tail) {
			iommu_print_event(iommu->evt_buf + head);
			head = (head + EVENT_ENTRY_SIZE) % iommu->evt_buf_size;
			}

			writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);

			spin_unlock_irqrestore(&iommu->lock, flags);
			}

			irqreturn_t amd_iommu_int_handler(int irq, void *data)
			{
			struct amd_iommu *iommu;

			list_for_each_entry(iommu, &amd_iommu_list, list)
			iommu_poll_events(iommu);

			return IRQ_HANDLED;
			}

	/****************************************************************************		/****************************************************************************
	*		*
	* IOMMU command queuing functions		* IOMMU command queuing functions
	@@ -213,6 +313,14 @@ static int iommu_flush_pages(struct amd_iommu *iommu, u16 domid,
	return 0;		return 0;
	}		}

			/* Flush the whole IO/TLB for a given protection domain */
			static void iommu_flush_tlb(struct amd_iommu *iommu, u16 domid)
			{
			u64 address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;

			iommu_queue_inv_iommu_pages(iommu, address, domid, 0, 1);
			}

	/****************************************************************************		/****************************************************************************
	*		*
	* The functions below are used the create the page table mappings for		* The functions below are used the create the page table mappings for
	@@ -372,11 +480,6 @@ static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,
	* efficient allocator.		* efficient allocator.
	*		*
	****************************************************************************/		****************************************************************************/
	static unsigned long dma_mask_to_pages(unsigned long mask)
	{
	return (mask >> PAGE_SHIFT) +
	(PAGE_ALIGN(mask & ~PAGE_MASK) >> PAGE_SHIFT);
	}

	/*		/*
	* The address allocator core function.		* The address allocator core function.
	@@ -385,25 +488,31 @@ static unsigned long dma_mask_to_pages(unsigned long mask)
	*/		*/
	static unsigned long dma_ops_alloc_addresses(struct device *dev,		static unsigned long dma_ops_alloc_addresses(struct device *dev,
	struct dma_ops_domain *dom,		struct dma_ops_domain *dom,
	unsigned int pages)		unsigned int pages,
			unsigned long align_mask,
			u64 dma_mask)
	{		{
	unsigned long limit = dma_mask_to_pages(*dev->dma_mask);		unsigned long limit;
	unsigned long address;		unsigned long address;
	unsigned long size = dom->aperture_size >> PAGE_SHIFT;
	unsigned long boundary_size;		unsigned long boundary_size;

	boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,		boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
	PAGE_SIZE) >> PAGE_SHIFT;		PAGE_SIZE) >> PAGE_SHIFT;
	limit = limit < size ? limit : size;		limit = iommu_device_max_index(dom->aperture_size >> PAGE_SHIFT, 0,
			dma_mask >> PAGE_SHIFT);

	if (dom->next_bit >= limit)		if (dom->next_bit >= limit) {
	dom->next_bit = 0;		dom->next_bit = 0;
			dom->need_flush = true;
			}

	address = iommu_area_alloc(dom->bitmap, limit, dom->next_bit, pages,		address = iommu_area_alloc(dom->bitmap, limit, dom->next_bit, pages,
	0 , boundary_size, 0);		0 , boundary_size, align_mask);
	if (address == -1)		if (address == -1) {
	address = iommu_area_alloc(dom->bitmap, limit, 0, pages,		address = iommu_area_alloc(dom->bitmap, limit, 0, pages,
	0, boundary_size, 0);		0, boundary_size, align_mask);
			dom->need_flush = true;
			}

	if (likely(address != -1)) {		if (likely(address != -1)) {
	dom->next_bit = address + pages;		dom->next_bit = address + pages;
	@@ -469,7 +578,7 @@ static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
	if (start_page + pages > last_page)		if (start_page + pages > last_page)
	pages = last_page - start_page;		pages = last_page - start_page;

	set_bit_string(dom->bitmap, start_page, pages);		iommu_area_reserve(dom->bitmap, start_page, pages);
	}		}

	static void dma_ops_free_pagetable(struct dma_ops_domain *dma_dom)		static void dma_ops_free_pagetable(struct dma_ops_domain *dma_dom)
	@@ -563,6 +672,9 @@ static struct dma_ops_domain dma_ops_domain_alloc(struct amd_iommu iommu,
	dma_dom->bitmap[0] = 1;		dma_dom->bitmap[0] = 1;
	dma_dom->next_bit = 0;		dma_dom->next_bit = 0;

			dma_dom->need_flush = false;
			dma_dom->target_dev = 0xffff;

	/* Intialize the exclusion range if necessary */		/* Intialize the exclusion range if necessary */
	if (iommu->exclusion_start &&		if (iommu->exclusion_start &&
	iommu->exclusion_start < dma_dom->aperture_size) {		iommu->exclusion_start < dma_dom->aperture_size) {
	@@ -633,12 +745,13 @@ static void set_device_domain(struct amd_iommu *iommu,

	u64 pte_root = virt_to_phys(domain->pt_root);		u64 pte_root = virt_to_phys(domain->pt_root);

	pte_root \|= (domain->mode & 0x07) << 9;		pte_root \|= (domain->mode & DEV_ENTRY_MODE_MASK)
	pte_root \|= IOMMU_PTE_IR \| IOMMU_PTE_IW \| IOMMU_PTE_P \| 2;		<< DEV_ENTRY_MODE_SHIFT;
			pte_root \|= IOMMU_PTE_IR \| IOMMU_PTE_IW \| IOMMU_PTE_P \| IOMMU_PTE_TV;

	write_lock_irqsave(&amd_iommu_devtable_lock, flags);		write_lock_irqsave(&amd_iommu_devtable_lock, flags);
	amd_iommu_dev_table[devid].data[0] = pte_root;		amd_iommu_dev_table[devid].data[0] = lower_32_bits(pte_root);
	amd_iommu_dev_table[devid].data[1] = pte_root >> 32;		amd_iommu_dev_table[devid].data[1] = upper_32_bits(pte_root);
	amd_iommu_dev_table[devid].data[2] = domain->id;		amd_iommu_dev_table[devid].data[2] = domain->id;

	amd_iommu_pd_table[devid] = domain;		amd_iommu_pd_table[devid] = domain;
	@@ -655,6 +768,45 @@ static void set_device_domain(struct amd_iommu *iommu,
	*		*
	*****************************************************************************/		*****************************************************************************/

			/*
			* This function checks if the driver got a valid device from the caller to
			* avoid dereferencing invalid pointers.
			*/
			static bool check_device(struct device *dev)
			{
			if (!dev \|\| !dev->dma_mask)
			return false;

			return true;
			}

			/*
			* In this function the list of preallocated protection domains is traversed to
			* find the domain for a specific device
			*/
			static struct dma_ops_domain *find_protection_domain(u16 devid)
			{
			struct dma_ops_domain entry, ret = NULL;
			unsigned long flags;

			if (list_empty(&iommu_pd_list))
			return NULL;

			spin_lock_irqsave(&iommu_pd_list_lock, flags);

			list_for_each_entry(entry, &iommu_pd_list, list) {
			if (entry->target_dev == devid) {
			ret = entry;
			list_del(&ret->list);
			break;
			}
			}

			spin_unlock_irqrestore(&iommu_pd_list_lock, flags);

			return ret;
			}

	/*		/*
	* In the dma_ops path we only have the struct device. This function		* In the dma_ops path we only have the struct device. This function
	* finds the corresponding IOMMU, the protection domain and the		* finds the corresponding IOMMU, the protection domain and the
	@@ -671,27 +823,30 @@ static int get_device_resources(struct device *dev,
	struct pci_dev *pcidev;		struct pci_dev *pcidev;
	u16 _bdf;		u16 _bdf;

	BUG_ON(!dev \|\| dev->bus != &pci_bus_type \|\| !dev->dma_mask);		*iommu = NULL;
			*domain = NULL;
			*bdf = 0xffff;

			if (dev->bus != &pci_bus_type)
			return 0;

	pcidev = to_pci_dev(dev);		pcidev = to_pci_dev(dev);
	_bdf = calc_devid(pcidev->bus->number, pcidev->devfn);		_bdf = calc_devid(pcidev->bus->number, pcidev->devfn);

	/* device not translated by any IOMMU in the system? */		/* device not translated by any IOMMU in the system? */
	if (_bdf > amd_iommu_last_bdf) {		if (_bdf > amd_iommu_last_bdf)
	*iommu = NULL;
	*domain = NULL;
	*bdf = 0xffff;
	return 0;		return 0;
	}

	*bdf = amd_iommu_alias_table[_bdf];		*bdf = amd_iommu_alias_table[_bdf];

	iommu = amd_iommu_rlookup_table[bdf];		iommu = amd_iommu_rlookup_table[bdf];
	if (*iommu == NULL)		if (*iommu == NULL)
	return 0;		return 0;
	dma_dom = (*iommu)->default_dom;
	domain = domain_for_device(bdf);		domain = domain_for_device(bdf);
	if (*domain == NULL) {		if (*domain == NULL) {
			dma_dom = find_protection_domain(*bdf);
			if (!dma_dom)
			dma_dom = (*iommu)->default_dom;
	*domain = &dma_dom->domain;		*domain = &dma_dom->domain;
	set_device_domain(iommu, domain, *bdf);		set_device_domain(iommu, domain, *bdf);
	printk(KERN_INFO "AMD IOMMU: Using protection domain %d for "		printk(KERN_INFO "AMD IOMMU: Using protection domain %d for "
	@@ -770,17 +925,24 @@ static dma_addr_t __map_single(struct device *dev,
	struct dma_ops_domain *dma_dom,		struct dma_ops_domain *dma_dom,
	phys_addr_t paddr,		phys_addr_t paddr,
	size_t size,		size_t size,
	int dir)		int dir,
			bool align,
			u64 dma_mask)
	{		{
	dma_addr_t offset = paddr & ~PAGE_MASK;		dma_addr_t offset = paddr & ~PAGE_MASK;
	dma_addr_t address, start;		dma_addr_t address, start;
	unsigned int pages;		unsigned int pages;
			unsigned long align_mask = 0;
	int i;		int i;

	pages = iommu_num_pages(paddr, size);		pages = iommu_num_pages(paddr, size);
	paddr &= PAGE_MASK;		paddr &= PAGE_MASK;

	address = dma_ops_alloc_addresses(dev, dma_dom, pages);		if (align)
			align_mask = (1UL << get_order(size)) - 1;

			address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,
			dma_mask);
	if (unlikely(address == bad_dma_address))		if (unlikely(address == bad_dma_address))
	goto out;		goto out;

	@@ -792,6 +954,12 @@ static dma_addr_t __map_single(struct device *dev,
	}		}
	address += offset;		address += offset;

			if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) {
			iommu_flush_tlb(iommu, dma_dom->domain.id);
			dma_dom->need_flush = false;
			} else if (unlikely(iommu_has_npcache(iommu)))
			iommu_flush_pages(iommu, dma_dom->domain.id, address, size);

	out:		out:
	return address;		return address;
	}		}
	@@ -822,6 +990,9 @@ static void __unmap_single(struct amd_iommu *iommu,
	}		}

	dma_ops_free_addresses(dma_dom, dma_addr, pages);		dma_ops_free_addresses(dma_dom, dma_addr, pages);

			if (amd_iommu_unmap_flush)
			iommu_flush_pages(iommu, dma_dom->domain.id, dma_addr, size);
	}		}

	/*		/*
	@@ -835,6 +1006,12 @@ static dma_addr_t map_single(struct device *dev, phys_addr_t paddr,
	struct protection_domain *domain;		struct protection_domain *domain;
	u16 devid;		u16 devid;
	dma_addr_t addr;		dma_addr_t addr;
			u64 dma_mask;

			if (!check_device(dev))
			return bad_dma_address;

			dma_mask = *dev->dma_mask;

	get_device_resources(dev, &iommu, &domain, &devid);		get_device_resources(dev, &iommu, &domain, &devid);

	@@ -843,14 +1020,12 @@ static dma_addr_t map_single(struct device *dev, phys_addr_t paddr,
	return (dma_addr_t)paddr;		return (dma_addr_t)paddr;

	spin_lock_irqsave(&domain->lock, flags);		spin_lock_irqsave(&domain->lock, flags);
	addr = __map_single(dev, iommu, domain->priv, paddr, size, dir);		addr = __map_single(dev, iommu, domain->priv, paddr, size, dir, false,
			dma_mask);
	if (addr == bad_dma_address)		if (addr == bad_dma_address)
	goto out;		goto out;

	if (iommu_has_npcache(iommu))		if (unlikely(iommu->need_sync))
	iommu_flush_pages(iommu, domain->id, addr, size);

	if (iommu->need_sync)
	iommu_completion_wait(iommu);		iommu_completion_wait(iommu);

	out:		out:
	@@ -870,7 +1045,8 @@ static void unmap_single(struct device *dev, dma_addr_t dma_addr,
	struct protection_domain *domain;		struct protection_domain *domain;
	u16 devid;		u16 devid;

	if (!get_device_resources(dev, &iommu, &domain, &devid))		if (!check_device(dev) \|\|
			!get_device_resources(dev, &iommu, &domain, &devid))
	/* device not handled by any AMD IOMMU */		/* device not handled by any AMD IOMMU */
	return;		return;

	@@ -878,9 +1054,7 @@ static void unmap_single(struct device *dev, dma_addr_t dma_addr,

	__unmap_single(iommu, domain->priv, dma_addr, size, dir);		__unmap_single(iommu, domain->priv, dma_addr, size, dir);

	iommu_flush_pages(iommu, domain->id, dma_addr, size);		if (unlikely(iommu->need_sync))

	if (iommu->need_sync)
	iommu_completion_wait(iommu);		iommu_completion_wait(iommu);

	spin_unlock_irqrestore(&domain->lock, flags);		spin_unlock_irqrestore(&domain->lock, flags);
	@@ -919,6 +1093,12 @@ static int map_sg(struct device dev, struct scatterlist sglist,
	struct scatterlist *s;		struct scatterlist *s;
	phys_addr_t paddr;		phys_addr_t paddr;
	int mapped_elems = 0;		int mapped_elems = 0;
			u64 dma_mask;

			if (!check_device(dev))
			return 0;

			dma_mask = *dev->dma_mask;

	get_device_resources(dev, &iommu, &domain, &devid);		get_device_resources(dev, &iommu, &domain, &devid);

	@@ -931,19 +1111,17 @@ static int map_sg(struct device dev, struct scatterlist sglist,
	paddr = sg_phys(s);		paddr = sg_phys(s);

	s->dma_address = __map_single(dev, iommu, domain->priv,		s->dma_address = __map_single(dev, iommu, domain->priv,
	paddr, s->length, dir);		paddr, s->length, dir, false,
			dma_mask);

	if (s->dma_address) {		if (s->dma_address) {
	s->dma_length = s->length;		s->dma_length = s->length;
	mapped_elems++;		mapped_elems++;
	} else		} else
	goto unmap;		goto unmap;
	if (iommu_has_npcache(iommu))
	iommu_flush_pages(iommu, domain->id, s->dma_address,
	s->dma_length);
	}		}

	if (iommu->need_sync)		if (unlikely(iommu->need_sync))
	iommu_completion_wait(iommu);		iommu_completion_wait(iommu);

	out:		out:
	@@ -977,7 +1155,8 @@ static void unmap_sg(struct device dev, struct scatterlist sglist,
	u16 devid;		u16 devid;
	int i;		int i;

	if (!get_device_resources(dev, &iommu, &domain, &devid))		if (!check_device(dev) \|\|
			!get_device_resources(dev, &iommu, &domain, &devid))
	return;		return;

	spin_lock_irqsave(&domain->lock, flags);		spin_lock_irqsave(&domain->lock, flags);
	@@ -985,12 +1164,10 @@ static void unmap_sg(struct device dev, struct scatterlist sglist,
	for_each_sg(sglist, s, nelems, i) {		for_each_sg(sglist, s, nelems, i) {
	__unmap_single(iommu, domain->priv, s->dma_address,		__unmap_single(iommu, domain->priv, s->dma_address,
	s->dma_length, dir);		s->dma_length, dir);
	iommu_flush_pages(iommu, domain->id, s->dma_address,
	s->dma_length);
	s->dma_address = s->dma_length = 0;		s->dma_address = s->dma_length = 0;
	}		}

	if (iommu->need_sync)		if (unlikely(iommu->need_sync))
	iommu_completion_wait(iommu);		iommu_completion_wait(iommu);

	spin_unlock_irqrestore(&domain->lock, flags);		spin_unlock_irqrestore(&domain->lock, flags);
	@@ -1008,25 +1185,33 @@ static void alloc_coherent(struct device dev, size_t size,
	struct protection_domain *domain;		struct protection_domain *domain;
	u16 devid;		u16 devid;
	phys_addr_t paddr;		phys_addr_t paddr;
			u64 dma_mask = dev->coherent_dma_mask;

			if (!check_device(dev))
			return NULL;

			if (!get_device_resources(dev, &iommu, &domain, &devid))
			flag &= ~(__GFP_DMA \| __GFP_HIGHMEM \| __GFP_DMA32);

			flag \|= __GFP_ZERO;
	virt_addr = (void *)__get_free_pages(flag, get_order(size));		virt_addr = (void *)__get_free_pages(flag, get_order(size));
	if (!virt_addr)		if (!virt_addr)
	return 0;		return 0;

	memset(virt_addr, 0, size);
	paddr = virt_to_phys(virt_addr);		paddr = virt_to_phys(virt_addr);

	get_device_resources(dev, &iommu, &domain, &devid);

	if (!iommu \|\| !domain) {		if (!iommu \|\| !domain) {
	*dma_addr = (dma_addr_t)paddr;		*dma_addr = (dma_addr_t)paddr;
	return virt_addr;		return virt_addr;
	}		}

			if (!dma_mask)
			dma_mask = *dev->dma_mask;

	spin_lock_irqsave(&domain->lock, flags);		spin_lock_irqsave(&domain->lock, flags);

	*dma_addr = __map_single(dev, iommu, domain->priv, paddr,		*dma_addr = __map_single(dev, iommu, domain->priv, paddr,
	size, DMA_BIDIRECTIONAL);		size, DMA_BIDIRECTIONAL, true, dma_mask);

	if (*dma_addr == bad_dma_address) {		if (*dma_addr == bad_dma_address) {
	free_pages((unsigned long)virt_addr, get_order(size));		free_pages((unsigned long)virt_addr, get_order(size));
	@@ -1034,10 +1219,7 @@ static void alloc_coherent(struct device dev, size_t size,
	goto out;		goto out;
	}		}

	if (iommu_has_npcache(iommu))		if (unlikely(iommu->need_sync))
	iommu_flush_pages(iommu, domain->id, *dma_addr, size);

	if (iommu->need_sync)
	iommu_completion_wait(iommu);		iommu_completion_wait(iommu);

	out:		out:
	@@ -1048,8 +1230,6 @@ static void alloc_coherent(struct device dev, size_t size,

	/*		/*
	* The exported free_coherent function for dma_ops.		* The exported free_coherent function for dma_ops.
	* FIXME: fix the generic x86 DMA layer so that it actually calls that
	* function.
	*/		*/
	static void free_coherent(struct device *dev, size_t size,		static void free_coherent(struct device *dev, size_t size,
	void *virt_addr, dma_addr_t dma_addr)		void *virt_addr, dma_addr_t dma_addr)
	@@ -1059,6 +1239,9 @@ static void free_coherent(struct device *dev, size_t size,
	struct protection_domain *domain;		struct protection_domain *domain;
	u16 devid;		u16 devid;

			if (!check_device(dev))
			return;

	get_device_resources(dev, &iommu, &domain, &devid);		get_device_resources(dev, &iommu, &domain, &devid);

	if (!iommu \|\| !domain)		if (!iommu \|\| !domain)
	@@ -1067,9 +1250,8 @@ static void free_coherent(struct device *dev, size_t size,
	spin_lock_irqsave(&domain->lock, flags);		spin_lock_irqsave(&domain->lock, flags);

	__unmap_single(iommu, domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);		__unmap_single(iommu, domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);
	iommu_flush_pages(iommu, domain->id, dma_addr, size);

	if (iommu->need_sync)		if (unlikely(iommu->need_sync))
	iommu_completion_wait(iommu);		iommu_completion_wait(iommu);

	spin_unlock_irqrestore(&domain->lock, flags);		spin_unlock_irqrestore(&domain->lock, flags);
	@@ -1078,6 +1260,30 @@ static void free_coherent(struct device *dev, size_t size,
	free_pages((unsigned long)virt_addr, get_order(size));		free_pages((unsigned long)virt_addr, get_order(size));
	}		}

			/*
			* This function is called by the DMA layer to find out if we can handle a
			* particular device. It is part of the dma_ops.
			*/
			static int amd_iommu_dma_supported(struct device *dev, u64 mask)
			{
			u16 bdf;
			struct pci_dev *pcidev;

			/* No device or no PCI device */
			if (!dev \|\| dev->bus != &pci_bus_type)
			return 0;

			pcidev = to_pci_dev(dev);

			bdf = calc_devid(pcidev->bus->number, pcidev->devfn);

			/* Out of our scope? */
			if (bdf > amd_iommu_last_bdf)
			return 0;

			return 1;
			}

	/*		/*
	* The function for pre-allocating protection domains.		* The function for pre-allocating protection domains.
	*		*
	@@ -1107,10 +1313,9 @@ void prealloc_protection_domains(void)
	if (!dma_dom)		if (!dma_dom)
	continue;		continue;
	init_unity_mappings_for_device(dma_dom, devid);		init_unity_mappings_for_device(dma_dom, devid);
	set_device_domain(iommu, &dma_dom->domain, devid);		dma_dom->target_dev = devid;
	printk(KERN_INFO "AMD IOMMU: Allocated domain %d for device ",
	dma_dom->domain.id);		list_add_tail(&dma_dom->list, &iommu_pd_list);
	print_devid(devid, 1);
	}		}
	}		}

	@@ -1121,6 +1326,7 @@ static struct dma_mapping_ops amd_iommu_dma_ops = {
	.unmap_single = unmap_single,		.unmap_single = unmap_single,
	.map_sg = map_sg,		.map_sg = map_sg,
	.unmap_sg = unmap_sg,		.unmap_sg = unmap_sg,
			.dma_supported = amd_iommu_dma_supported,
	};		};

	/*		/*