Merge tag 'iommu-updates-v4.2' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu

* ARM SMMUv3 Architecture Implementation

The SMMUv3 architecture is a significant deparature from previous

revisions, replacing the MMIO register interface with in-memory command

and event queues and adding support for the ATS and PRI components of

the PCIe specification.

** SMMUv3 required properties:

- compatible        : Should include:

                      * "arm,smmu-v3" for any SMMUv3 compliant

                        implementation. This entry should be last in the

                        compatible list.

- reg               : Base address and size of the SMMU.

- interrupts        : Non-secure interrupt list describing the wired

                      interrupt sources corresponding to entries in

                      interrupt-names. If no wired interrupts are

                      present then this property may be omitted.

- interrupt-names   : When the interrupts property is present, should

                      include the following:

                      * "eventq"    - Event Queue not empty

                      * "priq"      - PRI Queue not empty

                      * "cmdq-sync" - CMD_SYNC complete

                      * "gerror"    - Global Error activated

** SMMUv3 optional properties:

- dma-coherent      : Present if DMA operations made by the SMMU (page

                      table walks, stream table accesses etc) are cache

                      coherent with the CPU.

                      NOTE: this only applies to the SMMU itself, not

                      masters connected upstream of the SMMU.

F:	drivers/mmc/host/sdhci-of-arasan.c

F:	drivers/edac/synopsys_edac.c

ARM SMMU DRIVER

ARM SMMU DRIVERS

M:	Will Deacon <will.deacon@arm.com>

L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)

S:	Maintained

F:	drivers/iommu/arm-smmu.c

F:	drivers/iommu/arm-smmu-v3.c

F:	drivers/iommu/io-pgtable-arm.c

ARM64 PORT (AARCH64 ARCHITECTURE)

	  Enables bits of IOMMU API required by VFIO. The iommu_ops

	  is not implemented as it is not necessary for VFIO.

# ARM IOMMU support

config ARM_SMMU

	bool "ARM Ltd. System MMU (SMMU) Support"

	depends on (ARM64 || ARM) && MMU

	  Say Y here if your SoC includes an IOMMU device implementing

	  the ARM SMMU architecture.

config ARM_SMMU_V3

	bool "ARM Ltd. System MMU Version 3 (SMMUv3) Support"

	depends on ARM64 && PCI

	select IOMMU_API

	select IOMMU_IO_PGTABLE_LPAE

	help

	  Support for implementations of the ARM System MMU architecture

	  version 3 providing translation support to a PCIe root complex.

	  Say Y here if your system includes an IOMMU device implementing

	  the ARM SMMUv3 architecture.

endif # IOMMU_SUPPORT

obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o

obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o

obj-$(CONFIG_ARM_SMMU) += arm-smmu.o

obj-$(CONFIG_ARM_SMMU_V3) += arm-smmu-v3.o

obj-$(CONFIG_DMAR_TABLE) += dmar.o

obj-$(CONFIG_INTEL_IOMMU) += intel-iommu.o

obj-$(CONFIG_IPMMU_VMSA) += ipmmu-vmsa.o

static DEFINE_RWLOCK(amd_iommu_devtable_lock);

/* A list of preallocated protection domains */

static LIST_HEAD(iommu_pd_list);

static DEFINE_SPINLOCK(iommu_pd_list_lock);

/* List of all available dev_data structures */

static LIST_HEAD(dev_data_list);

static DEFINE_SPINLOCK(dev_data_list_lock);

struct kmem_cache *amd_iommu_irq_cache;

static void update_domain(struct protection_domain *domain);

static int __init alloc_passthrough_domain(void);

static int alloc_passthrough_domain(void);

/****************************************************************************

 *

}

/*

 * In this function the list of preallocated protection domains is traversed to

 * find the domain for a specific device

 * This function actually applies the mapping to the page table of the

 * dma_ops domain.

 */

static struct dma_ops_domain *find_protection_domain(u16 devid)

static void alloc_unity_mapping(struct dma_ops_domain *dma_dom,

				struct unity_map_entry *e)

{

	struct dma_ops_domain *entry, *ret = NULL;

	unsigned long flags;

	u16 alias = amd_iommu_alias_table[devid];

	if (list_empty(&iommu_pd_list))

		return NULL;

	spin_lock_irqsave(&iommu_pd_list_lock, flags);

	u64 addr;

	list_for_each_entry(entry, &iommu_pd_list, list) {

		if (entry->target_dev == devid ||

		    entry->target_dev == alias) {

			ret = entry;

			break;

	for (addr = e->address_start; addr < e->address_end;

	     addr += PAGE_SIZE) {

		if (addr < dma_dom->aperture_size)

			__set_bit(addr >> PAGE_SHIFT,

				  dma_dom->aperture[0]->bitmap);

	}

}

	spin_unlock_irqrestore(&iommu_pd_list_lock, flags);

/*

 * Inits the unity mappings required for a specific device

 */

static void init_unity_mappings_for_device(struct device *dev,

					   struct dma_ops_domain *dma_dom)

{

	struct unity_map_entry *e;

	u16 devid;

	devid = get_device_id(dev);

	return ret;

	list_for_each_entry(e, &amd_iommu_unity_map, list) {

		if (!(devid >= e->devid_start && devid <= e->devid_end))

			continue;

		alloc_unity_mapping(dma_dom, e);

	}

}

/*

static void init_iommu_group(struct device *dev)

{

	struct dma_ops_domain *dma_domain;

	struct iommu_domain *domain;

	struct iommu_group *group;

	group = iommu_group_get_for_dev(dev);

	if (!IS_ERR(group))

	if (IS_ERR(group))

		return;

	domain = iommu_group_default_domain(group);

	if (!domain)

		goto out;

	dma_domain = to_pdomain(domain)->priv;

	init_unity_mappings_for_device(dev, dma_domain);

out:

	iommu_group_put(group);

}

	/* Unlink from alias, it may change if another device is re-plugged */

	dev_data->alias_data = NULL;

	/* Remove dma-ops */

	dev->archdata.dma_ops = NULL;

	/*

	 * We keep dev_data around for unplugged devices and reuse it when the

	 * device is re-plugged - not doing so would introduce a ton of races.

	 */

}

void __init amd_iommu_uninit_devices(void)

{

	struct iommu_dev_data *dev_data, *n;

	struct pci_dev *pdev = NULL;

	for_each_pci_dev(pdev) {

		if (!check_device(&pdev->dev))

			continue;

		iommu_uninit_device(&pdev->dev);

	}

	/* Free all of our dev_data structures */

	list_for_each_entry_safe(dev_data, n, &dev_data_list, dev_data_list)

		free_dev_data(dev_data);

}

int __init amd_iommu_init_devices(void)

{

	struct pci_dev *pdev = NULL;

	int ret = 0;

	for_each_pci_dev(pdev) {

		if (!check_device(&pdev->dev))

			continue;

		ret = iommu_init_device(&pdev->dev);

		if (ret == -ENOTSUPP)

			iommu_ignore_device(&pdev->dev);

		else if (ret)

			goto out_free;

	}

	/*

	 * Initialize IOMMU groups only after iommu_init_device() has

	 * had a chance to populate any IVRS defined aliases.

	 */

	for_each_pci_dev(pdev) {

		if (check_device(&pdev->dev))

			init_iommu_group(&pdev->dev);

	}

	return 0;

out_free:

	amd_iommu_uninit_devices();

	return ret;

}

#ifdef CONFIG_AMD_IOMMU_STATS

/*

	return unmapped;

}

/*

 * This function checks if a specific unity mapping entry is needed for

 * this specific IOMMU.

 */

static int iommu_for_unity_map(struct amd_iommu *iommu,

			       struct unity_map_entry *entry)

{

	u16 bdf, i;

	for (i = entry->devid_start; i <= entry->devid_end; ++i) {

		bdf = amd_iommu_alias_table[i];

		if (amd_iommu_rlookup_table[bdf] == iommu)

			return 1;

	}

	return 0;

}

/*

 * This function actually applies the mapping to the page table of the

 * dma_ops domain.

 */

static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,

			     struct unity_map_entry *e)

{

	u64 addr;

	int ret;

	for (addr = e->address_start; addr < e->address_end;

	     addr += PAGE_SIZE) {

		ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot,

				     PAGE_SIZE);

		if (ret)

			return ret;

		/*

		 * if unity mapping is in aperture range mark the page

		 * as allocated in the aperture

		 */

		if (addr < dma_dom->aperture_size)

			__set_bit(addr >> PAGE_SHIFT,

				  dma_dom->aperture[0]->bitmap);

	}

	return 0;

}

/*

 * Init the unity mappings for a specific IOMMU in the system

 *

 * Basically iterates over all unity mapping entries and applies them to

 * the default domain DMA of that IOMMU if necessary.

 */

static int iommu_init_unity_mappings(struct amd_iommu *iommu)

{

	struct unity_map_entry *entry;

	int ret;

	list_for_each_entry(entry, &amd_iommu_unity_map, list) {

		if (!iommu_for_unity_map(iommu, entry))

			continue;

		ret = dma_ops_unity_map(iommu->default_dom, entry);

		if (ret)

			return ret;

	}

	return 0;

}

/*

 * Inits the unity mappings required for a specific device

 */

static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,

					  u16 devid)

{

	struct unity_map_entry *e;

	int ret;

	list_for_each_entry(e, &amd_iommu_unity_map, list) {

		if (!(devid >= e->devid_start && devid <= e->devid_end))

			continue;

		ret = dma_ops_unity_map(dma_dom, e);

		if (ret)

			return ret;

	}

	return 0;

}

/****************************************************************************

 *

 * The next functions belong to the address allocator for the dma_ops

	unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE;

	int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT;

	int i = start >> APERTURE_RANGE_SHIFT;

	unsigned long boundary_size;

	unsigned long boundary_size, mask;

	unsigned long address = -1;

	unsigned long limit;

	next_bit >>= PAGE_SHIFT;

	boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,

			PAGE_SIZE) >> PAGE_SHIFT;

	mask = dma_get_seg_boundary(dev);

	boundary_size = mask + 1 ? ALIGN(mask + 1, PAGE_SIZE) >> PAGE_SHIFT :

				   1UL << (BITS_PER_LONG - PAGE_SHIFT);

	for (;i < max_index; ++i) {

		unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT;

	pt = (u64 *)__pt;					\

								\

	for (i = 0; i < 512; ++i) {				\

		/* PTE present? */				\

		if (!IOMMU_PTE_PRESENT(pt[i]))			\

			continue;				\

								\

		/* Large PTE? */				\

		if (PM_PTE_LEVEL(pt[i]) == 0 ||			\

		    PM_PTE_LEVEL(pt[i]) == 7)			\

			continue;				\

								\

		p = (unsigned long)IOMMU_PTE_PAGE(pt[i]);	\

		FN(p);						\

	}							\

		goto free_dma_dom;

	dma_dom->need_flush = false;

	dma_dom->target_dev = 0xffff;

	add_domain_to_list(&dma_dom->domain);

	dev_data->ats.enabled = false;

}

/*

 * Find out the protection domain structure for a given PCI device. This

 * will give us the pointer to the page table root for example.

 */

static struct protection_domain *domain_for_device(struct device *dev)

static int amd_iommu_add_device(struct device *dev)

{

	struct iommu_dev_data *dev_data;

	struct protection_domain *dom = NULL;

	unsigned long flags;

	dev_data   = get_dev_data(dev);

	if (dev_data->domain)

		return dev_data->domain;

	if (dev_data->alias_data != NULL) {

		struct iommu_dev_data *alias_data = dev_data->alias_data;

		read_lock_irqsave(&amd_iommu_devtable_lock, flags);

		if (alias_data->domain != NULL) {

			__attach_device(dev_data, alias_data->domain);

			dom = alias_data->domain;

		}

		read_unlock_irqrestore(&amd_iommu_devtable_lock, flags);

	}

	return dom;

}

static int device_change_notifier(struct notifier_block *nb,

				  unsigned long action, void *data)

{

	struct dma_ops_domain *dma_domain;

	struct protection_domain *domain;

	struct iommu_dev_data *dev_data;

	struct device *dev = data;

	struct iommu_domain *domain;

	struct amd_iommu *iommu;

	unsigned long flags;

	u16 devid;

	int ret;

	if (!check_device(dev))

	if (!check_device(dev) || get_dev_data(dev))

		return 0;

	devid = get_device_id(dev);

	iommu = amd_iommu_rlookup_table[devid];

	dev_data = get_dev_data(dev);

	switch (action) {

	case BUS_NOTIFY_ADD_DEVICE:

	ret = iommu_init_device(dev);

	if (ret) {

		if (ret != -ENOTSUPP)

			pr_err("Failed to initialize device %s - trying to proceed anyway\n",

				dev_name(dev));

		iommu_init_device(dev);

		iommu_ignore_device(dev);

		dev->archdata.dma_ops = &nommu_dma_ops;

		goto out;

	}

	init_iommu_group(dev);

		/*

		 * dev_data is still NULL and

		 * got initialized in iommu_init_device

		 */

	dev_data = get_dev_data(dev);

		if (iommu_pass_through || dev_data->iommu_v2) {

			dev_data->passthrough = true;

			attach_device(dev, pt_domain);

			break;

		}

		domain = domain_for_device(dev);

		/* allocate a protection domain if a device is added */

		dma_domain = find_protection_domain(devid);

		if (!dma_domain) {

			dma_domain = dma_ops_domain_alloc();

			if (!dma_domain)

				goto out;

			dma_domain->target_dev = devid;

	BUG_ON(!dev_data);

			spin_lock_irqsave(&iommu_pd_list_lock, flags);

			list_add_tail(&dma_domain->list, &iommu_pd_list);

			spin_unlock_irqrestore(&iommu_pd_list_lock, flags);

		}

	if (dev_data->iommu_v2)

		iommu_request_dm_for_dev(dev);

	/* Domains are initialized for this device - have a look what we ended up with */

	domain = iommu_get_domain_for_dev(dev);

	if (domain->type == IOMMU_DOMAIN_IDENTITY) {

		dev_data->passthrough = true;

		dev->archdata.dma_ops = &nommu_dma_ops;

	} else {

		dev->archdata.dma_ops = &amd_iommu_dma_ops;

		break;

	case BUS_NOTIFY_REMOVED_DEVICE:

		iommu_uninit_device(dev);

	default:

		goto out;

	}

out:

	iommu_completion_wait(iommu);

out:

	return 0;

}

static struct notifier_block device_nb = {

	.notifier_call = device_change_notifier,

};

void amd_iommu_init_notifier(void)

static void amd_iommu_remove_device(struct device *dev)

{

	bus_register_notifier(&pci_bus_type, &device_nb);

	struct amd_iommu *iommu;

	u16 devid;

	if (!check_device(dev))

		return;

	devid = get_device_id(dev);

	iommu = amd_iommu_rlookup_table[devid];

	iommu_uninit_device(dev);

	iommu_completion_wait(iommu);

}

/*****************************************************************************

static struct protection_domain *get_domain(struct device *dev)

{

	struct protection_domain *domain;

	struct dma_ops_domain *dma_dom;

	u16 devid = get_device_id(dev);

	struct iommu_domain *io_domain;

	if (!check_device(dev))

		return ERR_PTR(-EINVAL);

	domain = domain_for_device(dev);

	if (domain != NULL && !dma_ops_domain(domain))

	io_domain = iommu_get_domain_for_dev(dev);

	if (!io_domain)

		return NULL;

	domain = to_pdomain(io_domain);

	if (!dma_ops_domain(domain))

		return ERR_PTR(-EBUSY);

	if (domain != NULL)

	return domain;

	/* Device not bound yet - bind it */

	dma_dom = find_protection_domain(devid);

	if (!dma_dom)

		dma_dom = amd_iommu_rlookup_table[devid]->default_dom;

	attach_device(dev, &dma_dom->domain);

	DUMP_printk("Using protection domain %d for device %s\n",

		    dma_dom->domain.id, dev_name(dev));

	return &dma_dom->domain;

}

static void update_device_table(struct protection_domain *domain)

	return check_device(dev);

}

/*

 * The function for pre-allocating protection domains.

 *

 * If the driver core informs the DMA layer if a driver grabs a device

 * we don't need to preallocate the protection domains anymore.

 * For now we have to.

 */

static void __init prealloc_protection_domains(void)

{

	struct iommu_dev_data *dev_data;

	struct dma_ops_domain *dma_dom;

	struct pci_dev *dev = NULL;

	u16 devid;

	for_each_pci_dev(dev) {

		/* Do we handle this device? */

		if (!check_device(&dev->dev))

			continue;

		dev_data = get_dev_data(&dev->dev);

		if (!amd_iommu_force_isolation && dev_data->iommu_v2) {

			/* Make sure passthrough domain is allocated */

			alloc_passthrough_domain();

			dev_data->passthrough = true;

			attach_device(&dev->dev, pt_domain);

			pr_info("AMD-Vi: Using passthrough domain for device %s\n",

				dev_name(&dev->dev));

		}

		/* Is there already any domain for it? */

		if (domain_for_device(&dev->dev))

			continue;

		devid = get_device_id(&dev->dev);

		dma_dom = dma_ops_domain_alloc();

		if (!dma_dom)

			continue;

		init_unity_mappings_for_device(dma_dom, devid);

		dma_dom->target_dev = devid;

		attach_device(&dev->dev, &dma_dom->domain);

		list_add_tail(&dma_dom->list, &iommu_pd_list);

	}

}

static struct dma_map_ops amd_iommu_dma_ops = {

	.alloc = alloc_coherent,

	.free = free_coherent,

	.dma_supported = amd_iommu_dma_supported,

};

static unsigned device_dma_ops_init(void)

int __init amd_iommu_init_api(void)

{

	struct iommu_dev_data *dev_data;

	struct pci_dev *pdev = NULL;

	unsigned unhandled = 0;

	for_each_pci_dev(pdev) {

		if (!check_device(&pdev->dev)) {

			iommu_ignore_device(&pdev->dev);

			unhandled += 1;

			continue;

		}

		dev_data = get_dev_data(&pdev->dev);

		if (!dev_data->passthrough)

			pdev->dev.archdata.dma_ops = &amd_iommu_dma_ops;

		else

			pdev->dev.archdata.dma_ops = &nommu_dma_ops;

	}

	return unhandled;

}

/*

 * The function which clues the AMD IOMMU driver into dma_ops.

 */

void __init amd_iommu_init_api(void)

{

	bus_set_iommu(&pci_bus_type, &amd_iommu_ops);

	return bus_set_iommu(&pci_bus_type, &amd_iommu_ops);

}

int __init amd_iommu_init_dma_ops(void)

{

	struct amd_iommu *iommu;

	int ret, unhandled;

	/*

	 * first allocate a default protection domain for every IOMMU we

	 * found in the system. Devices not assigned to any other

	 * protection domain will be assigned to the default one.

	 */

	for_each_iommu(iommu) {

		iommu->default_dom = dma_ops_domain_alloc();

		if (iommu->default_dom == NULL)

			return -ENOMEM;

		iommu->default_dom->domain.flags |= PD_DEFAULT_MASK;

		ret = iommu_init_unity_mappings(iommu);

		if (ret)

			goto free_domains;

	}

	/*

	 * Pre-allocate the protection domains for each device.

	 */

	prealloc_protection_domains();

	iommu_detected = 1;

	swiotlb = 0;

	/* Make the driver finally visible to the drivers */

	unhandled = device_dma_ops_init();

	if (unhandled && max_pfn > MAX_DMA32_PFN) {

		/* There are unhandled devices - initialize swiotlb for them */

		swiotlb = 1;

	}

	amd_iommu_stats_init();

	if (amd_iommu_unmap_flush)

		pr_info("AMD-Vi: Lazy IO/TLB flushing enabled\n");

	return 0;

free_domains:

	for_each_iommu(iommu) {

		dma_ops_domain_free(iommu->default_dom);

	}