Merge tag 'iommu-updates-v4.1' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu (79319a05) · Commits · e / devices / android_kernel_teracube_mt6765

drivers/iommu/amd_iommu.c

+122 −128

Original line number	Original line	Diff line number	Diff line
	@@ -33,6 +33,7 @@
	#include <linux/export.h>		#include <linux/export.h>
	#include <linux/irq.h>		#include <linux/irq.h>
	#include <linux/msi.h>		#include <linux/msi.h>
			#include <linux/dma-contiguous.h>
	#include <asm/irq_remapping.h>		#include <asm/irq_remapping.h>
	#include <asm/io_apic.h>		#include <asm/io_apic.h>
	#include <asm/apic.h>		#include <asm/apic.h>
	@@ -126,6 +127,11 @@ static int __init alloc_passthrough_domain(void);
	*		*
	****************************************************************************/		****************************************************************************/

			static struct protection_domain to_pdomain(struct iommu_domain dom)
			{
			return container_of(dom, struct protection_domain, domain);
			}

	static struct iommu_dev_data *alloc_dev_data(u16 devid)		static struct iommu_dev_data *alloc_dev_data(u16 devid)
	{		{
	struct iommu_dev_data *dev_data;		struct iommu_dev_data *dev_data;
	@@ -1321,7 +1327,9 @@ static u64 alloc_pte(struct protection_domain domain,
	* This function checks if there is a PTE for a given dma address. If		* This function checks if there is a PTE for a given dma address. If
	* there is one, it returns the pointer to it.		* there is one, it returns the pointer to it.
	*/		*/
	static u64 fetch_pte(struct protection_domain domain, unsigned long address)		static u64 fetch_pte(struct protection_domain domain,
			unsigned long address,
			unsigned long *page_size)
	{		{
	int level;		int level;
	u64 *pte;		u64 *pte;
	@@ -1331,6 +1339,7 @@ static u64 fetch_pte(struct protection_domain domain, unsigned long address)

	level = domain->mode - 1;		level = domain->mode - 1;
	pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)];		pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
			*page_size = PTE_LEVEL_PAGE_SIZE(level);

	while (level > 0) {		while (level > 0) {

	@@ -1339,19 +1348,9 @@ static u64 fetch_pte(struct protection_domain domain, unsigned long address)
	return NULL;		return NULL;

	/* Large PTE */		/* Large PTE */
	if (PM_PTE_LEVEL(*pte) == 0x07) {		if (PM_PTE_LEVEL(*pte) == 7 \|\|
	unsigned long pte_mask, __pte;		PM_PTE_LEVEL(*pte) == 0)
			break;
	/*
	* If we have a series of large PTEs, make
	* sure to return a pointer to the first one.
	*/
	pte_mask = PTE_PAGE_SIZE(*pte);
	pte_mask = ~((PAGE_SIZE_PTE_COUNT(pte_mask) << 3) - 1);
	__pte = ((unsigned long)pte) & pte_mask;

	return (u64 *)__pte;
	}

	/* No level skipping support yet */		/* No level skipping support yet */
	if (PM_PTE_LEVEL(*pte) != level)		if (PM_PTE_LEVEL(*pte) != level)
	@@ -1362,6 +1361,19 @@ static u64 fetch_pte(struct protection_domain domain, unsigned long address)
	/* Walk to the next level */		/* Walk to the next level */
	pte = IOMMU_PTE_PAGE(*pte);		pte = IOMMU_PTE_PAGE(*pte);
	pte = &pte[PM_LEVEL_INDEX(level, address)];		pte = &pte[PM_LEVEL_INDEX(level, address)];
			*page_size = PTE_LEVEL_PAGE_SIZE(level);
			}

			if (PM_PTE_LEVEL(*pte) == 0x07) {
			unsigned long pte_mask;

			/*
			* If we have a series of large PTEs, make
			* sure to return a pointer to the first one.
			*/
			page_size = pte_mask = PTE_PAGE_SIZE(pte);
			pte_mask = ~((PAGE_SIZE_PTE_COUNT(pte_mask) << 3) - 1);
			pte = (u64 *)(((unsigned long)pte) & pte_mask);
	}		}

	return pte;		return pte;
	@@ -1383,11 +1395,12 @@ static int iommu_map_page(struct protection_domain *dom,
	u64 __pte, *pte;		u64 __pte, *pte;
	int i, count;		int i, count;

			BUG_ON(!IS_ALIGNED(bus_addr, page_size));
			BUG_ON(!IS_ALIGNED(phys_addr, page_size));

	if (!(prot & IOMMU_PROT_MASK))		if (!(prot & IOMMU_PROT_MASK))
	return -EINVAL;		return -EINVAL;

	bus_addr = PAGE_ALIGN(bus_addr);
	phys_addr = PAGE_ALIGN(phys_addr);
	count = PAGE_SIZE_PTE_COUNT(page_size);		count = PAGE_SIZE_PTE_COUNT(page_size);
	pte = alloc_pte(dom, bus_addr, page_size, NULL, GFP_KERNEL);		pte = alloc_pte(dom, bus_addr, page_size, NULL, GFP_KERNEL);

	@@ -1398,7 +1411,7 @@ static int iommu_map_page(struct protection_domain *dom,
	if (IOMMU_PTE_PRESENT(pte[i]))		if (IOMMU_PTE_PRESENT(pte[i]))
	return -EBUSY;		return -EBUSY;

	if (page_size > PAGE_SIZE) {		if (count > 1) {
	__pte = PAGE_SIZE_PTE(phys_addr, page_size);		__pte = PAGE_SIZE_PTE(phys_addr, page_size);
	__pte \|= PM_LEVEL_ENC(7) \| IOMMU_PTE_P \| IOMMU_PTE_FC;		__pte \|= PM_LEVEL_ENC(7) \| IOMMU_PTE_P \| IOMMU_PTE_FC;
	} else		} else
	@@ -1421,7 +1434,8 @@ static unsigned long iommu_unmap_page(struct protection_domain *dom,
	unsigned long bus_addr,		unsigned long bus_addr,
	unsigned long page_size)		unsigned long page_size)
	{		{
	unsigned long long unmap_size, unmapped;		unsigned long long unmapped;
			unsigned long unmap_size;
	u64 *pte;		u64 *pte;

	BUG_ON(!is_power_of_2(page_size));		BUG_ON(!is_power_of_2(page_size));
	@@ -1430,27 +1444,11 @@ static unsigned long iommu_unmap_page(struct protection_domain *dom,

	while (unmapped < page_size) {		while (unmapped < page_size) {

	pte = fetch_pte(dom, bus_addr);		pte = fetch_pte(dom, bus_addr, &unmap_size);

	if (!pte) {
	/*
	* No PTE for this address
	* move forward in 4kb steps
	*/
	unmap_size = PAGE_SIZE;
	} else if (PM_PTE_LEVEL(*pte) == 0) {
	/* 4kb PTE found for this address */
	unmap_size = PAGE_SIZE;
	*pte = 0ULL;
	} else {
	int count, i;

	/* Large PTE found which maps this address */		if (pte) {
	unmap_size = PTE_PAGE_SIZE(*pte);		int i, count;

	/* Only unmap from the first pte in the page */
	if ((unmap_size - 1) & bus_addr)
	break;
	count = PAGE_SIZE_PTE_COUNT(unmap_size);		count = PAGE_SIZE_PTE_COUNT(unmap_size);
	for (i = 0; i < count; i++)		for (i = 0; i < count; i++)
	pte[i] = 0ULL;		pte[i] = 0ULL;
	@@ -1599,7 +1597,7 @@ static int alloc_new_range(struct dma_ops_domain *dma_dom,
	{		{
	int index = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;		int index = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;
	struct amd_iommu *iommu;		struct amd_iommu *iommu;
	unsigned long i, old_size;		unsigned long i, old_size, pte_pgsize;

	#ifdef CONFIG_IOMMU_STRESS		#ifdef CONFIG_IOMMU_STRESS
	populate = false;		populate = false;
	@@ -1672,12 +1670,13 @@ static int alloc_new_range(struct dma_ops_domain *dma_dom,
	*/		*/
	for (i = dma_dom->aperture[index]->offset;		for (i = dma_dom->aperture[index]->offset;
	i < dma_dom->aperture_size;		i < dma_dom->aperture_size;
	i += PAGE_SIZE) {		i += pte_pgsize) {
	u64 *pte = fetch_pte(&dma_dom->domain, i);		u64 *pte = fetch_pte(&dma_dom->domain, i, &pte_pgsize);
	if (!pte \|\| !IOMMU_PTE_PRESENT(*pte))		if (!pte \|\| !IOMMU_PTE_PRESENT(*pte))
	continue;		continue;

	dma_ops_reserve_addresses(dma_dom, i >> PAGE_SHIFT, 1);		dma_ops_reserve_addresses(dma_dom, i >> PAGE_SHIFT,
			pte_pgsize >> 12);
	}		}

	update_domain(&dma_dom->domain);		update_domain(&dma_dom->domain);
	@@ -2422,16 +2421,6 @@ static int device_change_notifier(struct notifier_block *nb,
	dev_data = get_dev_data(dev);		dev_data = get_dev_data(dev);

	switch (action) {		switch (action) {
	case BUS_NOTIFY_UNBOUND_DRIVER:

	domain = domain_for_device(dev);

	if (!domain)
	goto out;
	if (dev_data->passthrough)
	break;
	detach_device(dev);
	break;
	case BUS_NOTIFY_ADD_DEVICE:		case BUS_NOTIFY_ADD_DEVICE:

	iommu_init_device(dev);		iommu_init_device(dev);
	@@ -2467,7 +2456,7 @@ static int device_change_notifier(struct notifier_block *nb,
	dev->archdata.dma_ops = &amd_iommu_dma_ops;		dev->archdata.dma_ops = &amd_iommu_dma_ops;

	break;		break;
	case BUS_NOTIFY_DEL_DEVICE:		case BUS_NOTIFY_REMOVED_DEVICE:

	iommu_uninit_device(dev);		iommu_uninit_device(dev);

	@@ -2923,38 +2912,42 @@ static void alloc_coherent(struct device dev, size_t size,
	dma_addr_t *dma_addr, gfp_t flag,		dma_addr_t *dma_addr, gfp_t flag,
	struct dma_attrs *attrs)		struct dma_attrs *attrs)
	{		{
	unsigned long flags;
	void *virt_addr;
	struct protection_domain *domain;
	phys_addr_t paddr;
	u64 dma_mask = dev->coherent_dma_mask;		u64 dma_mask = dev->coherent_dma_mask;
			struct protection_domain *domain;
			unsigned long flags;
			struct page *page;

	INC_STATS_COUNTER(cnt_alloc_coherent);		INC_STATS_COUNTER(cnt_alloc_coherent);

	domain = get_domain(dev);		domain = get_domain(dev);
	if (PTR_ERR(domain) == -EINVAL) {		if (PTR_ERR(domain) == -EINVAL) {
	virt_addr = (void *)__get_free_pages(flag, get_order(size));		page = alloc_pages(flag, get_order(size));
	*dma_addr = __pa(virt_addr);		*dma_addr = page_to_phys(page);
	return virt_addr;		return page_address(page);
	} else if (IS_ERR(domain))		} else if (IS_ERR(domain))
	return NULL;		return NULL;

			size = PAGE_ALIGN(size);
	dma_mask = dev->coherent_dma_mask;		dma_mask = dev->coherent_dma_mask;
	flag &= ~(__GFP_DMA \| __GFP_HIGHMEM \| __GFP_DMA32);		flag &= ~(__GFP_DMA \| __GFP_HIGHMEM \| __GFP_DMA32);
	flag \|= __GFP_ZERO;

	virt_addr = (void *)__get_free_pages(flag, get_order(size));		page = alloc_pages(flag \| __GFP_NOWARN, get_order(size));
	if (!virt_addr)		if (!page) {
			if (!(flag & __GFP_WAIT))
	return NULL;		return NULL;

	paddr = virt_to_phys(virt_addr);		page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT,
			get_order(size));
			if (!page)
			return NULL;
			}

	if (!dma_mask)		if (!dma_mask)
	dma_mask = *dev->dma_mask;		dma_mask = *dev->dma_mask;

	spin_lock_irqsave(&domain->lock, flags);		spin_lock_irqsave(&domain->lock, flags);

	*dma_addr = __map_single(dev, domain->priv, paddr,		*dma_addr = __map_single(dev, domain->priv, page_to_phys(page),
	size, DMA_BIDIRECTIONAL, true, dma_mask);		size, DMA_BIDIRECTIONAL, true, dma_mask);

	if (*dma_addr == DMA_ERROR_CODE) {		if (*dma_addr == DMA_ERROR_CODE) {
	@@ -2966,11 +2959,12 @@ static void alloc_coherent(struct device dev, size_t size,

	spin_unlock_irqrestore(&domain->lock, flags);		spin_unlock_irqrestore(&domain->lock, flags);

	return virt_addr;		return page_address(page);

	out_free:		out_free:

	free_pages((unsigned long)virt_addr, get_order(size));		if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
			__free_pages(page, get_order(size));

	return NULL;		return NULL;
	}		}
	@@ -2982,11 +2976,15 @@ static void free_coherent(struct device *dev, size_t size,
	void *virt_addr, dma_addr_t dma_addr,		void *virt_addr, dma_addr_t dma_addr,
	struct dma_attrs *attrs)		struct dma_attrs *attrs)
	{		{
	unsigned long flags;
	struct protection_domain *domain;		struct protection_domain *domain;
			unsigned long flags;
			struct page *page;

	INC_STATS_COUNTER(cnt_free_coherent);		INC_STATS_COUNTER(cnt_free_coherent);

			page = virt_to_page(virt_addr);
			size = PAGE_ALIGN(size);

	domain = get_domain(dev);		domain = get_domain(dev);
	if (IS_ERR(domain))		if (IS_ERR(domain))
	goto free_mem;		goto free_mem;
	@@ -3000,7 +2998,8 @@ static void free_coherent(struct device *dev, size_t size,
	spin_unlock_irqrestore(&domain->lock, flags);		spin_unlock_irqrestore(&domain->lock, flags);

	free_mem:		free_mem:
	free_pages((unsigned long)virt_addr, get_order(size));		if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
			__free_pages(page, get_order(size));
	}		}

	/*		/*
	@@ -3236,42 +3235,45 @@ static int __init alloc_passthrough_domain(void)

	return 0;		return 0;
	}		}
	static int amd_iommu_domain_init(struct iommu_domain *dom)
			static struct iommu_domain *amd_iommu_domain_alloc(unsigned type)
	{		{
	struct protection_domain *domain;		struct protection_domain *pdomain;

	domain = protection_domain_alloc();		/* We only support unmanaged domains for now */
	if (!domain)		if (type != IOMMU_DOMAIN_UNMANAGED)
	goto out_free;		return NULL;

	domain->mode = PAGE_MODE_3_LEVEL;		pdomain = protection_domain_alloc();
	domain->pt_root = (void *)get_zeroed_page(GFP_KERNEL);		if (!pdomain)
	if (!domain->pt_root)
	goto out_free;		goto out_free;

	domain->iommu_domain = dom;		pdomain->mode = PAGE_MODE_3_LEVEL;
			pdomain->pt_root = (void *)get_zeroed_page(GFP_KERNEL);
			if (!pdomain->pt_root)
			goto out_free;

	dom->priv = domain;		pdomain->domain.geometry.aperture_start = 0;
			pdomain->domain.geometry.aperture_end = ~0ULL;
			pdomain->domain.geometry.force_aperture = true;

	dom->geometry.aperture_start = 0;		return &pdomain->domain;
	dom->geometry.aperture_end = ~0ULL;
	dom->geometry.force_aperture = true;

	return 0;

	out_free:		out_free:
	protection_domain_free(domain);		protection_domain_free(pdomain);

	return -ENOMEM;		return NULL;
	}		}

	static void amd_iommu_domain_destroy(struct iommu_domain *dom)		static void amd_iommu_domain_free(struct iommu_domain *dom)
	{		{
	struct protection_domain *domain = dom->priv;		struct protection_domain *domain;

	if (!domain)		if (!dom)
	return;		return;

			domain = to_pdomain(dom);

	if (domain->dev_cnt > 0)		if (domain->dev_cnt > 0)
	cleanup_domain(domain);		cleanup_domain(domain);

	@@ -3284,8 +3286,6 @@ static void amd_iommu_domain_destroy(struct iommu_domain *dom)
	free_gcr3_table(domain);		free_gcr3_table(domain);

	protection_domain_free(domain);		protection_domain_free(domain);

	dom->priv = NULL;
	}		}

	static void amd_iommu_detach_device(struct iommu_domain *dom,		static void amd_iommu_detach_device(struct iommu_domain *dom,
	@@ -3313,7 +3313,7 @@ static void amd_iommu_detach_device(struct iommu_domain *dom,
	static int amd_iommu_attach_device(struct iommu_domain *dom,		static int amd_iommu_attach_device(struct iommu_domain *dom,
	struct device *dev)		struct device *dev)
	{		{
	struct protection_domain *domain = dom->priv;		struct protection_domain *domain = to_pdomain(dom);
	struct iommu_dev_data *dev_data;		struct iommu_dev_data *dev_data;
	struct amd_iommu *iommu;		struct amd_iommu *iommu;
	int ret;		int ret;
	@@ -3340,7 +3340,7 @@ static int amd_iommu_attach_device(struct iommu_domain *dom,
	static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,		static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
	phys_addr_t paddr, size_t page_size, int iommu_prot)		phys_addr_t paddr, size_t page_size, int iommu_prot)
	{		{
	struct protection_domain *domain = dom->priv;		struct protection_domain *domain = to_pdomain(dom);
	int prot = 0;		int prot = 0;
	int ret;		int ret;

	@@ -3362,7 +3362,7 @@ static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
	static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,		static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,
	size_t page_size)		size_t page_size)
	{		{
	struct protection_domain *domain = dom->priv;		struct protection_domain *domain = to_pdomain(dom);
	size_t unmap_size;		size_t unmap_size;

	if (domain->mode == PAGE_MODE_NONE)		if (domain->mode == PAGE_MODE_NONE)
	@@ -3380,28 +3380,22 @@ static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,
	static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,		static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
	dma_addr_t iova)		dma_addr_t iova)
	{		{
	struct protection_domain *domain = dom->priv;		struct protection_domain *domain = to_pdomain(dom);
	unsigned long offset_mask;		unsigned long offset_mask, pte_pgsize;
	phys_addr_t paddr;
	u64 *pte, __pte;		u64 *pte, __pte;

	if (domain->mode == PAGE_MODE_NONE)		if (domain->mode == PAGE_MODE_NONE)
	return iova;		return iova;

	pte = fetch_pte(domain, iova);		pte = fetch_pte(domain, iova, &pte_pgsize);

	if (!pte \|\| !IOMMU_PTE_PRESENT(*pte))		if (!pte \|\| !IOMMU_PTE_PRESENT(*pte))
	return 0;		return 0;

	if (PM_PTE_LEVEL(*pte) == 0)		offset_mask = pte_pgsize - 1;
	offset_mask = PAGE_SIZE - 1;
	else
	offset_mask = PTE_PAGE_SIZE(*pte) - 1;

	__pte = *pte & PM_ADDR_MASK;		__pte = *pte & PM_ADDR_MASK;
	paddr = (__pte & ~offset_mask) \| (iova & offset_mask);

	return paddr;		return (__pte & ~offset_mask) \| (iova & offset_mask);
	}		}

	static bool amd_iommu_capable(enum iommu_cap cap)		static bool amd_iommu_capable(enum iommu_cap cap)
	@@ -3420,8 +3414,8 @@ static bool amd_iommu_capable(enum iommu_cap cap)

	static const struct iommu_ops amd_iommu_ops = {		static const struct iommu_ops amd_iommu_ops = {
	.capable = amd_iommu_capable,		.capable = amd_iommu_capable,
	.domain_init = amd_iommu_domain_init,		.domain_alloc = amd_iommu_domain_alloc,
	.domain_destroy = amd_iommu_domain_destroy,		.domain_free = amd_iommu_domain_free,
	.attach_dev = amd_iommu_attach_device,		.attach_dev = amd_iommu_attach_device,
	.detach_dev = amd_iommu_detach_device,		.detach_dev = amd_iommu_detach_device,
	.map = amd_iommu_map,		.map = amd_iommu_map,
	@@ -3483,7 +3477,7 @@ EXPORT_SYMBOL(amd_iommu_unregister_ppr_notifier);

	void amd_iommu_domain_direct_map(struct iommu_domain *dom)		void amd_iommu_domain_direct_map(struct iommu_domain *dom)
	{		{
	struct protection_domain *domain = dom->priv;		struct protection_domain *domain = to_pdomain(dom);
	unsigned long flags;		unsigned long flags;

	spin_lock_irqsave(&domain->lock, flags);		spin_lock_irqsave(&domain->lock, flags);
	@@ -3504,7 +3498,7 @@ EXPORT_SYMBOL(amd_iommu_domain_direct_map);

	int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids)		int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids)
	{		{
	struct protection_domain *domain = dom->priv;		struct protection_domain *domain = to_pdomain(dom);
	unsigned long flags;		unsigned long flags;
	int levels, ret;		int levels, ret;

	@@ -3616,7 +3610,7 @@ static int __amd_iommu_flush_page(struct protection_domain *domain, int pasid,
	int amd_iommu_flush_page(struct iommu_domain *dom, int pasid,		int amd_iommu_flush_page(struct iommu_domain *dom, int pasid,
	u64 address)		u64 address)
	{		{
	struct protection_domain *domain = dom->priv;		struct protection_domain *domain = to_pdomain(dom);
	unsigned long flags;		unsigned long flags;
	int ret;		int ret;

	@@ -3638,7 +3632,7 @@ static int __amd_iommu_flush_tlb(struct protection_domain *domain, int pasid)

	int amd_iommu_flush_tlb(struct iommu_domain *dom, int pasid)		int amd_iommu_flush_tlb(struct iommu_domain *dom, int pasid)
	{		{
	struct protection_domain *domain = dom->priv;		struct protection_domain *domain = to_pdomain(dom);
	unsigned long flags;		unsigned long flags;
	int ret;		int ret;

	@@ -3718,7 +3712,7 @@ static int __clear_gcr3(struct protection_domain *domain, int pasid)
	int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, int pasid,		int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, int pasid,
	unsigned long cr3)		unsigned long cr3)
	{		{
	struct protection_domain *domain = dom->priv;		struct protection_domain *domain = to_pdomain(dom);
	unsigned long flags;		unsigned long flags;
	int ret;		int ret;

	@@ -3732,7 +3726,7 @@ EXPORT_SYMBOL(amd_iommu_domain_set_gcr3);

	int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, int pasid)		int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, int pasid)
	{		{
	struct protection_domain *domain = dom->priv;		struct protection_domain *domain = to_pdomain(dom);
	unsigned long flags;		unsigned long flags;
	int ret;		int ret;

	@@ -3765,17 +3759,17 @@ EXPORT_SYMBOL(amd_iommu_complete_ppr);

	struct iommu_domain amd_iommu_get_v2_domain(struct pci_dev pdev)		struct iommu_domain amd_iommu_get_v2_domain(struct pci_dev pdev)
	{		{
	struct protection_domain *domain;		struct protection_domain *pdomain;

	domain = get_domain(&pdev->dev);		pdomain = get_domain(&pdev->dev);
	if (IS_ERR(domain))		if (IS_ERR(pdomain))
	return NULL;		return NULL;

	/* Only return IOMMUv2 domains */		/* Only return IOMMUv2 domains */
	if (!(domain->flags & PD_IOMMUV2_MASK))		if (!(pdomain->flags & PD_IOMMUV2_MASK))
	return NULL;		return NULL;

	return domain->iommu_domain;		return &pdomain->domain;
	}		}
	EXPORT_SYMBOL(amd_iommu_get_v2_domain);		EXPORT_SYMBOL(amd_iommu_get_v2_domain);

drivers/iommu/amd_iommu_types.h

+9 −4

Original line number	Original line	Diff line number	Diff line
	@@ -282,6 +282,12 @@
	#define PTE_PAGE_SIZE(pte) \		#define PTE_PAGE_SIZE(pte) \
	(1ULL << (1 + ffz(((pte) \| 0xfffULL))))		(1ULL << (1 + ffz(((pte) \| 0xfffULL))))

			/*
			* Takes a page-table level and returns the default page-size for this level
			*/
			#define PTE_LEVEL_PAGE_SIZE(level) \
			(1ULL << (12 + (9 * (level))))

	#define IOMMU_PTE_P (1ULL << 0)		#define IOMMU_PTE_P (1ULL << 0)
	#define IOMMU_PTE_TV (1ULL << 1)		#define IOMMU_PTE_TV (1ULL << 1)
	#define IOMMU_PTE_U (1ULL << 59)		#define IOMMU_PTE_U (1ULL << 59)
	@@ -400,6 +406,8 @@ struct iommu_domain;
	struct protection_domain {		struct protection_domain {
	struct list_head list; /* for list of all protection domains */		struct list_head list; /* for list of all protection domains */
	struct list_head dev_list; /* List of all devices in this domain */		struct list_head dev_list; /* List of all devices in this domain */
			struct iommu_domain domain; /* generic domain handle used by
			iommu core code */
	spinlock_t lock; /* mostly used to lock the page table*/		spinlock_t lock; /* mostly used to lock the page table*/
	struct mutex api_lock; /* protect page tables in the iommu-api path */		struct mutex api_lock; /* protect page tables in the iommu-api path */
	u16 id; /* the domain id written to the device table */		u16 id; /* the domain id written to the device table */
	@@ -412,9 +420,6 @@ struct protection_domain {
	unsigned dev_cnt; /* devices assigned to this domain */		unsigned dev_cnt; /* devices assigned to this domain */
	unsigned dev_iommu[MAX_IOMMUS]; /* per-IOMMU reference count */		unsigned dev_iommu[MAX_IOMMUS]; /* per-IOMMU reference count */
	void priv; / private data */		void priv; / private data */
	struct iommu_domain iommu_domain; / Pointer to generic
	domain structure */

	};		};

	/*		/*

drivers/iommu/amd_iommu_v2.c

+1 −1

Original line number	Original line	Diff line number	Diff line
	@@ -417,7 +417,7 @@ static void mn_release(struct mmu_notifier mn, struct mm_struct mm)
	dev_state = pasid_state->device_state;		dev_state = pasid_state->device_state;
	run_inv_ctx_cb = !pasid_state->invalid;		run_inv_ctx_cb = !pasid_state->invalid;

	if (run_inv_ctx_cb && pasid_state->device_state->inv_ctx_cb)		if (run_inv_ctx_cb && dev_state->inv_ctx_cb)
	dev_state->inv_ctx_cb(dev_state->pdev, pasid_state->pasid);		dev_state->inv_ctx_cb(dev_state->pdev, pasid_state->pasid);

	unbind_pasid(pasid_state);		unbind_pasid(pasid_state);

drivers/iommu/arm-smmu.c

+107 −64

Original line number	Original line	Diff line number	Diff line
	@@ -343,6 +343,7 @@ struct arm_smmu_domain {
	struct arm_smmu_cfg cfg;		struct arm_smmu_cfg cfg;
	enum arm_smmu_domain_stage stage;		enum arm_smmu_domain_stage stage;
	struct mutex init_mutex; /* Protects smmu pointer */		struct mutex init_mutex; /* Protects smmu pointer */
			struct iommu_domain domain;
	};		};

	static struct iommu_ops arm_smmu_ops;		static struct iommu_ops arm_smmu_ops;
	@@ -360,6 +361,11 @@ static struct arm_smmu_option_prop arm_smmu_options[] = {
	{ 0, NULL},		{ 0, NULL},
	};		};

			static struct arm_smmu_domain to_smmu_domain(struct iommu_domain dom)
			{
			return container_of(dom, struct arm_smmu_domain, domain);
			}

	static void parse_driver_options(struct arm_smmu_device *smmu)		static void parse_driver_options(struct arm_smmu_device *smmu)
	{		{
	int i = 0;		int i = 0;
	@@ -645,7 +651,7 @@ static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
	u32 fsr, far, fsynr, resume;		u32 fsr, far, fsynr, resume;
	unsigned long iova;		unsigned long iova;
	struct iommu_domain *domain = dev;		struct iommu_domain *domain = dev;
	struct arm_smmu_domain *smmu_domain = domain->priv;		struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;		struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
	struct arm_smmu_device *smmu = smmu_domain->smmu;		struct arm_smmu_device *smmu = smmu_domain->smmu;
	void __iomem *cb_base;		void __iomem *cb_base;
	@@ -730,6 +736,20 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
	stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;		stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
	cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);		cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);

			if (smmu->version > ARM_SMMU_V1) {
			/*
			* CBA2R.
			* Must be initialised before CBAR thanks to VMID16
			* architectural oversight affected some implementations.
			*/
			#ifdef CONFIG_64BIT
			reg = CBA2R_RW64_64BIT;
			#else
			reg = CBA2R_RW64_32BIT;
			#endif
			writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
			}

	/* CBAR */		/* CBAR */
	reg = cfg->cbar;		reg = cfg->cbar;
	if (smmu->version == ARM_SMMU_V1)		if (smmu->version == ARM_SMMU_V1)
	@@ -747,16 +767,6 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
	}		}
	writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));		writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));

	if (smmu->version > ARM_SMMU_V1) {
	/* CBA2R */
	#ifdef CONFIG_64BIT
	reg = CBA2R_RW64_64BIT;
	#else
	reg = CBA2R_RW64_32BIT;
	#endif
	writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
	}

	/* TTBRs */		/* TTBRs */
	if (stage1) {		if (stage1) {
	reg = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];		reg = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
	@@ -836,7 +846,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
	struct io_pgtable_ops *pgtbl_ops;		struct io_pgtable_ops *pgtbl_ops;
	struct io_pgtable_cfg pgtbl_cfg;		struct io_pgtable_cfg pgtbl_cfg;
	enum io_pgtable_fmt fmt;		enum io_pgtable_fmt fmt;
	struct arm_smmu_domain *smmu_domain = domain->priv;		struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;		struct arm_smmu_cfg *cfg = &smmu_domain->cfg;

	mutex_lock(&smmu_domain->init_mutex);		mutex_lock(&smmu_domain->init_mutex);
	@@ -958,7 +968,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,

	static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)		static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
	{		{
	struct arm_smmu_domain *smmu_domain = domain->priv;		struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
	struct arm_smmu_device *smmu = smmu_domain->smmu;		struct arm_smmu_device *smmu = smmu_domain->smmu;
	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;		struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
	void __iomem *cb_base;		void __iomem *cb_base;
	@@ -985,10 +995,12 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
	__arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);		__arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
	}		}

	static int arm_smmu_domain_init(struct iommu_domain *domain)		static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
	{		{
	struct arm_smmu_domain *smmu_domain;		struct arm_smmu_domain *smmu_domain;

			if (type != IOMMU_DOMAIN_UNMANAGED)
			return NULL;
	/*		/*
	* Allocate the domain and initialise some of its data structures.		* Allocate the domain and initialise some of its data structures.
	* We can't really do anything meaningful until we've added a		* We can't really do anything meaningful until we've added a
	@@ -996,17 +1008,17 @@ static int arm_smmu_domain_init(struct iommu_domain *domain)
	*/		*/
	smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);		smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
	if (!smmu_domain)		if (!smmu_domain)
	return -ENOMEM;		return NULL;

	mutex_init(&smmu_domain->init_mutex);		mutex_init(&smmu_domain->init_mutex);
	spin_lock_init(&smmu_domain->pgtbl_lock);		spin_lock_init(&smmu_domain->pgtbl_lock);
	domain->priv = smmu_domain;
	return 0;		return &smmu_domain->domain;
	}		}

	static void arm_smmu_domain_destroy(struct iommu_domain *domain)		static void arm_smmu_domain_free(struct iommu_domain *domain)
	{		{
	struct arm_smmu_domain *smmu_domain = domain->priv;		struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);

	/*		/*
	* Free the domain resources. We assume that all devices have		* Free the domain resources. We assume that all devices have
	@@ -1143,7 +1155,7 @@ static void arm_smmu_domain_remove_master(struct arm_smmu_domain *smmu_domain,
	static int arm_smmu_attach_dev(struct iommu_domain domain, struct device dev)		static int arm_smmu_attach_dev(struct iommu_domain domain, struct device dev)
	{		{
	int ret;		int ret;
	struct arm_smmu_domain *smmu_domain = domain->priv;		struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
	struct arm_smmu_device *smmu;		struct arm_smmu_device *smmu;
	struct arm_smmu_master_cfg *cfg;		struct arm_smmu_master_cfg *cfg;

	@@ -1187,7 +1199,7 @@ static int arm_smmu_attach_dev(struct iommu_domain domain, struct device dev)

	static void arm_smmu_detach_dev(struct iommu_domain domain, struct device dev)		static void arm_smmu_detach_dev(struct iommu_domain domain, struct device dev)
	{		{
	struct arm_smmu_domain *smmu_domain = domain->priv;		struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
	struct arm_smmu_master_cfg *cfg;		struct arm_smmu_master_cfg *cfg;

	cfg = find_smmu_master_cfg(dev);		cfg = find_smmu_master_cfg(dev);
	@@ -1203,7 +1215,7 @@ static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
	{		{
	int ret;		int ret;
	unsigned long flags;		unsigned long flags;
	struct arm_smmu_domain *smmu_domain = domain->priv;		struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
	struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;		struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;

	if (!ops)		if (!ops)
	@@ -1220,7 +1232,7 @@ static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
	{		{
	size_t ret;		size_t ret;
	unsigned long flags;		unsigned long flags;
	struct arm_smmu_domain *smmu_domain = domain->priv;		struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
	struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;		struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;

	if (!ops)		if (!ops)
	@@ -1235,7 +1247,7 @@ static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
	static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,		static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
	dma_addr_t iova)		dma_addr_t iova)
	{		{
	struct arm_smmu_domain *smmu_domain = domain->priv;		struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
	struct arm_smmu_device *smmu = smmu_domain->smmu;		struct arm_smmu_device *smmu = smmu_domain->smmu;
	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;		struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
	struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;		struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
	@@ -1281,7 +1293,7 @@ static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
	{		{
	phys_addr_t ret;		phys_addr_t ret;
	unsigned long flags;		unsigned long flags;
	struct arm_smmu_domain *smmu_domain = domain->priv;		struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
	struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;		struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;

	if (!ops)		if (!ops)
	@@ -1329,59 +1341,81 @@ static void __arm_smmu_release_pci_iommudata(void *data)
	kfree(data);		kfree(data);
	}		}

	static int arm_smmu_add_device(struct device *dev)		static int arm_smmu_add_pci_device(struct pci_dev *pdev)
	{		{
	struct arm_smmu_device *smmu;		int i, ret;
	struct arm_smmu_master_cfg *cfg;		u16 sid;
	struct iommu_group *group;		struct iommu_group *group;
	void (releasefn)(void ) = NULL;		struct arm_smmu_master_cfg *cfg;
	int ret;

	smmu = find_smmu_for_device(dev);
	if (!smmu)
	return -ENODEV;

	group = iommu_group_alloc();		group = iommu_group_get_for_dev(&pdev->dev);
	if (IS_ERR(group)) {		if (IS_ERR(group))
	dev_err(dev, "Failed to allocate IOMMU group\n");
	return PTR_ERR(group);		return PTR_ERR(group);
	}

	if (dev_is_pci(dev)) {
	struct pci_dev *pdev = to_pci_dev(dev);

			cfg = iommu_group_get_iommudata(group);
			if (!cfg) {
	cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);		cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
	if (!cfg) {		if (!cfg) {
	ret = -ENOMEM;		ret = -ENOMEM;
	goto out_put_group;		goto out_put_group;
	}		}

	cfg->num_streamids = 1;		iommu_group_set_iommudata(group, cfg,
			__arm_smmu_release_pci_iommudata);
			}

			if (cfg->num_streamids >= MAX_MASTER_STREAMIDS) {
			ret = -ENOSPC;
			goto out_put_group;
			}

	/*		/*
	* Assume Stream ID == Requester ID for now.		* Assume Stream ID == Requester ID for now.
	* We need a way to describe the ID mappings in FDT.		* We need a way to describe the ID mappings in FDT.
	*/		*/
	pci_for_each_dma_alias(pdev, __arm_smmu_get_pci_sid,		pci_for_each_dma_alias(pdev, __arm_smmu_get_pci_sid, &sid);
	&cfg->streamids[0]);		for (i = 0; i < cfg->num_streamids; ++i)
	releasefn = __arm_smmu_release_pci_iommudata;		if (cfg->streamids[i] == sid)
	} else {		break;

			/* Avoid duplicate SIDs, as this can lead to SMR conflicts */
			if (i == cfg->num_streamids)
			cfg->streamids[cfg->num_streamids++] = sid;

			return 0;
			out_put_group:
			iommu_group_put(group);
			return ret;
			}

			static int arm_smmu_add_platform_device(struct device *dev)
			{
			struct iommu_group *group;
	struct arm_smmu_master *master;		struct arm_smmu_master *master;
			struct arm_smmu_device *smmu = find_smmu_for_device(dev);

			if (!smmu)
			return -ENODEV;

	master = find_smmu_master(smmu, dev->of_node);		master = find_smmu_master(smmu, dev->of_node);
	if (!master) {		if (!master)
	ret = -ENODEV;		return -ENODEV;
	goto out_put_group;
	}		/* No automatic group creation for platform devices */
			group = iommu_group_alloc();
			if (IS_ERR(group))
			return PTR_ERR(group);

	cfg = &master->cfg;		iommu_group_set_iommudata(group, &master->cfg, NULL);
			return iommu_group_add_device(group, dev);
	}		}

	iommu_group_set_iommudata(group, cfg, releasefn);		static int arm_smmu_add_device(struct device *dev)
	ret = iommu_group_add_device(group, dev);		{
			if (dev_is_pci(dev))
			return arm_smmu_add_pci_device(to_pci_dev(dev));

	out_put_group:		return arm_smmu_add_platform_device(dev);
	iommu_group_put(group);
	return ret;
	}		}

	static void arm_smmu_remove_device(struct device *dev)		static void arm_smmu_remove_device(struct device *dev)
	@@ -1392,7 +1426,7 @@ static void arm_smmu_remove_device(struct device *dev)
	static int arm_smmu_domain_get_attr(struct iommu_domain *domain,		static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
	enum iommu_attr attr, void *data)		enum iommu_attr attr, void *data)
	{		{
	struct arm_smmu_domain *smmu_domain = domain->priv;		struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);

	switch (attr) {		switch (attr) {
	case DOMAIN_ATTR_NESTING:		case DOMAIN_ATTR_NESTING:
	@@ -1407,7 +1441,7 @@ static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
	enum iommu_attr attr, void *data)		enum iommu_attr attr, void *data)
	{		{
	int ret = 0;		int ret = 0;
	struct arm_smmu_domain *smmu_domain = domain->priv;		struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);

	mutex_lock(&smmu_domain->init_mutex);		mutex_lock(&smmu_domain->init_mutex);

	@@ -1435,8 +1469,8 @@ static int arm_smmu_domain_set_attr(struct iommu_domain *domain,

	static struct iommu_ops arm_smmu_ops = {		static struct iommu_ops arm_smmu_ops = {
	.capable = arm_smmu_capable,		.capable = arm_smmu_capable,
	.domain_init = arm_smmu_domain_init,		.domain_alloc = arm_smmu_domain_alloc,
	.domain_destroy = arm_smmu_domain_destroy,		.domain_free = arm_smmu_domain_free,
	.attach_dev = arm_smmu_attach_dev,		.attach_dev = arm_smmu_attach_dev,
	.detach_dev = arm_smmu_detach_dev,		.detach_dev = arm_smmu_detach_dev,
	.map = arm_smmu_map,		.map = arm_smmu_map,
	@@ -1633,6 +1667,15 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
	size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);		size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
	smmu->pa_size = size;		smmu->pa_size = size;

			/*
			* What the page table walker can address actually depends on which
			* descriptor format is in use, but since a) we don't know that yet,
			* and b) it can vary per context bank, this will have to do...
			*/
			if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
			dev_warn(smmu->dev,
			"failed to set DMA mask for table walker\n");

	if (smmu->version == ARM_SMMU_V1) {		if (smmu->version == ARM_SMMU_V1) {
	smmu->va_size = smmu->ipa_size;		smmu->va_size = smmu->ipa_size;
	size = SZ_4K \| SZ_2M \| SZ_1G;		size = SZ_4K \| SZ_2M \| SZ_1G;

drivers/iommu/exynos-iommu.c

+47 −40

File changed.

Preview size limit exceeded, changes collapsed.