Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c11d1080 authored by Patrick Daly's avatar Patrick Daly
Browse files

iommu/arm-smmu: Add support for page table donation 



For secure domains, the page tables need to be assigned
to the correct VMIDs. Add support for doing the assignment.

Change-Id: I60009ef96ae1c8965d57096b6a1b0658ae6acc9a
Signed-off-by: default avatarNeeti Desai <neetid@codeaurora.org>
Signed-off-by: default avatarPatrick Daly <pdaly@codeaurora.org>
parent 67be369f

drivers/iommu/arm-smmu.c

+165 −0
Original line number Diff line number Diff line
@@ -44,6 +44,7 @@ 
#include <linux/platform_device.h>

#include <linux/slab.h>

#include <linux/spinlock.h>

#include <soc/qcom/secure_buffer.h>

#include <linux/msm-bus.h>

#include <dt-bindings/msm/msm-bus-ids.h>
@@ -454,6 +455,12 @@ enum arm_smmu_domain_stage { 
	ARM_SMMU_DOMAIN_NESTED,

};



struct arm_smmu_pte_info {

	void *virt_addr;

	size_t size;

	struct list_head entry;

};



struct arm_smmu_domain {

	struct arm_smmu_device		*smmu;

	struct io_pgtable_ops		*pgtbl_ops;
@@ -463,6 +470,9 @@ struct arm_smmu_domain { 
	enum arm_smmu_domain_stage	stage;

	struct mutex			init_mutex; /* Protects smmu pointer */

	u32 attributes;

	u32				secure_vmid;

	struct list_head		pte_info_list;

	struct list_head		unassign_list;

	struct iommu_domain		domain;

};
@@ -502,6 +512,11 @@ static phys_addr_t arm_smmu_iova_to_phys_hard_no_halt( 
	struct iommu_domain *domain, dma_addr_t iova);

static void arm_smmu_destroy_domain_context(struct iommu_domain *domain);



static int arm_smmu_prepare_pgtable(void *addr, void *cookie);

static void arm_smmu_unprepare_pgtable(void *cookie, void *addr, size_t size);

static void arm_smmu_assign_table(struct arm_smmu_domain *smmu_domain);

static void arm_smmu_unassign_table(struct arm_smmu_domain *smmu_domain);



static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)

{

	return container_of(dom, struct arm_smmu_domain, domain);
@@ -1002,10 +1017,35 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, 
	}

}



static void *arm_smmu_alloc_pages_exact(void *cookie,

					size_t size, gfp_t gfp_mask)

{

	int ret;

	void *page = alloc_pages_exact(size, gfp_mask);



	if (likely(page)) {

		ret = arm_smmu_prepare_pgtable(page, cookie);

		if (ret) {

			free_pages_exact(page, size);

			return NULL;

		}

	}



	return page;

}



static void arm_smmu_free_pages_exact(void *cookie, void *virt, size_t size)

{

	arm_smmu_unprepare_pgtable(cookie, virt, size);

	/* unprepare also frees (possibly later), no need to free here */

}



static struct iommu_gather_ops arm_smmu_gather_ops = {

	.tlb_flush_all	= arm_smmu_tlb_inv_context,

	.tlb_add_flush	= arm_smmu_tlb_inv_range_nosync,

	.tlb_sync	= arm_smmu_tlb_sync,

	.alloc_pages_exact = arm_smmu_alloc_pages_exact,

	.free_pages_exact = arm_smmu_free_pages_exact,

};



static phys_addr_t arm_smmu_verify_fault(struct iommu_domain *domain,
@@ -1507,6 +1547,12 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, 
		goto out_clear_smmu;

	}



	/*

	 * assign any page table memory that might have been allocated

	 * during alloc_io_pgtable_ops

	 */

	arm_smmu_assign_table(smmu_domain);



	/* Update the domain's page sizes to reflect the page table format */

	domain->pgsize_bitmap = smmu_domain->pgtbl_cfg.pgsize_bitmap;
@@ -1576,6 +1622,7 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain) 
		arm_smmu_free_asid(domain);

		free_io_pgtable_ops(smmu_domain->pgtbl_ops);

		arm_smmu_power_off(smmu);

		arm_smmu_unassign_table(smmu_domain);

		return;

	}
@@ -1592,6 +1639,7 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain) 
	}



	free_io_pgtable_ops(smmu_domain->pgtbl_ops);

	arm_smmu_unassign_table(smmu_domain);

	__arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);



	arm_smmu_power_off(smmu);
@@ -1622,6 +1670,9 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) 
	mutex_init(&smmu_domain->init_mutex);

	spin_lock_init(&smmu_domain->pgtbl_lock);

	smmu_domain->cfg.cbndx = INVALID_CBNDX;

	smmu_domain->secure_vmid = VMID_INVAL;

	INIT_LIST_HEAD(&smmu_domain->pte_info_list);

	INIT_LIST_HEAD(&smmu_domain->unassign_list);



	return &smmu_domain->domain;

}
@@ -1803,6 +1854,95 @@ static void arm_smmu_detach_dev(struct iommu_domain *domain, 
	}

}



static void arm_smmu_assign_table(struct arm_smmu_domain *smmu_domain)

{

	int ret;

	int dest_vmids[2] = {VMID_HLOS, smmu_domain->secure_vmid};

	int dest_perms[2] = {PERM_READ | PERM_WRITE, PERM_READ};

	int source_vmid = VMID_HLOS;

	struct arm_smmu_pte_info *pte_info, *temp;



	if (smmu_domain->secure_vmid == VMID_INVAL)

		return;



	list_for_each_entry(pte_info, &smmu_domain->pte_info_list,

								entry) {

		ret = hyp_assign_phys(virt_to_phys(pte_info->virt_addr),

				      PAGE_SIZE, &source_vmid, 1,

				      dest_vmids, dest_perms, 2);

		if (WARN_ON(ret))

			break;

	}



	list_for_each_entry_safe(pte_info, temp, &smmu_domain->pte_info_list,

								entry) {

		list_del(&pte_info->entry);

		kfree(pte_info);

	}

}



static void arm_smmu_unassign_table(struct arm_smmu_domain *smmu_domain)

{

	int ret;

	int dest_vmids = VMID_HLOS;

	int dest_perms = PERM_READ | PERM_WRITE;

	int source_vmlist[2] = {VMID_HLOS, smmu_domain->secure_vmid};

	struct arm_smmu_pte_info *pte_info, *temp;



	if (smmu_domain->secure_vmid == VMID_INVAL)

		return;



	list_for_each_entry(pte_info, &smmu_domain->unassign_list, entry) {

		ret = hyp_assign_phys(virt_to_phys(pte_info->virt_addr),

				      PAGE_SIZE, source_vmlist, 2,

				      &dest_vmids, &dest_perms, 1);

		if (WARN_ON(ret))

			break;

		free_pages_exact(pte_info->virt_addr, pte_info->size);

	}



	list_for_each_entry_safe(pte_info, temp, &smmu_domain->unassign_list,

				 entry) {

		list_del(&pte_info->entry);

		kfree(pte_info);

	}

}



static void arm_smmu_unprepare_pgtable(void *cookie, void *addr, size_t size)

{

	struct arm_smmu_domain *smmu_domain = cookie;

	struct arm_smmu_pte_info *pte_info;



	if (smmu_domain->secure_vmid == VMID_INVAL) {

		free_pages_exact(addr, size);

		return;

	}



	pte_info = kzalloc(sizeof(struct arm_smmu_pte_info), GFP_ATOMIC);

	if (!pte_info)

		return;



	pte_info->virt_addr = addr;

	pte_info->size = size;

	list_add_tail(&pte_info->entry, &smmu_domain->unassign_list);

}



static int arm_smmu_prepare_pgtable(void *addr, void *cookie)

{

	struct arm_smmu_domain *smmu_domain = cookie;

	struct arm_smmu_pte_info *pte_info;



	if (smmu_domain->secure_vmid == VMID_INVAL)

		return -EINVAL;



	pte_info = kzalloc(sizeof(struct arm_smmu_pte_info), GFP_ATOMIC);

	if (!pte_info)

		return -ENOMEM;

	pte_info->virt_addr = addr;

	list_add_tail(&pte_info->entry, &smmu_domain->pte_info_list);

	return 0;

}



static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)

{

	int ret;
@@ -1892,6 +2032,9 @@ static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova, 
	spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);

	ret = ops->map(ops, iova, paddr, size, prot);

	spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);



	arm_smmu_assign_table(smmu_domain);



	return ret;

}
@@ -1915,6 +2058,14 @@ static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, 
	spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);



	arm_smmu_domain_power_off(domain, smmu_domain->smmu);

	/*

	 * While splitting up block mappings, we might allocate page table

	 * memory during unmap, so the vmids needs to be assigned to the

	 * memory here as well.

	 */

	arm_smmu_assign_table(smmu_domain);

	/* Also unassign any pages that were free'd during unmap */

	arm_smmu_unassign_table(smmu_domain);

	return ret;

}
@@ -1942,6 +2093,8 @@ static size_t arm_smmu_map_sg(struct iommu_domain *domain, unsigned long iova, 
		arm_smmu_unmap(domain, iova, size);



	arm_smmu_domain_power_off(domain, smmu_domain->smmu);

	arm_smmu_assign_table(smmu_domain);



	return ret;

}
@@ -2241,6 +2394,10 @@ static int arm_smmu_domain_get_attr(struct iommu_domain *domain, 
				    & (1 << DOMAIN_ATTR_S1_BYPASS));

		ret = 0;

		break;

	case DOMAIN_ATTR_SECURE_VMID:

		*((int *)data) = smmu_domain->secure_vmid;

		ret = 0;

		break;

	default:

		return -ENODEV;

	}
@@ -2355,6 +2512,14 @@ static int arm_smmu_domain_set_attr(struct iommu_domain *domain, 
			smmu_domain->attributes &= ~(1 << DOMAIN_ATTR_ATOMIC);

		break;

	}

	case DOMAIN_ATTR_SECURE_VMID:

		if (smmu_domain->secure_vmid != VMID_INVAL) {

			ret = -ENODEV;

			WARN(1, "secure vmid already set!");

			break;

		}

		smmu_domain->secure_vmid = *((int *)data);

		break;

	default:

		ret = -ENODEV;

	}

drivers/iommu/io-pgtable-arm.c

+14 −9
Original line number Diff line number Diff line
@@ -282,11 +282,12 @@ static dma_addr_t __arm_lpae_dma_addr(void *pages) 
}



static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp,

				    struct io_pgtable_cfg *cfg)

				    struct io_pgtable_cfg *cfg, void *cookie)

{

	struct device *dev = cfg->iommu_dev;

	dma_addr_t dma;

	void *pages = io_pgtable_alloc_pages_exact(size, gfp | __GFP_ZERO);

	void *pages = io_pgtable_alloc_pages_exact(cfg, cookie, size,

						   gfp | __GFP_ZERO);



	if (!pages)

		return NULL;
@@ -310,17 +311,17 @@ static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, 
	dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n");

	dma_unmap_single(dev, dma, size, DMA_TO_DEVICE);

out_free:

	io_pgtable_free_pages_exact(pages, size);

	io_pgtable_free_pages_exact(cfg, cookie, pages, size);

	return NULL;

}



static void __arm_lpae_free_pages(void *pages, size_t size,

				  struct io_pgtable_cfg *cfg)

				  struct io_pgtable_cfg *cfg, void *cookie)

{

	if (!selftest_running)

		dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages),

				 size, DMA_TO_DEVICE);

	io_pgtable_free_pages_exact(pages, size);

	io_pgtable_free_pages_exact(cfg, cookie, pages, size);

}



static void __arm_lpae_set_pte(arm_lpae_iopte *ptep, arm_lpae_iopte pte,
@@ -389,6 +390,7 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, 
	arm_lpae_iopte *cptep, pte;

	size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data);

	struct io_pgtable_cfg *cfg = &data->iop.cfg;

	void *cookie = data->iop.cookie;

	arm_lpae_iopte *pgtable = ptep;



	/* Find our entry at the current level */
@@ -442,7 +444,7 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, 
	pte = *ptep;

	if (!pte) {

		cptep = __arm_lpae_alloc_pages(ARM_LPAE_GRANULE(data),

					       GFP_ATOMIC, cfg);

					       GFP_ATOMIC, cfg, cookie);

		if (!cptep)

			return -ENOMEM;
@@ -607,6 +609,7 @@ static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl, 
{

	arm_lpae_iopte *start, *end;

	unsigned long table_size;

	void *cookie = data->iop.cookie;



	if (lvl == ARM_LPAE_START_LVL(data))

		table_size = data->pgd_size;
@@ -630,7 +633,7 @@ static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl, 
		__arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data));

	}



	__arm_lpae_free_pages(start, table_size, &data->iop.cfg);

	__arm_lpae_free_pages(start, table_size, &data->iop.cfg, cookie);

}



static void arm_lpae_free_pgtable(struct io_pgtable *iop)
@@ -983,7 +986,8 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) 
	cfg->arm_lpae_s1_cfg.mair[1] = 0;



	/* Looking good; allocate a pgd */

	data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg);

	data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL,

					   cfg, cookie);

	if (!data->pgd)

		goto out_free_data;
@@ -1077,7 +1081,8 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) 
	cfg->arm_lpae_s2_cfg.vtcr = reg;



	/* Allocate pgd pages */

	data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg);

	data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL,

					   cfg, cookie);

	if (!data->pgd)

		goto out_free_data;

drivers/iommu/io-pgtable.c

+16 −4
Original line number Diff line number Diff line
@@ -88,18 +88,30 @@ void free_io_pgtable_ops(struct io_pgtable_ops *ops) 


static atomic_t pages_allocated;



void *io_pgtable_alloc_pages_exact(size_t size, gfp_t gfp_mask)

void *io_pgtable_alloc_pages_exact(struct io_pgtable_cfg *cfg, void *cookie,

				   size_t size, gfp_t gfp_mask)

{

	void *ret = alloc_pages_exact(size, gfp_mask);

	void *ret;



	if (cfg->tlb->alloc_pages_exact)

		ret = cfg->tlb->alloc_pages_exact(cookie, size, gfp_mask);

	else

		ret = alloc_pages_exact(size, gfp_mask);



	if (likely(ret))

		atomic_add(1 << get_order(size), &pages_allocated);



	return ret;

}



void io_pgtable_free_pages_exact(void *virt, size_t size)

void io_pgtable_free_pages_exact(struct io_pgtable_cfg *cfg, void *cookie,

				 void *virt, size_t size)

{

	if (cfg->tlb->free_pages_exact)

		cfg->tlb->free_pages_exact(cookie, virt, size);

	else

		free_pages_exact(virt, size);



	atomic_sub(1 << get_order(size), &pages_allocated);

}

drivers/iommu/io-pgtable.h

+10 −2
Original line number Diff line number Diff line
@@ -24,6 +24,10 @@ enum io_pgtable_fmt { 
 * @tlb_sync:      Ensure any queued TLB invalidation has taken effect, and

 *                 any corresponding page table updates are visible to the

 *                 IOMMU.

 * @alloc_pages_exact: Allocate page table memory (optional, defaults to

 *                     alloc_pages_exact)

 * @free_pages_exact:  Free page table memory (optional, defaults to

 *                     free_pages_exact)

 *

 * Note that these can all be called in atomic context and must therefore

 * not block.
@@ -33,6 +37,8 @@ struct iommu_gather_ops { 
	void (*tlb_add_flush)(unsigned long iova, size_t size, size_t granule,

			      bool leaf, void *cookie);

	void (*tlb_sync)(void *cookie);

	void *(*alloc_pages_exact)(void *cookie, size_t size, gfp_t gfp_mask);

	void (*free_pages_exact)(void *cookie, void *virt, size_t size);

};



/**
@@ -222,7 +228,8 @@ extern struct io_pgtable_init_fns io_pgtable_arm_v7s_init_fns; 
 * Like alloc_pages_exact(), but with some additional accounting for debug

 * purposes.

 */

void *io_pgtable_alloc_pages_exact(size_t size, gfp_t gfp_mask);

void *io_pgtable_alloc_pages_exact(struct io_pgtable_cfg *cfg, void *cookie,

				   size_t size, gfp_t gfp_mask);



/**

 * io_pgtable_free_pages_exact:
@@ -233,6 +240,7 @@ void *io_pgtable_alloc_pages_exact(size_t size, gfp_t gfp_mask); 
 * Like free_pages_exact(), but with some additional accounting for debug

 * purposes.

 */

void io_pgtable_free_pages_exact(void *virt, size_t size);

void io_pgtable_free_pages_exact(struct io_pgtable_cfg *cfg, void *cookie,

				 void *virt, size_t size);



#endif /* __IO_PGTABLE_H */