Merge "iommu/arm-smmu: Add support for page table donation" into msm-4.8 (a1b1fc05) · Commits · e / devices / android_kernel_fairphone_FP3

drivers/iommu/arm-smmu.c

+165 −0

Original line number	Diff line number	Diff line
		@@ -44,6 +44,7 @@
		#include <linux/platform_device.h>
		#include <linux/slab.h>
		#include <linux/spinlock.h>
		#include <soc/qcom/secure_buffer.h>
		#include <linux/msm-bus.h>
		#include <dt-bindings/msm/msm-bus-ids.h>

		@@ -454,6 +455,12 @@ enum arm_smmu_domain_stage {
		ARM_SMMU_DOMAIN_NESTED,
		};

		struct arm_smmu_pte_info {
		void *virt_addr;
		size_t size;
		struct list_head entry;
		};

		struct arm_smmu_domain {
		struct arm_smmu_device *smmu;
		struct io_pgtable_ops *pgtbl_ops;
		@@ -463,6 +470,9 @@ struct arm_smmu_domain {
		enum arm_smmu_domain_stage stage;
		struct mutex init_mutex; /* Protects smmu pointer */
		u32 attributes;
		u32 secure_vmid;
		struct list_head pte_info_list;
		struct list_head unassign_list;
		struct iommu_domain domain;
		};

		@@ -502,6 +512,11 @@ static phys_addr_t arm_smmu_iova_to_phys_hard_no_halt(
		struct iommu_domain *domain, dma_addr_t iova);
		static void arm_smmu_destroy_domain_context(struct iommu_domain *domain);

		static int arm_smmu_prepare_pgtable(void addr, void cookie);
		static void arm_smmu_unprepare_pgtable(void cookie, void addr, size_t size);
		static void arm_smmu_assign_table(struct arm_smmu_domain *smmu_domain);
		static void arm_smmu_unassign_table(struct arm_smmu_domain *smmu_domain);

		static struct arm_smmu_domain to_smmu_domain(struct iommu_domain dom)
		{
		return container_of(dom, struct arm_smmu_domain, domain);
		@@ -1002,10 +1017,35 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
		}
		}

		static void arm_smmu_alloc_pages_exact(void cookie,
		size_t size, gfp_t gfp_mask)
		{
		int ret;
		void *page = alloc_pages_exact(size, gfp_mask);

		if (likely(page)) {
		ret = arm_smmu_prepare_pgtable(page, cookie);
		if (ret) {
		free_pages_exact(page, size);
		return NULL;
		}
		}

		return page;
		}

		static void arm_smmu_free_pages_exact(void cookie, void virt, size_t size)
		{
		arm_smmu_unprepare_pgtable(cookie, virt, size);
		/* unprepare also frees (possibly later), no need to free here */
		}

		static struct iommu_gather_ops arm_smmu_gather_ops = {
		.tlb_flush_all = arm_smmu_tlb_inv_context,
		.tlb_add_flush = arm_smmu_tlb_inv_range_nosync,
		.tlb_sync = arm_smmu_tlb_sync,
		.alloc_pages_exact = arm_smmu_alloc_pages_exact,
		.free_pages_exact = arm_smmu_free_pages_exact,
		};

		static phys_addr_t arm_smmu_verify_fault(struct iommu_domain *domain,
		@@ -1507,6 +1547,12 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
		goto out_clear_smmu;
		}

		/*
		* assign any page table memory that might have been allocated
		* during alloc_io_pgtable_ops
		*/
		arm_smmu_assign_table(smmu_domain);

		/* Update the domain's page sizes to reflect the page table format */
		domain->pgsize_bitmap = smmu_domain->pgtbl_cfg.pgsize_bitmap;

		@@ -1576,6 +1622,7 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
		arm_smmu_free_asid(domain);
		free_io_pgtable_ops(smmu_domain->pgtbl_ops);
		arm_smmu_power_off(smmu);
		arm_smmu_unassign_table(smmu_domain);
		return;
		}

		@@ -1592,6 +1639,7 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
		}

		free_io_pgtable_ops(smmu_domain->pgtbl_ops);
		arm_smmu_unassign_table(smmu_domain);
		__arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);

		arm_smmu_power_off(smmu);
		@@ -1622,6 +1670,9 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
		mutex_init(&smmu_domain->init_mutex);
		spin_lock_init(&smmu_domain->pgtbl_lock);
		smmu_domain->cfg.cbndx = INVALID_CBNDX;
		smmu_domain->secure_vmid = VMID_INVAL;
		INIT_LIST_HEAD(&smmu_domain->pte_info_list);
		INIT_LIST_HEAD(&smmu_domain->unassign_list);

		return &smmu_domain->domain;
		}
		@@ -1803,6 +1854,95 @@ static void arm_smmu_detach_dev(struct iommu_domain *domain,
		}
		}

		static void arm_smmu_assign_table(struct arm_smmu_domain *smmu_domain)
		{
		int ret;
		int dest_vmids[2] = {VMID_HLOS, smmu_domain->secure_vmid};
		int dest_perms[2] = {PERM_READ \| PERM_WRITE, PERM_READ};
		int source_vmid = VMID_HLOS;
		struct arm_smmu_pte_info pte_info, temp;

		if (smmu_domain->secure_vmid == VMID_INVAL)
		return;

		list_for_each_entry(pte_info, &smmu_domain->pte_info_list,
		entry) {
		ret = hyp_assign_phys(virt_to_phys(pte_info->virt_addr),
		PAGE_SIZE, &source_vmid, 1,
		dest_vmids, dest_perms, 2);
		if (WARN_ON(ret))
		break;
		}

		list_for_each_entry_safe(pte_info, temp, &smmu_domain->pte_info_list,
		entry) {
		list_del(&pte_info->entry);
		kfree(pte_info);
		}
		}

		static void arm_smmu_unassign_table(struct arm_smmu_domain *smmu_domain)
		{
		int ret;
		int dest_vmids = VMID_HLOS;
		int dest_perms = PERM_READ \| PERM_WRITE;
		int source_vmlist[2] = {VMID_HLOS, smmu_domain->secure_vmid};
		struct arm_smmu_pte_info pte_info, temp;

		if (smmu_domain->secure_vmid == VMID_INVAL)
		return;

		list_for_each_entry(pte_info, &smmu_domain->unassign_list, entry) {
		ret = hyp_assign_phys(virt_to_phys(pte_info->virt_addr),
		PAGE_SIZE, source_vmlist, 2,
		&dest_vmids, &dest_perms, 1);
		if (WARN_ON(ret))
		break;
		free_pages_exact(pte_info->virt_addr, pte_info->size);
		}

		list_for_each_entry_safe(pte_info, temp, &smmu_domain->unassign_list,
		entry) {
		list_del(&pte_info->entry);
		kfree(pte_info);
		}
		}

		static void arm_smmu_unprepare_pgtable(void cookie, void addr, size_t size)
		{
		struct arm_smmu_domain *smmu_domain = cookie;
		struct arm_smmu_pte_info *pte_info;

		if (smmu_domain->secure_vmid == VMID_INVAL) {
		free_pages_exact(addr, size);
		return;
		}

		pte_info = kzalloc(sizeof(struct arm_smmu_pte_info), GFP_ATOMIC);
		if (!pte_info)
		return;

		pte_info->virt_addr = addr;
		pte_info->size = size;
		list_add_tail(&pte_info->entry, &smmu_domain->unassign_list);
		}

		static int arm_smmu_prepare_pgtable(void addr, void cookie)
		{
		struct arm_smmu_domain *smmu_domain = cookie;
		struct arm_smmu_pte_info *pte_info;

		if (smmu_domain->secure_vmid == VMID_INVAL)
		return -EINVAL;

		pte_info = kzalloc(sizeof(struct arm_smmu_pte_info), GFP_ATOMIC);
		if (!pte_info)
		return -ENOMEM;
		pte_info->virt_addr = addr;
		list_add_tail(&pte_info->entry, &smmu_domain->pte_info_list);
		return 0;
		}

		static int arm_smmu_attach_dev(struct iommu_domain domain, struct device dev)
		{
		int ret;
		@@ -1892,6 +2032,9 @@ static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
		spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
		ret = ops->map(ops, iova, paddr, size, prot);
		spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);

		arm_smmu_assign_table(smmu_domain);

		return ret;
		}

		@@ -1915,6 +2058,14 @@ static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
		spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);

		arm_smmu_domain_power_off(domain, smmu_domain->smmu);
		/*
		* While splitting up block mappings, we might allocate page table
		* memory during unmap, so the vmids needs to be assigned to the
		* memory here as well.
		*/
		arm_smmu_assign_table(smmu_domain);
		/* Also unassign any pages that were free'd during unmap */
		arm_smmu_unassign_table(smmu_domain);
		return ret;
		}

		@@ -1942,6 +2093,8 @@ static size_t arm_smmu_map_sg(struct iommu_domain *domain, unsigned long iova,
		arm_smmu_unmap(domain, iova, size);

		arm_smmu_domain_power_off(domain, smmu_domain->smmu);
		arm_smmu_assign_table(smmu_domain);

		return ret;
		}

		@@ -2241,6 +2394,10 @@ static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
		& (1 << DOMAIN_ATTR_S1_BYPASS));
		ret = 0;
		break;
		case DOMAIN_ATTR_SECURE_VMID:
		((int )data) = smmu_domain->secure_vmid;
		ret = 0;
		break;
		default:
		return -ENODEV;
		}
		@@ -2355,6 +2512,14 @@ static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
		smmu_domain->attributes &= ~(1 << DOMAIN_ATTR_ATOMIC);
		break;
		}
		case DOMAIN_ATTR_SECURE_VMID:
		if (smmu_domain->secure_vmid != VMID_INVAL) {
		ret = -ENODEV;
		WARN(1, "secure vmid already set!");
		break;
		}
		smmu_domain->secure_vmid = ((int )data);
		break;
		default:
		ret = -ENODEV;
		}

drivers/iommu/io-pgtable-arm.c

+14 −9

Original line number	Diff line number	Diff line
		@@ -282,11 +282,12 @@ static dma_addr_t __arm_lpae_dma_addr(void *pages)
		}

		static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp,
		struct io_pgtable_cfg *cfg)
		struct io_pgtable_cfg cfg, void cookie)
		{
		struct device *dev = cfg->iommu_dev;
		dma_addr_t dma;
		void *pages = io_pgtable_alloc_pages_exact(size, gfp \| __GFP_ZERO);
		void *pages = io_pgtable_alloc_pages_exact(cfg, cookie, size,
		gfp \| __GFP_ZERO);

		if (!pages)
		return NULL;
		@@ -310,17 +311,17 @@ static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp,
		dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n");
		dma_unmap_single(dev, dma, size, DMA_TO_DEVICE);
		out_free:
		io_pgtable_free_pages_exact(pages, size);
		io_pgtable_free_pages_exact(cfg, cookie, pages, size);
		return NULL;
		}

		static void __arm_lpae_free_pages(void *pages, size_t size,
		struct io_pgtable_cfg *cfg)
		struct io_pgtable_cfg cfg, void cookie)
		{
		if (!selftest_running)
		dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages),
		size, DMA_TO_DEVICE);
		io_pgtable_free_pages_exact(pages, size);
		io_pgtable_free_pages_exact(cfg, cookie, pages, size);
		}

		static void __arm_lpae_set_pte(arm_lpae_iopte *ptep, arm_lpae_iopte pte,
		@@ -389,6 +390,7 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
		arm_lpae_iopte *cptep, pte;
		size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data);
		struct io_pgtable_cfg *cfg = &data->iop.cfg;
		void *cookie = data->iop.cookie;
		arm_lpae_iopte *pgtable = ptep;

		/* Find our entry at the current level */
		@@ -442,7 +444,7 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
		pte = *ptep;
		if (!pte) {
		cptep = __arm_lpae_alloc_pages(ARM_LPAE_GRANULE(data),
		GFP_ATOMIC, cfg);
		GFP_ATOMIC, cfg, cookie);
		if (!cptep)
		return -ENOMEM;

		@@ -607,6 +609,7 @@ static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,
		{
		arm_lpae_iopte start, end;
		unsigned long table_size;
		void *cookie = data->iop.cookie;

		if (lvl == ARM_LPAE_START_LVL(data))
		table_size = data->pgd_size;
		@@ -630,7 +633,7 @@ static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,
		__arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data));
		}

		__arm_lpae_free_pages(start, table_size, &data->iop.cfg);
		__arm_lpae_free_pages(start, table_size, &data->iop.cfg, cookie);
		}

		static void arm_lpae_free_pgtable(struct io_pgtable *iop)
		@@ -983,7 +986,8 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg cfg, void cookie)
		cfg->arm_lpae_s1_cfg.mair[1] = 0;

		/* Looking good; allocate a pgd */
		data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg);
		data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL,
		cfg, cookie);
		if (!data->pgd)
		goto out_free_data;

		@@ -1077,7 +1081,8 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg cfg, void cookie)
		cfg->arm_lpae_s2_cfg.vtcr = reg;

		/* Allocate pgd pages */
		data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg);
		data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL,
		cfg, cookie);
		if (!data->pgd)
		goto out_free_data;

drivers/iommu/io-pgtable.c

+16 −4

Original line number	Diff line number	Diff line
		@@ -88,18 +88,30 @@ void free_io_pgtable_ops(struct io_pgtable_ops *ops)

		static atomic_t pages_allocated;

		void *io_pgtable_alloc_pages_exact(size_t size, gfp_t gfp_mask)
		void io_pgtable_alloc_pages_exact(struct io_pgtable_cfg cfg, void *cookie,
		size_t size, gfp_t gfp_mask)
		{
		void *ret = alloc_pages_exact(size, gfp_mask);
		void *ret;

		if (cfg->tlb->alloc_pages_exact)
		ret = cfg->tlb->alloc_pages_exact(cookie, size, gfp_mask);
		else
		ret = alloc_pages_exact(size, gfp_mask);

		if (likely(ret))
		atomic_add(1 << get_order(size), &pages_allocated);

		return ret;
		}

		void io_pgtable_free_pages_exact(void *virt, size_t size)
		void io_pgtable_free_pages_exact(struct io_pgtable_cfg cfg, void cookie,
		void *virt, size_t size)
		{
		if (cfg->tlb->free_pages_exact)
		cfg->tlb->free_pages_exact(cookie, virt, size);
		else
		free_pages_exact(virt, size);

		atomic_sub(1 << get_order(size), &pages_allocated);
		}

drivers/iommu/io-pgtable.h

+10 −2

Original line number	Diff line number	Diff line
		@@ -24,6 +24,10 @@ enum io_pgtable_fmt {
		* @tlb_sync: Ensure any queued TLB invalidation has taken effect, and
		* any corresponding page table updates are visible to the
		* IOMMU.
		* @alloc_pages_exact: Allocate page table memory (optional, defaults to
		* alloc_pages_exact)
		* @free_pages_exact: Free page table memory (optional, defaults to
		* free_pages_exact)
		*
		* Note that these can all be called in atomic context and must therefore
		* not block.
		@@ -33,6 +37,8 @@ struct iommu_gather_ops {
		void (*tlb_add_flush)(unsigned long iova, size_t size, size_t granule,
		bool leaf, void *cookie);
		void (tlb_sync)(void cookie);
		void (alloc_pages_exact)(void *cookie, size_t size, gfp_t gfp_mask);
		void (free_pages_exact)(void cookie, void *virt, size_t size);
		};

		/**
		@@ -222,7 +228,8 @@ extern struct io_pgtable_init_fns io_pgtable_arm_v7s_init_fns;
		* Like alloc_pages_exact(), but with some additional accounting for debug
		* purposes.
		*/
		void *io_pgtable_alloc_pages_exact(size_t size, gfp_t gfp_mask);
		void io_pgtable_alloc_pages_exact(struct io_pgtable_cfg cfg, void *cookie,
		size_t size, gfp_t gfp_mask);

		/**
		* io_pgtable_free_pages_exact:
		@@ -233,6 +240,7 @@ void *io_pgtable_alloc_pages_exact(size_t size, gfp_t gfp_mask);
		* Like free_pages_exact(), but with some additional accounting for debug
		* purposes.
		*/
		void io_pgtable_free_pages_exact(void *virt, size_t size);
		void io_pgtable_free_pages_exact(struct io_pgtable_cfg cfg, void cookie,
		void *virt, size_t size);

		#endif /* __IO_PGTABLE_H */