Loading Documentation/devicetree/bindings/iommu/arm,smmu.txt +9 −0 Original line number Diff line number Diff line Loading @@ -118,6 +118,15 @@ conditions. a call back notifier on regulators in whcih SMMU can be halted or resumed when regulator is powered down/up. - qcom,enable-static-cb : Enables option to use pre-defined static context bank allocation programmed by TZ. Global register including SMR and S2CR registers are configured by TZ before kernel comes up and this programming is not altered throughout the life of system. We would be reading through these registers at run time to identify CB allocated for a particular sid. SID masking isn't supported as we are directly comparing client SID with ID bits of SMR registers. - clocks : List of clocks to be used during SMMU register access. See Documentation/devicetree/bindings/clock/clock-bindings.txt for information about the format. For each clock specified Loading drivers/iommu/Makefile +1 −0 Original line number Diff line number Diff line Loading @@ -4,6 +4,7 @@ obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o obj-$(CONFIG_IOMMU_API) += msm_dma_iommu_mapping.o obj-$(CONFIG_IOMMU_IO_PGTABLE) += io-pgtable.o obj-$(CONFIG_IOMMU_IO_PGTABLE_LPAE) += io-pgtable-arm.o obj-$(CONFIG_MSM_TZ_SMMU) += io-pgtable-msm-secure.o obj-$(CONFIG_OF_IOMMU) += of_iommu.o obj-$(CONFIG_IOMMU_DEBUG) += iommu-debug.o obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_domains.o msm_iommu_mapping.o Loading drivers/iommu/arm-smmu.c +169 −24 Original line number Diff line number Diff line Loading @@ -45,6 +45,7 @@ #include <linux/amba/bus.h> #include <soc/qcom/msm_tz_smmu.h> #include <soc/qcom/scm.h> #include <soc/qcom/secure_buffer.h> #include <linux/msm_pcie.h> #include <asm/cacheflush.h> Loading Loading @@ -361,6 +362,7 @@ struct arm_smmu_device { #define ARM_SMMU_OPT_NO_SMR_CHECK (1 << 9) #define ARM_SMMU_OPT_DYNAMIC (1 << 10) #define ARM_SMMU_OPT_HALT (1 << 11) #define ARM_SMMU_OPT_STATIC_CB (1 << 12) u32 options; enum arm_smmu_arch_version version; Loading Loading @@ -403,6 +405,8 @@ struct arm_smmu_device { struct msm_bus_client_handle *bus_client; char *bus_client_name; enum tz_smmu_device_id sec_id; }; struct arm_smmu_cfg { Loading Loading @@ -442,7 +446,8 @@ struct arm_smmu_domain { struct arm_smmu_device *smmu; struct io_pgtable_ops *pgtbl_ops; struct io_pgtable_cfg pgtbl_cfg; spinlock_t pgtbl_lock; spinlock_t pgtbl_spin_lock; struct mutex pgtbl_mutex_lock; struct arm_smmu_cfg cfg; enum arm_smmu_domain_stage stage; struct mutex init_mutex; /* Protects smmu pointer */ Loading Loading @@ -478,6 +483,7 @@ static struct arm_smmu_option_prop arm_smmu_options[] = { { ARM_SMMU_OPT_NO_SMR_CHECK, "qcom,no-smr-check" }, { ARM_SMMU_OPT_DYNAMIC, "qcom,dynamic" }, { ARM_SMMU_OPT_HALT, "qcom,enable-smmu-halt"}, { ARM_SMMU_OPT_STATIC_CB, "qcom,enable-static-cb"}, { 0, NULL}, }; Loading @@ -501,7 +507,9 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu); static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size); static bool arm_smmu_is_master_side_secure(struct arm_smmu_domain *smmu_domain); static bool arm_smmu_is_static_cb(struct arm_smmu_device *smmu); static bool arm_smmu_is_slave_side_secure(struct arm_smmu_domain *smmu_domain); static bool arm_smmu_has_secure_vmid(struct arm_smmu_domain *smmu_domain); static void parse_driver_options(struct arm_smmu_device *smmu) { Loading Loading @@ -1469,11 +1477,28 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR); } static bool arm_smmu_is_master_side_secure(struct arm_smmu_domain *smmu_domain) static bool arm_smmu_is_static_cb(struct arm_smmu_device *smmu) { return smmu->options & ARM_SMMU_OPT_STATIC_CB; } static bool arm_smmu_has_secure_vmid(struct arm_smmu_domain *smmu_domain) { return smmu_domain->secure_vmid != VMID_INVAL; } static bool arm_smmu_is_slave_side_secure(struct arm_smmu_domain *smmu_domain) { return arm_smmu_has_secure_vmid(smmu_domain) && arm_smmu_is_static_cb(smmu_domain->smmu); } static bool arm_smmu_is_master_side_secure(struct arm_smmu_domain *smmu_domain) { return arm_smmu_has_secure_vmid(smmu_domain) && !arm_smmu_is_static_cb(smmu_domain->smmu); } static void arm_smmu_secure_domain_lock(struct arm_smmu_domain *smmu_domain) { if (arm_smmu_is_master_side_secure(smmu_domain)) Loading @@ -1486,6 +1511,27 @@ static void arm_smmu_secure_domain_unlock(struct arm_smmu_domain *smmu_domain) mutex_unlock(&smmu_domain->assign_lock); } static unsigned long arm_smmu_pgtbl_lock(struct arm_smmu_domain *smmu_domain) { unsigned long flags; if (arm_smmu_is_slave_side_secure(smmu_domain)) mutex_lock(&smmu_domain->pgtbl_mutex_lock); else spin_lock_irqsave(&smmu_domain->pgtbl_spin_lock, flags); return flags; } static void arm_smmu_pgtbl_unlock(struct arm_smmu_domain *smmu_domain, unsigned long flags) { if (arm_smmu_is_slave_side_secure(smmu_domain)) mutex_unlock(&smmu_domain->pgtbl_mutex_lock); else spin_unlock_irqrestore(&smmu_domain->pgtbl_spin_lock, flags); } static int arm_smmu_init_domain_context(struct iommu_domain *domain, struct arm_smmu_device *smmu) { Loading Loading @@ -1553,12 +1599,17 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, goto out; } if (cfg->cbndx == INVALID_CBNDX) { ret = __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks); if (IS_ERR_VALUE(ret)) goto out; cfg->cbndx = ret; } else { if (test_and_set_bit(cfg->cbndx, smmu->context_map)) goto out; } if (smmu->version == ARM_SMMU_V1) { cfg->irptndx = atomic_inc_return(&smmu->irptndx); cfg->irptndx %= smmu->num_context_irqs; Loading @@ -1566,16 +1617,28 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, cfg->irptndx = cfg->cbndx; } smmu_domain->smmu = smmu; if (arm_smmu_is_slave_side_secure(smmu_domain)) { smmu_domain->pgtbl_cfg = (struct io_pgtable_cfg) { .pgsize_bitmap = arm_smmu_ops.pgsize_bitmap, .arm_msm_secure_cfg = { .sec_id = smmu->sec_id, .cbndx = cfg->cbndx, }, }; fmt = ARM_MSM_SECURE; } else { smmu_domain->pgtbl_cfg = (struct io_pgtable_cfg) { .pgsize_bitmap = arm_smmu_ops.pgsize_bitmap, .ias = ias, .oas = oas, .tlb = &arm_smmu_gather_ops, }; } cfg->asid = cfg->cbndx + 1; cfg->vmid = cfg->cbndx + 2; smmu_domain->smmu = smmu; pgtbl_ops = alloc_io_pgtable_ops(fmt, &smmu_domain->pgtbl_cfg, smmu_domain); if (!pgtbl_ops) { Loading Loading @@ -1678,8 +1741,9 @@ static int arm_smmu_domain_init(struct iommu_domain *domain) smmu_domain->cfg.vmid = INVALID_VMID; mutex_init(&smmu_domain->init_mutex); spin_lock_init(&smmu_domain->pgtbl_lock); spin_lock_init(&smmu_domain->pgtbl_spin_lock); mutex_init(&smmu_domain->assign_lock); mutex_init(&smmu_domain->pgtbl_mutex_lock); domain->priv = smmu_domain; return 0; } Loading Loading @@ -1929,6 +1993,40 @@ out: return ret; } static int arm_smmu_populate_cb(struct arm_smmu_device *smmu, struct arm_smmu_domain *smmu_domain, struct device *dev) { void __iomem *gr0_base; struct arm_smmu_master_cfg *cfg; struct arm_smmu_cfg *smmu_cfg = &smmu_domain->cfg; int i; u32 sid; gr0_base = ARM_SMMU_GR0(smmu); cfg = find_smmu_master_cfg(dev); if (!cfg) return -ENODEV; sid = cfg->streamids[0]; for (i = 0; i < smmu->num_mapping_groups; i++) { u32 smr, s2cr; u8 cbndx; smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(i)); if (sid == ((smr >> SMR_ID_SHIFT) & SMR_ID_MASK)) { s2cr = readl_relaxed(gr0_base + ARM_SMMU_GR0_S2CR(i)); cbndx = (s2cr >> S2CR_CBNDX_SHIFT) & S2CR_CBNDX_MASK; smmu_cfg->cbndx = cbndx; return 0; } } return -EINVAL; } static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) { int ret; Loading Loading @@ -1987,6 +2085,15 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) } smmu->attach_count++; if (arm_smmu_is_static_cb(smmu)) { ret = arm_smmu_populate_cb(smmu, smmu_domain, dev); if (ret) { dev_err(dev, "Failed to get valid context bank\n"); goto err_disable_clocks; } } /* Ensure that the domain is finalised */ ret = arm_smmu_init_domain_context(domain, smmu); if (IS_ERR_VALUE(ret)) Loading Loading @@ -2215,9 +2322,9 @@ static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova, arm_smmu_secure_domain_lock(smmu_domain); spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags); flags = arm_smmu_pgtbl_lock(smmu_domain); ret = ops->map(ops, iova, paddr, size, prot); spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags); arm_smmu_pgtbl_unlock(smmu_domain, flags); if (!ret) ret = arm_smmu_assign_table(smmu_domain); Loading @@ -2235,15 +2342,31 @@ static size_t arm_smmu_map_sg(struct iommu_domain *domain, unsigned long iova, unsigned long flags; struct arm_smmu_domain *smmu_domain = domain->priv; struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; struct arm_smmu_device *smmu = smmu_domain->smmu; int atomic_ctx = smmu_domain->attributes & (1 << DOMAIN_ATTR_ATOMIC); if (!ops) return -ENODEV; if (arm_smmu_is_slave_side_secure(smmu_domain) && atomic_ctx) { dev_err(smmu->dev, "Slave side atomic context not supported\n"); return 0; } if (arm_smmu_is_slave_side_secure(smmu_domain)) { mutex_lock(&smmu_domain->init_mutex); if (arm_smmu_enable_clocks(smmu)) { mutex_unlock(&smmu_domain->init_mutex); return 0; } } arm_smmu_secure_domain_lock(smmu_domain); spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags); flags = arm_smmu_pgtbl_lock(smmu_domain); ret = ops->map_sg(ops, iova, sg, nents, prot, &size); spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags); arm_smmu_pgtbl_unlock(smmu_domain, flags); if (ret) { if (arm_smmu_assign_table(smmu_domain)) { Loading @@ -2257,6 +2380,10 @@ static size_t arm_smmu_map_sg(struct iommu_domain *domain, unsigned long iova, out: arm_smmu_secure_domain_unlock(smmu_domain); if (arm_smmu_is_slave_side_secure(smmu_domain)) { arm_smmu_disable_clocks(smmu_domain->smmu); mutex_unlock(&smmu_domain->init_mutex); } return ret; } Loading @@ -2272,6 +2399,12 @@ static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, if (!ops) return 0; if (arm_smmu_is_slave_side_secure(smmu_domain) && atomic_ctx) { dev_err(smmu_domain->smmu->dev, "Slave side atomic context not supported\n"); return 0; } /* * The contract here is that if you set DOMAIN_ATTR_ATOMIC your * domain *must* must be attached an SMMU during unmap. This Loading Loading @@ -2299,9 +2432,9 @@ static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, } } spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags); flags = arm_smmu_pgtbl_lock(smmu_domain); ret = ops->unmap(ops, iova, size); spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags); arm_smmu_pgtbl_unlock(smmu_domain, flags); /* * While splitting up block mappings, we might allocate page table Loading Loading @@ -2341,9 +2474,9 @@ static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain, if (!ops) return 0; spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags); flags = arm_smmu_pgtbl_lock(smmu_domain); ret = ops->iova_to_phys(ops, iova); spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags); arm_smmu_pgtbl_unlock(smmu_domain, flags); return ret; } Loading Loading @@ -2389,6 +2522,16 @@ static void arm_smmu_resume(struct arm_smmu_device *smmu) void __iomem *impl_def1_base = ARM_SMMU_IMPL_DEF1(smmu); u32 reg; if (arm_smmu_is_static_cb(smmu)) { int ret, scm_ret; ret = scm_restore_sec_cfg(smmu->sec_id, 0x0, &scm_ret); if (ret || scm_ret) { pr_err("scm call IOMMU_SECURE_CFG failed\n"); return; } } reg = readl_relaxed(impl_def1_base + IMPL_DEF1_MICRO_MMU_CTRL); reg &= ~MICRO_MMU_CTRL_LOCAL_HALT_REQ; writel_relaxed(reg, impl_def1_base + IMPL_DEF1_MICRO_MMU_CTRL); Loading Loading @@ -3376,6 +3519,8 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) err = arm_smmu_enable_clocks(smmu); if (err) goto out_put_masters; smmu->sec_id = msm_dev_to_device_id(dev); err = arm_smmu_device_cfg_probe(smmu); arm_smmu_disable_clocks(smmu); if (err) Loading drivers/iommu/io-pgtable-msm-secure.c 0 → 100644 +243 −0 Original line number Diff line number Diff line /* Copyright (c) 2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and * only version 2 as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. */ #define pr_fmt(fmt) "io-pgtable-msm-secure: " fmt #include <linux/iommu.h> #include <linux/kernel.h> #include <linux/scatterlist.h> #include <linux/sizes.h> #include <linux/slab.h> #include <linux/types.h> #include <soc/qcom/scm.h> #include <asm/cacheflush.h> #include "io-pgtable.h" #define IOMMU_SECURE_MAP2_FLAT 0x12 #define IOMMU_SECURE_UNMAP2_FLAT 0x13 #define IOMMU_TLBINVAL_FLAG 0x00000001 #define io_pgtable_to_data(x) \ container_of((x), struct msm_secure_io_pgtable, iop) #define io_pgtable_ops_to_pgtable(x) \ container_of((x), struct io_pgtable, ops) #define io_pgtable_ops_to_data(x) \ io_pgtable_to_data(io_pgtable_ops_to_pgtable(x)) struct msm_secure_io_pgtable { struct io_pgtable iop; }; static int msm_secure_map(struct io_pgtable_ops *ops, unsigned long iova, phys_addr_t paddr, size_t size, int iommu_prot) { return -EINVAL; } static dma_addr_t msm_secure_get_phys_addr(struct scatterlist *sg) { /* * Try sg_dma_address first so that we can * map carveout regions that do not have a * struct page associated with them. */ dma_addr_t pa = sg_dma_address(sg); if (pa == 0) pa = sg_phys(sg); return pa; } static int msm_secure_map_sg(struct io_pgtable_ops *ops, unsigned long iova, struct scatterlist *sg, unsigned int nents, int iommu_prot, size_t *size) { struct msm_secure_io_pgtable *data = io_pgtable_ops_to_data(ops); struct io_pgtable_cfg *cfg = &data->iop.cfg; int ret = -EINVAL; struct scatterlist *tmp, *sgiter; dma_addr_t *pa_list = 0; unsigned int cnt, offset = 0, chunk_offset = 0; dma_addr_t pa; void *flush_va, *flush_va_end; unsigned long len = 0; struct scm_desc desc = {0}; int i; u32 resp; for_each_sg(sg, tmp, nents, i) len += tmp->length; if (!IS_ALIGNED(iova, SZ_1M) || !IS_ALIGNED(len, SZ_1M)) return -EINVAL; if (sg->length == len) { cnt = 1; pa = msm_secure_get_phys_addr(sg); if (!IS_ALIGNED(pa, SZ_1M)) return -EINVAL; desc.args[0] = virt_to_phys(&pa); desc.args[1] = cnt; desc.args[2] = len; flush_va = &pa; } else { sgiter = sg; if (!IS_ALIGNED(sgiter->length, SZ_1M)) return -EINVAL; cnt = sg->length / SZ_1M; while ((sgiter = sg_next(sgiter))) { if (!IS_ALIGNED(sgiter->length, SZ_1M)) return -EINVAL; cnt += sgiter->length / SZ_1M; } pa_list = kmalloc_array(cnt, sizeof(*pa_list), GFP_KERNEL); if (!pa_list) return -ENOMEM; sgiter = sg; cnt = 0; pa = msm_secure_get_phys_addr(sgiter); while (offset < len) { if (!IS_ALIGNED(pa, SZ_1M)) { kfree(pa_list); return -EINVAL; } pa_list[cnt] = pa + chunk_offset; chunk_offset += SZ_1M; offset += SZ_1M; cnt++; if (chunk_offset >= sgiter->length && offset < len) { chunk_offset = 0; sgiter = sg_next(sgiter); pa = msm_secure_get_phys_addr(sgiter); } } desc.args[0] = virt_to_phys(pa_list); desc.args[1] = cnt; desc.args[2] = SZ_1M; flush_va = pa_list; } desc.args[3] = cfg->arm_msm_secure_cfg.sec_id; desc.args[4] = cfg->arm_msm_secure_cfg.cbndx; desc.args[5] = iova; desc.args[6] = len; desc.args[7] = 0; desc.arginfo = SCM_ARGS(8, SCM_RW, SCM_VAL, SCM_VAL, SCM_VAL, SCM_VAL, SCM_VAL, SCM_VAL, SCM_VAL); /* * Ensure that the buffer is in RAM by the time it gets to TZ */ flush_va_end = (void *) (((unsigned long) flush_va) + (cnt * sizeof(*pa_list))); dmac_clean_range(flush_va, flush_va_end); if (is_scm_armv8()) { ret = scm_call2(SCM_SIP_FNID(SCM_SVC_MP, IOMMU_SECURE_MAP2_FLAT), &desc); resp = desc.ret[0]; if (ret || resp) ret = -EINVAL; else ret = len; } kfree(pa_list); return ret; } static size_t msm_secure_unmap(struct io_pgtable_ops *ops, unsigned long iova, size_t len) { struct msm_secure_io_pgtable *data = io_pgtable_ops_to_data(ops); struct io_pgtable_cfg *cfg = &data->iop.cfg; int ret = -EINVAL; struct scm_desc desc = {0}; if (!IS_ALIGNED(iova, SZ_1M) || !IS_ALIGNED(len, SZ_1M)) return ret; desc.args[0] = cfg->arm_msm_secure_cfg.sec_id; desc.args[1] = cfg->arm_msm_secure_cfg.cbndx; desc.args[2] = iova; desc.args[3] = len; desc.args[4] = IOMMU_TLBINVAL_FLAG; desc.arginfo = SCM_ARGS(5); if (is_scm_armv8()) { ret = scm_call2(SCM_SIP_FNID(SCM_SVC_MP, IOMMU_SECURE_UNMAP2_FLAT), &desc); if (!ret) ret = len; } return ret; } static phys_addr_t msm_secure_iova_to_phys(struct io_pgtable_ops *ops, unsigned long iova) { return -EINVAL; } static struct msm_secure_io_pgtable * msm_secure_alloc_pgtable_data(struct io_pgtable_cfg *cfg) { struct msm_secure_io_pgtable *data; data = kmalloc(sizeof(*data), GFP_KERNEL); if (!data) return NULL; data->iop.ops = (struct io_pgtable_ops) { .map = msm_secure_map, .map_sg = msm_secure_map_sg, .unmap = msm_secure_unmap, .iova_to_phys = msm_secure_iova_to_phys, }; return data; } static struct io_pgtable * msm_secure_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) { struct msm_secure_io_pgtable *data = msm_secure_alloc_pgtable_data(cfg); return &data->iop; } static void msm_secure_free_pgtable(struct io_pgtable *iop) { struct msm_secure_io_pgtable *data = io_pgtable_to_data(iop); kfree(data); } struct io_pgtable_init_fns io_pgtable_arm_msm_secure_init_fns = { .alloc = msm_secure_alloc_pgtable, .free = msm_secure_free_pgtable, }; drivers/iommu/io-pgtable.c +4 −0 Original line number Diff line number Diff line Loading @@ -33,6 +33,7 @@ extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns; extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns; extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns; extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns; extern struct io_pgtable_init_fns io_pgtable_arm_msm_secure_init_fns; static const struct io_pgtable_init_fns * io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = Loading @@ -43,6 +44,9 @@ io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = [ARM_64_LPAE_S1] = &io_pgtable_arm_64_lpae_s1_init_fns, [ARM_64_LPAE_S2] = &io_pgtable_arm_64_lpae_s2_init_fns, #endif #ifdef CONFIG_MSM_TZ_SMMU [ARM_MSM_SECURE] = &io_pgtable_arm_msm_secure_init_fns, #endif }; static struct dentry *io_pgtable_top; Loading Loading
Documentation/devicetree/bindings/iommu/arm,smmu.txt +9 −0 Original line number Diff line number Diff line Loading @@ -118,6 +118,15 @@ conditions. a call back notifier on regulators in whcih SMMU can be halted or resumed when regulator is powered down/up. - qcom,enable-static-cb : Enables option to use pre-defined static context bank allocation programmed by TZ. Global register including SMR and S2CR registers are configured by TZ before kernel comes up and this programming is not altered throughout the life of system. We would be reading through these registers at run time to identify CB allocated for a particular sid. SID masking isn't supported as we are directly comparing client SID with ID bits of SMR registers. - clocks : List of clocks to be used during SMMU register access. See Documentation/devicetree/bindings/clock/clock-bindings.txt for information about the format. For each clock specified Loading
drivers/iommu/Makefile +1 −0 Original line number Diff line number Diff line Loading @@ -4,6 +4,7 @@ obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o obj-$(CONFIG_IOMMU_API) += msm_dma_iommu_mapping.o obj-$(CONFIG_IOMMU_IO_PGTABLE) += io-pgtable.o obj-$(CONFIG_IOMMU_IO_PGTABLE_LPAE) += io-pgtable-arm.o obj-$(CONFIG_MSM_TZ_SMMU) += io-pgtable-msm-secure.o obj-$(CONFIG_OF_IOMMU) += of_iommu.o obj-$(CONFIG_IOMMU_DEBUG) += iommu-debug.o obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_domains.o msm_iommu_mapping.o Loading
drivers/iommu/arm-smmu.c +169 −24 Original line number Diff line number Diff line Loading @@ -45,6 +45,7 @@ #include <linux/amba/bus.h> #include <soc/qcom/msm_tz_smmu.h> #include <soc/qcom/scm.h> #include <soc/qcom/secure_buffer.h> #include <linux/msm_pcie.h> #include <asm/cacheflush.h> Loading Loading @@ -361,6 +362,7 @@ struct arm_smmu_device { #define ARM_SMMU_OPT_NO_SMR_CHECK (1 << 9) #define ARM_SMMU_OPT_DYNAMIC (1 << 10) #define ARM_SMMU_OPT_HALT (1 << 11) #define ARM_SMMU_OPT_STATIC_CB (1 << 12) u32 options; enum arm_smmu_arch_version version; Loading Loading @@ -403,6 +405,8 @@ struct arm_smmu_device { struct msm_bus_client_handle *bus_client; char *bus_client_name; enum tz_smmu_device_id sec_id; }; struct arm_smmu_cfg { Loading Loading @@ -442,7 +446,8 @@ struct arm_smmu_domain { struct arm_smmu_device *smmu; struct io_pgtable_ops *pgtbl_ops; struct io_pgtable_cfg pgtbl_cfg; spinlock_t pgtbl_lock; spinlock_t pgtbl_spin_lock; struct mutex pgtbl_mutex_lock; struct arm_smmu_cfg cfg; enum arm_smmu_domain_stage stage; struct mutex init_mutex; /* Protects smmu pointer */ Loading Loading @@ -478,6 +483,7 @@ static struct arm_smmu_option_prop arm_smmu_options[] = { { ARM_SMMU_OPT_NO_SMR_CHECK, "qcom,no-smr-check" }, { ARM_SMMU_OPT_DYNAMIC, "qcom,dynamic" }, { ARM_SMMU_OPT_HALT, "qcom,enable-smmu-halt"}, { ARM_SMMU_OPT_STATIC_CB, "qcom,enable-static-cb"}, { 0, NULL}, }; Loading @@ -501,7 +507,9 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu); static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size); static bool arm_smmu_is_master_side_secure(struct arm_smmu_domain *smmu_domain); static bool arm_smmu_is_static_cb(struct arm_smmu_device *smmu); static bool arm_smmu_is_slave_side_secure(struct arm_smmu_domain *smmu_domain); static bool arm_smmu_has_secure_vmid(struct arm_smmu_domain *smmu_domain); static void parse_driver_options(struct arm_smmu_device *smmu) { Loading Loading @@ -1469,11 +1477,28 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR); } static bool arm_smmu_is_master_side_secure(struct arm_smmu_domain *smmu_domain) static bool arm_smmu_is_static_cb(struct arm_smmu_device *smmu) { return smmu->options & ARM_SMMU_OPT_STATIC_CB; } static bool arm_smmu_has_secure_vmid(struct arm_smmu_domain *smmu_domain) { return smmu_domain->secure_vmid != VMID_INVAL; } static bool arm_smmu_is_slave_side_secure(struct arm_smmu_domain *smmu_domain) { return arm_smmu_has_secure_vmid(smmu_domain) && arm_smmu_is_static_cb(smmu_domain->smmu); } static bool arm_smmu_is_master_side_secure(struct arm_smmu_domain *smmu_domain) { return arm_smmu_has_secure_vmid(smmu_domain) && !arm_smmu_is_static_cb(smmu_domain->smmu); } static void arm_smmu_secure_domain_lock(struct arm_smmu_domain *smmu_domain) { if (arm_smmu_is_master_side_secure(smmu_domain)) Loading @@ -1486,6 +1511,27 @@ static void arm_smmu_secure_domain_unlock(struct arm_smmu_domain *smmu_domain) mutex_unlock(&smmu_domain->assign_lock); } static unsigned long arm_smmu_pgtbl_lock(struct arm_smmu_domain *smmu_domain) { unsigned long flags; if (arm_smmu_is_slave_side_secure(smmu_domain)) mutex_lock(&smmu_domain->pgtbl_mutex_lock); else spin_lock_irqsave(&smmu_domain->pgtbl_spin_lock, flags); return flags; } static void arm_smmu_pgtbl_unlock(struct arm_smmu_domain *smmu_domain, unsigned long flags) { if (arm_smmu_is_slave_side_secure(smmu_domain)) mutex_unlock(&smmu_domain->pgtbl_mutex_lock); else spin_unlock_irqrestore(&smmu_domain->pgtbl_spin_lock, flags); } static int arm_smmu_init_domain_context(struct iommu_domain *domain, struct arm_smmu_device *smmu) { Loading Loading @@ -1553,12 +1599,17 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, goto out; } if (cfg->cbndx == INVALID_CBNDX) { ret = __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks); if (IS_ERR_VALUE(ret)) goto out; cfg->cbndx = ret; } else { if (test_and_set_bit(cfg->cbndx, smmu->context_map)) goto out; } if (smmu->version == ARM_SMMU_V1) { cfg->irptndx = atomic_inc_return(&smmu->irptndx); cfg->irptndx %= smmu->num_context_irqs; Loading @@ -1566,16 +1617,28 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, cfg->irptndx = cfg->cbndx; } smmu_domain->smmu = smmu; if (arm_smmu_is_slave_side_secure(smmu_domain)) { smmu_domain->pgtbl_cfg = (struct io_pgtable_cfg) { .pgsize_bitmap = arm_smmu_ops.pgsize_bitmap, .arm_msm_secure_cfg = { .sec_id = smmu->sec_id, .cbndx = cfg->cbndx, }, }; fmt = ARM_MSM_SECURE; } else { smmu_domain->pgtbl_cfg = (struct io_pgtable_cfg) { .pgsize_bitmap = arm_smmu_ops.pgsize_bitmap, .ias = ias, .oas = oas, .tlb = &arm_smmu_gather_ops, }; } cfg->asid = cfg->cbndx + 1; cfg->vmid = cfg->cbndx + 2; smmu_domain->smmu = smmu; pgtbl_ops = alloc_io_pgtable_ops(fmt, &smmu_domain->pgtbl_cfg, smmu_domain); if (!pgtbl_ops) { Loading Loading @@ -1678,8 +1741,9 @@ static int arm_smmu_domain_init(struct iommu_domain *domain) smmu_domain->cfg.vmid = INVALID_VMID; mutex_init(&smmu_domain->init_mutex); spin_lock_init(&smmu_domain->pgtbl_lock); spin_lock_init(&smmu_domain->pgtbl_spin_lock); mutex_init(&smmu_domain->assign_lock); mutex_init(&smmu_domain->pgtbl_mutex_lock); domain->priv = smmu_domain; return 0; } Loading Loading @@ -1929,6 +1993,40 @@ out: return ret; } static int arm_smmu_populate_cb(struct arm_smmu_device *smmu, struct arm_smmu_domain *smmu_domain, struct device *dev) { void __iomem *gr0_base; struct arm_smmu_master_cfg *cfg; struct arm_smmu_cfg *smmu_cfg = &smmu_domain->cfg; int i; u32 sid; gr0_base = ARM_SMMU_GR0(smmu); cfg = find_smmu_master_cfg(dev); if (!cfg) return -ENODEV; sid = cfg->streamids[0]; for (i = 0; i < smmu->num_mapping_groups; i++) { u32 smr, s2cr; u8 cbndx; smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(i)); if (sid == ((smr >> SMR_ID_SHIFT) & SMR_ID_MASK)) { s2cr = readl_relaxed(gr0_base + ARM_SMMU_GR0_S2CR(i)); cbndx = (s2cr >> S2CR_CBNDX_SHIFT) & S2CR_CBNDX_MASK; smmu_cfg->cbndx = cbndx; return 0; } } return -EINVAL; } static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) { int ret; Loading Loading @@ -1987,6 +2085,15 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) } smmu->attach_count++; if (arm_smmu_is_static_cb(smmu)) { ret = arm_smmu_populate_cb(smmu, smmu_domain, dev); if (ret) { dev_err(dev, "Failed to get valid context bank\n"); goto err_disable_clocks; } } /* Ensure that the domain is finalised */ ret = arm_smmu_init_domain_context(domain, smmu); if (IS_ERR_VALUE(ret)) Loading Loading @@ -2215,9 +2322,9 @@ static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova, arm_smmu_secure_domain_lock(smmu_domain); spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags); flags = arm_smmu_pgtbl_lock(smmu_domain); ret = ops->map(ops, iova, paddr, size, prot); spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags); arm_smmu_pgtbl_unlock(smmu_domain, flags); if (!ret) ret = arm_smmu_assign_table(smmu_domain); Loading @@ -2235,15 +2342,31 @@ static size_t arm_smmu_map_sg(struct iommu_domain *domain, unsigned long iova, unsigned long flags; struct arm_smmu_domain *smmu_domain = domain->priv; struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; struct arm_smmu_device *smmu = smmu_domain->smmu; int atomic_ctx = smmu_domain->attributes & (1 << DOMAIN_ATTR_ATOMIC); if (!ops) return -ENODEV; if (arm_smmu_is_slave_side_secure(smmu_domain) && atomic_ctx) { dev_err(smmu->dev, "Slave side atomic context not supported\n"); return 0; } if (arm_smmu_is_slave_side_secure(smmu_domain)) { mutex_lock(&smmu_domain->init_mutex); if (arm_smmu_enable_clocks(smmu)) { mutex_unlock(&smmu_domain->init_mutex); return 0; } } arm_smmu_secure_domain_lock(smmu_domain); spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags); flags = arm_smmu_pgtbl_lock(smmu_domain); ret = ops->map_sg(ops, iova, sg, nents, prot, &size); spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags); arm_smmu_pgtbl_unlock(smmu_domain, flags); if (ret) { if (arm_smmu_assign_table(smmu_domain)) { Loading @@ -2257,6 +2380,10 @@ static size_t arm_smmu_map_sg(struct iommu_domain *domain, unsigned long iova, out: arm_smmu_secure_domain_unlock(smmu_domain); if (arm_smmu_is_slave_side_secure(smmu_domain)) { arm_smmu_disable_clocks(smmu_domain->smmu); mutex_unlock(&smmu_domain->init_mutex); } return ret; } Loading @@ -2272,6 +2399,12 @@ static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, if (!ops) return 0; if (arm_smmu_is_slave_side_secure(smmu_domain) && atomic_ctx) { dev_err(smmu_domain->smmu->dev, "Slave side atomic context not supported\n"); return 0; } /* * The contract here is that if you set DOMAIN_ATTR_ATOMIC your * domain *must* must be attached an SMMU during unmap. This Loading Loading @@ -2299,9 +2432,9 @@ static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, } } spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags); flags = arm_smmu_pgtbl_lock(smmu_domain); ret = ops->unmap(ops, iova, size); spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags); arm_smmu_pgtbl_unlock(smmu_domain, flags); /* * While splitting up block mappings, we might allocate page table Loading Loading @@ -2341,9 +2474,9 @@ static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain, if (!ops) return 0; spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags); flags = arm_smmu_pgtbl_lock(smmu_domain); ret = ops->iova_to_phys(ops, iova); spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags); arm_smmu_pgtbl_unlock(smmu_domain, flags); return ret; } Loading Loading @@ -2389,6 +2522,16 @@ static void arm_smmu_resume(struct arm_smmu_device *smmu) void __iomem *impl_def1_base = ARM_SMMU_IMPL_DEF1(smmu); u32 reg; if (arm_smmu_is_static_cb(smmu)) { int ret, scm_ret; ret = scm_restore_sec_cfg(smmu->sec_id, 0x0, &scm_ret); if (ret || scm_ret) { pr_err("scm call IOMMU_SECURE_CFG failed\n"); return; } } reg = readl_relaxed(impl_def1_base + IMPL_DEF1_MICRO_MMU_CTRL); reg &= ~MICRO_MMU_CTRL_LOCAL_HALT_REQ; writel_relaxed(reg, impl_def1_base + IMPL_DEF1_MICRO_MMU_CTRL); Loading Loading @@ -3376,6 +3519,8 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) err = arm_smmu_enable_clocks(smmu); if (err) goto out_put_masters; smmu->sec_id = msm_dev_to_device_id(dev); err = arm_smmu_device_cfg_probe(smmu); arm_smmu_disable_clocks(smmu); if (err) Loading
drivers/iommu/io-pgtable-msm-secure.c 0 → 100644 +243 −0 Original line number Diff line number Diff line /* Copyright (c) 2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and * only version 2 as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. */ #define pr_fmt(fmt) "io-pgtable-msm-secure: " fmt #include <linux/iommu.h> #include <linux/kernel.h> #include <linux/scatterlist.h> #include <linux/sizes.h> #include <linux/slab.h> #include <linux/types.h> #include <soc/qcom/scm.h> #include <asm/cacheflush.h> #include "io-pgtable.h" #define IOMMU_SECURE_MAP2_FLAT 0x12 #define IOMMU_SECURE_UNMAP2_FLAT 0x13 #define IOMMU_TLBINVAL_FLAG 0x00000001 #define io_pgtable_to_data(x) \ container_of((x), struct msm_secure_io_pgtable, iop) #define io_pgtable_ops_to_pgtable(x) \ container_of((x), struct io_pgtable, ops) #define io_pgtable_ops_to_data(x) \ io_pgtable_to_data(io_pgtable_ops_to_pgtable(x)) struct msm_secure_io_pgtable { struct io_pgtable iop; }; static int msm_secure_map(struct io_pgtable_ops *ops, unsigned long iova, phys_addr_t paddr, size_t size, int iommu_prot) { return -EINVAL; } static dma_addr_t msm_secure_get_phys_addr(struct scatterlist *sg) { /* * Try sg_dma_address first so that we can * map carveout regions that do not have a * struct page associated with them. */ dma_addr_t pa = sg_dma_address(sg); if (pa == 0) pa = sg_phys(sg); return pa; } static int msm_secure_map_sg(struct io_pgtable_ops *ops, unsigned long iova, struct scatterlist *sg, unsigned int nents, int iommu_prot, size_t *size) { struct msm_secure_io_pgtable *data = io_pgtable_ops_to_data(ops); struct io_pgtable_cfg *cfg = &data->iop.cfg; int ret = -EINVAL; struct scatterlist *tmp, *sgiter; dma_addr_t *pa_list = 0; unsigned int cnt, offset = 0, chunk_offset = 0; dma_addr_t pa; void *flush_va, *flush_va_end; unsigned long len = 0; struct scm_desc desc = {0}; int i; u32 resp; for_each_sg(sg, tmp, nents, i) len += tmp->length; if (!IS_ALIGNED(iova, SZ_1M) || !IS_ALIGNED(len, SZ_1M)) return -EINVAL; if (sg->length == len) { cnt = 1; pa = msm_secure_get_phys_addr(sg); if (!IS_ALIGNED(pa, SZ_1M)) return -EINVAL; desc.args[0] = virt_to_phys(&pa); desc.args[1] = cnt; desc.args[2] = len; flush_va = &pa; } else { sgiter = sg; if (!IS_ALIGNED(sgiter->length, SZ_1M)) return -EINVAL; cnt = sg->length / SZ_1M; while ((sgiter = sg_next(sgiter))) { if (!IS_ALIGNED(sgiter->length, SZ_1M)) return -EINVAL; cnt += sgiter->length / SZ_1M; } pa_list = kmalloc_array(cnt, sizeof(*pa_list), GFP_KERNEL); if (!pa_list) return -ENOMEM; sgiter = sg; cnt = 0; pa = msm_secure_get_phys_addr(sgiter); while (offset < len) { if (!IS_ALIGNED(pa, SZ_1M)) { kfree(pa_list); return -EINVAL; } pa_list[cnt] = pa + chunk_offset; chunk_offset += SZ_1M; offset += SZ_1M; cnt++; if (chunk_offset >= sgiter->length && offset < len) { chunk_offset = 0; sgiter = sg_next(sgiter); pa = msm_secure_get_phys_addr(sgiter); } } desc.args[0] = virt_to_phys(pa_list); desc.args[1] = cnt; desc.args[2] = SZ_1M; flush_va = pa_list; } desc.args[3] = cfg->arm_msm_secure_cfg.sec_id; desc.args[4] = cfg->arm_msm_secure_cfg.cbndx; desc.args[5] = iova; desc.args[6] = len; desc.args[7] = 0; desc.arginfo = SCM_ARGS(8, SCM_RW, SCM_VAL, SCM_VAL, SCM_VAL, SCM_VAL, SCM_VAL, SCM_VAL, SCM_VAL); /* * Ensure that the buffer is in RAM by the time it gets to TZ */ flush_va_end = (void *) (((unsigned long) flush_va) + (cnt * sizeof(*pa_list))); dmac_clean_range(flush_va, flush_va_end); if (is_scm_armv8()) { ret = scm_call2(SCM_SIP_FNID(SCM_SVC_MP, IOMMU_SECURE_MAP2_FLAT), &desc); resp = desc.ret[0]; if (ret || resp) ret = -EINVAL; else ret = len; } kfree(pa_list); return ret; } static size_t msm_secure_unmap(struct io_pgtable_ops *ops, unsigned long iova, size_t len) { struct msm_secure_io_pgtable *data = io_pgtable_ops_to_data(ops); struct io_pgtable_cfg *cfg = &data->iop.cfg; int ret = -EINVAL; struct scm_desc desc = {0}; if (!IS_ALIGNED(iova, SZ_1M) || !IS_ALIGNED(len, SZ_1M)) return ret; desc.args[0] = cfg->arm_msm_secure_cfg.sec_id; desc.args[1] = cfg->arm_msm_secure_cfg.cbndx; desc.args[2] = iova; desc.args[3] = len; desc.args[4] = IOMMU_TLBINVAL_FLAG; desc.arginfo = SCM_ARGS(5); if (is_scm_armv8()) { ret = scm_call2(SCM_SIP_FNID(SCM_SVC_MP, IOMMU_SECURE_UNMAP2_FLAT), &desc); if (!ret) ret = len; } return ret; } static phys_addr_t msm_secure_iova_to_phys(struct io_pgtable_ops *ops, unsigned long iova) { return -EINVAL; } static struct msm_secure_io_pgtable * msm_secure_alloc_pgtable_data(struct io_pgtable_cfg *cfg) { struct msm_secure_io_pgtable *data; data = kmalloc(sizeof(*data), GFP_KERNEL); if (!data) return NULL; data->iop.ops = (struct io_pgtable_ops) { .map = msm_secure_map, .map_sg = msm_secure_map_sg, .unmap = msm_secure_unmap, .iova_to_phys = msm_secure_iova_to_phys, }; return data; } static struct io_pgtable * msm_secure_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) { struct msm_secure_io_pgtable *data = msm_secure_alloc_pgtable_data(cfg); return &data->iop; } static void msm_secure_free_pgtable(struct io_pgtable *iop) { struct msm_secure_io_pgtable *data = io_pgtable_to_data(iop); kfree(data); } struct io_pgtable_init_fns io_pgtable_arm_msm_secure_init_fns = { .alloc = msm_secure_alloc_pgtable, .free = msm_secure_free_pgtable, };
drivers/iommu/io-pgtable.c +4 −0 Original line number Diff line number Diff line Loading @@ -33,6 +33,7 @@ extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns; extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns; extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns; extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns; extern struct io_pgtable_init_fns io_pgtable_arm_msm_secure_init_fns; static const struct io_pgtable_init_fns * io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = Loading @@ -43,6 +44,9 @@ io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = [ARM_64_LPAE_S1] = &io_pgtable_arm_64_lpae_s1_init_fns, [ARM_64_LPAE_S2] = &io_pgtable_arm_64_lpae_s2_init_fns, #endif #ifdef CONFIG_MSM_TZ_SMMU [ARM_MSM_SECURE] = &io_pgtable_arm_msm_secure_init_fns, #endif }; static struct dentry *io_pgtable_top; Loading