msm: kgsl: Allocate gpu virtual memory from user virtual space (fef41593) · Commits · e / devices / android_kernel_sony_msm8994

drivers/gpu/msm/kgsl.c

+154 −53

Original line number	Diff line number	Diff line
		@@ -89,6 +89,11 @@ static int kgsl_setup_dma_buf(struct kgsl_mem_entry *entry,

		static const struct file_operations kgsl_fops;

		static int __kgsl_check_collision(struct kgsl_process_private *private,
		struct kgsl_mem_entry *entry,
		unsigned long *gpuaddr, unsigned long len,
		int flag_top_down);

		static int kgsl_memfree_hist_init(void)
		{
		void *base;
		@@ -297,7 +302,25 @@ kgsl_mem_entry_track_gpuaddr(struct kgsl_process_private *process,
		goto done;
		}
		if (!kgsl_memdesc_use_cpu_map(&entry->memdesc)) {
		ret = kgsl_mmu_get_gpuaddr(process->pagetable, &entry->memdesc);
		ret = -ENOMEM;
		/*
		* For external allocations use mmu gen pool to assign
		* virtual address
		*/
		if (KGSL_MEMFLAGS_USERMEM_MASK & entry->memdesc.flags)
		ret = kgsl_mmu_get_gpuaddr(process->pagetable,
		&entry->memdesc);
		if (-ENOMEM == ret) {
		unsigned long gpuaddr = 0;
		size_t size = entry->memdesc.size;
		if (kgsl_memdesc_has_guard_page(&entry->memdesc))
		size += PAGE_SIZE;

		ret = __kgsl_check_collision(process, NULL, &gpuaddr,
		size, 0);
		if (!ret)
		entry->memdesc.gpuaddr = gpuaddr;
		}
		if (ret)
		goto done;
		}
		@@ -337,7 +360,9 @@ kgsl_mem_entry_untrack_gpuaddr(struct kgsl_process_private *process,
		{
		assert_spin_locked(&process->mem_lock);
		if (entry->memdesc.gpuaddr) {
		kgsl_mmu_put_gpuaddr(process->pagetable, &entry->memdesc);
		if (KGSL_MEMFLAGS_USERMEM_MASK & entry->memdesc.flags)
		kgsl_mmu_put_gpuaddr(process->pagetable,
		&entry->memdesc);
		rb_erase(&entry->node, &entry->priv->mem_rb);
		}
		}
		@@ -1210,6 +1235,19 @@ kgsl_sharedmem_find_id(struct kgsl_process_private *process, unsigned int id)
		return entry;
		}

		/**
		* kgsl_mem_entry_unset_pend() - Unset the pending free flag of an entry
		* @entry - The memory entry
		*/
		static inline void kgsl_mem_entry_unset_pend(struct kgsl_mem_entry *entry)
		{
		if (entry == NULL)
		return;
		spin_lock(&entry->priv->mem_lock);
		entry->pending_free = 0;
		spin_unlock(&entry->priv->mem_lock);
		}

		/**
		* kgsl_mem_entry_set_pend() - Set the pending free flag of a memory entry
		* @entry - The memory entry
		@@ -2293,6 +2331,10 @@ long kgsl_ioctl_cmdstream_freememontimestamp_ctxtid(
		param->timestamp);
		result = kgsl_add_event(dev_priv->device, &context->events,
		param->timestamp, kgsl_freemem_event_cb, entry);

		if (result)
		kgsl_mem_entry_unset_pend(entry);

		kgsl_mem_entry_put(entry);

		out:
		@@ -3650,10 +3692,20 @@ get_mmap_entry(struct kgsl_process_private *private,
		goto err_put;
		}

		if (kgsl_memdesc_use_cpu_map(&entry->memdesc)) {
		if (len != kgsl_memdesc_mmapsize(&entry->memdesc)) {
		ret = -ERANGE;
		goto err_put;
		}
		} else if (len != kgsl_memdesc_mmapsize(&entry->memdesc) &&
		len != entry->memdesc.size) {
		/*
		* If cpu_map != gpumap then user can map either the
		* mmapsize or the entry size
		*/
		ret = -ERANGE;
		goto err_put;
		}

		*out_entry = entry;
		return 0;
		@@ -3669,6 +3721,98 @@ mmap_range_valid(unsigned long addr, unsigned long len)
		KGSL_SVM_UPPER_BOUND);
		}

		/**
		* __kgsl_check_collision() - Find a non colliding gpuaddr for the process
		* @private: Process private pointer contaning the list of allocations
		* @entry: The entry colliding with given address
		* @gpuaddr: In out parameter. The In parameter contains the desired gpuaddr
		* if the gpuaddr collides then the out parameter contains the non colliding
		* address
		* @len: Length of address range
		* @flag_top_down: Indicates whether free address range should be checked in
		* top down or bottom up fashion
		*/
		static int __kgsl_check_collision(struct kgsl_process_private *private,
		struct kgsl_mem_entry *entry,
		unsigned long *gpuaddr, unsigned long len,
		int flag_top_down)
		{
		int ret = 0;
		unsigned long addr = *gpuaddr;
		struct kgsl_mem_entry *collision_entry = entry;

		if (!addr) {
		addr = flag_top_down ? (KGSL_SVM_UPPER_BOUND - len) : PAGE_SIZE;
		collision_entry = NULL;
		}

		do {
		/*
		* If address collided with an existing entry then find a new
		* one
		*/
		if (collision_entry) {
		/*
		* If top down search then next address to consider
		* is lower. The highest lower address possible is the
		* colliding entry address - the length of
		* allocation
		*/
		if (flag_top_down) {
		addr = collision_entry->memdesc.gpuaddr - len;
		/* Check for loopback */
		if (addr > collision_entry->memdesc.gpuaddr) {
		KGSL_CORE_ERR_ONCE(
		"Underflow err ent:%x/%zx, addr:%lx/%lx\n",
		collision_entry->memdesc.gpuaddr,
		kgsl_memdesc_mmapsize(
		&collision_entry->memdesc),
		addr, len);
		*gpuaddr =
		KGSL_SVM_UPPER_BOUND;
		ret = -EAGAIN;
		break;
		}
		} else {
		/*
		* Bottom up mode the next address to consider
		* is higher. The lowest higher address possible
		* colliding entry address + the size of the
		* colliding entry
		*/
		addr = collision_entry->memdesc.gpuaddr +
		kgsl_memdesc_mmapsize(
		&collision_entry->memdesc);
		/* overflow check */
		if (addr < collision_entry->memdesc.gpuaddr \|\|
		!mmap_range_valid(addr, len)) {
		KGSL_CORE_ERR_ONCE(
		"Overflow err ent:%x/%zx, addr:%lx/%lx\n",
		collision_entry->memdesc.gpuaddr,
		kgsl_memdesc_mmapsize(
		&collision_entry->memdesc),
		addr, len);
		*gpuaddr =
		KGSL_SVM_UPPER_BOUND;
		ret = -EAGAIN;
		break;
		}
		}
		}
		if (kgsl_sharedmem_region_empty(private, addr, len,
		&collision_entry)) {
		/* no collision with addr then return */
		*gpuaddr = addr;
		break;
		} else if (!collision_entry) {
		ret = -ENOENT;
		break;
		}
		} while (1);

		return ret;
		}

		/**
		* kgsl_check_gpu_addr_collision() - Check if an address range collides with
		* existing allocations of a process
		@@ -3730,56 +3874,13 @@ static int kgsl_check_gpu_addr_collision(
		*/
		len += 1 << align;

		/*
		* Loop through the gpu map address space to find an unmapped
		* region of size len, lopping is done either top down or bottom
		* up based on flag_top_down setting
		*/
		do {
		if (!collision_entry) {
		ret = -ENOENT;
		break;
		}
		if (flag_top_down) {
		addr = collision_entry->memdesc.gpuaddr - len;
		if (addr > collision_entry->memdesc.gpuaddr) {
		KGSL_CORE_ERR_ONCE(
		"Underflow err ent:%x/%zx, addr:%lx/%lx align:%u",
		collision_entry->memdesc.gpuaddr,
		kgsl_memdesc_mmapsize(
		&collision_entry->memdesc),
		addr, len, align);
		*gpumap_free_addr =
		KGSL_SVM_UPPER_BOUND;
		ret = -EAGAIN;
		break;
		}
		} else {
		addr = collision_entry->memdesc.gpuaddr +
		kgsl_memdesc_mmapsize(
		&collision_entry->memdesc);
		/* overflow check */
		if (addr < collision_entry->memdesc.gpuaddr \|\|
		!mmap_range_valid(addr, len)) {
		KGSL_CORE_ERR_ONCE(
		"Overflow err ent:%x/%zx, addr:%lx/%lx align:%u",
		collision_entry->memdesc.gpuaddr,
		kgsl_memdesc_mmapsize(
		&collision_entry->memdesc),
		addr, len, align);
		*gpumap_free_addr =
		KGSL_SVM_UPPER_BOUND;
		ret = -EAGAIN;
		break;
		}
		}
		collision_entry = NULL;
		if (kgsl_sharedmem_region_empty(private, addr, len,
		&collision_entry)) {
		ret = __kgsl_check_collision(private, collision_entry, &addr,
		len, flag_top_down);
		if (!ret \|\| -EAGAIN == ret) {
		*gpumap_free_addr = addr;
		break;
		ret = -EAGAIN;
		}
		} while (1);

		spin_unlock(&private->mem_lock);
		}
		return ret;

drivers/gpu/msm/kgsl.h

+2 −1

Original line number	Diff line number	Diff line
		@@ -134,7 +134,8 @@ struct kgsl_memdesc_ops {
		#define KGSL_MEMDESC_FROZEN BIT(2)
		/* The memdesc is mapped into a pagetable */
		#define KGSL_MEMDESC_MAPPED BIT(3)

		/* Indicates gpuaddr is assigned via gen pool */
		#define KGSL_MEMDESC_GENPOOL_ALLOC BIT(4)
		/* shared memory allocation */
		struct kgsl_memdesc {
		struct kgsl_pagetable *pagetable;

drivers/gpu/msm/kgsl_iommu.c

+3 −26

Original line number	Diff line number	Diff line
		@@ -1341,13 +1341,6 @@ static int kgsl_iommu_init(struct kgsl_mmu *mmu)
		if (!iommu)
		return -ENOMEM;

		/*
		* These are constant for all cases so set them early so
		* kgsl_set_register_map() can use them
		*/
		mmu->global_pt_base = KGSL_IOMMU_GLOBAL_MEM_BASE;
		mmu->global_pt_size = KGSL_GLOBAL_PT_SIZE;

		mmu->priv = iommu;
		status = kgsl_get_iommu_ctxt(mmu);
		if (status)
		@@ -1370,25 +1363,9 @@ static int kgsl_iommu_init(struct kgsl_mmu *mmu)
		"gtcu_iface_clk") >= 0)
		iommu->gtcu_iface_clk = clk_get(&pdev->dev, "gtcu_iface_clk");

		/*
		* For IOMMU per-process pagetables, the allocatable range
		* and the kernel global range must both be outside
		* the userspace address range. There is a 1Mb gap
		* between these address ranges to make overrun
		* detection easier.
		* For the shared pagetable case use 2GB and because
		* mirroring the CPU address space is not possible and
		* we're better off with extra room.
		*/
		if (mmu->pt_per_process) {
		mmu->pt_base = KGSL_PER_PROCESS_PT_BASE;
		mmu->pt_size = KGSL_PER_PROCESS_PT_SIZE;
		mmu->use_cpu_map = true;
		} else {
		mmu->pt_base = KGSL_PAGETABLE_BASE;
		mmu->pt_size = SZ_2G;
		mmu->use_cpu_map = false;
		}
		mmu->pt_base = KGSL_MMU_MAPPED_MEM_BASE;
		mmu->pt_size = KGSL_MMU_MAPPED_MEM_SIZE;
		mmu->use_cpu_map = mmu->pt_per_process;

		status = kgsl_iommu_init_sync_lock(mmu);
		if (status)

drivers/gpu/msm/kgsl_iommu.h

+0 −5

Original line number	Diff line number	Diff line
		@@ -16,11 +16,6 @@
		#include <linux/qcom_iommu.h>

		/* Pagetable virtual base */
		#define KGSL_PAGETABLE_BASE 0x10000000
		#define KGSL_PER_PROCESS_PT_BASE 0xC0000000
		#define KGSL_PER_PROCESS_PT_SIZE (KGSL_IOMMU_GLOBAL_MEM_BASE - \
		KGSL_PER_PROCESS_PT_BASE - SZ_1M)

		#define KGSL_IOMMU_CTX_OFFSET_V0 0
		#define KGSL_IOMMU_CTX_OFFSET_V1 0x8000
		#define KGSL_IOMMU_CTX_OFFSET_V2 0x9000

drivers/gpu/msm/kgsl_mmu.c

+16 −53

Original line number	Diff line number	Diff line
		@@ -176,10 +176,9 @@ EXPORT_SYMBOL(kgsl_remove_global_pt_entry);
		int kgsl_add_global_pt_entry(struct kgsl_device *device,
		struct kgsl_memdesc *memdesc)
		{
		struct kgsl_mmu *mmu = &device->mmu;
		int i;
		int index = 0;
		unsigned int gaddr = mmu->global_pt_base;
		unsigned int gaddr = KGSL_MMU_GLOBAL_MEM_BASE;
		unsigned int size = ALIGN(memdesc->size, PAGE_SIZE);

		if (memdesc->priv & KGSL_MEMDESC_GLOBAL)
		@@ -204,7 +203,8 @@ int kgsl_add_global_pt_entry(struct kgsl_device *device,
		break;
		}
		index = i;
		if ((gaddr + size) >= (mmu->global_pt_base + mmu->global_pt_size))
		if ((gaddr + size) >= (KGSL_MMU_GLOBAL_MEM_BASE +
		KGSL_GLOBAL_PT_SIZE))
		return -ENOMEM;

		memdesc->priv \|= KGSL_MEMDESC_GLOBAL;
		@@ -354,25 +354,6 @@ sysfs_show_mapped(struct kobject *kobj,
		return ret;
		}

		static ssize_t
		sysfs_show_va_range(struct kobject *kobj,
		struct kobj_attribute *attr,
		char *buf)
		{
		struct kgsl_pagetable *pt;
		int ret = 0;

		pt = _get_pt_from_kobj(kobj);

		if (pt) {
		ret += snprintf(buf, PAGE_SIZE, "0x%x\n",
		kgsl_mmu_get_ptsize(pt->mmu));
		}

		kgsl_put_pagetable(pt);
		return ret;
		}

		static ssize_t
		sysfs_show_max_mapped(struct kobject *kobj,
		struct kobj_attribute *attr,
		@@ -402,12 +383,6 @@ static struct kobj_attribute attr_mapped = {
		.store = NULL,
		};

		static struct kobj_attribute attr_va_range = {
		.attr = { .name = "va_range", .mode = 0444 },
		.show = sysfs_show_va_range,
		.store = NULL,
		};

		static struct kobj_attribute attr_max_mapped = {
		.attr = { .name = "max_mapped", .mode = 0444 },
		.show = sysfs_show_max_mapped,
		@@ -417,7 +392,6 @@ static struct kobj_attribute attr_max_mapped = {
		static struct attribute *pagetable_attrs[] = {
		&attr_entries.attr,
		&attr_mapped.attr,
		&attr_va_range.attr,
		&attr_max_mapped.attr,
		NULL,
		};
		@@ -588,7 +562,6 @@ kgsl_mmu_createpagetableobject(struct kgsl_mmu *mmu,
		int status = 0;
		struct kgsl_pagetable *pagetable = NULL;
		unsigned long flags;
		unsigned int ptsize;

		pagetable = kzalloc(sizeof(struct kgsl_pagetable), GFP_KERNEL);
		if (pagetable == NULL)
		@@ -598,7 +571,6 @@ kgsl_mmu_createpagetableobject(struct kgsl_mmu *mmu,

		spin_lock_init(&pagetable->lock);

		ptsize = kgsl_mmu_get_ptsize(mmu);
		pagetable->mmu = mmu;
		pagetable->name = name;
		pagetable->fault_addr = 0xFFFFFFFF;
		@@ -610,8 +582,8 @@ kgsl_mmu_createpagetableobject(struct kgsl_mmu *mmu,
		goto err_alloc;
		}

		if (gen_pool_add(pagetable->pool, kgsl_mmu_get_base_addr(mmu),
		ptsize, -1)) {
		if (gen_pool_add(pagetable->pool, mmu->pt_base,
		mmu->pt_size, -1)) {
		KGSL_CORE_ERR("gen_pool_add failed\n");
		goto err_pool;
		}
		@@ -747,15 +719,10 @@ kgsl_mmu_get_gpuaddr(struct kgsl_pagetable *pagetable,
		memdesc->gpuaddr = gen_pool_alloc_aligned(pagetable->pool, size,
		page_align);

		if (memdesc->gpuaddr == 0) {
		KGSL_CORE_ERR("gen_pool_alloc(%d) failed\n", size);
		KGSL_CORE_ERR(" [%d] allocated=%d, entries=%d\n",
		pagetable->name,
		pagetable->stats.mapped,
		pagetable->stats.entries);
		if (memdesc->gpuaddr == 0)
		return -ENOMEM;
		}

		memdesc->priv \|= KGSL_MEMDESC_GENPOOL_ALLOC;
		return 0;
		}
		EXPORT_SYMBOL(kgsl_mmu_get_gpuaddr);
		@@ -827,6 +794,9 @@ kgsl_mmu_put_gpuaddr(struct kgsl_pagetable *pagetable,
		if (kgsl_mmu_type == KGSL_MMU_TYPE_NONE)
		goto done;

		if (!(KGSL_MEMDESC_GENPOOL_ALLOC & memdesc->priv))
		goto done;

		/* Add space for the guard page when freeing the mmu VA. */
		size = memdesc->size;
		if (kgsl_memdesc_has_guard_page(memdesc))
		@@ -834,10 +804,6 @@ kgsl_mmu_put_gpuaddr(struct kgsl_pagetable *pagetable,

		pool = pagetable->pool;

		if (KGSL_MMU_TYPE_IOMMU == kgsl_mmu_get_mmutype()) {
		if (kgsl_memdesc_use_cpu_map(memdesc))
		pool = NULL;
		}
		if (pool)
		gen_pool_free(pool, memdesc->gpuaddr, size);
		/*
		@@ -845,8 +811,10 @@ kgsl_mmu_put_gpuaddr(struct kgsl_pagetable *pagetable,
		* may be in use by other pagetables
		*/
		done:
		if (!kgsl_memdesc_is_global(memdesc))
		if (!kgsl_memdesc_is_global(memdesc)) {
		memdesc->gpuaddr = 0;
		memdesc->priv &= ~KGSL_MEMDESC_GENPOOL_ALLOC;
		}
		return 0;
		}
		EXPORT_SYMBOL(kgsl_mmu_put_gpuaddr);
		@@ -958,13 +926,8 @@ int kgsl_mmu_gpuaddr_in_range(struct kgsl_pagetable *pt, unsigned int gpuaddr)
		{
		if (KGSL_MMU_TYPE_NONE == kgsl_mmu_type)
		return (gpuaddr != 0);
		if (gpuaddr >= kgsl_mmu_get_base_addr(pt->mmu) &&
		gpuaddr < kgsl_mmu_get_base_addr(pt->mmu) +
		kgsl_mmu_get_ptsize(pt->mmu))
		return 1;
		if (kgsl_mmu_use_cpu_map(pt->mmu))
		return (gpuaddr > 0 && gpuaddr < KGSL_SVM_UPPER_BOUND);
		return 0;

		return (gpuaddr > 0 && gpuaddr < KGSL_MMU_GLOBAL_MEM_BASE);
		}
		EXPORT_SYMBOL(kgsl_mmu_gpuaddr_in_range);