Merge "msm: kgsl: Optimize virtual memory allocation" (28b7c31e) · Commits · e / devices / android_kernel_sony_msm8994

drivers/gpu/msm/kgsl.c

+65 −87

Original line number	Diff line number	Diff line
		@@ -318,8 +318,12 @@ kgsl_mem_entry_track_gpuaddr(struct kgsl_process_private *process,
		struct rb_node **node;
		struct rb_node *parent = NULL;
		struct kgsl_pagetable *pagetable = process->pagetable;
		size_t size = entry->memdesc.size;

		assert_spin_locked(&process->mem_lock);

		if (kgsl_memdesc_has_guard_page(&entry->memdesc))
		size += PAGE_SIZE;
		/*
		* If cpu=gpu map is used then caller needs to set the
		* gpu address
		@@ -332,34 +336,12 @@ kgsl_mem_entry_track_gpuaddr(struct kgsl_process_private *process,
		ret = -EINVAL;
		goto done;
		}
		if (!kgsl_memdesc_use_cpu_map(&entry->memdesc)) {
		ret = -ENOMEM;
		/*
		* For external allocations use mmu gen pool to assign
		* virtual address
		*/
		if (KGSL_MEMFLAGS_USERMEM_MASK & entry->memdesc.flags) {
		/* Get secured buffer gpuaddr from secured pool */
		if (kgsl_memdesc_is_secured(&entry->memdesc))
		pagetable = pagetable->mmu->securepagetable;
		ret = kgsl_mmu_get_gpuaddr(pagetable, &entry->memdesc);
		}
		if ((kgsl_memdesc_is_secured(&entry->memdesc)) && (ret))
		goto done;
		if (-ENOMEM == ret) {
		unsigned long gpuaddr = 0;
		size_t size = entry->memdesc.size;
		if (kgsl_memdesc_has_guard_page(&entry->memdesc))
		size += PAGE_SIZE;

		ret = __kgsl_check_collision(process, NULL, &gpuaddr,
		size, 0);
		if (!ret)
		entry->memdesc.gpuaddr = gpuaddr;
		}
		ret = kgsl_mmu_get_gpuaddr(pagetable, &entry->memdesc);
		if (ret)
		goto done;
		}

		node = &process->mem_rb.rb_node;

		@@ -396,7 +378,6 @@ kgsl_mem_entry_untrack_gpuaddr(struct kgsl_process_private *process,
		{
		assert_spin_locked(&process->mem_lock);
		if (entry->memdesc.gpuaddr) {
		if (KGSL_MEMFLAGS_USERMEM_MASK & entry->memdesc.flags)
		kgsl_mmu_put_gpuaddr(entry->memdesc.pagetable,
		&entry->memdesc);
		rb_erase(&entry->node, &entry->priv->mem_rb);
		@@ -3809,18 +3790,16 @@ static int __kgsl_check_collision(struct kgsl_process_private *private,
		int ret = 0;
		unsigned long addr = *gpuaddr;
		struct kgsl_mem_entry *collision_entry = entry;
		struct rb_node node, node_first, *node_last;

		if (!addr) {
		addr = flag_top_down ? (KGSL_SVM_UPPER_BOUND - len) : PAGE_SIZE;
		collision_entry = NULL;
		}
		if (!collision_entry)
		return -ENOENT;

		do {
		/*
		* If address collided with an existing entry then find a new
		* one
		*/
		if (collision_entry) {
		node = &(collision_entry->node);
		node_first = rb_first(&private->mem_rb);
		node_last = rb_last(&private->mem_rb);

		while (1) {
		/*
		* If top down search then next address to consider
		* is lower. The highest lower address possible is the
		@@ -3830,18 +3809,18 @@ static int __kgsl_check_collision(struct kgsl_process_private *private,
		if (flag_top_down) {
		addr = collision_entry->memdesc.gpuaddr - len;
		/* Check for loopback */
		if (addr > collision_entry->memdesc.gpuaddr) {
		KGSL_CORE_ERR_ONCE(
		"Underflow err ent:%x/%zx, addr:%lx/%lx\n",
		collision_entry->memdesc.gpuaddr,
		kgsl_memdesc_mmapsize(
		&collision_entry->memdesc),
		addr, len);
		*gpuaddr =
		KGSL_SVM_UPPER_BOUND;
		if (addr > collision_entry->memdesc.gpuaddr \|\| !addr) {
		*gpuaddr = KGSL_SVM_UPPER_BOUND;
		ret = -EAGAIN;
		break;
		}
		if (node == node_first) {
		collision_entry = NULL;
		} else {
		node = rb_prev(&collision_entry->node);
		collision_entry = container_of(node,
		struct kgsl_mem_entry, node);
		}
		} else {
		/*
		* Bottom up mode the next address to consider
		@@ -3855,29 +3834,28 @@ static int __kgsl_check_collision(struct kgsl_process_private *private,
		/* overflow check */
		if (addr < collision_entry->memdesc.gpuaddr \|\|
		!mmap_range_valid(addr, len)) {
		KGSL_CORE_ERR_ONCE(
		"Overflow err ent:%x/%zx, addr:%lx/%lx\n",
		collision_entry->memdesc.gpuaddr,
		kgsl_memdesc_mmapsize(
		&collision_entry->memdesc),
		addr, len);
		*gpuaddr =
		KGSL_SVM_UPPER_BOUND;
		*gpuaddr = KGSL_SVM_UPPER_BOUND;
		ret = -EAGAIN;
		break;
		}
		if (node == node_last) {
		collision_entry = NULL;
		} else {
		node = rb_next(&collision_entry->node);
		collision_entry = container_of(node,
		struct kgsl_mem_entry, node);
		}
		}
		if (kgsl_sharedmem_region_empty(private, addr, len,
		&collision_entry)) {
		/* no collision with addr then return */

		if (!collision_entry \|\|
		!kgsl_addr_range_overlap(addr, len,
		collision_entry->memdesc.gpuaddr,
		kgsl_memdesc_mmapsize(&collision_entry->memdesc))) {
		/* success */
		*gpuaddr = addr;
		break;
		} else if (!collision_entry) {
		ret = -ENOENT;
		break;
		}
		} while (1);
		}

		return ret;
		}

drivers/gpu/msm/kgsl.h

+2 −0

Original line number	Diff line number	Diff line
		@@ -120,6 +120,8 @@ struct kgsl_memdesc_ops {
		#define KGSL_MEMDESC_GENPOOL_ALLOC BIT(4)
		/* The memdesc is secured for content protection */
		#define KGSL_MEMDESC_SECURE BIT(5)
		/* Indicates gpuaddr is assigned via bimap */
		#define KGSL_MEMDESC_BITMAP_ALLOC BIT(6)

		/* shared memory allocation */
		struct kgsl_memdesc {

drivers/gpu/msm/kgsl_mmu.c

+51 −10

Original line number	Diff line number	Diff line
		@@ -250,6 +250,9 @@ static void kgsl_destroy_pagetable(struct kref *kref)

		pagetable->pt_ops->mmu_destroy_pagetable(pagetable);

		if (pagetable->mem_bitmap)
		vfree(pagetable->mem_bitmap);

		kfree(pagetable);
		}

		@@ -524,6 +527,7 @@ kgsl_mmu_createpagetableobject(struct kgsl_mmu *mmu,
		unsigned long flags;
		unsigned int ptbase, ptsize;
		char *pool_name;
		int nbits;

		pagetable = kzalloc(sizeof(struct kgsl_pagetable), GFP_KERNEL);
		if (pagetable == NULL)
		@@ -560,6 +564,13 @@ kgsl_mmu_createpagetableobject(struct kgsl_mmu *mmu,
		goto err;
		}

		/* allocate bitmap for virtual memory management */
		nbits = KGSL_SVM_UPPER_BOUND >> PAGE_SHIFT;
		pagetable->mem_bitmap = vmalloc(BITS_TO_LONGS(nbits) * sizeof(long));
		if (!pagetable->mem_bitmap)
		goto err;
		memset(pagetable->mem_bitmap, 0, BITS_TO_LONGS(nbits) * sizeof(long));

		if (KGSL_MMU_TYPE_IOMMU == kgsl_mmu_type)
		pagetable->pt_ops = &iommu_pt_ops;

		@@ -585,6 +596,8 @@ err:
		pagetable->pt_ops->mmu_destroy_pagetable(pagetable);
		if (pagetable->pool)
		gen_pool_destroy(pagetable->pool);
		if (pagetable->mem_bitmap)
		vfree(pagetable->mem_bitmap);

		kfree(pagetable);

		@@ -682,17 +695,37 @@ kgsl_mmu_get_gpuaddr(struct kgsl_pagetable *pagetable,
		if (kgsl_memdesc_use_cpu_map(memdesc)) {
		if (memdesc->gpuaddr == 0)
		return -EINVAL;
		bitmap_set(pagetable->mem_bitmap,
		memdesc->gpuaddr >> PAGE_SHIFT,
		size >> PAGE_SHIFT);
		memdesc->priv \|= KGSL_MEMDESC_BITMAP_ALLOC;
		return 0;
		}
		}

		if (KGSL_MEMFLAGS_USERMEM_MASK & memdesc->flags) {
		memdesc->gpuaddr = gen_pool_alloc_aligned(pagetable->pool, size,
		page_align);
		if (memdesc->gpuaddr)
		memdesc->priv \|= KGSL_MEMDESC_GENPOOL_ALLOC;
		} else {
		unsigned int gpuaddr = bitmap_find_next_zero_area(
		pagetable->mem_bitmap,
		KGSL_SVM_UPPER_BOUND >> PAGE_SHIFT, 1,
		size >> PAGE_SHIFT, 0);

		if (gpuaddr < (KGSL_SVM_UPPER_BOUND >> PAGE_SHIFT)) {
		bitmap_set(pagetable->mem_bitmap,
		gpuaddr, size >> PAGE_SHIFT);
		memdesc->gpuaddr = gpuaddr << PAGE_SHIFT;
		}
		if (memdesc->gpuaddr)
		memdesc->priv \|= KGSL_MEMDESC_BITMAP_ALLOC;
		}

		if (memdesc->gpuaddr == 0)
		return -ENOMEM;

		memdesc->priv \|= KGSL_MEMDESC_GENPOOL_ALLOC;
		return 0;
		}
		EXPORT_SYMBOL(kgsl_mmu_get_gpuaddr);
		@@ -764,27 +797,35 @@ kgsl_mmu_put_gpuaddr(struct kgsl_pagetable *pagetable,
		if (kgsl_mmu_type == KGSL_MMU_TYPE_NONE)
		goto done;

		if (!(KGSL_MEMDESC_GENPOOL_ALLOC & memdesc->priv))
		goto done;

		/* Add space for the guard page when freeing the mmu VA. */
		size = memdesc->size;
		if (kgsl_memdesc_has_guard_page(memdesc))
		size += PAGE_SIZE;

		if (KGSL_MEMDESC_BITMAP_ALLOC & memdesc->priv) {
		bitmap_clear(pagetable->mem_bitmap,
		memdesc->gpuaddr >> PAGE_SHIFT,
		size >> PAGE_SHIFT);
		memdesc->priv &= ~KGSL_MEMDESC_BITMAP_ALLOC;
		goto done;
		}

		if (!(KGSL_MEMDESC_GENPOOL_ALLOC & memdesc->priv))
		goto done;

		pool = pagetable->pool;

		if (pool)
		if (pool) {
		gen_pool_free(pool, memdesc->gpuaddr, size);
		memdesc->priv &= ~KGSL_MEMDESC_GENPOOL_ALLOC;
		}
		/*
		* Don't clear the gpuaddr on global mappings because they
		* may be in use by other pagetables
		*/
		done:
		if (!kgsl_memdesc_is_global(memdesc)) {
		if (!kgsl_memdesc_is_global(memdesc))
		memdesc->gpuaddr = 0;
		memdesc->priv &= ~KGSL_MEMDESC_GENPOOL_ALLOC;
		}
		return 0;
		}
		EXPORT_SYMBOL(kgsl_mmu_put_gpuaddr);

drivers/gpu/msm/kgsl_mmu.h

+1 −0

Original line number	Diff line number	Diff line
		@@ -79,6 +79,7 @@ struct kgsl_pagetable {
		unsigned int fault_addr;
		void *priv;
		struct kgsl_mmu *mmu;
		unsigned long *mem_bitmap;
		};

		struct kgsl_mmu;