Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit ad7adec6 authored by Shubhraprakash Das's avatar Shubhraprakash Das
Browse files

msm: kgsl: Optimize virtual memory allocation 



Create a bitmap to track virtual memory allocations for
each pagetable and use this bitmap to efficiently allocate
virtual memory instead of walking the rb tree of the process.
Also, when checking for address collisions for a process
use rb tree macros to traverse the rb list when looking
for next address collisions.

CRs-Fixed: 672081, 675418
Change-Id: I9d0d9da773d112bb9f83b56714109237da66ce6e
Signed-off-by: default avatarShubhraprakash Das <sadas@codeaurora.org>
parent 15e3d7d4

drivers/gpu/msm/kgsl.c

+65 −87
Original line number Diff line number Diff line
@@ -318,8 +318,12 @@ kgsl_mem_entry_track_gpuaddr(struct kgsl_process_private *process, 
	struct rb_node **node;

	struct rb_node *parent = NULL;

	struct kgsl_pagetable *pagetable = process->pagetable;

	size_t size = entry->memdesc.size;



	assert_spin_locked(&process->mem_lock);



	if (kgsl_memdesc_has_guard_page(&entry->memdesc))

		size += PAGE_SIZE;

	/*

	 * If cpu=gpu map is used then caller needs to set the

	 * gpu address
@@ -332,34 +336,12 @@ kgsl_mem_entry_track_gpuaddr(struct kgsl_process_private *process, 
		ret = -EINVAL;

		goto done;

	}

	if (!kgsl_memdesc_use_cpu_map(&entry->memdesc)) {

		ret = -ENOMEM;

		/*

		 * For external allocations use mmu gen pool to assign

		 * virtual address

		 */

		if (KGSL_MEMFLAGS_USERMEM_MASK & entry->memdesc.flags) {

			/* Get secured buffer gpuaddr from secured pool */

	if (kgsl_memdesc_is_secured(&entry->memdesc))

		pagetable = pagetable->mmu->securepagetable;

			ret = kgsl_mmu_get_gpuaddr(pagetable, &entry->memdesc);

		}

		if ((kgsl_memdesc_is_secured(&entry->memdesc)) && (ret))

			goto done;

		if (-ENOMEM == ret) {

			unsigned long gpuaddr = 0;

			size_t size = entry->memdesc.size;

			if (kgsl_memdesc_has_guard_page(&entry->memdesc))

				size += PAGE_SIZE;



			ret = __kgsl_check_collision(process, NULL, &gpuaddr,

						size, 0);

			if (!ret)

				entry->memdesc.gpuaddr = gpuaddr;

		}

	ret = kgsl_mmu_get_gpuaddr(pagetable, &entry->memdesc);

	if (ret)

		goto done;

	}



	node = &process->mem_rb.rb_node;
@@ -396,7 +378,6 @@ kgsl_mem_entry_untrack_gpuaddr(struct kgsl_process_private *process, 
{

	assert_spin_locked(&process->mem_lock);

	if (entry->memdesc.gpuaddr) {

		if (KGSL_MEMFLAGS_USERMEM_MASK & entry->memdesc.flags)

		kgsl_mmu_put_gpuaddr(entry->memdesc.pagetable,

					&entry->memdesc);

		rb_erase(&entry->node, &entry->priv->mem_rb);
@@ -3807,18 +3788,16 @@ static int __kgsl_check_collision(struct kgsl_process_private *private, 
	int ret = 0;

	unsigned long addr = *gpuaddr;

	struct kgsl_mem_entry *collision_entry = entry;

	struct rb_node *node, *node_first, *node_last;



	if (!addr) {

		addr = flag_top_down ? (KGSL_SVM_UPPER_BOUND - len) : PAGE_SIZE;

		collision_entry = NULL;

	}

	if (!collision_entry)

		return -ENOENT;



	do {

		/*

		 * If address collided with an existing entry then find a new

		 * one

		 */

		if (collision_entry) {

	node = &(collision_entry->node);

	node_first = rb_first(&private->mem_rb);

	node_last = rb_last(&private->mem_rb);



	while (1) {

		/*

		 * If top down search then next address to consider

		 * is lower. The highest lower address possible is the
@@ -3828,18 +3807,18 @@ static int __kgsl_check_collision(struct kgsl_process_private *private, 
		if (flag_top_down) {

			addr = collision_entry->memdesc.gpuaddr - len;

			/* Check for loopback */

				if (addr > collision_entry->memdesc.gpuaddr) {

					KGSL_CORE_ERR_ONCE(

					"Underflow err ent:%x/%zx, addr:%lx/%lx\n",

					collision_entry->memdesc.gpuaddr,

					kgsl_memdesc_mmapsize(

						&collision_entry->memdesc),

					addr, len);

					*gpuaddr =

						KGSL_SVM_UPPER_BOUND;

			if (addr > collision_entry->memdesc.gpuaddr || !addr) {

				*gpuaddr = KGSL_SVM_UPPER_BOUND;

				ret = -EAGAIN;

				break;

			}

			if (node == node_first) {

				collision_entry = NULL;

			} else {

				node = rb_prev(&collision_entry->node);

				collision_entry = container_of(node,

					struct kgsl_mem_entry, node);

			}

		} else {

			/*

			 * Bottom up mode the next address to consider
@@ -3853,29 +3832,28 @@ static int __kgsl_check_collision(struct kgsl_process_private *private, 
			/* overflow check */

			if (addr < collision_entry->memdesc.gpuaddr ||

				!mmap_range_valid(addr, len)) {

					KGSL_CORE_ERR_ONCE(

					"Overflow err ent:%x/%zx, addr:%lx/%lx\n",

					collision_entry->memdesc.gpuaddr,

					kgsl_memdesc_mmapsize(

						&collision_entry->memdesc),

					addr, len);

					*gpuaddr =

						KGSL_SVM_UPPER_BOUND;

				*gpuaddr = KGSL_SVM_UPPER_BOUND;

				ret = -EAGAIN;

				break;

			}

			if (node == node_last) {

				collision_entry = NULL;

			} else {

				node = rb_next(&collision_entry->node);

				collision_entry = container_of(node,

					struct kgsl_mem_entry, node);

			}

		}

		if (kgsl_sharedmem_region_empty(private, addr, len,

						&collision_entry)) {

			/* no collision with addr then return */



		if (!collision_entry ||

			!kgsl_addr_range_overlap(addr, len,

			collision_entry->memdesc.gpuaddr,

			kgsl_memdesc_mmapsize(&collision_entry->memdesc))) {

			/* success */

			*gpuaddr = addr;

			break;

		} else if (!collision_entry) {

			ret = -ENOENT;

			break;

		}

	} while (1);

	}



	return ret;

}

drivers/gpu/msm/kgsl.h

+2 −0
Original line number Diff line number Diff line
@@ -120,6 +120,8 @@ struct kgsl_memdesc_ops { 
#define KGSL_MEMDESC_GENPOOL_ALLOC BIT(4)

/* The memdesc is secured for content protection */

#define KGSL_MEMDESC_SECURE BIT(5)

/* Indicates gpuaddr is assigned via bimap */

#define KGSL_MEMDESC_BITMAP_ALLOC BIT(6)



/* shared memory allocation */

struct kgsl_memdesc {

drivers/gpu/msm/kgsl_mmu.c

+51 −10
Original line number Diff line number Diff line
@@ -250,6 +250,9 @@ static void kgsl_destroy_pagetable(struct kref *kref) 


	pagetable->pt_ops->mmu_destroy_pagetable(pagetable);



	if (pagetable->mem_bitmap)

		vfree(pagetable->mem_bitmap);



	kfree(pagetable);

}
@@ -524,6 +527,7 @@ kgsl_mmu_createpagetableobject(struct kgsl_mmu *mmu, 
	unsigned long flags;

	unsigned int ptbase, ptsize;

	char *pool_name;

	int nbits;



	pagetable = kzalloc(sizeof(struct kgsl_pagetable), GFP_KERNEL);

	if (pagetable == NULL)
@@ -560,6 +564,13 @@ kgsl_mmu_createpagetableobject(struct kgsl_mmu *mmu, 
		goto err;

	}



	/* allocate bitmap for virtual memory management */

	nbits = KGSL_SVM_UPPER_BOUND >> PAGE_SHIFT;

	pagetable->mem_bitmap = vmalloc(BITS_TO_LONGS(nbits) * sizeof(long));

	if (!pagetable->mem_bitmap)

		goto err;

	memset(pagetable->mem_bitmap, 0, BITS_TO_LONGS(nbits) * sizeof(long));



	if (KGSL_MMU_TYPE_IOMMU == kgsl_mmu_type)

		pagetable->pt_ops = &iommu_pt_ops;
@@ -585,6 +596,8 @@ err: 
		pagetable->pt_ops->mmu_destroy_pagetable(pagetable);

	if (pagetable->pool)

		gen_pool_destroy(pagetable->pool);

	if (pagetable->mem_bitmap)

		vfree(pagetable->mem_bitmap);



	kfree(pagetable);
@@ -682,17 +695,37 @@ kgsl_mmu_get_gpuaddr(struct kgsl_pagetable *pagetable, 
		if (kgsl_memdesc_use_cpu_map(memdesc)) {

			if (memdesc->gpuaddr == 0)

				return -EINVAL;

			bitmap_set(pagetable->mem_bitmap,

				memdesc->gpuaddr >> PAGE_SHIFT,

				size >> PAGE_SHIFT);

			memdesc->priv |= KGSL_MEMDESC_BITMAP_ALLOC;

			return 0;

		}

	}



	if (KGSL_MEMFLAGS_USERMEM_MASK & memdesc->flags) {

		memdesc->gpuaddr = gen_pool_alloc_aligned(pagetable->pool, size,

						page_align);

		if (memdesc->gpuaddr)

			memdesc->priv |= KGSL_MEMDESC_GENPOOL_ALLOC;

	} else {

		unsigned int gpuaddr = bitmap_find_next_zero_area(

				pagetable->mem_bitmap,

				KGSL_SVM_UPPER_BOUND >> PAGE_SHIFT, 1,

				size >> PAGE_SHIFT, 0);



		if (gpuaddr < (KGSL_SVM_UPPER_BOUND >> PAGE_SHIFT)) {

			bitmap_set(pagetable->mem_bitmap,

				gpuaddr, size >> PAGE_SHIFT);

			memdesc->gpuaddr = gpuaddr << PAGE_SHIFT;

		}

		if (memdesc->gpuaddr)

			memdesc->priv |= KGSL_MEMDESC_BITMAP_ALLOC;

	}



	if (memdesc->gpuaddr == 0)

		return -ENOMEM;



	memdesc->priv |= KGSL_MEMDESC_GENPOOL_ALLOC;

	return 0;

}

EXPORT_SYMBOL(kgsl_mmu_get_gpuaddr);
@@ -764,27 +797,35 @@ kgsl_mmu_put_gpuaddr(struct kgsl_pagetable *pagetable, 
	if (kgsl_mmu_type == KGSL_MMU_TYPE_NONE)

		goto done;



	if (!(KGSL_MEMDESC_GENPOOL_ALLOC & memdesc->priv))

		goto done;



	/* Add space for the guard page when freeing the mmu VA. */

	size = memdesc->size;

	if (kgsl_memdesc_has_guard_page(memdesc))

		size += PAGE_SIZE;



	if (KGSL_MEMDESC_BITMAP_ALLOC & memdesc->priv) {

		bitmap_clear(pagetable->mem_bitmap,

			memdesc->gpuaddr >> PAGE_SHIFT,

			size >> PAGE_SHIFT);

		memdesc->priv &= ~KGSL_MEMDESC_BITMAP_ALLOC;

		goto done;

	}



	if (!(KGSL_MEMDESC_GENPOOL_ALLOC & memdesc->priv))

		goto done;



	pool = pagetable->pool;



	if (pool)

	if (pool) {

		gen_pool_free(pool, memdesc->gpuaddr, size);

		memdesc->priv &= ~KGSL_MEMDESC_GENPOOL_ALLOC;

	}

	/*

	 * Don't clear the gpuaddr on global mappings because they

	 * may be in use by other pagetables

	 */

done:

	if (!kgsl_memdesc_is_global(memdesc)) {

	if (!kgsl_memdesc_is_global(memdesc))

		memdesc->gpuaddr = 0;

		memdesc->priv &= ~KGSL_MEMDESC_GENPOOL_ALLOC;

	}

	return 0;

}

EXPORT_SYMBOL(kgsl_mmu_put_gpuaddr);

drivers/gpu/msm/kgsl_mmu.h

+1 −0
Original line number Diff line number Diff line
@@ -79,6 +79,7 @@ struct kgsl_pagetable { 
	unsigned int fault_addr;

	void *priv;

	struct kgsl_mmu *mmu;

	unsigned long *mem_bitmap;

};



struct kgsl_mmu;