Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit eb8ec7b9 authored by Jordan Crouse's avatar Jordan Crouse Committed by Tarun Karra
Browse files

msm: kgsl: Fix address generation for SVM buffers 



Fix various corner cases and various mathmatical mistakes in the
SVM address path.  End result should be clearer and (hopefully)
more correct code.

Change-Id: Ic0dedbad1418609913b4cec7781680bbe0be78ff
Signed-off-by: default avatarJordan Crouse <jcrouse@codeaurora.org>
parent 7c1d3047

drivers/gpu/msm/kgsl.c

+47 −21
Original line number Diff line number Diff line
@@ -3343,15 +3343,14 @@ static unsigned long _gpu_find_svm(struct kgsl_process_private *private, 


/* Search top down in the CPU VM region for a free address */

static unsigned long _cpu_get_unmapped_area(unsigned long bottom,

		unsigned long top, unsigned long start,

		unsigned long len, unsigned int align)

		unsigned long top, unsigned long len, unsigned long align)

{

	struct vm_unmapped_area_info info;

	unsigned long addr, err;



	info.flags = VM_UNMAPPED_AREA_TOPDOWN;

	info.low_limit = bottom;

	info.high_limit = min_t(unsigned long, top, start);

	info.high_limit = top;

	info.length = len;

	info.align_offset = 0;

	info.align_mask = align - 1;
@@ -3368,13 +3367,14 @@ static unsigned long _cpu_get_unmapped_area(unsigned long bottom, 
static unsigned long _search_range(struct kgsl_process_private *private,

		struct kgsl_mem_entry *entry,

		unsigned long start, unsigned long end,

		unsigned long len, unsigned int align)

		unsigned long len, uint64_t align)

{

	unsigned long cpu = end, gpu = end, result = -ENOMEM;

	unsigned long cpu, gpu = end, result = -ENOMEM;



	while (gpu > start) {

		/* find a new empty spot on the CPU below the last one */

		cpu = _cpu_get_unmapped_area(start, cpu, gpu, len, align);

		cpu = _cpu_get_unmapped_area(start, gpu, len,

			(unsigned long) align);

		if (IS_ERR_VALUE(cpu)) {

			result = cpu;

			break;
@@ -3386,12 +3386,32 @@ static unsigned long _search_range(struct kgsl_process_private *private, 


		trace_kgsl_mem_unmapped_area_collision(entry, cpu, len);



		if (cpu <= start) {

			result = -ENOMEM;

			break;

		}



		/* move downward to the next empty spot on the GPU */

		gpu = _gpu_find_svm(private, start, cpu, len, align);

		if (IS_ERR_VALUE(gpu)) {

			result = gpu;

			break;

		}



		/* Check that_gpu_find_svm doesn't put us in a loop */

		BUG_ON(gpu >= cpu);



		/* Break if the recommended GPU address is out of range */

		if (gpu < start) {

			result = -ENOMEM;

			break;

		}



		/*

		 * Add the length of the chunk to the GPU address to yield the

		 * upper bound for the CPU search

		 */

		gpu += len;

	}

	return result;

}
@@ -3404,6 +3424,7 @@ static unsigned long _get_svm_area(struct kgsl_process_private *private, 
	int align_shift = kgsl_memdesc_get_align(&entry->memdesc);

	uint64_t align;

	unsigned long result;

	unsigned long addr;



	if (align_shift >= ilog2(SZ_2M))

		align = SZ_2M;
@@ -3435,13 +3456,13 @@ static unsigned long _get_svm_area(struct kgsl_process_private *private, 
		 * See if the hint is usable, if not we will use

		 * it as the start point for searching.

		 */

		hint = clamp_t(unsigned long, hint & ~(align - 1),

		addr = clamp_t(unsigned long, hint & ~(align - 1),

				start, end);



		vma = find_vma(current->mm, hint);

		vma = find_vma(current->mm, addr);



		if (vma == NULL || ((hint + len) <= vma->vm_start)) {

			result = _gpu_set_svm_region(private, entry, hint, len);

		if (vma == NULL || ((addr + len) <= vma->vm_start)) {

			result = _gpu_set_svm_region(private, entry, addr, len);



			/* On failure drop down to keep searching */

			if (!IS_ERR_VALUE(result))
@@ -3449,16 +3470,17 @@ static unsigned long _get_svm_area(struct kgsl_process_private *private, 
		}

	} else {

		/* no hint, start search at the top and work down */

		hint = end & ~(align - 1);

		addr = end & ~(align - 1);

	}



	/*

	 * Search downwards from the hint first. If that fails we

	 * must try to search above it.

	 */

	result = _search_range(private, entry, start, hint, len, align);

	if (IS_ERR_VALUE(result))

		result = _search_range(private, entry, hint, end, len, align);

	result = _search_range(private, entry, start, addr, len, align);

	if (IS_ERR_VALUE(result) && hint != 0)

		result = _search_range(private, entry, addr, end, len, align);



	return result;

}
@@ -3482,15 +3504,19 @@ kgsl_get_unmapped_area(struct file *file, unsigned long addr, 
	if (ret)

		return ret;



	if (!kgsl_memdesc_use_cpu_map(&entry->memdesc))

	if (!kgsl_memdesc_use_cpu_map(&entry->memdesc)) {

		val = get_unmapped_area(NULL, addr, len, 0, flags);

	else

		if (IS_ERR_VALUE(val))

			KGSL_MEM_ERR(device,

				"get_unmapped_area: pid %d addr %lx pgoff %lx len %ld failed error %d\n",

				private->pid, addr, pgoff, len, (int) val);

	} else {

		 val = _get_svm_area(private, entry, addr, len, flags);



		 if (IS_ERR_VALUE(val))

			KGSL_MEM_ERR(device,

			"pid %d addr %lx pgoff %lx len %ld failed error %d\n",

				"_get_svm_area: pid %d addr %lx pgoff %lx len %ld failed error %d\n",

				private->pid, addr, pgoff, len, (int) val);

	}



	kgsl_mem_entry_put(entry);

	return val;

drivers/gpu/msm/kgsl_iommu.c

+42 −15
Original line number Diff line number Diff line
@@ -1804,52 +1804,79 @@ static uint64_t _get_unmapped_area_topdown(struct kgsl_pagetable *pagetable, 
	struct kgsl_iommu_pt *pt = pagetable->priv;

	struct rb_node *node = rb_last(&pt->rbtree);

	uint64_t end = top;



	uint64_t mask = ~(align - 1);

	struct kgsl_iommu_addr_entry *entry;



	if (node == NULL)

		return (top - size) & ~(align - 1);

	/* Make sure that the bottom is correctly aligned */

	bottom = ALIGN(bottom, align);



	entry = rb_entry(node, struct kgsl_iommu_addr_entry, node);

	/* Make sure the requested size will fit in the range */

	if (size > (top - bottom))

		return -ENOMEM;



	if (ALIGN(entry->base + entry->size, align) < top) {

		if (top - ALIGN(entry->base + entry->size, align) >= size)

			return (top - size) & ~(align - 1);

	/* Walk back through the list to find the highest entry in the range */

	for (node = rb_last(&pt->rbtree); node != NULL; node = rb_prev(node)) {

		entry = rb_entry(node, struct kgsl_iommu_addr_entry, node);

		if (entry->base < top)

			break;

	}



	while (node != NULL) {

		uint64_t gap;

		uint64_t offset;



		entry = rb_entry(node, struct kgsl_iommu_addr_entry, node);



		/* If the entire entry is below the range the search is over */

		if ((entry->base + entry->size) < bottom)

			break;



		if ((entry->base + entry->size) < end) {

			gap = end - ALIGN(entry->base + entry->size, align);

			if (gap >= size)

				return ALIGN(entry->base + entry->size, align);

		/* Get the top of the entry properly aligned */

		offset = ALIGN(entry->base + entry->size, align);



		/*

		 * Try to allocate the memory from the top of the gap,

		 * making sure that it fits between the top of this entry and

		 * the bottom of the previous one

		 */



		if (offset < end) {

			uint64_t chunk = (end - size) & mask;



			if (chunk >= offset)

				return chunk;

		}



		/*

		 * If we get here and the current entry is outside of the range

		 * then we are officially out of room

		 */



		if (entry->base < bottom)

			return (uint64_t) -ENOMEM;



		/* Set the top of the gap to the current entry->base */

		end = entry->base;



		/* And move on to the next lower entry */

		node = rb_prev(node);

	}



	if (((end - size) & ~(align - 1)) >= bottom)

		return (end - size) & ~(align - 1);

	/* If we get here then there are no more entries in the region */

	if (((end - size) & mask) >= bottom)

		return (end - size) & mask;



	return (uint64_t) -ENOMEM;

}



static uint64_t kgsl_iommu_find_svm_region(struct kgsl_pagetable *pagetable,

		uint64_t start, uint64_t end, uint64_t size,

		unsigned int alignment)

		uint64_t alignment)

{

	uint64_t addr;



	/* Avoid black holes */

	BUG_ON(end <= start);



	spin_lock(&pagetable->lock);

	addr = _get_unmapped_area_topdown(pagetable,

			start, end, size, alignment);

drivers/gpu/msm/kgsl_mmu.c

+1 −1
Original line number Diff line number Diff line
@@ -682,7 +682,7 @@ static int _nommu_get_gpuaddr(struct kgsl_memdesc *memdesc) 
 */

uint64_t kgsl_mmu_find_svm_region(struct kgsl_pagetable *pagetable,

		uint64_t start, uint64_t end, uint64_t size,

		unsigned int align)

		uint64_t align)

{

	BUG_ON(pagetable == NULL || pagetable->pt_ops->find_svm_region == NULL);

	return pagetable->pt_ops->find_svm_region(pagetable, start, end, size,

drivers/gpu/msm/kgsl_mmu.h

+2 −2
Original line number Diff line number Diff line
@@ -98,7 +98,7 @@ struct kgsl_mmu_pt_ops { 
	int (*get_gpuaddr)(struct kgsl_pagetable *, struct kgsl_memdesc *);

	void (*put_gpuaddr)(struct kgsl_pagetable *, struct kgsl_memdesc *);

	uint64_t (*find_svm_region)(struct kgsl_pagetable *, uint64_t, uint64_t,

		uint64_t, unsigned int);

		uint64_t, uint64_t);

	int (*set_svm_region)(struct kgsl_pagetable *, uint64_t, uint64_t);

	int (*svm_range)(struct kgsl_pagetable *, uint64_t *, uint64_t *,

			uint64_t);
@@ -193,7 +193,7 @@ struct kgsl_pagetable *kgsl_mmu_get_pt_from_ptname(struct kgsl_mmu *mmu, 


uint64_t kgsl_mmu_find_svm_region(struct kgsl_pagetable *pagetable,

		uint64_t start, uint64_t end, uint64_t size,

		unsigned int alignment);

		uint64_t alignment);



int kgsl_mmu_set_svm_region(struct kgsl_pagetable *pagetable, uint64_t gpuaddr,

		uint64_t size);

drivers/gpu/msm/kgsl_trace.h

+1 −1
Original line number Diff line number Diff line
@@ -468,7 +468,7 @@ TRACE_EVENT(kgsl_mem_unmapped_area_collision, 
		 unsigned long addr,

		 unsigned long len),



	TP_ARGS(mem_entry, len, addr),

	TP_ARGS(mem_entry, addr, len),



	TP_STRUCT__entry(

		__field(unsigned int, id)