Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 0191b643 authored by Chris Goldsworthy's avatar Chris Goldsworthy
Browse files

ion: Reduce secure system heap allocation time by stealing pages 



When performing a secure system heap allocation, and allocating from a
secure pool fails, attempt to steal pages from the uncached system heap
pools, before falling back to the buddy allocator.

Additionally, remove redundant code that was once used for zeroing out
pages allocated from buddy.

Change-Id: Id0b08f49b1fb037b26cd09e6336a0b0dc5768d82
Signed-off-by: default avatarChris Goldsworthy <cgoldswo@codeaurora.org>
parent 0fc8b8b2

drivers/staging/android/ion/heaps/ion_system_heap.c

+53 −70
Original line number Diff line number Diff line
@@ -51,11 +51,6 @@ static inline unsigned int order_to_size(int order) 
	return PAGE_SIZE << order;

}



struct pages_mem {

	struct page **pages;

	u32 size;

};



int ion_heap_is_system_heap_type(enum ion_heap_type type)

{

	return type == ((enum ion_heap_type)ION_HEAP_TYPE_SYSTEM);
@@ -71,14 +66,52 @@ static struct page *alloc_buffer_page(struct ion_system_heap *sys_heap, 
	struct ion_page_pool *pool;

	int vmid = get_secure_vmid(buffer->flags);

	struct device *dev = sys_heap->heap.dev;

	int order_ind = order_to_index(order);

	struct task_struct *worker;



	if (vmid > 0)

		pool = sys_heap->secure_pools[vmid][order_to_index(order)];

	else if (!cached)

		pool = sys_heap->uncached_pools[order_to_index(order)];

	else

		pool = sys_heap->cached_pools[order_to_index(order)];

	if (vmid > 0) {

		pool = sys_heap->secure_pools[vmid][order_ind];



		/*

		 * We should skip stealing pages if (1) we're focing our

		 * allocations to come from buddy; or (2) pool refilling is

		 * disabled, in which case stealing pages could deplete the

		 * uncached pools.

		 */

		if (!(*from_pool && pool_auto_refill_en))

			goto normal_alloc;



		page = ion_page_pool_alloc_pool_only(pool);

		if (!IS_ERR(page))

			return page;



		pool = sys_heap->uncached_pools[order_ind];

		page = ion_page_pool_alloc_pool_only(pool);

		if (IS_ERR(page)) {

			pool = sys_heap->secure_pools[vmid][order_ind];

			goto normal_alloc;

		}



		/*

		 * Here, setting `from_pool = false` indicates that the

		 * page didn't come from the secure pool, and causes

		 * the page to be hyp-assigned.

		 */

		*from_pool = false;



		if (pool_auto_refill_en &&

		    pool_count_below_lowmark(pool)) {

			worker = sys_heap->kworker[ION_KTHREAD_UNCACHED];

			wake_up_process(worker);

		}

		return page;

	} else if (!cached) {

		pool = sys_heap->uncached_pools[order_ind];

	} else {

		pool = sys_heap->cached_pools[order_ind];

	}



normal_alloc:

	page = ion_page_pool_alloc(pool, from_pool);



	if (pool_auto_refill_en &&
@@ -209,13 +242,11 @@ alloc_from_pool_preferred(struct ion_system_heap *heap, 
	return alloc_largest_available(heap, buffer, size, max_order);

}



static unsigned int process_info(struct page_info *info,

static void process_info(struct page_info *info,

			 struct scatterlist *sg,

				 struct scatterlist *sg_sync,

				 struct pages_mem *data, unsigned int i)

			 struct scatterlist *sg_sync)

{

	struct page *page = info->page;

	unsigned int j;



	if (sg_sync) {

		sg_set_page(sg_sync, page, (1 << info->order) * PAGE_SIZE, 0);
@@ -228,45 +259,9 @@ static unsigned int process_info(struct page_info *info, 
	 * on the currently targeted hardware.

	 */

	sg_dma_address(sg) = page_to_phys(page);

	if (data) {

		for (j = 0; j < (1 << info->order); ++j)

			data->pages[i++] = nth_page(page, j);

	}



	list_del(&info->list);

	kfree(info);

	return i;

}



static int ion_heap_alloc_pages_mem(struct pages_mem *pages_mem)

{

	struct page **pages;

	unsigned int page_tbl_size;



	page_tbl_size = sizeof(struct page *) * (pages_mem->size >> PAGE_SHIFT);

	if (page_tbl_size > SZ_8K) {

		/*

		 * Do fallback to ensure we have a balance between

		 * performance and availability.

		 */

		pages = kmalloc(page_tbl_size,

				__GFP_COMP | __GFP_NORETRY |

				__GFP_NOWARN);

		if (!pages)

			pages = vmalloc(page_tbl_size);

	} else {

		pages = kmalloc(page_tbl_size, GFP_KERNEL);

	}



	if (!pages)

		return -ENOMEM;



	pages_mem->pages = pages;

	return 0;

}



static void ion_heap_free_pages_mem(struct pages_mem *pages_mem)

{

	kvfree(pages_mem->pages);

}



static int ion_system_heap_allocate(struct ion_heap *heap,
@@ -287,7 +282,6 @@ static int ion_system_heap_allocate(struct ion_heap *heap, 
	unsigned int nents_sync = 0;

	unsigned long size_remaining = PAGE_ALIGN(size);

	unsigned int max_order = orders[0];

	struct pages_mem data;

	unsigned int sz;

	int vmid = get_secure_vmid(buffer->flags);
@@ -301,7 +295,6 @@ static int ion_system_heap_allocate(struct ion_heap *heap, 
		return -EINVAL;

	}



	data.size = 0;

	INIT_LIST_HEAD(&pages);

	INIT_LIST_HEAD(&pages_from_pool);
@@ -326,7 +319,6 @@ static int ion_system_heap_allocate(struct ion_heap *heap, 
			list_add_tail(&info->list, &pages_from_pool);

		} else {

			list_add_tail(&info->list, &pages);

			data.size += sz;

			++nents_sync;

		}

		size_remaining -= sz;
@@ -334,15 +326,10 @@ static int ion_system_heap_allocate(struct ion_heap *heap, 
		i++;

	}



	ret = ion_heap_alloc_pages_mem(&data);



	if (ret)

		goto err;



	table = kzalloc(sizeof(*table), GFP_KERNEL);

	if (!table) {

		ret = -ENOMEM;

		goto err_free_data_pages;

		goto err;

	}



	ret = sg_alloc_table(table, i, GFP_KERNEL);
@@ -355,7 +342,6 @@ static int ion_system_heap_allocate(struct ion_heap *heap, 
			goto err_free_sg;

	}



	i = 0;

	sg = table->sgl;

	sg_sync = table_sync.sgl;
@@ -371,16 +357,16 @@ static int ion_system_heap_allocate(struct ion_heap *heap, 
						    struct page_info, list);

		if (info && tmp_info) {

			if (info->order >= tmp_info->order) {

				i = process_info(info, sg, sg_sync, &data, i);

				process_info(info, sg, sg_sync);

				sg_sync = sg_next(sg_sync);

			} else {

				i = process_info(tmp_info, sg, 0, 0, i);

				process_info(tmp_info, sg, NULL);

			}

		} else if (info) {

			i = process_info(info, sg, sg_sync, &data, i);

			process_info(info, sg, sg_sync);

			sg_sync = sg_next(sg_sync);

		} else if (tmp_info) {

			i = process_info(tmp_info, sg, 0, 0, i);

			process_info(tmp_info, sg, NULL);

		}

		sg = sg_next(sg);
@@ -397,7 +383,6 @@ static int ion_system_heap_allocate(struct ion_heap *heap, 
	buffer->sg_table = table;

	if (nents_sync)

		sg_free_table(&table_sync);

	ion_heap_free_pages_mem(&data);

	ion_prepare_sgl_for_force_dma_sync(buffer->sg_table);

	return 0;
@@ -417,8 +402,6 @@ static int ion_system_heap_allocate(struct ion_heap *heap, 
	sg_free_table(table);

err1:

	kfree(table);

err_free_data_pages:

	ion_heap_free_pages_mem(&data);

err:

	list_for_each_entry_safe(info, tmp_info, &pages, list) {

		free_buffer_page(sys_heap, buffer, info->page, info->order);