Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 8f081087 authored by Linux Build Service Account's avatar Linux Build Service Account Committed by Gerrit - the friendly Code Review server
Browse files

Merge "staging: ion: Optimize zeroing when allocating from system heap"

parents acaa37d8 7548cfc5

drivers/staging/android/ion/ion_heap.c

+21 −31
Original line number Diff line number Diff line
@@ -25,6 +25,7 @@ 
#include <linux/vmalloc.h>

#include <linux/slab.h>

#include <linux/highmem.h>

#include <linux/dma-mapping.h>

#include "ion.h"

#include "ion_priv.h"
@@ -112,16 +113,15 @@ int ion_heap_map_user(struct ion_heap *heap, struct ion_buffer *buffer, 
 * chunks to minimize the number of memsets and vmaps/vunmaps.

 *

 * Note that the `pages' array should be composed of all 4K pages.

 *

 * NOTE: This function does not guarantee synchronization of the caches

 * and thus caller is responsible for handling any cache maintenance

 * operations needed.

 */

int ion_heap_pages_zero(struct page **pages, int num_pages)

{

	int i, j, k, npages_to_vmap;

	int i, j, npages_to_vmap;

	void *ptr = NULL;

	/*

	 * It's cheaper just to use writecombine memory and skip the

	 * cache vs. using a cache memory and trying to flush it afterwards

	 */

	pgprot_t pgprot = pgprot_writecombine(PAGE_KERNEL);



	/*

	 * As an optimization, we manually zero out all of the pages
@@ -137,7 +137,7 @@ int ion_heap_pages_zero(struct page **pages, int num_pages) 
		for (j = 0; j < MAX_VMAP_RETRIES && npages_to_vmap;

			++j) {

			ptr = vmap(&pages[i], npages_to_vmap,

					VM_IOREMAP, pgprot);

					VM_IOREMAP, PAGE_KERNEL);

			if (ptr)

				break;

			else
@@ -146,18 +146,6 @@ int ion_heap_pages_zero(struct page **pages, int num_pages) 
		if (!ptr)

			return -ENOMEM;



		/*

		 * We have to invalidate the cache here because there

		 * might be dirty lines to these physical pages (which

		 * we don't care about) that could get written out at

		 * any moment.

		 */

		for (k = 0; k < npages_to_vmap; k++) {

			void *p = kmap_atomic(pages[i + k]);



			dmac_inv_range(p, p + PAGE_SIZE);

			kunmap_atomic(p);

		}

		memset(ptr, 0, npages_to_vmap * PAGE_SIZE);

		vunmap(ptr);

	}
@@ -165,11 +153,13 @@ int ion_heap_pages_zero(struct page **pages, int num_pages) 
	return 0;

}



static int ion_heap_alloc_pages_mem(int page_tbl_size,

				struct pages_mem *pages_mem)

int ion_heap_alloc_pages_mem(struct pages_mem *pages_mem)

{

	struct page **pages;

	unsigned int page_tbl_size;



	pages_mem->free_fn = kfree;

	page_tbl_size = sizeof(struct page *) * (pages_mem->size >> PAGE_SHIFT);

	if (page_tbl_size > SZ_8K) {

		/*

		 * Do fallback to ensure we have a balance between
@@ -193,7 +183,7 @@ static int ion_heap_alloc_pages_mem(int page_tbl_size, 
	return 0;

}



static void ion_heap_free_pages_mem(struct pages_mem *pages_mem)

void ion_heap_free_pages_mem(struct pages_mem *pages_mem)

{

	pages_mem->free_fn(pages_mem->pages);

}
@@ -203,15 +193,17 @@ int ion_heap_high_order_page_zero(struct page *page, int order) 
	int i, ret;

	struct pages_mem pages_mem;

	int npages = 1 << order;

	int page_tbl_size = sizeof(struct page *) * npages;

	pages_mem.size = npages * PAGE_SIZE;



	if (ion_heap_alloc_pages_mem(page_tbl_size, &pages_mem))

	if (ion_heap_alloc_pages_mem(&pages_mem))

		return -ENOMEM;



	for (i = 0; i < (1 << order); ++i)

		pages_mem.pages[i] = page + i;



	ret = ion_heap_pages_zero(pages_mem.pages, npages);

	dma_sync_single_for_device(NULL, page_to_phys(page), pages_mem.size,

				   DMA_BIDIRECTIONAL);

	ion_heap_free_pages_mem(&pages_mem);

	return ret;

}
@@ -220,16 +212,12 @@ int ion_heap_buffer_zero(struct ion_buffer *buffer) 
{

	struct sg_table *table = buffer->sg_table;

	struct scatterlist *sg;

	int i, j, ret = 0, npages = 0, page_tbl_size = 0;

	int i, j, ret = 0, npages = 0;

	struct pages_mem pages_mem;



	for_each_sg(table->sgl, sg, table->nents, i) {

		unsigned long len = sg->length;

		int nrpages = len >> PAGE_SHIFT;

		page_tbl_size += sizeof(struct page *) * nrpages;

	}

	pages_mem.size = PAGE_ALIGN(buffer->size);



	if (ion_heap_alloc_pages_mem(page_tbl_size, &pages_mem))

	if (ion_heap_alloc_pages_mem(&pages_mem))

		return -ENOMEM;



	for_each_sg(table->sgl, sg, table->nents, i) {
@@ -241,6 +229,8 @@ int ion_heap_buffer_zero(struct ion_buffer *buffer) 
	}



	ret = ion_heap_pages_zero(pages_mem.pages, npages);

	dma_sync_sg_for_device(NULL, table->sgl, table->nents,

			       DMA_BIDIRECTIONAL);

	ion_heap_free_pages_mem(&pages_mem);

	return ret;

}

drivers/staging/android/ion/ion_page_pool.c

+0 −3
Original line number Diff line number Diff line
@@ -42,9 +42,6 @@ static void *ion_page_pool_alloc_pages(struct ion_page_pool *pool) 
		if (ion_heap_high_order_page_zero(page, pool->order))

			goto error_free_pages;



	ion_pages_sync_for_device(NULL, page, PAGE_SIZE << pool->order,

						DMA_BIDIRECTIONAL);



	return page;

error_free_pages:

	__free_pages(page, pool->order);

drivers/staging/android/ion/ion_priv.h

+3 −0
Original line number Diff line number Diff line
@@ -223,6 +223,7 @@ void ion_device_add_heap(struct ion_device *dev, struct ion_heap *heap); 


struct pages_mem {

	struct page **pages;

	u32 size;

	void (*free_fn) (const void *);

};
@@ -237,6 +238,8 @@ int ion_heap_map_user(struct ion_heap *, struct ion_buffer *, 
int ion_heap_pages_zero(struct page **pages, int num_pages);

int ion_heap_buffer_zero(struct ion_buffer *buffer);

int ion_heap_high_order_page_zero(struct page *page, int order);

int ion_heap_alloc_pages_mem(struct pages_mem *pages_mem);

void ion_heap_free_pages_mem(struct pages_mem *pages_mem);



/**

 * ion_heap_init_deferred_free -- initialize deferred free functionality

drivers/staging/android/ion/ion_system_heap.c

+121 −11
Original line number Diff line number Diff line
@@ -29,10 +29,10 @@ 
#include <linux/dma-mapping.h>

#include <trace/events/kmem.h>



static gfp_t high_order_gfp_flags = (GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN |

static gfp_t high_order_gfp_flags = (GFP_HIGHUSER | __GFP_NOWARN |

				     __GFP_NO_KSWAPD | __GFP_NORETRY)

				     & ~__GFP_WAIT;

static gfp_t low_order_gfp_flags  = (GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN);

static gfp_t low_order_gfp_flags  = (GFP_HIGHUSER | __GFP_NOWARN);



#ifndef CONFIG_ALLOC_BUFFERS_IN_4K_CHUNKS

static const unsigned int orders[] = {9, 8, 4, 0};
@@ -138,6 +138,33 @@ static struct page_info *alloc_largest_available(struct ion_system_heap *heap, 
	}

	return NULL;

}

static unsigned int process_info(struct page_info *info,

				 struct scatterlist *sg,

				 struct scatterlist *sg_sync,

				 struct pages_mem *data, unsigned int i)

{

	struct page *page = info->page;

	unsigned int j;



	if (sg_sync) {

		sg_set_page(sg_sync, page, (1 << info->order) * PAGE_SIZE, 0);

		sg_dma_address(sg_sync) = page_to_phys(page);

	}

	sg_set_page(sg, page, (1 << info->order) * PAGE_SIZE, 0);

	/*

	 * This is not correct - sg_dma_address needs a dma_addr_t

	 * that is valid for the the targeted device, but this works

	 * on the currently targeted hardware.

	 */

	sg_dma_address(sg) = page_to_phys(page);

	if (data) {

		for (j = 0; j < (1 << info->order); ++j)

			data->pages[i++] = nth_page(page, j);

	}

	list_del(&info->list);

	kfree(info);

	return i;

}



static int ion_system_heap_allocate(struct ion_heap *heap,

				     struct ion_buffer *buffer,
@@ -148,54 +175,137 @@ static int ion_system_heap_allocate(struct ion_heap *heap, 
							struct ion_system_heap,

							heap);

	struct sg_table *table;

	struct sg_table table_sync;

	struct scatterlist *sg;

	struct scatterlist *sg_sync;

	int ret;

	struct list_head pages;

	struct list_head pages_from_pool;

	struct page_info *info, *tmp_info;

	int i = 0;

	unsigned int nents_sync = 0;

	unsigned long size_remaining = PAGE_ALIGN(size);

	unsigned int max_order = orders[0];

	struct pages_mem data;

	unsigned int sz;



	if (align > PAGE_SIZE)

		return -EINVAL;



	data.size = 0;

	INIT_LIST_HEAD(&pages);

	INIT_LIST_HEAD(&pages_from_pool);

	while (size_remaining > 0) {

		info = alloc_largest_available(sys_heap, buffer, size_remaining,

						max_order);

		if (!info)

			goto err;



		sz = (1 << info->order) * PAGE_SIZE;



		if (info->from_pool) {

			list_add_tail(&info->list, &pages_from_pool);

		} else {

			list_add_tail(&info->list, &pages);

		size_remaining -= (1 << info->order) * PAGE_SIZE;

			data.size += sz;

			++nents_sync;

		}

		size_remaining -= sz;

		max_order = info->order;

		i++;

	}



	ret = ion_heap_alloc_pages_mem(&data);



	if (ret)

		goto err;



	table = kzalloc(sizeof(struct sg_table), GFP_KERNEL);

	if (!table)

		goto err;

		goto err_free_data_pages;



	ret = sg_alloc_table(table, i, GFP_KERNEL);

	if (ret)

		goto err1;



	if (nents_sync) {

		ret = sg_alloc_table(&table_sync, nents_sync, GFP_KERNEL);

		if (ret)

			goto err_free_sg;

	}



	i = 0;

	sg = table->sgl;

	list_for_each_entry_safe(info, tmp_info, &pages, list) {

		struct page *page = info->page;

		sg_set_page(sg, page, (1 << info->order) * PAGE_SIZE, 0);

	sg_sync = table_sync.sgl;



	/*

	 * We now have two separate lists. One list contains pages from the

	 * pool and the other pages from buddy. We want to merge these

	 * together while preserving the ordering of the pages (higher order

	 * first).

	 */

	do {

		info = list_first_entry_or_null(&pages, struct page_info, list);

		tmp_info = list_first_entry_or_null(&pages_from_pool,

							struct page_info, list);

		if (info && tmp_info) {

			if (info->order >= tmp_info->order) {

				i = process_info(info, sg, sg_sync, &data, i);

				sg_sync = sg_next(sg_sync);

			} else {

				i = process_info(tmp_info, sg, 0, 0, i);

			}

		} else if (info) {

			i = process_info(info, sg, sg_sync, &data, i);

			sg_sync = sg_next(sg_sync);

		} else if (tmp_info) {

			i = process_info(tmp_info, sg, 0, 0, i);

		} else {

			BUG();

		}

		sg = sg_next(sg);

		list_del(&info->list);

		kfree(info);



	} while (sg);



	ret = ion_heap_pages_zero(data.pages, data.size >> PAGE_SHIFT);

	if (ret) {

		pr_err("Unable to zero pages\n");

		goto err_free_sg2;

	}



	if (nents_sync)

		dma_sync_sg_for_device(NULL, table_sync.sgl, table_sync.nents,

				       DMA_BIDIRECTIONAL);



	buffer->priv_virt = table;

	if (nents_sync)

		sg_free_table(&table_sync);

	ion_heap_free_pages_mem(&data);

	return 0;

err_free_sg2:

	/* We failed to zero buffers. Bypass pool */

	buffer->flags |= ION_FLAG_FREED_FROM_SHRINKER;



	for_each_sg(table->sgl, sg, table->nents, i)

		free_buffer_page(sys_heap, buffer, sg_page(sg),

				get_order(sg->length));

	if (nents_sync)

		sg_free_table(&table_sync);

err_free_sg:

	sg_free_table(table);

err1:

	kfree(table);

err_free_data_pages:

	ion_heap_free_pages_mem(&data);

err:

	list_for_each_entry_safe(info, tmp_info, &pages, list) {

		free_buffer_page(sys_heap, buffer, info->page, info->order);

		kfree(info);

	}

	list_for_each_entry_safe(info, tmp_info, &pages_from_pool, list) {

		free_buffer_page(sys_heap, buffer, info->page, info->order);

		kfree(info);

	}

	return -ENOMEM;

}
@@ -435,7 +545,7 @@ static int ion_system_contig_heap_allocate(struct ion_heap *heap, 
	if (align > (PAGE_SIZE << order))

		return -EINVAL;



	page = alloc_pages(low_order_gfp_flags, order);

	page = alloc_pages(low_order_gfp_flags | __GFP_ZERO, order);

	if (!page)

		return -ENOMEM;