Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit eeaa9b10 authored by Shrenuj Bansal's avatar Shrenuj Bansal
Browse files

msm: kgsl: Add 1M and 8K pools to the allocator 



This change includes the below:
- Add 1M and 8k pools and structure the allocator to use all pools
from the largest page to the smallest
- Reserve a set number of pages for each of these pools at init time
- When allocating, use the reserved pools and then fall back to
allocating from system memory using only 8k and 4k pages
- Remove maximums on the pool sizes
- Zero the memory when we create the pool initially and add pages
back to the pool on free

CRs-Fixed: 995735
Change-Id: I9440bad62d3e13b434902f167c9d23467b1c4235
Signed-off-by: default avatarShrenuj Bansal <shrenujb@codeaurora.org>
parent b6a4104f

drivers/gpu/msm/kgsl_pool.c

+117 −27
Original line number Diff line number Diff line
@@ -21,18 +21,21 @@ 
#include "kgsl_device.h"

#include "kgsl_pool.h"



/*

 * Maximum pool size in terms of pages

 * = (Number of pools * Max size per pool)

/**

 * struct kgsl_page_pool - Structure to hold information for the pool

 * @pool_order: Page order describing the size of the page

 * @page_count: Number of pages currently present in the pool

 * @reserved_pages: Number of pages reserved at init for the pool

 * @allocation_allowed: Tells if reserved pool gets exhausted, can we allocate

 * from system memory

 * @list_lock: Spinlock for page list in the pool

 * @page_list: List of pages held/reserved in this pool

 */

#define KGSL_POOL_MAX_PAGES (2 * 4096)



/* Set the max pool size to 8192 pages */

static unsigned int kgsl_pool_max_pages = KGSL_POOL_MAX_PAGES;



struct kgsl_page_pool {

	unsigned int pool_order;

	int page_count;

	unsigned int reserved_pages;

	bool allocation_allowed;

	spinlock_t list_lock;

	struct list_head page_list;

};
@@ -40,15 +43,34 @@ struct kgsl_page_pool { 
static struct kgsl_page_pool kgsl_pools[] = {

	{

		.pool_order = 0,

		.reserved_pages = 2048,

		.allocation_allowed = true,

		.list_lock = __SPIN_LOCK_UNLOCKED(kgsl_pools[0].list_lock),

		.page_list = LIST_HEAD_INIT(kgsl_pools[0].page_list),

	},

#ifndef CONFIG_ALLOC_BUFFERS_IN_4K_CHUNKS

	{

		.pool_order = 4,

		.pool_order = 1,

		.reserved_pages = 1024,

		.allocation_allowed = true,

		.list_lock = __SPIN_LOCK_UNLOCKED(kgsl_pools[1].list_lock),

		.page_list = LIST_HEAD_INIT(kgsl_pools[1].page_list),

	},

	{

		.pool_order = 4,

		.reserved_pages = 256,

		.allocation_allowed = false,

		.list_lock = __SPIN_LOCK_UNLOCKED(kgsl_pools[2].list_lock),

		.page_list = LIST_HEAD_INIT(kgsl_pools[2].page_list),

	},

	{

		.pool_order = 8,

		.reserved_pages = 32,

		.allocation_allowed = false,

		.list_lock = __SPIN_LOCK_UNLOCKED(kgsl_pools[3].list_lock),

		.page_list = LIST_HEAD_INIT(kgsl_pools[3].page_list),

	},



#endif

};
@@ -68,10 +90,28 @@ _kgsl_get_pool_from_order(unsigned int order) 
	return NULL;

}



/* Map the page into kernel and zero it out */

static void

_kgsl_pool_zero_page(struct page *p, unsigned int pool_order)

{

	int i;



	for (i = 0; i < (1 << pool_order); i++) {

		struct page *page = nth_page(p, i);

		void *addr = kmap_atomic(page);



		memset(addr, 0, PAGE_SIZE);

		dmac_flush_range(addr, addr + PAGE_SIZE);

		kunmap_atomic(addr);

	}

}



/* Add a page to specified pool */

static void

_kgsl_pool_add_page(struct kgsl_page_pool *pool, struct page *p)

{

	_kgsl_pool_zero_page(p, pool->pool_order);



	spin_lock(&pool->list_lock);

	list_add_tail(&p->lru, &pool->page_list);

	pool->page_count++;
@@ -156,14 +196,14 @@ _kgsl_pool_shrink(struct kgsl_page_pool *pool, int num_pages) 
 * (current_pool_size - target_pages) pages from pool

 * starting from higher order pool.

 */

static int

static unsigned long

kgsl_pool_reduce(unsigned int target_pages)

{

	int total_pages = 0;

	int i;

	int nr_removed;

	struct kgsl_page_pool *pool;

	unsigned int pcount = 0;

	unsigned long pcount = 0;



	total_pages = kgsl_pool_size_total();
@@ -223,6 +263,17 @@ void kgsl_pool_free_sgt(struct sg_table *sgt) 
	}

}



static int kgsl_pool_idx_lookup(unsigned int order)

{

	int i;



	for (i = 0; i < KGSL_NUM_POOLS; i++)

		if (order == kgsl_pools[i].pool_order)

			return i;



	return -ENOMEM;

}



/**

 * kgsl_pool_alloc_page() - Allocate a page of requested size

 * @page_size: Size of the page to be allocated
@@ -232,35 +283,55 @@ void kgsl_pool_free_sgt(struct sg_table *sgt) 
 *

 * Return total page count on success and negative value on failure

 */

int kgsl_pool_alloc_page(int page_size, struct page **pages,

					unsigned int pages_len)

int kgsl_pool_alloc_page(int *page_size, struct page **pages,

			unsigned int pages_len, unsigned int *align)

{

	int j;

	int pcount = 0;

	struct kgsl_page_pool *pool;

	struct page *page = NULL;

	struct page *p = NULL;

	int order = get_order(*page_size);

	int pool_idx;



	if ((pages == NULL) || pages_len < (page_size >> PAGE_SHIFT))

	if ((pages == NULL) || pages_len < (*page_size >> PAGE_SHIFT))

		return -EINVAL;



	pool = _kgsl_get_pool_from_order(get_order(page_size));

	pool = _kgsl_get_pool_from_order(order);

	pool_idx = kgsl_pool_idx_lookup(order);



	if (pool != NULL)

		page = _kgsl_pool_get_page(pool);



	/* Allocate a new page if not allocated from pool */

	if (page == NULL) {

		gfp_t gfp_mask = kgsl_gfp_mask(get_order(page_size));

		gfp_t gfp_mask = kgsl_gfp_mask(order);



		/* Only allocate non-reserved memory for certain pools */

		if (!pool->allocation_allowed) {

			*page_size = PAGE_SIZE <<

					kgsl_pools[pool_idx-1].pool_order;

			*align = ilog2(*page_size);

			return -EAGAIN;

		}



		page = alloc_pages(gfp_mask,

					get_order(page_size));

		page = alloc_pages(gfp_mask, order);



		if (!page)

		if (!page) {

			if (pool_idx > 0) {

				/* Retry with lower order pages */

				*page_size = PAGE_SIZE <<

					kgsl_pools[pool_idx-1].pool_order;

				*align = ilog2(*page_size);

				return -EAGAIN;

			} else

				return -ENOMEM;

		}



	for (j = 0; j < (page_size >> PAGE_SHIFT); j++) {

		_kgsl_pool_zero_page(page, order);

	}



	for (j = 0; j < (*page_size >> PAGE_SHIFT); j++) {

		p = nth_page(page, j);

		pages[pcount] = p;

		pcount++;
@@ -279,18 +350,34 @@ void kgsl_pool_free_page(struct page *page) 


	page_order = compound_order(page);



	if (kgsl_pool_size_total() < kgsl_pool_max_pages) {

	pool = _kgsl_get_pool_from_order(page_order);

	if (pool != NULL) {

		_kgsl_pool_add_page(pool, page);

		return;

	}

	}



	/* Give back to system as not added to pool */

	__free_pages(page, page_order);

}



static void kgsl_pool_reserve_pages(void)

{

	int i, j;



	for (i = 0; i < KGSL_NUM_POOLS; i++) {

		struct page *page;



		for (j = 0; j < kgsl_pools[i].reserved_pages; j++) {

			int order = kgsl_pools[i].pool_order;

			gfp_t gfp_mask = kgsl_gfp_mask(order);



			page = alloc_pages(gfp_mask, order);

			if (page != NULL)

				_kgsl_pool_add_page(&kgsl_pools[i], page);

		}

	}

}



/* Functions for the shrinker */



static unsigned long
@@ -326,6 +413,9 @@ static struct shrinker kgsl_pool_shrinker = { 


void kgsl_init_page_pools(void)

{

	/* Reserve the appropriate number of pages for each pool */

	kgsl_pool_reserve_pages();



	/* Initialize shrinker */

	register_shrinker(&kgsl_pool_shrinker);

}

drivers/gpu/msm/kgsl_pool.h

+2 −2
Original line number Diff line number Diff line
@@ -36,8 +36,8 @@ kgsl_gfp_mask(unsigned int page_order) 
void kgsl_pool_free_sgt(struct sg_table *sgt);

void kgsl_init_page_pools(void);

void kgsl_exit_page_pools(void);

int kgsl_pool_alloc_page(int page_size, struct page **pages,

						unsigned int pages_len);

int kgsl_pool_alloc_page(int *page_size, struct page **pages,

			unsigned int pages_len, unsigned int *align);

void kgsl_pool_free_page(struct page *p);

#endif /* __KGSL_POOL_H */

drivers/gpu/msm/kgsl_sharedmem.c

+18 −67
Original line number Diff line number Diff line
@@ -656,8 +656,14 @@ EXPORT_SYMBOL(kgsl_cache_range_op); 
#ifndef CONFIG_ALLOC_BUFFERS_IN_4K_CHUNKS

static inline int get_page_size(size_t size, unsigned int align)

{

	return (align >= ilog2(SZ_64K) && size >= SZ_64K)

					? SZ_64K : PAGE_SIZE;

	if (align >= ilog2(SZ_1M) && size >= SZ_1M)

		return SZ_1M;

	else if (align >= ilog2(SZ_64K) && size >= SZ_64K)

		return SZ_64K;

	else if (align >= ilog2(SZ_8K) && size >= SZ_8K)

		return SZ_8K;

	else

		return PAGE_SIZE;

}

#else

static inline int get_page_size(size_t size, unsigned int align)
@@ -666,56 +672,6 @@ static inline int get_page_size(size_t size, unsigned int align) 
}

#endif



static void kgsl_zero_pages(struct page **pages, unsigned int pcount)

{

	unsigned int j;

	unsigned int step = ((VMALLOC_END - VMALLOC_START)/8) >> PAGE_SHIFT;

	pgprot_t page_prot = pgprot_writecombine(PAGE_KERNEL);

	void *ptr;



	/*

	 * All memory that goes to the user has to be zeroed out before it gets

	 * exposed to userspace. This means that the memory has to be mapped in

	 * the kernel, zeroed (memset) and then unmapped.  This also means that

	 * the dcache has to be flushed to ensure coherency between the kernel

	 * and user pages. We used to pass __GFP_ZERO to alloc_page which mapped

	 * zeroed and unmaped each individual page, and then we had to turn

	 * around and call flush_dcache_page() on that page to clear the caches.

	 * This was killing us for performance. Instead, we found it is much

	 * faster to allocate the pages without GFP_ZERO, map a chunk of the

	 * range ('step' pages), memset it, flush it and then unmap

	 * - this results in a factor of 4 improvement for speed for large

	 * buffers. There is a small decrease in speed for small buffers,

	 * but only on the order of a few microseconds at best. The 'step'

	 * size is based on a guess at the amount of free vmalloc space, but

	 * will scale down if there's not enough free space.

	 */

	for (j = 0; j < pcount; j += step) {

		step = min(step, pcount - j);



		ptr = vmap(&pages[j], step, VM_IOREMAP, page_prot);



		if (ptr != NULL) {

			memset(ptr, 0, step * PAGE_SIZE);

			dmac_flush_range(ptr, ptr + step * PAGE_SIZE);

			vunmap(ptr);

		} else {

			int k;

			/* Very, very, very slow path */



			for (k = j; k < j + step; k++) {

				ptr = kmap_atomic(pages[k]);

				memset(ptr, 0, PAGE_SIZE);

				dmac_flush_range(ptr, ptr + PAGE_SIZE);

				kunmap_atomic(ptr);

			}

			/* scale down the step size to avoid this path */

			if (step > 1)

				step >>= 1;

		}

	}

}



static int

kgsl_sharedmem_page_alloc_user(struct kgsl_memdesc *memdesc,

			struct kgsl_pagetable *pagetable,
@@ -741,8 +697,10 @@ kgsl_sharedmem_page_alloc_user(struct kgsl_memdesc *memdesc, 
	 * larger however to accomodate hardware quirks

	 */



	if (align < ilog2(page_size))

	if (align < ilog2(page_size)) {

		kgsl_memdesc_set_align(memdesc, ilog2(page_size));

		align = ilog2(page_size);

	}



	/*

	 * There needs to be enough room in the page array to be able to
@@ -775,18 +733,13 @@ kgsl_sharedmem_page_alloc_user(struct kgsl_memdesc *memdesc, 
	while (len > 0) {

		int page_count;



		/* don't waste space at the end of the allocation*/

		if (len < page_size)

			page_size = PAGE_SIZE;



		page_count = kgsl_pool_alloc_page(page_size,

					pages + pcount, len_alloc - pcount);

		page_count = kgsl_pool_alloc_page(&page_size,

					pages + pcount, len_alloc - pcount,

					&align);



		if (page_count <= 0) {

			if (page_size != PAGE_SIZE) {

				page_size = PAGE_SIZE;

			if (page_count == -EAGAIN)

				continue;

			}



			/*

			 * Update sglen and memdesc size,as requested allocation
@@ -807,6 +760,9 @@ kgsl_sharedmem_page_alloc_user(struct kgsl_memdesc *memdesc, 
		pcount += page_count;

		len -= page_size;

		memdesc->size += page_size;



		/* Get the needed page size for the next iteration */

		page_size = get_page_size(len, align);

	}



	ret = sg_alloc_table_from_pages(memdesc->sgt, pages, pcount, 0,
@@ -844,11 +800,6 @@ kgsl_sharedmem_page_alloc_user(struct kgsl_memdesc *memdesc, 
	KGSL_STATS_ADD(memdesc->size, &kgsl_driver.stats.page_alloc,

		&kgsl_driver.stats.page_alloc_max);



	/*

	 * Zero out the pages.

	 */

	kgsl_zero_pages(pages, pcount);



done:

	if (ret) {

		if (pages) {