Merge "msm: kgsl: Add 1M and 8K pools to the allocator" (d129a237) · Commits · e / devices / android_kernel_xiaomi_markw

drivers/gpu/msm/kgsl_pool.c

+117 −27

Original line number	Diff line number	Diff line
		@@ -21,18 +21,21 @@
		#include "kgsl_device.h"
		#include "kgsl_pool.h"

		/*
		* Maximum pool size in terms of pages
		* = (Number of pools * Max size per pool)
		/**
		* struct kgsl_page_pool - Structure to hold information for the pool
		* @pool_order: Page order describing the size of the page
		* @page_count: Number of pages currently present in the pool
		* @reserved_pages: Number of pages reserved at init for the pool
		* @allocation_allowed: Tells if reserved pool gets exhausted, can we allocate
		* from system memory
		* @list_lock: Spinlock for page list in the pool
		* @page_list: List of pages held/reserved in this pool
		*/
		#define KGSL_POOL_MAX_PAGES (2 * 4096)

		/* Set the max pool size to 8192 pages */
		static unsigned int kgsl_pool_max_pages = KGSL_POOL_MAX_PAGES;

		struct kgsl_page_pool {
		unsigned int pool_order;
		int page_count;
		unsigned int reserved_pages;
		bool allocation_allowed;
		spinlock_t list_lock;
		struct list_head page_list;
		};
		@@ -40,15 +43,34 @@ struct kgsl_page_pool {
		static struct kgsl_page_pool kgsl_pools[] = {
		{
		.pool_order = 0,
		.reserved_pages = 2048,
		.allocation_allowed = true,
		.list_lock = __SPIN_LOCK_UNLOCKED(kgsl_pools[0].list_lock),
		.page_list = LIST_HEAD_INIT(kgsl_pools[0].page_list),
		},
		#ifndef CONFIG_ALLOC_BUFFERS_IN_4K_CHUNKS
		{
		.pool_order = 4,
		.pool_order = 1,
		.reserved_pages = 1024,
		.allocation_allowed = true,
		.list_lock = __SPIN_LOCK_UNLOCKED(kgsl_pools[1].list_lock),
		.page_list = LIST_HEAD_INIT(kgsl_pools[1].page_list),
		},
		{
		.pool_order = 4,
		.reserved_pages = 256,
		.allocation_allowed = false,
		.list_lock = __SPIN_LOCK_UNLOCKED(kgsl_pools[2].list_lock),
		.page_list = LIST_HEAD_INIT(kgsl_pools[2].page_list),
		},
		{
		.pool_order = 8,
		.reserved_pages = 32,
		.allocation_allowed = false,
		.list_lock = __SPIN_LOCK_UNLOCKED(kgsl_pools[3].list_lock),
		.page_list = LIST_HEAD_INIT(kgsl_pools[3].page_list),
		},

		#endif
		};

		@@ -68,10 +90,28 @@ _kgsl_get_pool_from_order(unsigned int order)
		return NULL;
		}

		/* Map the page into kernel and zero it out */
		static void
		_kgsl_pool_zero_page(struct page *p, unsigned int pool_order)
		{
		int i;

		for (i = 0; i < (1 << pool_order); i++) {
		struct page *page = nth_page(p, i);
		void *addr = kmap_atomic(page);

		memset(addr, 0, PAGE_SIZE);
		dmac_flush_range(addr, addr + PAGE_SIZE);
		kunmap_atomic(addr);
		}
		}

		/* Add a page to specified pool */
		static void
		_kgsl_pool_add_page(struct kgsl_page_pool pool, struct page p)
		{
		_kgsl_pool_zero_page(p, pool->pool_order);

		spin_lock(&pool->list_lock);
		list_add_tail(&p->lru, &pool->page_list);
		pool->page_count++;
		@@ -156,14 +196,14 @@ _kgsl_pool_shrink(struct kgsl_page_pool *pool, int num_pages)
		* (current_pool_size - target_pages) pages from pool
		* starting from higher order pool.
		*/
		static int
		static unsigned long
		kgsl_pool_reduce(unsigned int target_pages)
		{
		int total_pages = 0;
		int i;
		int nr_removed;
		struct kgsl_page_pool *pool;
		unsigned int pcount = 0;
		unsigned long pcount = 0;

		total_pages = kgsl_pool_size_total();

		@@ -223,6 +263,17 @@ void kgsl_pool_free_sgt(struct sg_table *sgt)
		}
		}

		static int kgsl_pool_idx_lookup(unsigned int order)
		{
		int i;

		for (i = 0; i < KGSL_NUM_POOLS; i++)
		if (order == kgsl_pools[i].pool_order)
		return i;

		return -ENOMEM;
		}

		/**
		* kgsl_pool_alloc_page() - Allocate a page of requested size
		* @page_size: Size of the page to be allocated
		@@ -232,35 +283,55 @@ void kgsl_pool_free_sgt(struct sg_table *sgt)
		*
		* Return total page count on success and negative value on failure
		*/
		int kgsl_pool_alloc_page(int page_size, struct page **pages,
		unsigned int pages_len)
		int kgsl_pool_alloc_page(int page_size, struct page *pages,
		unsigned int pages_len, unsigned int *align)
		{
		int j;
		int pcount = 0;
		struct kgsl_page_pool *pool;
		struct page *page = NULL;
		struct page *p = NULL;
		int order = get_order(*page_size);
		int pool_idx;

		if ((pages == NULL) \|\| pages_len < (page_size >> PAGE_SHIFT))
		if ((pages == NULL) \|\| pages_len < (*page_size >> PAGE_SHIFT))
		return -EINVAL;

		pool = _kgsl_get_pool_from_order(get_order(page_size));
		pool = _kgsl_get_pool_from_order(order);
		pool_idx = kgsl_pool_idx_lookup(order);

		if (pool != NULL)
		page = _kgsl_pool_get_page(pool);

		/* Allocate a new page if not allocated from pool */
		if (page == NULL) {
		gfp_t gfp_mask = kgsl_gfp_mask(get_order(page_size));
		gfp_t gfp_mask = kgsl_gfp_mask(order);

		/* Only allocate non-reserved memory for certain pools */
		if (!pool->allocation_allowed) {
		*page_size = PAGE_SIZE <<
		kgsl_pools[pool_idx-1].pool_order;
		align = ilog2(page_size);
		return -EAGAIN;
		}

		page = alloc_pages(gfp_mask,
		get_order(page_size));
		page = alloc_pages(gfp_mask, order);

		if (!page)
		if (!page) {
		if (pool_idx > 0) {
		/* Retry with lower order pages */
		*page_size = PAGE_SIZE <<
		kgsl_pools[pool_idx-1].pool_order;
		align = ilog2(page_size);
		return -EAGAIN;
		} else
		return -ENOMEM;
		}

		for (j = 0; j < (page_size >> PAGE_SHIFT); j++) {
		_kgsl_pool_zero_page(page, order);
		}

		for (j = 0; j < (*page_size >> PAGE_SHIFT); j++) {
		p = nth_page(page, j);
		pages[pcount] = p;
		pcount++;
		@@ -279,18 +350,34 @@ void kgsl_pool_free_page(struct page *page)

		page_order = compound_order(page);

		if (kgsl_pool_size_total() < kgsl_pool_max_pages) {
		pool = _kgsl_get_pool_from_order(page_order);
		if (pool != NULL) {
		_kgsl_pool_add_page(pool, page);
		return;
		}
		}

		/* Give back to system as not added to pool */
		__free_pages(page, page_order);
		}

		static void kgsl_pool_reserve_pages(void)
		{
		int i, j;

		for (i = 0; i < KGSL_NUM_POOLS; i++) {
		struct page *page;

		for (j = 0; j < kgsl_pools[i].reserved_pages; j++) {
		int order = kgsl_pools[i].pool_order;
		gfp_t gfp_mask = kgsl_gfp_mask(order);

		page = alloc_pages(gfp_mask, order);
		if (page != NULL)
		_kgsl_pool_add_page(&kgsl_pools[i], page);
		}
		}
		}

		/* Functions for the shrinker */

		static unsigned long
		@@ -326,6 +413,9 @@ static struct shrinker kgsl_pool_shrinker = {

		void kgsl_init_page_pools(void)
		{
		/* Reserve the appropriate number of pages for each pool */
		kgsl_pool_reserve_pages();

		/* Initialize shrinker */
		register_shrinker(&kgsl_pool_shrinker);
		}

drivers/gpu/msm/kgsl_pool.h

+2 −2

Original line number	Diff line number	Diff line
		@@ -36,8 +36,8 @@ kgsl_gfp_mask(unsigned int page_order)
		void kgsl_pool_free_sgt(struct sg_table *sgt);
		void kgsl_init_page_pools(void);
		void kgsl_exit_page_pools(void);
		int kgsl_pool_alloc_page(int page_size, struct page **pages,
		unsigned int pages_len);
		int kgsl_pool_alloc_page(int page_size, struct page *pages,
		unsigned int pages_len, unsigned int *align);
		void kgsl_pool_free_page(struct page *p);
		#endif /* __KGSL_POOL_H */

drivers/gpu/msm/kgsl_sharedmem.c

+18 −67

Original line number	Diff line number	Diff line
		@@ -656,8 +656,14 @@ EXPORT_SYMBOL(kgsl_cache_range_op);
		#ifndef CONFIG_ALLOC_BUFFERS_IN_4K_CHUNKS
		static inline int get_page_size(size_t size, unsigned int align)
		{
		return (align >= ilog2(SZ_64K) && size >= SZ_64K)
		? SZ_64K : PAGE_SIZE;
		if (align >= ilog2(SZ_1M) && size >= SZ_1M)
		return SZ_1M;
		else if (align >= ilog2(SZ_64K) && size >= SZ_64K)
		return SZ_64K;
		else if (align >= ilog2(SZ_8K) && size >= SZ_8K)
		return SZ_8K;
		else
		return PAGE_SIZE;
		}
		#else
		static inline int get_page_size(size_t size, unsigned int align)
		@@ -666,56 +672,6 @@ static inline int get_page_size(size_t size, unsigned int align)
		}
		#endif

		static void kgsl_zero_pages(struct page **pages, unsigned int pcount)
		{
		unsigned int j;
		unsigned int step = ((VMALLOC_END - VMALLOC_START)/8) >> PAGE_SHIFT;
		pgprot_t page_prot = pgprot_writecombine(PAGE_KERNEL);
		void *ptr;

		/*
		* All memory that goes to the user has to be zeroed out before it gets
		* exposed to userspace. This means that the memory has to be mapped in
		* the kernel, zeroed (memset) and then unmapped. This also means that
		* the dcache has to be flushed to ensure coherency between the kernel
		* and user pages. We used to pass __GFP_ZERO to alloc_page which mapped
		* zeroed and unmaped each individual page, and then we had to turn
		* around and call flush_dcache_page() on that page to clear the caches.
		* This was killing us for performance. Instead, we found it is much
		* faster to allocate the pages without GFP_ZERO, map a chunk of the
		* range ('step' pages), memset it, flush it and then unmap
		* - this results in a factor of 4 improvement for speed for large
		* buffers. There is a small decrease in speed for small buffers,
		* but only on the order of a few microseconds at best. The 'step'
		* size is based on a guess at the amount of free vmalloc space, but
		* will scale down if there's not enough free space.
		*/
		for (j = 0; j < pcount; j += step) {
		step = min(step, pcount - j);

		ptr = vmap(&pages[j], step, VM_IOREMAP, page_prot);

		if (ptr != NULL) {
		memset(ptr, 0, step * PAGE_SIZE);
		dmac_flush_range(ptr, ptr + step * PAGE_SIZE);
		vunmap(ptr);
		} else {
		int k;
		/* Very, very, very slow path */

		for (k = j; k < j + step; k++) {
		ptr = kmap_atomic(pages[k]);
		memset(ptr, 0, PAGE_SIZE);
		dmac_flush_range(ptr, ptr + PAGE_SIZE);
		kunmap_atomic(ptr);
		}
		/* scale down the step size to avoid this path */
		if (step > 1)
		step >>= 1;
		}
		}
		}

		static int
		kgsl_sharedmem_page_alloc_user(struct kgsl_memdesc *memdesc,
		struct kgsl_pagetable *pagetable,
		@@ -741,8 +697,10 @@ kgsl_sharedmem_page_alloc_user(struct kgsl_memdesc *memdesc,
		* larger however to accomodate hardware quirks
		*/

		if (align < ilog2(page_size))
		if (align < ilog2(page_size)) {
		kgsl_memdesc_set_align(memdesc, ilog2(page_size));
		align = ilog2(page_size);
		}

		/*
		* There needs to be enough room in the page array to be able to
		@@ -775,18 +733,13 @@ kgsl_sharedmem_page_alloc_user(struct kgsl_memdesc *memdesc,
		while (len > 0) {
		int page_count;

		/* don't waste space at the end of the allocation*/
		if (len < page_size)
		page_size = PAGE_SIZE;

		page_count = kgsl_pool_alloc_page(page_size,
		pages + pcount, len_alloc - pcount);
		page_count = kgsl_pool_alloc_page(&page_size,
		pages + pcount, len_alloc - pcount,
		&align);

		if (page_count <= 0) {
		if (page_size != PAGE_SIZE) {
		page_size = PAGE_SIZE;
		if (page_count == -EAGAIN)
		continue;
		}

		/*
		* Update sglen and memdesc size,as requested allocation
		@@ -807,6 +760,9 @@ kgsl_sharedmem_page_alloc_user(struct kgsl_memdesc *memdesc,
		pcount += page_count;
		len -= page_size;
		memdesc->size += page_size;

		/* Get the needed page size for the next iteration */
		page_size = get_page_size(len, align);
		}

		ret = sg_alloc_table_from_pages(memdesc->sgt, pages, pcount, 0,
		@@ -844,11 +800,6 @@ kgsl_sharedmem_page_alloc_user(struct kgsl_memdesc *memdesc,
		KGSL_STATS_ADD(memdesc->size, &kgsl_driver.stats.page_alloc,
		&kgsl_driver.stats.page_alloc_max);

		/*
		* Zero out the pages.
		*/
		kgsl_zero_pages(pages, pcount);

		done:
		if (ret) {
		if (pages) {