Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 75bbf593 authored by Linux Build Service Account's avatar Linux Build Service Account Committed by Gerrit - the friendly Code Review server
Browse files

Merge "gpu: ion: Also shrink memory cached in the deferred free list"

parents 0c313f7c 58aacff2

drivers/gpu/ion/ion.c

+48 −71
Original line number Diff line number Diff line
@@ -30,8 +30,6 @@ 
#include <linux/mm.h>

#include <linux/mm_types.h>

#include <linux/rbtree.h>

#include <linux/rtmutex.h>

#include <linux/sched.h>

#include <linux/slab.h>

#include <linux/seq_file.h>

#include <linux/uaccess.h>
@@ -147,7 +145,6 @@ static void ion_buffer_add(struct ion_device *dev, 


static int ion_buffer_alloc_dirty(struct ion_buffer *buffer);



static bool ion_heap_drain_freelist(struct ion_heap *heap);

/* this function should only be called while dev->lock is held */

static struct ion_buffer *ion_buffer_create(struct ion_heap *heap,

				     struct ion_device *dev,
@@ -174,7 +171,7 @@ static struct ion_buffer *ion_buffer_create(struct ion_heap *heap, 
		if (!(heap->flags & ION_HEAP_FLAG_DEFER_FREE))

			goto err2;



		ion_heap_drain_freelist(heap);

		ion_heap_freelist_drain(heap, 0);

		ret = heap->ops->allocate(heap, buffer, len, align,

					  flags);

		if (ret)
@@ -235,7 +232,7 @@ err2: 
	return ERR_PTR(ret);

}



static void _ion_buffer_destroy(struct ion_buffer *buffer)

void ion_buffer_destroy(struct ion_buffer *buffer)

{

	if (WARN_ON(buffer->kmap_cnt > 0))

		buffer->heap->ops->unmap_kernel(buffer->heap, buffer);
@@ -247,7 +244,7 @@ static void _ion_buffer_destroy(struct ion_buffer *buffer) 
	kfree(buffer);

}



static void ion_buffer_destroy(struct kref *kref)

static void _ion_buffer_destroy(struct kref *kref)

{

	struct ion_buffer *buffer = container_of(kref, struct ion_buffer, ref);

	struct ion_heap *heap = buffer->heap;
@@ -257,14 +254,10 @@ static void ion_buffer_destroy(struct kref *kref) 
	rb_erase(&buffer->node, &dev->buffers);

	mutex_unlock(&dev->buffer_lock);



	if (heap->flags & ION_HEAP_FLAG_DEFER_FREE) {

		rt_mutex_lock(&heap->lock);

		list_add(&buffer->list, &heap->free_list);

		rt_mutex_unlock(&heap->lock);

		wake_up(&heap->waitqueue);

		return;

	}

	_ion_buffer_destroy(buffer);

	if (heap->flags & ION_HEAP_FLAG_DEFER_FREE)

		ion_heap_freelist_add(heap, buffer);

	else

		ion_buffer_destroy(buffer);

}



static void ion_buffer_get(struct ion_buffer *buffer)
@@ -274,7 +267,7 @@ static void ion_buffer_get(struct ion_buffer *buffer) 


static int ion_buffer_put(struct ion_buffer *buffer)

{

	return kref_put(&buffer->ref, ion_buffer_destroy);

	return kref_put(&buffer->ref, _ion_buffer_destroy);

}



static void ion_buffer_add_to_handle(struct ion_buffer *buffer)
@@ -1602,87 +1595,71 @@ static const struct file_operations debug_heap_fops = { 
	.release = single_release,

};



static size_t ion_heap_free_list_is_empty(struct ion_heap *heap)

{

	bool is_empty;



	rt_mutex_lock(&heap->lock);

	is_empty = list_empty(&heap->free_list);

	rt_mutex_unlock(&heap->lock);



	return is_empty;

}



static int ion_heap_deferred_free(void *data)

#ifdef DEBUG_HEAP_SHRINKER

static int debug_shrink_set(void *data, u64 val)

{

        struct ion_heap *heap = data;

        struct shrink_control sc;

        int objs;



	while (true) {

		struct ion_buffer *buffer;

        sc.gfp_mask = -1;

        sc.nr_to_scan = 0;



		wait_event_freezable(heap->waitqueue,

				     !ion_heap_free_list_is_empty(heap));

        if (!val)

                return 0;



		rt_mutex_lock(&heap->lock);

		if (list_empty(&heap->free_list)) {

			rt_mutex_unlock(&heap->lock);

			continue;

		}

		buffer = list_first_entry(&heap->free_list, struct ion_buffer,

					  list);

		list_del(&buffer->list);

		rt_mutex_unlock(&heap->lock);

		_ion_buffer_destroy(buffer);

	}

        objs = heap->shrinker.shrink(&heap->shrinker, &sc);

        sc.nr_to_scan = objs;



        heap->shrinker.shrink(&heap->shrinker, &sc);

        return 0;

}



static bool ion_heap_drain_freelist(struct ion_heap *heap)

static int debug_shrink_get(void *data, u64 *val)

{

	struct ion_buffer *buffer, *tmp;



	if (ion_heap_free_list_is_empty(heap))

		return false;

	rt_mutex_lock(&heap->lock);

	list_for_each_entry_safe(buffer, tmp, &heap->free_list, list) {

		list_del(&buffer->list);

		_ion_buffer_destroy(buffer);

	}

	BUG_ON(!list_empty(&heap->free_list));

	rt_mutex_unlock(&heap->lock);

        struct ion_heap *heap = data;

        struct shrink_control sc;

        int objs;



        sc.gfp_mask = -1;

        sc.nr_to_scan = 0;



	return true;

        objs = heap->shrinker.shrink(&heap->shrinker, &sc);

        *val = objs;

        return 0;

}



DEFINE_SIMPLE_ATTRIBUTE(debug_shrink_fops, debug_shrink_get,

                        debug_shrink_set, "%llu\n");

#endif



void ion_device_add_heap(struct ion_device *dev, struct ion_heap *heap)

{

	struct sched_param param = { .sched_priority = 0 };



	if (!heap->ops->allocate || !heap->ops->free || !heap->ops->map_dma ||

	    !heap->ops->unmap_dma)

		pr_err("%s: can not add heap with invalid ops struct.\n",

		       __func__);



	if (heap->flags & ION_HEAP_FLAG_DEFER_FREE) {

		INIT_LIST_HEAD(&heap->free_list);

		rt_mutex_init(&heap->lock);

		init_waitqueue_head(&heap->waitqueue);

		heap->task = kthread_run(ion_heap_deferred_free, heap,

					 "%s", heap->name);

		sched_setscheduler(heap->task, SCHED_IDLE, &param);

		if (IS_ERR(heap->task))

			pr_err("%s: creating thread for deferred free failed\n",

			       __func__);

	}

	if (heap->flags & ION_HEAP_FLAG_DEFER_FREE)

		ion_heap_init_deferred_free(heap);



	heap->dev = dev;

	down_write(&dev->lock);

	plist_node_init(&heap->node, heap->id);

	/* use negative heap->id to reverse the priority -- when traversing

	   the list later attempt higher id numbers first */

	plist_node_init(&heap->node, -heap->id);

	plist_add(&heap->node, &dev->heaps);

	debugfs_create_file(heap->name, 0664, dev->debug_root, heap,

			    &debug_heap_fops);

#ifdef DEBUG_HEAP_SHRINKER

	if (heap->shrinker.shrink) {

		char debug_name[64];



		snprintf(debug_name, 64, "%s_shrink", heap->name);

		debugfs_create_file(debug_name, 0644, dev->debug_root, heap,

				    &debug_shrink_fops);

	}

#endif

	up_write(&dev->lock);

}

drivers/gpu/ion/ion_heap.c

+107 −0
Original line number Diff line number Diff line
@@ -16,8 +16,12 @@ 
 */



#include <linux/err.h>

#include <linux/freezer.h>

#include <linux/ion.h>

#include <linux/kthread.h>

#include <linux/mm.h>

#include <linux/rtmutex.h>

#include <linux/sched.h>

#include <linux/scatterlist.h>

#include <linux/vmalloc.h>

#include "ion_priv.h"
@@ -131,6 +135,109 @@ end: 
	return ret;

}



void ion_heap_free_page(struct ion_buffer *buffer, struct page *page,

		       unsigned int order)

{

	int i;



	if (!ion_buffer_fault_user_mappings(buffer)) {

		__free_pages(page, order);

		return;

	}

	for (i = 0; i < (1 << order); i++)

		__free_page(page + i);

}



void ion_heap_freelist_add(struct ion_heap *heap, struct ion_buffer * buffer)

{

	rt_mutex_lock(&heap->lock);

	list_add(&buffer->list, &heap->free_list);

	heap->free_list_size += buffer->size;

	rt_mutex_unlock(&heap->lock);

	wake_up(&heap->waitqueue);

}



size_t ion_heap_freelist_size(struct ion_heap *heap)

{

	size_t size;



	rt_mutex_lock(&heap->lock);

	size = heap->free_list_size;

	rt_mutex_unlock(&heap->lock);



	return size;

}



size_t ion_heap_freelist_drain(struct ion_heap *heap, size_t size)

{

	struct ion_buffer *buffer, *tmp;

	size_t total_drained = 0;



	if (ion_heap_freelist_size(heap) == 0)

		return 0;



	rt_mutex_lock(&heap->lock);

	if (size == 0)

		size = heap->free_list_size;



	list_for_each_entry_safe(buffer, tmp, &heap->free_list, list) {

		if (total_drained >= size)

			break;

		list_del(&buffer->list);

		ion_buffer_destroy(buffer);

		heap->free_list_size -= buffer->size;

		total_drained += buffer->size;

	}

	rt_mutex_unlock(&heap->lock);



	return total_drained;

}



int ion_heap_deferred_free(void *data)

{

	struct ion_heap *heap = data;



	while (true) {

		struct ion_buffer *buffer;



		wait_event_freezable(heap->waitqueue,

				     ion_heap_freelist_size(heap) > 0);



		rt_mutex_lock(&heap->lock);

		if (list_empty(&heap->free_list)) {

			rt_mutex_unlock(&heap->lock);

			continue;

		}

		buffer = list_first_entry(&heap->free_list, struct ion_buffer,

					  list);

		list_del(&buffer->list);

		heap->free_list_size -= buffer->size;

		rt_mutex_unlock(&heap->lock);

		ion_buffer_destroy(buffer);

	}



	return 0;

}



int ion_heap_init_deferred_free(struct ion_heap *heap)

{

	struct sched_param param = { .sched_priority = 0 };



	INIT_LIST_HEAD(&heap->free_list);

	heap->free_list_size = 0;

	rt_mutex_init(&heap->lock);

	init_waitqueue_head(&heap->waitqueue);

	heap->task = kthread_run(ion_heap_deferred_free, heap,

				 "%s", heap->name);

	sched_setscheduler(heap->task, SCHED_IDLE, &param);

	if (IS_ERR(heap->task)) {

		pr_err("%s: creating thread for deferred free failed\n",

		       __func__);

		return PTR_RET(heap->task);

	}

	return 0;

}



struct ion_heap *ion_heap_create(struct ion_platform_heap *heap_data)

{

	struct ion_heap *heap = NULL;

drivers/gpu/ion/ion_page_pool.c

+22 −104
Original line number Diff line number Diff line
@@ -21,14 +21,8 @@ 
#include <linux/list.h>

#include <linux/module.h>

#include <linux/slab.h>

#include <linux/shrinker.h>

#include "ion_priv.h"



/* #define DEBUG_PAGE_POOL_SHRINKER */



static struct plist_head pools = PLIST_HEAD_INIT(pools);

static struct shrinker shrinker;



struct ion_page_pool_item {

	struct page *page;

	struct list_head list;
@@ -128,89 +122,28 @@ void ion_page_pool_free(struct ion_page_pool *pool, struct page* page) 
		ion_page_pool_free_pages(pool, page);

}



#ifdef DEBUG_PAGE_POOL_SHRINKER

static int debug_drop_pools_set(void *data, u64 val)

{

	struct shrink_control sc;

	int objs;



	sc.gfp_mask = -1;

	sc.nr_to_scan = 0;



	if (!val)

		return 0;



	objs = shrinker.shrink(&shrinker, &sc);

	sc.nr_to_scan = objs;



	shrinker.shrink(&shrinker, &sc);

	return 0;

}



static int debug_drop_pools_get(void *data, u64 *val)

{

	struct shrink_control sc;

	int objs;



	sc.gfp_mask = -1;

	sc.nr_to_scan = 0;



	objs = shrinker.shrink(&shrinker, &sc);

	*val = objs;

	return 0;

}



DEFINE_SIMPLE_ATTRIBUTE(debug_drop_pools_fops, debug_drop_pools_get,

                        debug_drop_pools_set, "%llu\n");



static int debug_grow_pools_set(void *data, u64 val)

static int ion_page_pool_total(struct ion_page_pool *pool, bool high)

{

	struct ion_page_pool *pool;

	struct page *page;



	plist_for_each_entry(pool, &pools, list) {

		if (val != pool->list.prio)

			continue;

		page = ion_page_pool_alloc_pages(pool);

		if (page)

			ion_page_pool_add(pool, page);

	}



	return 0;

}



DEFINE_SIMPLE_ATTRIBUTE(debug_grow_pools_fops, debug_drop_pools_get,

			debug_grow_pools_set, "%llu\n");

#endif



static int ion_page_pool_total(bool high)

{

	struct ion_page_pool *pool;

	int total = 0;



	plist_for_each_entry(pool, &pools, list) {

	total += high ? (pool->high_count + pool->low_count) *

		(1 << pool->order) :

			pool->low_count * (1 << pool->order);

	}

	return total;

}



static int ion_page_pool_shrink(struct shrinker *shrinker,

				 struct shrink_control *sc)

int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask,

				int nr_to_scan)

{

	struct ion_page_pool *pool;

	int nr_freed = 0;

	int i;

	bool high;

	int nr_to_scan = sc->nr_to_scan;



	high = sc->gfp_mask & __GFP_HIGHMEM;

	high = gfp_mask & __GFP_HIGHMEM;



	if (nr_to_scan == 0)

		return ion_page_pool_total(high);

		return ion_page_pool_total(pool, high);



	plist_for_each_entry(pool, &pools, list) {

	for (i = 0; i < nr_to_scan; i++) {

		struct page *page;
@@ -227,10 +160,8 @@ static int ion_page_pool_shrink(struct shrinker *shrinker, 
		ion_page_pool_free_pages(pool, page);

		nr_freed += (1 << pool->order);

	}

		nr_to_scan -= i;

	}



	return ion_page_pool_total(high);

	return nr_freed;

}



struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order)
@@ -247,35 +178,22 @@ struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order) 
	pool->order = order;

	mutex_init(&pool->mutex);

	plist_node_init(&pool->list, order);

	plist_add(&pool->list, &pools);



	return pool;

}



void ion_page_pool_destroy(struct ion_page_pool *pool)

{

	plist_del(&pool->list, &pools);

	kfree(pool);

}



static int __init ion_page_pool_init(void)

{

	shrinker.shrink = ion_page_pool_shrink;

	shrinker.seeks = DEFAULT_SEEKS;

	shrinker.batch = 0;

	register_shrinker(&shrinker);

#ifdef DEBUG_PAGE_POOL_SHRINKER

	debugfs_create_file("ion_pools_shrink", 0644, NULL, NULL,

			    &debug_drop_pools_fops);

	debugfs_create_file("ion_pools_grow", 0644, NULL, NULL,

			    &debug_grow_pools_fops);

#endif

	return 0;

}



static void __exit ion_page_pool_exit(void)

{

	unregister_shrinker(&shrinker);

}



module_init(ion_page_pool_init);

drivers/gpu/ion/ion_priv.h

+55 −2
Original line number Diff line number Diff line
@@ -85,6 +85,7 @@ struct ion_buffer { 
	char task_comm[TASK_COMM_LEN];

	pid_t pid;

};

void ion_buffer_destroy(struct ion_buffer *buffer);



/**

 * struct ion_heap_ops - ops to operate on a given heap
@@ -136,8 +137,13 @@ struct ion_heap_ops { 
 *			allocating.  These are specified by platform data and

 *			MUST be unique

 * @name:		used for debugging

 * @shrinker:		a shrinker for the heap, if the heap caches system

 *			memory, it must define a shrinker to return it on low

 *			memory conditions, this includes system memory cached

 *			in the deferred free lists for heaps that support it

 * @priv:		private heap data

 * @free_list:		free list head if deferred free is used

 * @free_list_size	size of the deferred free list in bytes

 * @lock:		protects the free list

 * @waitqueue:		queue to wait on from deferred free thread

 * @task:		task struct of deferred free thread
@@ -157,8 +163,10 @@ struct ion_heap { 
	unsigned long flags;

	unsigned int id;

	const char *name;

	struct shrinker shrinker;

	void *priv;

	struct list_head free_list;

	size_t free_list_size;

	struct rt_mutex lock;

	wait_queue_head_t waitqueue;

	struct task_struct *task;
@@ -216,6 +224,43 @@ int ion_heap_map_user(struct ion_heap *, struct ion_buffer *, 
			struct vm_area_struct *);

int ion_heap_buffer_zero(struct ion_buffer *buffer);



/**

 * ion_heap_init_deferred_free -- initialize deferred free functionality

 * @heap:		the heap

 *

 * If a heap sets the ION_HEAP_FLAG_DEFER_FREE flag this function will

 * be called to setup deferred frees. Calls to free the buffer will

 * return immediately and the actual free will occur some time later

 */

int ion_heap_init_deferred_free(struct ion_heap *heap);



/**

 * ion_heap_freelist_add - add a buffer to the deferred free list

 * @heap:		the heap

 * @buffer: 		the buffer

 *

 * Adds an item to the deferred freelist.

 */

void ion_heap_freelist_add(struct ion_heap *heap, struct ion_buffer *buffer);



/**

 * ion_heap_freelist_drain - drain the deferred free list

 * @heap:		the heap

 * @size:		ammount of memory to drain in bytes

 *

 * Drains the indicated amount of memory from the deferred freelist immediately.

 * Returns the total amount freed.  The total freed may be higher depending

 * on the size of the items in the list, or lower if there is insufficient

 * total memory on the freelist.

 */

size_t ion_heap_freelist_drain(struct ion_heap *heap, size_t size);



/**

 * ion_heap_freelist_size - returns the size of the freelist in bytes

 * @heap:		the heap

 */

size_t ion_heap_freelist_size(struct ion_heap *heap);





/**

 * functions for creating and destroying the built in ion heaps.
@@ -285,8 +330,6 @@ struct ion_page_pool { 
	struct list_head high_items;

	struct list_head low_items;

	struct mutex mutex;

	void *(*alloc)(struct ion_page_pool *pool);

	void (*free)(struct ion_page_pool *pool, struct page *page);

	gfp_t gfp_mask;

	unsigned int order;

	struct plist_node list;
@@ -297,6 +340,16 @@ void ion_page_pool_destroy(struct ion_page_pool *); 
void *ion_page_pool_alloc(struct ion_page_pool *);

void ion_page_pool_free(struct ion_page_pool *, struct page *);



/** ion_page_pool_shrink - shrinks the size of the memory cached in the pool

 * @pool:		the pool

 * @gfp_mask:		the memory type to reclaim

 * @nr_to_scan:		number of items to shrink in pages

 *

 * returns the number of items freed in pages

 */

int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask,

			  int nr_to_scan);



int ion_walk_heaps(struct ion_client *client, int heap_id, void *data,

			int (*f)(struct ion_heap *heap, void *data));

drivers/gpu/ion/ion_system_heap.c

+49 −0
Original line number Diff line number Diff line
@@ -265,6 +265,50 @@ static struct ion_heap_ops system_heap_ops = { 
	.map_user = ion_heap_map_user,

};



static int ion_system_heap_shrink(struct shrinker *shrinker,

				  struct shrink_control *sc) {



	struct ion_heap *heap = container_of(shrinker, struct ion_heap,

					     shrinker);

	struct ion_system_heap *sys_heap = container_of(heap,

							struct ion_system_heap,

							heap);

	int nr_total = 0;

	int nr_freed = 0;

	int i;



	if (sc->nr_to_scan == 0)

		goto end;



	/* shrink the free list first, no point in zeroing the memory if

	   we're just going to reclaim it */

	nr_freed += ion_heap_freelist_drain(heap, sc->nr_to_scan * PAGE_SIZE) /

		PAGE_SIZE;



	if (nr_freed >= sc->nr_to_scan)

		goto end;



	for (i = 0; i < num_orders; i++) {

		struct ion_page_pool *pool = sys_heap->pools[i];



		nr_freed += ion_page_pool_shrink(pool, sc->gfp_mask,

						 sc->nr_to_scan);

		if (nr_freed >= sc->nr_to_scan)

			break;

	}



end:

	/* total number of items is whatever the page pools are holding

	   plus whatever's in the freelist */

	for (i = 0; i < num_orders; i++) {

		struct ion_page_pool *pool = sys_heap->pools[i];

		nr_total += ion_page_pool_shrink(pool, sc->gfp_mask, 0);

	}

	nr_total += ion_heap_freelist_size(heap) / PAGE_SIZE;

	return nr_total;



}



static int ion_system_heap_debug_show(struct ion_heap *heap, struct seq_file *s,

				      void *unused)

{
@@ -311,6 +355,11 @@ struct ion_heap *ion_system_heap_create(struct ion_platform_heap *unused) 
			goto err_create_pool;

		heap->pools[i] = pool;

	}



	heap->heap.shrinker.shrink = ion_system_heap_shrink;

	heap->heap.shrinker.seeks = DEFAULT_SEEKS;

	heap->heap.shrinker.batch = 0;

	register_shrinker(&heap->heap.shrinker);

	heap->heap.debug_show = ion_system_heap_debug_show;

	return &heap->heap;

err_create_pool: