ion: Restore GKI system heap implementation

# SPDX-License-Identifier: GPL-2.0

obj-$(CONFIG_ION_SYSTEM_HEAP) += ion_sys_heap.o

ion_sys_heap-y := ion_system_heap.o ion_page_pool.o

obj-$(CONFIG_ION_MSM_HEAPS) += msm_ion_heaps.o

msm_ion_heaps-objs += msm_ion.o msm_ion_dma_buf.o ion_msm_page_pool.o \

		ion_msm_system_heap.o ion_carveout_heap.o \

// SPDX-License-Identifier: GPL-2.0

/*

 * ION Memory Allocator page pool helpers

 *

 * Copyright (C) 2011 Google, Inc.

 */

#include <linux/list.h>

#include <linux/slab.h>

#include <linux/swap.h>

#include <linux/sched/signal.h>

#include "ion_page_pool.h"

static inline struct page *ion_page_pool_alloc_pages(struct ion_page_pool *pool)

{

	if (fatal_signal_pending(current))

		return NULL;

	return alloc_pages(pool->gfp_mask, pool->order);

}

static void ion_page_pool_free_pages(struct ion_page_pool *pool,

				     struct page *page)

{

	__free_pages(page, pool->order);

}

static void ion_page_pool_add(struct ion_page_pool *pool, struct page *page)

{

	mutex_lock(&pool->mutex);

	if (PageHighMem(page)) {

		list_add_tail(&page->lru, &pool->high_items);

		pool->high_count++;

	} else {

		list_add_tail(&page->lru, &pool->low_items);

		pool->low_count++;

	}

	mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE,

							1 << pool->order);

	mutex_unlock(&pool->mutex);

}

static struct page *ion_page_pool_remove(struct ion_page_pool *pool, bool high)

{

	struct page *page;

	if (high) {

		BUG_ON(!pool->high_count);

		page = list_first_entry(&pool->high_items, struct page, lru);

		pool->high_count--;

	} else {

		BUG_ON(!pool->low_count);

		page = list_first_entry(&pool->low_items, struct page, lru);

		pool->low_count--;

	}

	list_del(&page->lru);

	mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE,

							-(1 << pool->order));

	return page;

}

struct page *ion_page_pool_alloc(struct ion_page_pool *pool)

{

	struct page *page = NULL;

	BUG_ON(!pool);

	mutex_lock(&pool->mutex);

	if (pool->high_count)

		page = ion_page_pool_remove(pool, true);

	else if (pool->low_count)

		page = ion_page_pool_remove(pool, false);

	mutex_unlock(&pool->mutex);

	if (!page)

		page = ion_page_pool_alloc_pages(pool);

	return page;

}

void ion_page_pool_free(struct ion_page_pool *pool, struct page *page)

{

	BUG_ON(pool->order != compound_order(page));

	ion_page_pool_add(pool, page);

}

static int ion_page_pool_total(struct ion_page_pool *pool, bool high)

{

	int count = pool->low_count;

	if (high)

		count += pool->high_count;

	return count << pool->order;

}

int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask,

			 int nr_to_scan)

{

	int freed = 0;

	bool high;

	if (current_is_kswapd())

		high = true;

	else

		high = !!(gfp_mask & __GFP_HIGHMEM);

	if (nr_to_scan == 0)

		return ion_page_pool_total(pool, high);

	while (freed < nr_to_scan) {

		struct page *page;

		mutex_lock(&pool->mutex);

		if (pool->low_count) {

			page = ion_page_pool_remove(pool, false);

		} else if (high && pool->high_count) {

			page = ion_page_pool_remove(pool, true);

		} else {

			mutex_unlock(&pool->mutex);

			break;

		}

		mutex_unlock(&pool->mutex);

		ion_page_pool_free_pages(pool, page);

		freed += (1 << pool->order);

	}

	return freed;

}

struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order)

{

	struct ion_page_pool *pool = kmalloc(sizeof(*pool), GFP_KERNEL);

	if (!pool)

		return NULL;

	pool->high_count = 0;

	pool->low_count = 0;

	INIT_LIST_HEAD(&pool->low_items);

	INIT_LIST_HEAD(&pool->high_items);

	pool->gfp_mask = gfp_mask | __GFP_COMP;

	pool->order = order;

	mutex_init(&pool->mutex);

	plist_node_init(&pool->list, order);

	return pool;

}

void ion_page_pool_destroy(struct ion_page_pool *pool)

{

	kfree(pool);

}

/* SPDX-License-Identifier: GPL-2.0 */

/*

 * ION Page Pool kernel interface header

 *

 * Copyright (C) 2011 Google, Inc.

 */

#ifndef _ION_PAGE_POOL_H

#define _ION_PAGE_POOL_H

#include <linux/mm_types.h>

#include <linux/mutex.h>

#include <linux/shrinker.h>

#include <linux/types.h>

/**

 * functions for creating and destroying a heap pool -- allows you

 * to keep a pool of pre allocated memory to use from your heap.  Keeping

 * a pool of memory that is ready for dma, ie any cached mapping have been

 * invalidated from the cache, provides a significant performance benefit on

 * many systems

 */

/**

 * struct ion_page_pool - pagepool struct

 * @high_count:		number of highmem items in the pool

 * @low_count:		number of lowmem items in the pool

 * @high_items:		list of highmem items

 * @low_items:		list of lowmem items

 * @mutex:		lock protecting this struct and especially the count

 *			item list

 * @gfp_mask:		gfp_mask to use from alloc

 * @order:		order of pages in the pool

 * @list:		plist node for list of pools

 *

 * Allows you to keep a pool of pre allocated pages to use from your heap.

 * Keeping a pool of pages that is ready for dma, ie any cached mapping have

 * been invalidated from the cache, provides a significant performance benefit

 * on many systems

 */

struct ion_page_pool {

	int high_count;

	int low_count;

	struct list_head high_items;

	struct list_head low_items;

	struct mutex mutex;

	gfp_t gfp_mask;

	unsigned int order;

	struct plist_node list;

};

struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order);

void ion_page_pool_destroy(struct ion_page_pool *pool);

struct page *ion_page_pool_alloc(struct ion_page_pool *pool);

void ion_page_pool_free(struct ion_page_pool *pool, struct page *page);

/** ion_page_pool_shrink - shrinks the size of the memory cached in the pool

 * @pool:		the pool

 * @gfp_mask:		the memory type to reclaim

 * @nr_to_scan:		number of items to shrink in pages

 *

 * returns the number of items freed in pages

 */

int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask,

			 int nr_to_scan);

#endif /* _ION_PAGE_POOL_H */

// SPDX-License-Identifier: GPL-2.0

/*

 * ION Memory Allocator system heap exporter

 *

 * Copyright (C) 2011 Google, Inc.

 */

#include <asm/page.h>

#include <linux/dma-mapping.h>

#include <linux/err.h>

#include <linux/highmem.h>

#include <linux/ion.h>

#include <linux/mm.h>

#include <linux/module.h>

#include <linux/scatterlist.h>

#include <linux/slab.h>

#include <linux/vmalloc.h>

#include "ion_page_pool.h"

#define NUM_ORDERS ARRAY_SIZE(orders)

static gfp_t high_order_gfp_flags = (GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN |

				     __GFP_NORETRY) & ~__GFP_RECLAIM;

static gfp_t low_order_gfp_flags  = GFP_HIGHUSER | __GFP_ZERO;

static const unsigned int orders[] = {8, 4, 0};

static int order_to_index(unsigned int order)

{

	int i;

	for (i = 0; i < NUM_ORDERS; i++)

		if (order == orders[i])

			return i;

	BUG();

	return -1;

}

static inline unsigned int order_to_size(int order)

{

	return PAGE_SIZE << order;

}

struct ion_system_heap {

	struct ion_heap heap;

	struct ion_page_pool *pools[NUM_ORDERS];

};

static struct page *alloc_buffer_page(struct ion_system_heap *heap,

				      struct ion_buffer *buffer,

				      unsigned long order)

{

	struct ion_page_pool *pool = heap->pools[order_to_index(order)];

	return ion_page_pool_alloc(pool);

}

static void free_buffer_page(struct ion_system_heap *heap,

			     struct ion_buffer *buffer, struct page *page)

{

	struct ion_page_pool *pool;

	unsigned int order = compound_order(page);

	/* go to system */

	if (buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE) {

		__free_pages(page, order);

		return;

	}

	pool = heap->pools[order_to_index(order)];

	ion_page_pool_free(pool, page);

}

static struct page *alloc_largest_available(struct ion_system_heap *heap,

					    struct ion_buffer *buffer,

					    unsigned long size,

					    unsigned int max_order)

{

	struct page *page;

	int i;

	for (i = 0; i < NUM_ORDERS; i++) {

		if (size < order_to_size(orders[i]))

			continue;

		if (max_order < orders[i])

			continue;

		page = alloc_buffer_page(heap, buffer, orders[i]);

		if (!page)

			continue;

		return page;

	}

	return NULL;

}

static int ion_system_heap_allocate(struct ion_heap *heap,

				    struct ion_buffer *buffer,

				    unsigned long size,

				    unsigned long flags)

{

	struct ion_system_heap *sys_heap = container_of(heap,

							struct ion_system_heap,

							heap);

	struct sg_table *table;

	struct scatterlist *sg;

	struct list_head pages;

	struct page *page, *tmp_page;

	int i = 0;

	unsigned long size_remaining = PAGE_ALIGN(size);

	unsigned int max_order = orders[0];

	if (size / PAGE_SIZE > totalram_pages() / 2)

		return -ENOMEM;

	INIT_LIST_HEAD(&pages);

	while (size_remaining > 0) {

		page = alloc_largest_available(sys_heap, buffer, size_remaining,

					       max_order);

		if (!page)

			goto free_pages;

		list_add_tail(&page->lru, &pages);

		size_remaining -= page_size(page);

		max_order = compound_order(page);

		i++;

	}

	table = kmalloc(sizeof(*table), GFP_KERNEL);

	if (!table)

		goto free_pages;

	if (sg_alloc_table(table, i, GFP_KERNEL))

		goto free_table;

	sg = table->sgl;

	list_for_each_entry_safe(page, tmp_page, &pages, lru) {

		sg_set_page(sg, page, page_size(page), 0);

		sg = sg_next(sg);

		list_del(&page->lru);

	}

	buffer->sg_table = table;

	ion_buffer_prep_noncached(buffer);

	return 0;

free_table:

	kfree(table);

free_pages:

	list_for_each_entry_safe(page, tmp_page, &pages, lru)

		free_buffer_page(sys_heap, buffer, page);

	return -ENOMEM;

}

static void ion_system_heap_free(struct ion_buffer *buffer)

{

	struct ion_system_heap *sys_heap = container_of(buffer->heap,

							struct ion_system_heap,

							heap);

	struct sg_table *table = buffer->sg_table;

	struct scatterlist *sg;

	int i;

	/* zero the buffer before goto page pool */

	if (!(buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE))

		ion_buffer_zero(buffer);

	for_each_sg(table->sgl, sg, table->nents, i)

		free_buffer_page(sys_heap, buffer, sg_page(sg));

	sg_free_table(table);

	kfree(table);

}

static int ion_system_heap_shrink(struct ion_heap *heap, gfp_t gfp_mask,

				  int nr_to_scan)

{

	struct ion_page_pool *pool;

	struct ion_system_heap *sys_heap;

	int nr_total = 0;

	int i, nr_freed;

	int only_scan = 0;

	sys_heap = container_of(heap, struct ion_system_heap, heap);

	if (!nr_to_scan)

		only_scan = 1;

	for (i = 0; i < NUM_ORDERS; i++) {

		pool = sys_heap->pools[i];

		if (only_scan) {

			nr_total += ion_page_pool_shrink(pool,

							 gfp_mask,

							 nr_to_scan);

		} else {

			nr_freed = ion_page_pool_shrink(pool,

							gfp_mask,

							nr_to_scan);

			nr_to_scan -= nr_freed;

			nr_total += nr_freed;

			if (nr_to_scan <= 0)

				break;

		}

	}

	return nr_total;

}

static void ion_system_heap_destroy_pools(struct ion_page_pool **pools)

{

	int i;

	for (i = 0; i < NUM_ORDERS; i++)

		if (pools[i])

			ion_page_pool_destroy(pools[i]);

}

static int ion_system_heap_create_pools(struct ion_page_pool **pools)

{

	int i;

	for (i = 0; i < NUM_ORDERS; i++) {

		struct ion_page_pool *pool;

		gfp_t gfp_flags = low_order_gfp_flags;

		if (orders[i] > 4)

			gfp_flags = high_order_gfp_flags;

		pool = ion_page_pool_create(gfp_flags, orders[i]);

		if (!pool)

			goto err_create_pool;

		pools[i] = pool;

	}

	return 0;

err_create_pool:

	ion_system_heap_destroy_pools(pools);

	return -ENOMEM;

}

static struct ion_heap_ops system_heap_ops = {

	.allocate = ion_system_heap_allocate,

	.free = ion_system_heap_free,

	.shrink = ion_system_heap_shrink,

};

static struct ion_system_heap system_heap = {

	.heap = {

		.ops = &system_heap_ops,

		.type = ION_HEAP_TYPE_SYSTEM,

		.flags = ION_HEAP_FLAG_DEFER_FREE,

		.name = "ion_system_heap",

	}

};

static int __init ion_system_heap_init(void)

{

	int ret = ion_system_heap_create_pools(system_heap.pools);

	if (ret)

		return ret;

	return ion_device_add_heap(&system_heap.heap);

}

static void __exit ion_system_heap_exit(void)

{

	ion_device_remove_heap(&system_heap.heap);

	ion_system_heap_destroy_pools(system_heap.pools);

}

module_init(ion_system_heap_init);

module_exit(ion_system_heap_exit);

MODULE_LICENSE("GPL v2");