Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c45ee1fa authored by Rebecca Schultz Zavin's avatar Rebecca Schultz Zavin Committed by Mitchel Humpherys
Browse files

gpu: ion: Fix performance issue in faulting code 



Previously the code to fault ion buffers in one page at a time had a
performance problem caused by the requirement to traverse the sg list
looking for the right page to load in (a result of the fact that the items in
the list may not be of uniform size).  To fix the problem, for buffers
that will be faulted in, also keep a flat array of all the pages in the buffer
to use from the fault handler.  To recover some of the additional memory
footprint this creates per buffer, dirty bits used to indicate which
pages have been faulted in to the cpu are now stored in the low bit of each
page struct pointer in the page array.

Change-Id: I891b077dc0c88ed6d416b256626d8778fd67be84
Signed-off-by: default avatarRebecca Schultz Zavin <rebecca@android.com>
Git-commit: bbf5538ee0e4c30ba9b4556f9699ed5f31df67da
Git-repo: https://android.googlesource.com/kernel/common/


[mitchelh@codeaurora.org: conflicts due to msm modifications. Changed a
 private dma API (__dma_page_cpu_to_dev) to public (ARM-specific)
 wrapper (arm_dma_ops.sync_single_for_device).]
Signed-off-by: default avatarMitchel Humpherys <mitchelh@codeaurora.org>
parent 1b5243e7

drivers/gpu/ion/ion.c

+62 −46
Original line number Diff line number Diff line
@@ -3,7 +3,7 @@ 
 * drivers/gpu/ion/ion.c

 *

 * Copyright (C) 2011 Google, Inc.

 * Copyright (c) 2011-2013, The Linux Foundation. All rights reserved.

 * Copyright (c) 2011-2014, The Linux Foundation. All rights reserved.

 *

 * This software is licensed under the terms of the GNU General Public

 * License version 2, as published by the Free Software Foundation, and
@@ -33,6 +33,7 @@ 
#include <linux/slab.h>

#include <linux/seq_file.h>

#include <linux/uaccess.h>

#include <linux/vmalloc.h>

#include <linux/debugfs.h>

#include <linux/dma-buf.h>

#include <linux/idr.h>
@@ -123,6 +124,26 @@ bool ion_buffer_cached(struct ion_buffer *buffer) 
	return !!(buffer->flags & ION_FLAG_CACHED);

}



static inline struct page *ion_buffer_page(struct page *page)

{

	return (struct page *)((unsigned long)page & ~(1UL));

}



static inline bool ion_buffer_page_is_dirty(struct page *page)

{

	return !!((unsigned long)page & 1UL);

}



static inline void ion_buffer_page_dirty(struct page **page)

{

	*page = (struct page *)((unsigned long)(*page) | 1UL);

}



static inline void ion_buffer_page_clean(struct page **page)

{

	*page = (struct page *)((unsigned long)(*page) & ~(1UL));

}



/* this function should only be called while dev->lock is held */

static void ion_buffer_add(struct ion_device *dev,

			   struct ion_buffer *buffer)
@@ -149,8 +170,6 @@ static void ion_buffer_add(struct ion_device *dev, 
	rb_insert_color(&buffer->node, &dev->buffers);

}



static int ion_buffer_alloc_dirty(struct ion_buffer *buffer);



/* this function should only be called while dev->lock is held */

static struct ion_buffer *ion_buffer_create(struct ion_heap *heap,

				     struct ion_device *dev,
@@ -199,17 +218,23 @@ static struct ion_buffer *ion_buffer_create(struct ion_heap *heap, 
	}

	buffer->sg_table = table;

	if (ion_buffer_fault_user_mappings(buffer)) {

		for_each_sg(buffer->sg_table->sgl, sg, buffer->sg_table->nents,

			    i) {

			if (sg_dma_len(sg) == PAGE_SIZE)

				continue;

			pr_err("%s: cached mappings that will be faulted in "

			       "must have pagewise sg_lists\n", __func__);

			ret = -EINVAL;

			goto err;

		int num_pages = PAGE_ALIGN(buffer->size) / PAGE_SIZE;

		struct scatterlist *sg;

		int i, j, k = 0;



		buffer->pages = vmalloc(sizeof(struct page *) * num_pages);

		if (!buffer->pages) {

			ret = -ENOMEM;

			goto err1;

		}



		for_each_sg(table->sgl, sg, table->nents, i) {

			struct page *page = sg_page(sg);



			for (j = 0; j < sg_dma_len(sg) / PAGE_SIZE; j++)

				buffer->pages[k++] = page++;

		}



		ret = ion_buffer_alloc_dirty(buffer);

		if (ret)

			goto err;

	}
@@ -235,6 +260,9 @@ static struct ion_buffer *ion_buffer_create(struct ion_heap *heap, 
err:

	heap->ops->unmap_dma(heap, buffer);

	heap->ops->free(buffer);

err1:

	if (buffer->pages)

		vfree(buffer->pages);

err2:

	kfree(buffer);

	return ERR_PTR(ret);
@@ -247,8 +275,8 @@ void ion_buffer_destroy(struct ion_buffer *buffer) 
	buffer->heap->ops->unmap_dma(buffer->heap, buffer);



	buffer->heap->ops->free(buffer);

	if (buffer->flags & ION_FLAG_CACHED)

		kfree(buffer->dirty);

	if (buffer->pages)

		vfree(buffer->pages);

	kfree(buffer);

}
@@ -982,17 +1010,6 @@ static void ion_unmap_dma_buf(struct dma_buf_attachment *attachment, 
{

}



static int ion_buffer_alloc_dirty(struct ion_buffer *buffer)

{

	unsigned long pages = buffer->sg_table->nents;

	unsigned long length = (pages + BITS_PER_LONG - 1)/BITS_PER_LONG;



	buffer->dirty = kzalloc(length * sizeof(unsigned long), GFP_KERNEL);

	if (!buffer->dirty)

		return -ENOMEM;

	return 0;

}



struct ion_vma_list {

	struct list_head list;

	struct vm_area_struct *vma;
@@ -1002,9 +1019,9 @@ static void ion_buffer_sync_for_device(struct ion_buffer *buffer, 
				       struct device *dev,

				       enum dma_data_direction dir)

{

	struct scatterlist *sg;

	int i;

	struct ion_vma_list *vma_list;

	int pages = PAGE_ALIGN(buffer->size) / PAGE_SIZE;

	int i;



	pr_debug("%s: syncing for device %s\n", __func__,

		 dev ? dev_name(dev) : "null");
@@ -1013,11 +1030,13 @@ static void ion_buffer_sync_for_device(struct ion_buffer *buffer, 
		return;



	mutex_lock(&buffer->lock);

	for_each_sg(buffer->sg_table->sgl, sg, buffer->sg_table->nents, i) {

		if (!test_bit(i, buffer->dirty))

			continue;

		dma_sync_sg_for_device(dev, sg, 1, dir);

		clear_bit(i, buffer->dirty);

	for (i = 0; i < pages; i++) {

		struct page *page = buffer->pages[i];



		if (ion_buffer_page_is_dirty(page))

			arm_dma_ops.sync_single_for_device(

				dev, page_to_phys(page), PAGE_SIZE, dir);

		ion_buffer_page_clean(buffer->pages + i);

	}

	list_for_each_entry(vma_list, &buffer->vmas, list) {

		struct vm_area_struct *vma = vma_list->vma;
@@ -1031,21 +1050,18 @@ static void ion_buffer_sync_for_device(struct ion_buffer *buffer, 
int ion_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)

{

	struct ion_buffer *buffer = vma->vm_private_data;

	struct scatterlist *sg;

	int i;

	int ret;



	mutex_lock(&buffer->lock);

	set_bit(vmf->pgoff, buffer->dirty);

	ion_buffer_page_dirty(buffer->pages + vmf->pgoff);



	for_each_sg(buffer->sg_table->sgl, sg, buffer->sg_table->nents, i) {

		if (i != vmf->pgoff)

			continue;

		dma_sync_sg_for_cpu(NULL, sg, 1, DMA_BIDIRECTIONAL);

		vm_insert_page(vma, (unsigned long)vmf->virtual_address,

			       sg_page(sg));

		break;

	}

	BUG_ON(!buffer->pages || !buffer->pages[vmf->pgoff]);

	ret = vm_insert_page(vma, (unsigned long)vmf->virtual_address,

			     ion_buffer_page(buffer->pages[vmf->pgoff]));

	mutex_unlock(&buffer->lock);

	if (ret)

		return VM_FAULT_ERROR;



	return VM_FAULT_NOPAGE;

}

drivers/gpu/ion/ion_heap.c

+17 −3
Original line number Diff line number Diff line
@@ -2,7 +2,7 @@ 
 * drivers/gpu/ion/ion_heap.c

 *

 * Copyright (C) 2011 Google, Inc.

 * Copyright (c) 2011-2013, The Linux Foundation. All rights reserved.

 * Copyright (c) 2011-2014, The Linux Foundation. All rights reserved.

 *

 * This software is licensed under the terms of the GNU General Public

 * License version 2, as published by the Free Software Foundation, and
@@ -240,7 +240,21 @@ int ion_heap_buffer_zero(struct ion_buffer *buffer) 
	return ret;

}



void ion_heap_free_page(struct ion_buffer *buffer, struct page *page,

struct page *ion_heap_alloc_pages(struct ion_buffer *buffer, gfp_t gfp_flags,

				  unsigned int order)

{

	struct page *page = alloc_pages(gfp_flags, order);



	if (!page)

		return page;



	if (ion_buffer_fault_user_mappings(buffer))

		split_page(page, order);



	return page;

}



void ion_heap_free_pages(struct ion_buffer *buffer, struct page *page,

			 unsigned int order)

{

	int i;

drivers/gpu/ion/ion_priv.h

+17 −5
Original line number Diff line number Diff line
@@ -2,7 +2,7 @@ 
 * drivers/gpu/ion/ion_priv.h

 *

 * Copyright (C) 2011 Google, Inc.

 * Copyright (c) 2011-2013, The Linux Foundation. All rights reserved.

 * Copyright (c) 2011-2014, The Linux Foundation. All rights reserved.

 *

 * This software is licensed under the terms of the GNU General Public

 * License version 2, as published by the Free Software Foundation, and
@@ -49,9 +49,8 @@ struct ion_buffer *ion_handle_buffer(struct ion_handle *handle); 
 * @vaddr:		the kenrel mapping if kmap_cnt is not zero

 * @dmap_cnt:		number of times the buffer is mapped for dma

 * @sg_table:		the sg table for the buffer if dmap_cnt is not zero

 * @dirty:		bitmask representing which pages of this buffer have

 *			been dirtied by the cpu and need cache maintenance

 *			before dma

 * @pages:		flat array of pages in the buffer -- used by fault

 *			handler and only valid for buffers that are faulted in

 * @vmas:		list of vma's mapping this buffer

 * @handle_count:	count of handles referencing this buffer

 * @task_comm:		taskcomm of last client to reference this buffer in a
@@ -78,7 +77,7 @@ struct ion_buffer { 
	void *vaddr;

	int dmap_cnt;

	struct sg_table *sg_table;

	unsigned long *dirty;

	struct page **pages;

	struct list_head vmas;

	/* used to track orphaned buffers */

	int handle_count;
@@ -238,6 +237,19 @@ int ion_heap_pages_zero(struct page **pages, int num_pages); 
int ion_heap_buffer_zero(struct ion_buffer *buffer);

int ion_heap_high_order_page_zero(struct page *page, int order);



/**

 * ion_heap_alloc_pages - allocate pages from alloc_pages

 * @buffer:		the buffer to allocate for, used to extract the flags

 * @gfp_flags:		the gfp_t for the allocation

 * @order:		the order of the allocatoin

 *

 * This funciton allocations from alloc pages and also does any other

 * necessary operations based on the buffer->flags.  For buffers which

 * will be faulted in the pages are split using split_page

 */

struct page *ion_heap_alloc_pages(struct ion_buffer *buffer, gfp_t gfp_flags,

				  unsigned int order);



/**

 * ion_heap_init_deferred_free -- initialize deferred free functionality

 * @heap:		the heap

drivers/gpu/ion/ion_system_heap.c

+3 −20
Original line number Diff line number Diff line
@@ -68,7 +68,6 @@ static struct page *alloc_buffer_page(struct ion_system_heap *heap, 
				      unsigned long order)

{

	bool cached = ion_buffer_cached(buffer);

	bool split_pages = ion_buffer_fault_user_mappings(buffer);

	struct page *page;

	struct ion_page_pool *pool;
@@ -80,8 +79,6 @@ static struct page *alloc_buffer_page(struct ion_system_heap *heap, 
	if (!page)

		return 0;



	if (split_pages)

		split_page(page, order);

	return page;

}
@@ -156,7 +153,6 @@ static int ion_system_heap_allocate(struct ion_heap *heap, 
	int i = 0;

	unsigned long size_remaining = PAGE_ALIGN(size);

	unsigned int max_order = orders[0];

	bool split_pages = ion_buffer_fault_user_mappings(buffer);



	INIT_LIST_HEAD(&pages);

	while (size_remaining > 0) {
@@ -173,28 +169,15 @@ static int ion_system_heap_allocate(struct ion_heap *heap, 
	if (!table)

		goto err;



	if (split_pages)

		ret = sg_alloc_table(table, PAGE_ALIGN(size) / PAGE_SIZE,

				     GFP_KERNEL);

	else

	ret = sg_alloc_table(table, i, GFP_KERNEL);



	if (ret)

		goto err1;



	sg = table->sgl;

	list_for_each_entry_safe(info, tmp_info, &pages, list) {

		struct page *page = info->page;

		if (split_pages) {

			for (i = 0; i < (1 << info->order); i++) {

				sg_set_page(sg, page + i, PAGE_SIZE, 0);

		sg_set_page(sg, page, (1 << info->order) * PAGE_SIZE, 0);

		sg = sg_next(sg);

			}

		} else {

			sg_set_page(sg, page, (1 << info->order) * PAGE_SIZE,

				    0);

			sg = sg_next(sg);

		}

		list_del(&info->list);

		kfree(info);

	}