drivers: add Contiguous Memory Allocator

			Also note the kernel might malfunction if you disable

			some critical bits.

	cma=nn[MG]	[ARM,KNL]

			Sets the size of kernel global memory area for contiguous

			memory allocations. For more information, see

			include/linux/dma-contiguous.h

	cmo_free_hint=	[PPC] Format: { yes | no }

			Specify whether pages are marked as being inactive

			when they are freed.  This is used in CMO environments

config HAVE_DMA_ATTRS

	bool

config HAVE_DMA_CONTIGUOUS

	bool

config USE_GENERIC_SMP_HELPERS

	bool

	  APIs extension; the file's descriptor can then be passed on to other

	  driver.

config CMA

	bool "Contiguous Memory Allocator (EXPERIMENTAL)"

	depends on HAVE_DMA_CONTIGUOUS && HAVE_MEMBLOCK && EXPERIMENTAL

	select MIGRATION

	help

	  This enables the Contiguous Memory Allocator which allows drivers

	  to allocate big physically-contiguous blocks of memory for use with

	  hardware components that do not support I/O map nor scatter-gather.

	  For more information see <include/linux/dma-contiguous.h>.

	  If unsure, say "n".

if CMA

config CMA_DEBUG

	bool "CMA debug messages (DEVELOPMENT)"

	depends on DEBUG_KERNEL

	help

	  Turns on debug messages in CMA.  This produces KERN_DEBUG

	  messages for every CMA call as well as various messages while

	  processing calls such as dma_alloc_from_contiguous().

	  This option does not affect warning and error messages.

comment "Default contiguous memory area size:"

config CMA_SIZE_MBYTES

	int "Size in Mega Bytes"

	depends on !CMA_SIZE_SEL_PERCENTAGE

	default 16

	help

	  Defines the size (in MiB) of the default memory area for Contiguous

	  Memory Allocator.

config CMA_SIZE_PERCENTAGE

	int "Percentage of total memory"

	depends on !CMA_SIZE_SEL_MBYTES

	default 10

	help

	  Defines the size of the default memory area for Contiguous Memory

	  Allocator as a percentage of the total memory in the system.

choice

	prompt "Selected region size"

	default CMA_SIZE_SEL_ABSOLUTE

config CMA_SIZE_SEL_MBYTES

	bool "Use mega bytes value only"

config CMA_SIZE_SEL_PERCENTAGE

	bool "Use percentage value only"

config CMA_SIZE_SEL_MIN

	bool "Use lower value (minimum)"

config CMA_SIZE_SEL_MAX

	bool "Use higher value (maximum)"

endchoice

config CMA_ALIGNMENT

	int "Maximum PAGE_SIZE order of alignment for contiguous buffers"

	range 4 9

	default 8

	help

	  DMA mapping framework by default aligns all buffers to the smallest

	  PAGE_SIZE order which is greater than or equal to the requested buffer

	  size. This works well for buffers up to a few hundreds kilobytes, but

	  for larger buffers it just a memory waste. With this parameter you can

	  specify the maximum PAGE_SIZE order for contiguous buffers. Larger

	  buffers will be aligned only to this specified order. The order is

	  expressed as a power of two multiplied by the PAGE_SIZE.

	  For example, if your system defaults to 4KiB pages, the order value

	  of 8 means that the buffers will be aligned up to 1MiB only.

	  If unsure, leave the default value "8".

config CMA_AREAS

	int "Maximum count of the CMA device-private areas"

	default 7

	help

	  CMA allows to create CMA areas for particular devices. This parameter

	  sets the maximum number of such device private CMA areas in the

	  system.

	  If unsure, leave the default value "7".

endif

endmenu

			   attribute_container.o transport_class.o \

			   topology.o

obj-$(CONFIG_DEVTMPFS)	+= devtmpfs.o

obj-$(CONFIG_CMA) += dma-contiguous.o

obj-y			+= power/

obj-$(CONFIG_HAS_DMA)	+= dma-mapping.o

obj-$(CONFIG_HAVE_GENERIC_DMA_COHERENT) += dma-coherent.o

/*

 * Contiguous Memory Allocator for DMA mapping framework

 * Copyright (c) 2010-2011 by Samsung Electronics.

 * Written by:

 *	Marek Szyprowski <m.szyprowski@samsung.com>

 *	Michal Nazarewicz <mina86@mina86.com>

 *

 * This program is free software; you can redistribute it and/or

 * modify it under the terms of the GNU General Public License as

 * published by the Free Software Foundation; either version 2 of the

 * License or (at your optional) any later version of the license.

 */

#define pr_fmt(fmt) "cma: " fmt

#ifdef CONFIG_CMA_DEBUG

#ifndef DEBUG

#  define DEBUG

#endif

#endif

#include <asm/page.h>

#include <asm/dma-contiguous.h>

#include <linux/memblock.h>

#include <linux/err.h>

#include <linux/mm.h>

#include <linux/mutex.h>

#include <linux/page-isolation.h>

#include <linux/slab.h>

#include <linux/swap.h>

#include <linux/mm_types.h>

#include <linux/dma-contiguous.h>

#ifndef SZ_1M

#define SZ_1M (1 << 20)

#endif

struct cma {

	unsigned long	base_pfn;

	unsigned long	count;

	unsigned long	*bitmap;

};

struct cma *dma_contiguous_default_area;

#ifdef CONFIG_CMA_SIZE_MBYTES

#define CMA_SIZE_MBYTES CONFIG_CMA_SIZE_MBYTES

#else

#define CMA_SIZE_MBYTES 0

#endif

/*

 * Default global CMA area size can be defined in kernel's .config.

 * This is usefull mainly for distro maintainers to create a kernel

 * that works correctly for most supported systems.

 * The size can be set in bytes or as a percentage of the total memory

 * in the system.

 *

 * Users, who want to set the size of global CMA area for their system

 * should use cma= kernel parameter.

 */

static const unsigned long size_bytes = CMA_SIZE_MBYTES * SZ_1M;

static long size_cmdline = -1;

static int __init early_cma(char *p)

{

	pr_debug("%s(%s)\n", __func__, p);

	size_cmdline = memparse(p, &p);

	return 0;

}

early_param("cma", early_cma);

#ifdef CONFIG_CMA_SIZE_PERCENTAGE

static unsigned long __init __maybe_unused cma_early_percent_memory(void)

{

	struct memblock_region *reg;

	unsigned long total_pages = 0;

	/*

	 * We cannot use memblock_phys_mem_size() here, because

	 * memblock_analyze() has not been called yet.

	 */

	for_each_memblock(memory, reg)

		total_pages += memblock_region_memory_end_pfn(reg) -

			       memblock_region_memory_base_pfn(reg);

	return (total_pages * CONFIG_CMA_SIZE_PERCENTAGE / 100) << PAGE_SHIFT;

}

#else

static inline __maybe_unused unsigned long cma_early_percent_memory(void)

{

	return 0;

}

#endif

/**

 * dma_contiguous_reserve() - reserve area for contiguous memory handling

 * @limit: End address of the reserved memory (optional, 0 for any).

 *

 * This function reserves memory from early allocator. It should be

 * called by arch specific code once the early allocator (memblock or bootmem)

 * has been activated and all other subsystems have already allocated/reserved

 * memory.

 */

void __init dma_contiguous_reserve(phys_addr_t limit)

{

	unsigned long selected_size = 0;

	pr_debug("%s(limit %08lx)\n", __func__, (unsigned long)limit);

	if (size_cmdline != -1) {

		selected_size = size_cmdline;

	} else {

#ifdef CONFIG_CMA_SIZE_SEL_MBYTES

		selected_size = size_bytes;

#elif defined(CONFIG_CMA_SIZE_SEL_PERCENTAGE)

		selected_size = cma_early_percent_memory();

#elif defined(CONFIG_CMA_SIZE_SEL_MIN)

		selected_size = min(size_bytes, cma_early_percent_memory());

#elif defined(CONFIG_CMA_SIZE_SEL_MAX)

		selected_size = max(size_bytes, cma_early_percent_memory());

#endif

	}

	if (selected_size) {

		pr_debug("%s: reserving %ld MiB for global area\n", __func__,

			 selected_size / SZ_1M);

		dma_declare_contiguous(NULL, selected_size, 0, limit);

	}

};

static DEFINE_MUTEX(cma_mutex);

static __init int cma_activate_area(unsigned long base_pfn, unsigned long count)

{

	unsigned long pfn = base_pfn;

	unsigned i = count >> pageblock_order;

	struct zone *zone;

	WARN_ON_ONCE(!pfn_valid(pfn));

	zone = page_zone(pfn_to_page(pfn));

	do {

		unsigned j;

		base_pfn = pfn;

		for (j = pageblock_nr_pages; j; --j, pfn++) {

			WARN_ON_ONCE(!pfn_valid(pfn));

			if (page_zone(pfn_to_page(pfn)) != zone)

				return -EINVAL;

		}

		init_cma_reserved_pageblock(pfn_to_page(base_pfn));

	} while (--i);

	return 0;

}

static __init struct cma *cma_create_area(unsigned long base_pfn,

				     unsigned long count)

{

	int bitmap_size = BITS_TO_LONGS(count) * sizeof(long);

	struct cma *cma;

	int ret = -ENOMEM;

	pr_debug("%s(base %08lx, count %lx)\n", __func__, base_pfn, count);

	cma = kmalloc(sizeof *cma, GFP_KERNEL);

	if (!cma)

		return ERR_PTR(-ENOMEM);

	cma->base_pfn = base_pfn;

	cma->count = count;

	cma->bitmap = kzalloc(bitmap_size, GFP_KERNEL);

	if (!cma->bitmap)

		goto no_mem;

	ret = cma_activate_area(base_pfn, count);

	if (ret)

		goto error;

	pr_debug("%s: returned %p\n", __func__, (void *)cma);

	return cma;

error:

	kfree(cma->bitmap);

no_mem:

	kfree(cma);

	return ERR_PTR(ret);

}

static struct cma_reserved {

	phys_addr_t start;

	unsigned long size;

	struct device *dev;

} cma_reserved[MAX_CMA_AREAS] __initdata;

static unsigned cma_reserved_count __initdata;

static int __init cma_init_reserved_areas(void)

{

	struct cma_reserved *r = cma_reserved;

	unsigned i = cma_reserved_count;

	pr_debug("%s()\n", __func__);

	for (; i; --i, ++r) {

		struct cma *cma;

		cma = cma_create_area(PFN_DOWN(r->start),

				      r->size >> PAGE_SHIFT);

		if (!IS_ERR(cma))

			dev_set_cma_area(r->dev, cma);

	}

	return 0;

}

core_initcall(cma_init_reserved_areas);

/**

 * dma_declare_contiguous() - reserve area for contiguous memory handling

 *			      for particular device

 * @dev:   Pointer to device structure.

 * @size:  Size of the reserved memory.

 * @base:  Start address of the reserved memory (optional, 0 for any).

 * @limit: End address of the reserved memory (optional, 0 for any).

 *

 * This function reserves memory for specified device. It should be

 * called by board specific code when early allocator (memblock or bootmem)

 * is still activate.

 */

int __init dma_declare_contiguous(struct device *dev, unsigned long size,

				  phys_addr_t base, phys_addr_t limit)

{

	struct cma_reserved *r = &cma_reserved[cma_reserved_count];

	unsigned long alignment;

	pr_debug("%s(size %lx, base %08lx, limit %08lx)\n", __func__,

		 (unsigned long)size, (unsigned long)base,

		 (unsigned long)limit);

	/* Sanity checks */

	if (cma_reserved_count == ARRAY_SIZE(cma_reserved)) {

		pr_err("Not enough slots for CMA reserved regions!\n");

		return -ENOSPC;

	}

	if (!size)

		return -EINVAL;

	/* Sanitise input arguments */

	alignment = PAGE_SIZE << max(MAX_ORDER, pageblock_order);

	base = ALIGN(base, alignment);

	size = ALIGN(size, alignment);

	limit &= ~(alignment - 1);

	/* Reserve memory */

	if (base) {

		if (memblock_is_region_reserved(base, size) ||

		    memblock_reserve(base, size) < 0) {

			base = -EBUSY;

			goto err;

		}

	} else {

		/*

		 * Use __memblock_alloc_base() since

		 * memblock_alloc_base() panic()s.

		 */

		phys_addr_t addr = __memblock_alloc_base(size, alignment, limit);

		if (!addr) {

			base = -ENOMEM;

			goto err;

		} else if (addr + size > ~(unsigned long)0) {

			memblock_free(addr, size);

			base = -EINVAL;

			goto err;

		} else {

			base = addr;

		}

	}

	/*

	 * Each reserved area must be initialised later, when more kernel

	 * subsystems (like slab allocator) are available.

	 */

	r->start = base;

	r->size = size;

	r->dev = dev;

	cma_reserved_count++;

	pr_info("CMA: reserved %ld MiB at %08lx\n", size / SZ_1M,

		(unsigned long)base);

	/* Architecture specific contiguous memory fixup. */

	dma_contiguous_early_fixup(base, size);

	return 0;

err:

	pr_err("CMA: failed to reserve %ld MiB\n", size / SZ_1M);

	return base;

}

/**

 * dma_alloc_from_contiguous() - allocate pages from contiguous area

 * @dev:   Pointer to device for which the allocation is performed.

 * @count: Requested number of pages.

 * @align: Requested alignment of pages (in PAGE_SIZE order).

 *

 * This function allocates memory buffer for specified device. It uses

 * device specific contiguous memory area if available or the default

 * global one. Requires architecture specific get_dev_cma_area() helper

 * function.

 */

struct page *dma_alloc_from_contiguous(struct device *dev, int count,

				       unsigned int align)

{

	unsigned long mask, pfn, pageno, start = 0;

	struct cma *cma = dev_get_cma_area(dev);

	int ret;

	if (!cma || !cma->count)

		return NULL;

	if (align > CONFIG_CMA_ALIGNMENT)

		align = CONFIG_CMA_ALIGNMENT;

	pr_debug("%s(cma %p, count %d, align %d)\n", __func__, (void *)cma,

		 count, align);

	if (!count)

		return NULL;

	mask = (1 << align) - 1;

	mutex_lock(&cma_mutex);

	for (;;) {

		pageno = bitmap_find_next_zero_area(cma->bitmap, cma->count,

						    start, count, mask);

		if (pageno >= cma->count) {

			ret = -ENOMEM;

			goto error;

		}

		pfn = cma->base_pfn + pageno;

		ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA);

		if (ret == 0) {

			bitmap_set(cma->bitmap, pageno, count);

			break;

		} else if (ret != -EBUSY) {

			goto error;

		}

		pr_debug("%s(): memory range at %p is busy, retrying\n",

			 __func__, pfn_to_page(pfn));

		/* try again with a bit different memory target */

		start = pageno + mask + 1;

	}

	mutex_unlock(&cma_mutex);

	pr_debug("%s(): returned %p\n", __func__, pfn_to_page(pfn));

	return pfn_to_page(pfn);

error:

	mutex_unlock(&cma_mutex);

	return NULL;

}

/**

 * dma_release_from_contiguous() - release allocated pages

 * @dev:   Pointer to device for which the pages were allocated.

 * @pages: Allocated pages.

 * @count: Number of allocated pages.

 *

 * This function releases memory allocated by dma_alloc_from_contiguous().

 * It returns false when provided pages do not belong to contiguous area and

 * true otherwise.

 */

bool dma_release_from_contiguous(struct device *dev, struct page *pages,

				 int count)

{

	struct cma *cma = dev_get_cma_area(dev);

	unsigned long pfn;

	if (!cma || !pages)

		return false;

	pr_debug("%s(page %p)\n", __func__, (void *)pages);

	pfn = page_to_pfn(pages);

	if (pfn < cma->base_pfn || pfn >= cma->base_pfn + cma->count)

		return false;

	VM_BUG_ON(pfn + count > cma->base_pfn + cma->count);

	mutex_lock(&cma_mutex);

	bitmap_clear(cma->bitmap, pfn - cma->base_pfn, count);

	free_contig_range(pfn, count);

	mutex_unlock(&cma_mutex);

	return true;

}