Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit a558f6c1 authored by Heesub Shin's avatar Heesub Shin Committed by Matt Wagantall
Browse files

cma: redirect page allocation to CMA 



CMA pages are designed to be used as fallback for movable allocations
and cannot be used for non-movable allocations. If CMA pages are
utilized poorly, non-movable allocations may end up getting starved if
all regular movable pages are allocated and the only pages left are
CMA. Always using CMA pages first creates unacceptable performance
problems. As a midway alternative, use CMA pages for certain
userspace allocations. The userspace pages can be migrated or dropped
quickly which giving decent utilization.

Change-Id: I6165dda01b705309eebabc6dfa67146b7a95c174
CRs-Fixed: 452508
Signed-off-by: default avatarKyungmin Park <kyungmin.park@samsung.com>
Signed-off-by: default avatarHeesub Shin <heesub.shin@samsung.com>
[lauraa@codeaurora.org: Missing CONFIG_CMA guards, add commit text]
Signed-off-by: default avatarLaura Abbott <lauraa@codeaurora.org>
parent a5e790aa

include/linux/gfp.h

+11 −2
Original line number Diff line number Diff line
@@ -34,6 +34,7 @@ struct vm_area_struct; 
#define ___GFP_NO_KSWAPD	0x400000u

#define ___GFP_OTHER_NODE	0x800000u

#define ___GFP_WRITE		0x1000000u

#define ___GFP_CMA		0x2000000u

/* If the above are modified, __GFP_BITS_SHIFT may need updating */



/*
@@ -49,7 +50,9 @@ struct vm_area_struct; 
#define __GFP_HIGHMEM	((__force gfp_t)___GFP_HIGHMEM)

#define __GFP_DMA32	((__force gfp_t)___GFP_DMA32)

#define __GFP_MOVABLE	((__force gfp_t)___GFP_MOVABLE)  /* Page is movable */

#define GFP_ZONEMASK	(__GFP_DMA|__GFP_HIGHMEM|__GFP_DMA32|__GFP_MOVABLE)

#define __GFP_CMA	((__force gfp_t)___GFP_CMA)

#define GFP_ZONEMASK	(__GFP_DMA|__GFP_HIGHMEM|__GFP_DMA32|__GFP_MOVABLE| \

			__GFP_CMA)

/*

 * Action modifiers - doesn't change the zoning

 *
@@ -131,7 +134,7 @@ struct vm_area_struct; 
#endif



/* This mask makes up all the page movable related flags */

#define GFP_MOVABLE_MASK (__GFP_RECLAIMABLE|__GFP_MOVABLE)

#define GFP_MOVABLE_MASK (__GFP_RECLAIMABLE|__GFP_MOVABLE|__GFP_CMA)



/* Control page allocator reclaim behavior */

#define GFP_RECLAIM_MASK (__GFP_WAIT|__GFP_HIGH|__GFP_IO|__GFP_FS|\
@@ -164,8 +167,14 @@ static inline int gfpflags_to_migratetype(const gfp_t gfp_flags) 
		return MIGRATE_UNMOVABLE;



	/* Group based on mobility */

#ifndef CONFIG_CMA

	return (((gfp_flags & __GFP_MOVABLE) != 0) << 1) |

		((gfp_flags & __GFP_RECLAIMABLE) != 0);

#else

	return (((gfp_flags & __GFP_MOVABLE) != 0) << 1) |

		(((gfp_flags & __GFP_CMA) != 0) << 1) |

		((gfp_flags & __GFP_RECLAIMABLE) != 0);

#endif

}



#ifdef CONFIG_HIGHMEM

include/linux/highmem.h

+15 −0
Original line number Diff line number Diff line
@@ -186,8 +186,23 @@ static inline struct page * 
alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma,

					unsigned long vaddr)

{

#ifndef CONFIG_CMA

	return __alloc_zeroed_user_highpage(__GFP_MOVABLE, vma, vaddr);

#else

	return __alloc_zeroed_user_highpage(__GFP_MOVABLE|__GFP_CMA, vma,

						vaddr);

#endif

}



#ifdef CONFIG_CMA

static inline struct page *

alloc_zeroed_user_highpage_movable_cma(struct vm_area_struct *vma,

						unsigned long vaddr)

{

	return __alloc_zeroed_user_highpage(__GFP_MOVABLE|__GFP_CMA, vma,

						vaddr);

}

#endif



static inline void clear_highpage(struct page *page)

{

include/linux/mmzone.h

+3 −0
Original line number Diff line number Diff line
@@ -366,6 +366,9 @@ struct zone { 
	 * considered dirtyable memory.

	 */

	unsigned long		dirty_balance_reserve;

#ifdef CONFIG_CMA

	bool			cma_alloc;

#endif



#ifndef CONFIG_SPARSEMEM

	/*

mm/page_alloc.c

+46 −4
Original line number Diff line number Diff line
@@ -1222,6 +1222,37 @@ retry_reserve: 
	return page;

}



static struct page *__rmqueue_cma(struct zone *zone, unsigned int order,

							int migratetype)

{

	struct page *page = 0;

#ifdef CONFIG_CMA

	if (migratetype == MIGRATE_MOVABLE && !zone->cma_alloc)

		page = __rmqueue_smallest(zone, order, MIGRATE_CMA);

	else

#endif

retry_reserve :

		page = __rmqueue_smallest(zone, order, migratetype);





	if (unlikely(!page) && migratetype != MIGRATE_RESERVE) {

		page = __rmqueue_fallback(zone, order, migratetype);



		/*

		 * Use MIGRATE_RESERVE rather than fail an allocation. goto

		 * is used because __rmqueue_smallest is an inline function

		 * and we want just one call site

		 */

		if (!page) {

			migratetype = MIGRATE_RESERVE;

			goto retry_reserve;

		}

	}



	trace_mm_page_alloc_zone_locked(page, order, migratetype);

	return page;

}



/*

 * Obtain a specified number of elements from the buddy allocator, all under

 * a single hold of the lock, for efficiency.  Add them to the supplied list.
@@ -1229,13 +1260,17 @@ retry_reserve: 
 */

static int rmqueue_bulk(struct zone *zone, unsigned int order,

			unsigned long count, struct list_head *list,

			int migratetype, bool cold)

			int migratetype, bool cold, int cma)

{

	int i;



	spin_lock(&zone->lock);

	for (i = 0; i < count; ++i) {

		struct page *page = __rmqueue(zone, order, migratetype);

		struct page *page;

		if (cma)

			page = __rmqueue_cma(zone, order, migratetype);

		else

			page = __rmqueue(zone, order, migratetype);

		if (unlikely(page == NULL))

			break;
@@ -1587,7 +1622,8 @@ again: 
		if (list_empty(list)) {

			pcp->count += rmqueue_bulk(zone, 0,

					pcp->batch, list,

					migratetype, cold);

					migratetype, cold,

					gfp_flags & __GFP_CMA);

			if (unlikely(list_empty(list)))

				goto failed;

		}
@@ -1614,6 +1650,9 @@ again: 
			WARN_ON_ONCE(order > 1);

		}

		spin_lock_irqsave(&zone->lock, flags);

		if (gfp_flags & __GFP_CMA)

			page = __rmqueue_cma(zone, order, migratetype);

		else

			page = __rmqueue(zone, order, migratetype);

		spin_unlock(&zone->lock);

		if (!page)
@@ -6416,6 +6455,8 @@ int alloc_contig_range(unsigned long start, unsigned long end, 
	if (ret)

		return ret;



	cc.zone->cma_alloc = 1;



	ret = __alloc_contig_migrate_range(&cc, start, end);

	if (ret)

		goto done;
@@ -6474,6 +6515,7 @@ int alloc_contig_range(unsigned long start, unsigned long end, 
done:

	undo_isolate_page_range(pfn_max_align_down(start),

				pfn_max_align_up(end), migratetype);

	cc.zone->cma_alloc = 0;

	return ret;

}