Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 095c51f2 authored by Heesub Shin's avatar Heesub Shin Committed by Isaac J. Manjarres
Browse files

cma: redirect page allocation to CMA 



CMA pages are designed to be used as fallback for movable allocations
and cannot be used for non-movable allocations. If CMA pages are
utilized poorly, non-movable allocations may end up getting starved if
all regular movable pages are allocated and the only pages left are
CMA. Always using CMA pages first creates unacceptable performance
problems. As a midway alternative, use CMA pages for certain
userspace allocations. The userspace pages can be migrated or dropped
quickly which giving decent utilization.

Change-Id: I6165dda01b705309eebabc6dfa67146b7a95c174
Signed-off-by: default avatarKyungmin Park <kyungmin.park@samsung.com>
Signed-off-by: default avatarHeesub Shin <heesub.shin@samsung.com>
[lauraa@codeaurora.org: Missing CONFIG_CMA guards, add commit text]
Signed-off-by: default avatarLaura Abbott <lauraa@codeaurora.org>
[lmark@codeaurora.org: resolve conflicts relating to MIGRATE_HIGHATOMIC]
Signed-off-by: default avatarLiam Mark <lmark@codeaurora.org>
Signed-off-by: default avatarVinayak Menon <vinmenon@codeaurora.org>
Signed-off-by: default avatarCharan Teja Reddy <charante@codeaurora.org>
[swatsrid@codeaurora.org: Fix merge conflicts]
Signed-off-by: default avatarSwathi Sridhar <swatsrid@codeaurora.org>
[isaacm@codeaurora.org: Introduce CONFIG_CMA_DIRECT_UTILIZATION]
Signed-off-by: default avatarIsaac J. Manjarres <isaacm@codeaurora.org>
parent 059aa413

include/linux/gfp.h

+15 −1
Original line number Diff line number Diff line
@@ -39,8 +39,17 @@ struct vm_area_struct; 
#define ___GFP_HARDWALL		0x100000u

#define ___GFP_THISNODE		0x200000u

#define ___GFP_ACCOUNT		0x400000u

#ifdef CONFIG_CMA_DIRECT_UTILIZATION

#define ___GFP_CMA		0x800000u

#else

#define ___GFP_CMA		0

#endif

#ifdef CONFIG_LOCKDEP

#define ___GFP_NOLOCKDEP	0x800000u

#ifdef CONFIG_CMA_DIRECT_UTILIZATION

#define ___GFP_NOLOCKDEP	0x1000000u

#else

#define __GFP_NOLOCKDEP		0x800000u

#endif

#else

#define ___GFP_NOLOCKDEP	0

#endif
@@ -57,6 +66,7 @@ struct vm_area_struct; 
#define __GFP_HIGHMEM	((__force gfp_t)___GFP_HIGHMEM)

#define __GFP_DMA32	((__force gfp_t)___GFP_DMA32)

#define __GFP_MOVABLE	((__force gfp_t)___GFP_MOVABLE)  /* ZONE_MOVABLE allowed */

#define __GFP_CMA	((__force gfp_t)___GFP_CMA)

#define GFP_ZONEMASK	(__GFP_DMA|__GFP_HIGHMEM|__GFP_DMA32|__GFP_MOVABLE)



/**
@@ -217,7 +227,11 @@ struct vm_area_struct; 
#define __GFP_NOLOCKDEP ((__force gfp_t)___GFP_NOLOCKDEP)



/* Room for N __GFP_FOO bits */

#ifdef CONFIG_CMA_DIRECT_UTILIZATION

#define __GFP_BITS_SHIFT (24 + IS_ENABLED(CONFIG_LOCKDEP))

#else

#define __GFP_BITS_SHIFT (23 + IS_ENABLED(CONFIG_LOCKDEP))

#endif

#define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1))



/**

include/linux/highmem.h

+5 −0
Original line number Diff line number Diff line
@@ -205,7 +205,12 @@ static inline struct page * 
alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma,

					unsigned long vaddr)

{

#ifndef CONFIG_CMA_DIRECT_UTILIZATION

	return __alloc_zeroed_user_highpage(__GFP_MOVABLE, vma, vaddr);

#else

	return __alloc_zeroed_user_highpage(__GFP_MOVABLE|__GFP_CMA, vma,

						vaddr);

#endif

}



static inline void clear_highpage(struct page *page)

include/linux/mmzone.h

+4 −0
Original line number Diff line number Diff line
@@ -441,6 +441,10 @@ struct zone { 
	struct pglist_data	*zone_pgdat;

	struct per_cpu_pageset __percpu *pageset;



#ifdef CONFIG_CMA_DIRECT_UTILIZATION

	bool			cma_alloc;

#endif



#ifndef CONFIG_SPARSEMEM

	/*

	 * Flags for a pageblock_nr_pages block. See pageblock-flags.h.

mm/Kconfig

+17 −0
Original line number Diff line number Diff line
@@ -539,6 +539,23 @@ config CMA_AREAS 


	  If unsure, leave the default value "7".



config CMA_DIRECT_UTILIZATION

	bool "Allow clients to directly use CMA for page allocations"

	depends on CMA && QGKI

	help

	  CMA pages are currently used as a fallback for moveable page

	  allocations only when all regular moveable pages have been

	  allocated. This can cause stalls for non-movable allocations,

	  as non-movable allocations will not consider CMA memory as free

	  memory, and thus, will cause the system to reclaim movable pages

	  to satisfy non-movable allocations, ignoring the CMA pages. Since

	  the system now has movable memory, CMA pages will also be ignored,

	  since they are used only when there is not enough movable memory

	  to satisfy a movable request. Enable this option to specify to the

	  page allocator that movable allocations should be satisfied with

	  CMA pages, so that CMA pages can be used directly for allocations,

	  instead of as a fallback, thus, improving CMA utilization.



config MEM_SOFT_DIRTY

	bool "Track memory changes"

	depends on CHECKPOINT_RESTORE && HAVE_ARCH_SOFT_DIRTY && PROC_FS

mm/page_alloc.c

+53 −8
Original line number Diff line number Diff line
@@ -2737,9 +2737,12 @@ __rmqueue(struct zone *zone, unsigned int order, int migratetype, 


retry:

	page = __rmqueue_smallest(zone, order, migratetype);



	if (unlikely(!page)) {

#ifndef CONFIG_CMA_DIRECT_UTILIZATION

		if (migratetype == MIGRATE_MOVABLE)

			page = __rmqueue_cma_fallback(zone, order);

#endif



		if (!page && __rmqueue_fallback(zone, order, migratetype,

								alloc_flags))
@@ -2750,6 +2753,28 @@ __rmqueue(struct zone *zone, unsigned int order, int migratetype, 
	return page;

}



static struct page *__rmqueue_cma(struct zone *zone, unsigned int order,

					int migratetype,

					unsigned int alloc_flags)

{

	struct page *page = 0;



retry:

#ifdef CONFIG_CMA_DIRECT_UTILIZATION

	if (migratetype == MIGRATE_MOVABLE && !zone->cma_alloc)

		page = __rmqueue_cma_fallback(zone, order);

	else

#endif

		page = __rmqueue_smallest(zone, order, migratetype);



	if (unlikely(!page) && __rmqueue_fallback(zone, order, migratetype,

						  alloc_flags))

		goto retry;



	trace_mm_page_alloc_zone_locked(page, order, migratetype);

	return page;

}



/*

 * Obtain a specified number of elements from the buddy allocator, all under

 * a single hold of the lock, for efficiency.  Add them to the supplied list.
@@ -2757,14 +2782,20 @@ __rmqueue(struct zone *zone, unsigned int order, int migratetype, 
 */

static int rmqueue_bulk(struct zone *zone, unsigned int order,

			unsigned long count, struct list_head *list,

			int migratetype, unsigned int alloc_flags)

			int migratetype, unsigned int alloc_flags, int cma)

{

	int i, alloced = 0;



	spin_lock(&zone->lock);

	for (i = 0; i < count; ++i) {

		struct page *page = __rmqueue(zone, order, migratetype,

		struct page *page;



		if (cma)

			page = __rmqueue_cma(zone, order, migratetype,

					     alloc_flags);

		else

			page = __rmqueue(zone, order, migratetype, alloc_flags);



		if (unlikely(page == NULL))

			break;
@@ -3228,7 +3259,7 @@ static inline void zone_statistics(struct zone *preferred_zone, struct zone *z) 
static struct page *__rmqueue_pcplist(struct zone *zone, int migratetype,

			unsigned int alloc_flags,

			struct per_cpu_pages *pcp,

			struct list_head *list)

			struct list_head *list, gfp_t gfp_flags)

{

	struct page *page;
@@ -3236,7 +3267,8 @@ static struct page *__rmqueue_pcplist(struct zone *zone, int migratetype, 
		if (list_empty(list)) {

			pcp->count += rmqueue_bulk(zone, 0,

					pcp->batch, list,

					migratetype, alloc_flags);

					migratetype, alloc_flags,

					gfp_flags & __GFP_CMA);

			if (unlikely(list_empty(list)))

				return NULL;

		}
@@ -3262,7 +3294,8 @@ static struct page *rmqueue_pcplist(struct zone *preferred_zone, 
	local_irq_save(flags);

	pcp = &this_cpu_ptr(zone->pageset)->pcp;

	list = &pcp->lists[migratetype];

	page = __rmqueue_pcplist(zone,  migratetype, alloc_flags, pcp, list);

	page = __rmqueue_pcplist(zone,  migratetype, alloc_flags, pcp, list,

				 gfp_flags);

	if (page) {

		__count_zid_vm_events(PGALLOC, page_zonenum(page), 1);

		zone_statistics(preferred_zone, zone);
@@ -3303,8 +3336,14 @@ struct page *rmqueue(struct zone *preferred_zone, 
			if (page)

				trace_mm_page_alloc_zone_locked(page, order, migratetype);

		}

		if (!page)

			page = __rmqueue(zone, order, migratetype, alloc_flags);

		if (!page) {

			if (gfp_flags & __GFP_CMA)

				page = __rmqueue_cma(zone, order, migratetype,

						     alloc_flags);

			else

				page = __rmqueue(zone, order, migratetype,

						 alloc_flags);

		}

	} while (page && check_new_pages(page, order));

	spin_unlock(&zone->lock);

	if (!page)
@@ -8438,6 +8477,9 @@ int alloc_contig_range(unsigned long start, unsigned long end, 
	if (ret < 0)

		return ret;



#ifdef CONFIG_CMA_DIRECT_UTILIZATION

	cc.zone->cma_alloc = 1;

#endif

	/*

	 * In case of -EBUSY, we'd like to know which page causes problem.

	 * So, just fall through. test_pages_isolated() has a tracepoint
@@ -8519,6 +8561,9 @@ int alloc_contig_range(unsigned long start, unsigned long end, 
done:

	undo_isolate_page_range(pfn_max_align_down(start),

				pfn_max_align_up(end), migratetype);

#ifdef CONFIG_CMA_DIRECT_UTILIZATION

	cc.zone->cma_alloc = 0;

#endif

	return ret;

}

#endif /* CONFIG_CONTIG_ALLOC */