Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 26234f36 authored by Xiao Guangrong's avatar Xiao Guangrong Committed by Linus Torvalds
Browse files

thp: introduce khugepaged_prealloc_page and khugepaged_alloc_page 



They are used to abstract the difference between NUMA enabled and NUMA
disabled to make the code more readable

Signed-off-by: default avatarXiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 420256ef

mm/huge_memory.c

+98 −68
Original line number Diff line number Diff line
@@ -1827,28 +1827,34 @@ static void __collapse_huge_page_copy(pte_t *pte, struct page *page, 
	}

}



static void collapse_huge_page(struct mm_struct *mm,

			       unsigned long address,

			       struct page **hpage,

			       struct vm_area_struct *vma,

			       int node)

static void khugepaged_alloc_sleep(void)

{

	pgd_t *pgd;

	pud_t *pud;

	pmd_t *pmd, _pmd;

	pte_t *pte;

	pgtable_t pgtable;

	struct page *new_page;

	spinlock_t *ptl;

	int isolated;

	unsigned long hstart, hend;

	wait_event_freezable_timeout(khugepaged_wait, false,

			msecs_to_jiffies(khugepaged_alloc_sleep_millisecs));

}



	VM_BUG_ON(address & ~HPAGE_PMD_MASK);

#ifndef CONFIG_NUMA

	up_read(&mm->mmap_sem);

	VM_BUG_ON(!*hpage);

	new_page = *hpage;

#else

#ifdef CONFIG_NUMA

static bool khugepaged_prealloc_page(struct page **hpage, bool *wait)

{

	if (IS_ERR(*hpage)) {

		if (!*wait)

			return false;



		*wait = false;

		khugepaged_alloc_sleep();

	} else if (*hpage) {

		put_page(*hpage);

		*hpage = NULL;

	}



	return true;

}



static struct page

*khugepaged_alloc_page(struct page **hpage, struct mm_struct *mm,

		       struct vm_area_struct *vma, unsigned long address,

		       int node)

{

	VM_BUG_ON(*hpage);

	/*

	 * Allocate the page while the vma is still valid and under
@@ -1860,7 +1866,7 @@ static void collapse_huge_page(struct mm_struct *mm, 
	 * mmap_sem in read mode is good idea also to allow greater

	 * scalability.

	 */

	new_page = alloc_hugepage_vma(khugepaged_defrag(), vma, address,

	*hpage  = alloc_hugepage_vma(khugepaged_defrag(), vma, address,

				      node, __GFP_OTHER_NODE);



	/*
@@ -1868,15 +1874,81 @@ static void collapse_huge_page(struct mm_struct *mm, 
	 * preparation for taking it in write mode.

	 */

	up_read(&mm->mmap_sem);

	if (unlikely(!new_page)) {

	if (unlikely(!*hpage)) {

		count_vm_event(THP_COLLAPSE_ALLOC_FAILED);

		*hpage = ERR_PTR(-ENOMEM);

		return;

		return NULL;

	}



	count_vm_event(THP_COLLAPSE_ALLOC);

	return *hpage;

}

	*hpage = new_page;

#else

static struct page *khugepaged_alloc_hugepage(bool *wait)

{

	struct page *hpage;



	do {

		hpage = alloc_hugepage(khugepaged_defrag());

		if (!hpage) {

			count_vm_event(THP_COLLAPSE_ALLOC_FAILED);

			if (!*wait)

				return NULL;



			*wait = false;

			khugepaged_alloc_sleep();

		} else

			count_vm_event(THP_COLLAPSE_ALLOC);

	} while (unlikely(!hpage) && likely(khugepaged_enabled()));



	return hpage;

}



static bool khugepaged_prealloc_page(struct page **hpage, bool *wait)

{

	if (!*hpage)

		*hpage = khugepaged_alloc_hugepage(wait);



	if (unlikely(!*hpage))

		return false;



	return true;

}



static struct page

*khugepaged_alloc_page(struct page **hpage, struct mm_struct *mm,

		       struct vm_area_struct *vma, unsigned long address,

		       int node)

{

	up_read(&mm->mmap_sem);

	VM_BUG_ON(!*hpage);

	return  *hpage;

}

#endif



static void collapse_huge_page(struct mm_struct *mm,

				   unsigned long address,

				   struct page **hpage,

				   struct vm_area_struct *vma,

				   int node)

{

	pgd_t *pgd;

	pud_t *pud;

	pmd_t *pmd, _pmd;

	pte_t *pte;

	pgtable_t pgtable;

	struct page *new_page;

	spinlock_t *ptl;

	int isolated;

	unsigned long hstart, hend;



	VM_BUG_ON(address & ~HPAGE_PMD_MASK);



	/* release the mmap_sem read lock. */

	new_page = khugepaged_alloc_page(hpage, mm, vma, address, node);

	if (!new_page)

		return;



	if (unlikely(mem_cgroup_newpage_charge(new_page, mm, GFP_KERNEL)))

		return;
@@ -2215,34 +2287,6 @@ static int khugepaged_wait_event(void) 
		kthread_should_stop();

}



static void khugepaged_alloc_sleep(void)

{

	wait_event_freezable_timeout(khugepaged_wait, false,

			msecs_to_jiffies(khugepaged_alloc_sleep_millisecs));

}



#ifndef CONFIG_NUMA

static struct page *khugepaged_alloc_hugepage(bool *wait)

{

	struct page *hpage;



	do {

		hpage = alloc_hugepage(khugepaged_defrag());

		if (!hpage) {

			count_vm_event(THP_COLLAPSE_ALLOC_FAILED);

			if (!*wait)

				return NULL;



			*wait = false;

			khugepaged_alloc_sleep();

		} else

			count_vm_event(THP_COLLAPSE_ALLOC);

	} while (unlikely(!hpage) && likely(khugepaged_enabled()));



	return hpage;

}

#endif



static void khugepaged_do_scan(void)

{

	struct page *hpage = NULL;
@@ -2253,23 +2297,9 @@ static void khugepaged_do_scan(void) 
	barrier(); /* write khugepaged_pages_to_scan to local stack */



	while (progress < pages) {

#ifndef CONFIG_NUMA

		if (!hpage)

			hpage = khugepaged_alloc_hugepage(&wait);



		if (unlikely(!hpage))

			break;

#else

		if (IS_ERR(hpage)) {

			if (!wait)

		if (!khugepaged_prealloc_page(&hpage, &wait))

			break;

			wait = false;

			khugepaged_alloc_sleep();

		} else if (hpage) {

			put_page(hpage);

			hpage = NULL;

		}

#endif



		cond_resched();



		if (unlikely(kthread_should_stop() || freezing(current)))