Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 10102459 authored by Kirill A. Shutemov's avatar Kirill A. Shutemov Committed by Linus Torvalds
Browse files

mm: introduce do_set_pmd() 

With postponed page table allocation we have chance to setup huge pages.
do_set_pte() calls do_set_pmd() if following criteria met:

 - page is compound;
 - pmd entry in pmd_none();
 - vma has suitable size and alignment;

Link: http://lkml.kernel.org/r/1466021202-61880-12-git-send-email-kirill.shutemov@linux.intel.com


Signed-off-by: default avatarKirill A. Shutemov <kirill.shutemov@linux.intel.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent dd78fedd

include/linux/huge_mm.h

+2 −0
Original line number Diff line number Diff line
@@ -143,6 +143,8 @@ static inline bool is_huge_zero_pmd(pmd_t pmd) 
struct page *get_huge_zero_page(void);

void put_huge_zero_page(void);



#define mk_huge_pmd(page, prot) pmd_mkhuge(mk_pmd(page, prot))



#else /* CONFIG_TRANSPARENT_HUGEPAGE */

#define HPAGE_PMD_SHIFT ({ BUILD_BUG(); 0; })

#define HPAGE_PMD_MASK ({ BUILD_BUG(); 0; })

mm/huge_memory.c

+0 −5
Original line number Diff line number Diff line
@@ -796,11 +796,6 @@ pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) 
	return pmd;

}



static inline pmd_t mk_huge_pmd(struct page *page, pgprot_t prot)

{

	return pmd_mkhuge(mk_pmd(page, prot));

}



static inline struct list_head *page_deferred_list(struct page *page)

{

	/*

mm/memory.c

+71 −1
Original line number Diff line number Diff line
@@ -2920,6 +2920,66 @@ static int pte_alloc_one_map(struct fault_env *fe) 
	return 0;

}



#ifdef CONFIG_TRANSPARENT_HUGEPAGE



#define HPAGE_CACHE_INDEX_MASK (HPAGE_PMD_NR - 1)

static inline bool transhuge_vma_suitable(struct vm_area_struct *vma,

		unsigned long haddr)

{

	if (((vma->vm_start >> PAGE_SHIFT) & HPAGE_CACHE_INDEX_MASK) !=

			(vma->vm_pgoff & HPAGE_CACHE_INDEX_MASK))

		return false;

	if (haddr < vma->vm_start || haddr + HPAGE_PMD_SIZE > vma->vm_end)

		return false;

	return true;

}



static int do_set_pmd(struct fault_env *fe, struct page *page)

{

	struct vm_area_struct *vma = fe->vma;

	bool write = fe->flags & FAULT_FLAG_WRITE;

	unsigned long haddr = fe->address & HPAGE_PMD_MASK;

	pmd_t entry;

	int i, ret;



	if (!transhuge_vma_suitable(vma, haddr))

		return VM_FAULT_FALLBACK;



	ret = VM_FAULT_FALLBACK;

	page = compound_head(page);



	fe->ptl = pmd_lock(vma->vm_mm, fe->pmd);

	if (unlikely(!pmd_none(*fe->pmd)))

		goto out;



	for (i = 0; i < HPAGE_PMD_NR; i++)

		flush_icache_page(vma, page + i);



	entry = mk_huge_pmd(page, vma->vm_page_prot);

	if (write)

		entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);



	add_mm_counter(vma->vm_mm, MM_FILEPAGES, HPAGE_PMD_NR);

	page_add_file_rmap(page, true);



	set_pmd_at(vma->vm_mm, haddr, fe->pmd, entry);



	update_mmu_cache_pmd(vma, haddr, fe->pmd);



	/* fault is handled */

	ret = 0;

out:

	spin_unlock(fe->ptl);

	return ret;

}

#else

static int do_set_pmd(struct fault_env *fe, struct page *page)

{

	BUILD_BUG();

	return 0;

}

#endif



/**

 * alloc_set_pte - setup new PTE entry for given page and add reverse page

 * mapping. If needed, the fucntion allocates page table or use pre-allocated.
@@ -2939,9 +2999,19 @@ int alloc_set_pte(struct fault_env *fe, struct mem_cgroup *memcg, 
	struct vm_area_struct *vma = fe->vma;

	bool write = fe->flags & FAULT_FLAG_WRITE;

	pte_t entry;

	int ret;



	if (pmd_none(*fe->pmd) && PageTransCompound(page)) {

		/* THP on COW? */

		VM_BUG_ON_PAGE(memcg, page);



		ret = do_set_pmd(fe, page);

		if (ret != VM_FAULT_FALLBACK)

			return ret;

	}



	if (!fe->pte) {

		int ret = pte_alloc_one_map(fe);

		ret = pte_alloc_one_map(fe);

		if (ret)

			return ret;

	}

mm/migrate.c

+1 −2
Original line number Diff line number Diff line
@@ -1986,8 +1986,7 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm, 
	}



	orig_entry = *pmd;

	entry = mk_pmd(new_page, vma->vm_page_prot);

	entry = pmd_mkhuge(entry);

	entry = mk_huge_pmd(new_page, vma->vm_page_prot);

	entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);



	/*