Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit deb94544 authored by Konstantin Khlebnikov's avatar Konstantin Khlebnikov Committed by Linus Torvalds
Browse files

pagemap: switch to the new format and do some cleanup 



This patch removes page-shift bits (scheduled to remove since 3.11) and
completes migration to the new bit layout.  Also it cleans messy macro.

Signed-off-by: default avatarKonstantin Khlebnikov <khlebnikov@yandex-team.ru>
Reviewed-by: default avatarNaoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Mark Williamson <mwilliamson@undo-software.com>
Tested-by: default avatarMark Williamson <mwilliamson@undo-software.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent a06db751

fs/proc/task_mmu.c

+53 −97
Original line number Diff line number Diff line
@@ -712,23 +712,6 @@ const struct file_operations proc_tid_smaps_operations = { 
	.release	= proc_map_release,

};



/*

 * We do not want to have constant page-shift bits sitting in

 * pagemap entries and are about to reuse them some time soon.

 *

 * Here's the "migration strategy":

 * 1. when the system boots these bits remain what they are,

 *    but a warning about future change is printed in log;

 * 2. once anyone clears soft-dirty bits via clear_refs file,

 *    these flag is set to denote, that user is aware of the

 *    new API and those page-shift bits change their meaning.

 *    The respective warning is printed in dmesg;

 * 3. In a couple of releases we will remove all the mentions

 *    of page-shift in pagemap entries.

 */



static bool soft_dirty_cleared __read_mostly;



enum clear_refs_types {

	CLEAR_REFS_ALL = 1,

	CLEAR_REFS_ANON,
@@ -889,13 +872,6 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, 
	if (type < CLEAR_REFS_ALL || type >= CLEAR_REFS_LAST)

		return -EINVAL;



	if (type == CLEAR_REFS_SOFT_DIRTY) {

		soft_dirty_cleared = true;

		pr_warn_once("The pagemap bits 55-60 has changed their meaning!"

			     " See the linux/Documentation/vm/pagemap.txt for "

			     "details.\n");

	}



	task = get_proc_task(file_inode(file));

	if (!task)

		return -ESRCH;
@@ -963,36 +939,24 @@ typedef struct { 
struct pagemapread {

	int pos, len;		/* units: PM_ENTRY_BYTES, not bytes */

	pagemap_entry_t *buffer;

	bool v2;

};



#define PAGEMAP_WALK_SIZE	(PMD_SIZE)

#define PAGEMAP_WALK_MASK	(PMD_MASK)



#define PM_ENTRY_BYTES		sizeof(pagemap_entry_t)

#define PM_STATUS_BITS      3

#define PM_STATUS_OFFSET    (64 - PM_STATUS_BITS)

#define PM_STATUS_MASK      (((1LL << PM_STATUS_BITS) - 1) << PM_STATUS_OFFSET)

#define PM_STATUS(nr)       (((nr) << PM_STATUS_OFFSET) & PM_STATUS_MASK)

#define PM_PSHIFT_BITS      6

#define PM_PSHIFT_OFFSET    (PM_STATUS_OFFSET - PM_PSHIFT_BITS)

#define PM_PSHIFT_MASK      (((1LL << PM_PSHIFT_BITS) - 1) << PM_PSHIFT_OFFSET)

#define __PM_PSHIFT(x)      (((u64) (x) << PM_PSHIFT_OFFSET) & PM_PSHIFT_MASK)

#define PM_PFRAME_MASK      ((1LL << PM_PSHIFT_OFFSET) - 1)

#define PM_PFRAME(x)        ((x) & PM_PFRAME_MASK)

/* in "new" pagemap pshift bits are occupied with more status bits */

#define PM_STATUS2(v2, x)   (__PM_PSHIFT(v2 ? x : PAGE_SHIFT))



#define __PM_SOFT_DIRTY      (1LL)

#define PM_PRESENT          PM_STATUS(4LL)

#define PM_SWAP             PM_STATUS(2LL)

#define PM_FILE             PM_STATUS(1LL)

#define PM_NOT_PRESENT(v2)  PM_STATUS2(v2, 0)

#define PM_PFRAME_BITS		55

#define PM_PFRAME_MASK		GENMASK_ULL(PM_PFRAME_BITS - 1, 0)

#define PM_SOFT_DIRTY		BIT_ULL(55)

#define PM_FILE			BIT_ULL(61)

#define PM_SWAP			BIT_ULL(62)

#define PM_PRESENT		BIT_ULL(63)



#define PM_END_OF_BUFFER    1



static inline pagemap_entry_t make_pme(u64 val)

static inline pagemap_entry_t make_pme(u64 frame, u64 flags)

{

	return (pagemap_entry_t) { .pme = val };

	return (pagemap_entry_t) { .pme = (frame & PM_PFRAME_MASK) | flags };

}



static int add_to_pagemap(unsigned long addr, pagemap_entry_t *pme,
@@ -1013,7 +977,7 @@ static int pagemap_pte_hole(unsigned long start, unsigned long end, 


	while (addr < end) {

		struct vm_area_struct *vma = find_vma(walk->mm, addr);

		pagemap_entry_t pme = make_pme(PM_NOT_PRESENT(pm->v2));

		pagemap_entry_t pme = make_pme(0, 0);

		/* End of address space hole, which we mark as non-present. */

		unsigned long hole_end;
@@ -1033,7 +997,7 @@ static int pagemap_pte_hole(unsigned long start, unsigned long end, 


		/* Addresses in the VMA. */

		if (vma->vm_flags & VM_SOFTDIRTY)

			pme.pme |= PM_STATUS2(pm->v2, __PM_SOFT_DIRTY);

			pme = make_pme(0, PM_SOFT_DIRTY);

		for (; addr < min(end, vma->vm_end); addr += PAGE_SIZE) {

			err = add_to_pagemap(addr, &pme, pm);

			if (err)
@@ -1044,63 +1008,61 @@ static int pagemap_pte_hole(unsigned long start, unsigned long end, 
	return err;

}



static void pte_to_pagemap_entry(pagemap_entry_t *pme, struct pagemapread *pm,

static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm,

		struct vm_area_struct *vma, unsigned long addr, pte_t pte)

{

	u64 frame, flags;

	u64 frame = 0, flags = 0;

	struct page *page = NULL;

	int flags2 = 0;



	if (pte_present(pte)) {

		frame = pte_pfn(pte);

		flags = PM_PRESENT;

		flags |= PM_PRESENT;

		page = vm_normal_page(vma, addr, pte);

		if (pte_soft_dirty(pte))

			flags2 |= __PM_SOFT_DIRTY;

			flags |= PM_SOFT_DIRTY;

	} else if (is_swap_pte(pte)) {

		swp_entry_t entry;

		if (pte_swp_soft_dirty(pte))

			flags2 |= __PM_SOFT_DIRTY;

			flags |= PM_SOFT_DIRTY;

		entry = pte_to_swp_entry(pte);

		frame = swp_type(entry) |

			(swp_offset(entry) << MAX_SWAPFILES_SHIFT);

		flags = PM_SWAP;

		flags |= PM_SWAP;

		if (is_migration_entry(entry))

			page = migration_entry_to_page(entry);

	} else {

		if (vma->vm_flags & VM_SOFTDIRTY)

			flags2 |= __PM_SOFT_DIRTY;

		*pme = make_pme(PM_NOT_PRESENT(pm->v2) | PM_STATUS2(pm->v2, flags2));

		return;

	}



	if (page && !PageAnon(page))

		flags |= PM_FILE;

	if ((vma->vm_flags & VM_SOFTDIRTY))

		flags2 |= __PM_SOFT_DIRTY;

	if (vma->vm_flags & VM_SOFTDIRTY)

		flags |= PM_SOFT_DIRTY;



	*pme = make_pme(PM_PFRAME(frame) | PM_STATUS2(pm->v2, flags2) | flags);

	return make_pme(frame, flags);

}



#ifdef CONFIG_TRANSPARENT_HUGEPAGE

static void thp_pmd_to_pagemap_entry(pagemap_entry_t *pme, struct pagemapread *pm,

		pmd_t pmd, int offset, int pmd_flags2)

static pagemap_entry_t thp_pmd_to_pagemap_entry(struct pagemapread *pm,

		pmd_t pmd, int offset, u64 flags)

{

	u64 frame = 0;



	/*

	 * Currently pmd for thp is always present because thp can not be

	 * swapped-out, migrated, or HWPOISONed (split in such cases instead.)

	 * This if-check is just to prepare for future implementation.

	 */

	if (pmd_present(pmd))

		*pme = make_pme(PM_PFRAME(pmd_pfn(pmd) + offset)

				| PM_STATUS2(pm->v2, pmd_flags2) | PM_PRESENT);

	else

		*pme = make_pme(PM_NOT_PRESENT(pm->v2) | PM_STATUS2(pm->v2, pmd_flags2));

	if (pmd_present(pmd)) {

		frame = pmd_pfn(pmd) + offset;

		flags |= PM_PRESENT;

	}



	return make_pme(frame, flags);

}

#else

static inline void thp_pmd_to_pagemap_entry(pagemap_entry_t *pme, struct pagemapread *pm,

		pmd_t pmd, int offset, int pmd_flags2)

static pagemap_entry_t thp_pmd_to_pagemap_entry(struct pagemapread *pm,

		pmd_t pmd, int offset, u64 flags)

{

	return make_pme(0, 0);

}

#endif
@@ -1114,12 +1076,10 @@ static int pagemap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, 
	int err = 0;



	if (pmd_trans_huge_lock(pmd, vma, &ptl) == 1) {

		int pmd_flags2;

		u64 flags = 0;



		if ((vma->vm_flags & VM_SOFTDIRTY) || pmd_soft_dirty(*pmd))

			pmd_flags2 = __PM_SOFT_DIRTY;

		else

			pmd_flags2 = 0;

			flags |= PM_SOFT_DIRTY;



		for (; addr != end; addr += PAGE_SIZE) {

			unsigned long offset;
@@ -1127,7 +1087,7 @@ static int pagemap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, 


			offset = (addr & ~PAGEMAP_WALK_MASK) >>

					PAGE_SHIFT;

			thp_pmd_to_pagemap_entry(&pme, pm, *pmd, offset, pmd_flags2);

			pme = thp_pmd_to_pagemap_entry(pm, *pmd, offset, flags);

			err = add_to_pagemap(addr, &pme, pm);

			if (err)

				break;
@@ -1147,7 +1107,7 @@ static int pagemap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, 
	for (; addr < end; pte++, addr += PAGE_SIZE) {

		pagemap_entry_t pme;



		pte_to_pagemap_entry(&pme, pm, vma, addr, *pte);

		pme = pte_to_pagemap_entry(pm, vma, addr, *pte);

		err = add_to_pagemap(addr, &pme, pm);

		if (err)

			break;
@@ -1160,16 +1120,17 @@ static int pagemap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, 
}



#ifdef CONFIG_HUGETLB_PAGE

static void huge_pte_to_pagemap_entry(pagemap_entry_t *pme, struct pagemapread *pm,

					pte_t pte, int offset, int flags2)

static pagemap_entry_t huge_pte_to_pagemap_entry(struct pagemapread *pm,

					pte_t pte, int offset, u64 flags)

{

	if (pte_present(pte))

		*pme = make_pme(PM_PFRAME(pte_pfn(pte) + offset)	|

				PM_STATUS2(pm->v2, flags2)		|

				PM_PRESENT);

	else

		*pme = make_pme(PM_NOT_PRESENT(pm->v2)			|

				PM_STATUS2(pm->v2, flags2));

	u64 frame = 0;



	if (pte_present(pte)) {

		frame = pte_pfn(pte) + offset;

		flags |= PM_PRESENT;

	}



	return make_pme(frame, flags);

}



/* This function walks within one hugetlb entry in the single call */
@@ -1180,17 +1141,15 @@ static int pagemap_hugetlb_range(pte_t *pte, unsigned long hmask, 
	struct pagemapread *pm = walk->private;

	struct vm_area_struct *vma = walk->vma;

	int err = 0;

	int flags2;

	u64 flags = 0;

	pagemap_entry_t pme;



	if (vma->vm_flags & VM_SOFTDIRTY)

		flags2 = __PM_SOFT_DIRTY;

	else

		flags2 = 0;

		flags |= PM_SOFT_DIRTY;



	for (; addr != end; addr += PAGE_SIZE) {

		int offset = (addr & ~hmask) >> PAGE_SHIFT;

		huge_pte_to_pagemap_entry(&pme, pm, *pte, offset, flags2);

		pme = huge_pte_to_pagemap_entry(pm, *pte, offset, flags);

		err = add_to_pagemap(addr, &pme, pm);

		if (err)

			return err;
@@ -1211,7 +1170,8 @@ static int pagemap_hugetlb_range(pte_t *pte, unsigned long hmask, 
 * Bits 0-54  page frame number (PFN) if present

 * Bits 0-4   swap type if swapped

 * Bits 5-54  swap offset if swapped

 * Bits 55-60 page shift (page size = 1<<page shift)

 * Bit  55    pte is soft-dirty (see Documentation/vm/soft-dirty.txt)

 * Bits 56-60 zero

 * Bit  61    page is file-page or shared-anon

 * Bit  62    page swapped

 * Bit  63    page present
@@ -1250,7 +1210,6 @@ static ssize_t pagemap_read(struct file *file, char __user *buf, 
	if (!count)

		goto out_mm;



	pm.v2 = soft_dirty_cleared;

	pm.len = (PAGEMAP_WALK_SIZE >> PAGE_SHIFT);

	pm.buffer = kmalloc(pm.len * PM_ENTRY_BYTES, GFP_TEMPORARY);

	ret = -ENOMEM;
@@ -1323,9 +1282,6 @@ static int pagemap_open(struct inode *inode, struct file *file) 
	/* do not disclose physical addresses: attack vector */

	if (!capable(CAP_SYS_ADMIN))

		return -EPERM;

	pr_warn_once("Bits 55-60 of /proc/PID/pagemap entries are about "

			"to stop being page-shift some time soon. See the "

			"linux/Documentation/vm/pagemap.txt for details.\n");



	mm = proc_mem_open(inode, PTRACE_MODE_READ);

	if (IS_ERR(mm))

tools/vm/page-types.c

+8 −17
Original line number Diff line number Diff line
@@ -57,23 +57,14 @@ 
 * pagemap kernel ABI bits

 */



#define PM_ENTRY_BYTES      sizeof(uint64_t)

#define PM_STATUS_BITS      3

#define PM_STATUS_OFFSET    (64 - PM_STATUS_BITS)

#define PM_STATUS_MASK      (((1LL << PM_STATUS_BITS) - 1) << PM_STATUS_OFFSET)

#define PM_STATUS(nr)       (((nr) << PM_STATUS_OFFSET) & PM_STATUS_MASK)

#define PM_PSHIFT_BITS      6

#define PM_PSHIFT_OFFSET    (PM_STATUS_OFFSET - PM_PSHIFT_BITS)

#define PM_PSHIFT_MASK      (((1LL << PM_PSHIFT_BITS) - 1) << PM_PSHIFT_OFFSET)

#define __PM_PSHIFT(x)      (((uint64_t) (x) << PM_PSHIFT_OFFSET) & PM_PSHIFT_MASK)

#define PM_PFRAME_MASK      ((1LL << PM_PSHIFT_OFFSET) - 1)

#define PM_ENTRY_BYTES		8

#define PM_PFRAME_BITS		55

#define PM_PFRAME_MASK		((1LL << PM_PFRAME_BITS) - 1)

#define PM_PFRAME(x)		((x) & PM_PFRAME_MASK)



#define __PM_SOFT_DIRTY      (1LL)

#define PM_PRESENT          PM_STATUS(4LL)

#define PM_SWAP             PM_STATUS(2LL)

#define PM_SOFT_DIRTY       __PM_PSHIFT(__PM_SOFT_DIRTY)



#define PM_SOFT_DIRTY		(1ULL << 55)

#define PM_FILE			(1ULL << 61)

#define PM_SWAP			(1ULL << 62)

#define PM_PRESENT		(1ULL << 63)



/*

 * kernel page flags