Merge "mm: swap: clean up swap readahead" (870f5d92) · Commits · e / devices / android_kernel_oneplus_sm8150

include/linux/swap.h

+20 −14

Original line number	Diff line number	Diff line
		@@ -417,12 +417,8 @@ extern struct page *__read_swap_cache_async(swp_entry_t, gfp_t,
		bool *new_page_allocated);
		extern struct page *swapin_readahead(swp_entry_t, gfp_t,
		struct vm_area_struct *vma, unsigned long addr);

		extern struct page swap_readahead_detect(struct vm_fault vmf,
		struct vma_swap_readahead *swap_ra);
		extern struct page *do_swap_page_readahead(swp_entry_t fentry, gfp_t gfp_mask,
		struct vm_fault *vmf,
		struct vma_swap_readahead *swap_ra);
		struct vm_fault *vmf);

		/* linux/mm/swapfile.c */
		extern atomic_long_t nr_swap_pages;
		@@ -463,9 +459,11 @@ extern unsigned int count_swap_pages(int, int);
		extern sector_t map_swap_page(struct page , struct block_device *);
		extern sector_t swapdev_block(int, pgoff_t);
		extern int page_swapcount(struct page *);
		extern int __swap_count(struct swap_info_struct *si, swp_entry_t entry);
		extern int __swp_swapcount(swp_entry_t entry);
		extern int swp_swapcount(swp_entry_t entry);
		extern struct swap_info_struct page_swap_info(struct page );
		extern struct swap_info_struct *swp_swap_info(swp_entry_t entry);
		extern bool reuse_swap_page(struct page , int );
		extern int try_to_free_swap(struct page *);
		struct backing_dev_info;
		@@ -474,6 +472,16 @@ extern void exit_swap_address_space(unsigned int type);

		#else /* CONFIG_SWAP */

		static inline int swap_readpage(struct page *page, bool do_poll)
		{
		return 0;
		}

		static inline struct swap_info_struct *swp_swap_info(swp_entry_t entry)
		{
		return NULL;
		}

		#define swap_address_space(entry) (NULL)
		#define get_nr_swap_pages() 0L
		#define total_swap_pages 0L
		@@ -529,15 +537,8 @@ static inline bool swap_use_vma_readahead(void)
		return false;
		}

		static inline struct page *swap_readahead_detect(
		struct vm_fault vmf, struct vma_swap_readahead swap_ra)
		{
		return NULL;
		}

		static inline struct page *do_swap_page_readahead(
		swp_entry_t fentry, gfp_t gfp_mask,
		struct vm_fault vmf, struct vma_swap_readahead swap_ra)
		static inline struct page *do_swap_page_readahead(swp_entry_t fentry,
		gfp_t gfp_mask, struct vm_fault *vmf)
		{
		return NULL;
		}
		@@ -578,6 +579,11 @@ static inline int page_swapcount(struct page *page)
		return 0;
		}

		static inline int __swap_count(struct swap_info_struct *si, swp_entry_t entry)
		{
		return 0;
		}

		static inline int __swp_swapcount(swp_entry_t entry)
		{
		return 0;

mm/memory.c

+40 −27

Original line number	Diff line number	Diff line
		@@ -2844,21 +2844,14 @@ int do_swap_page(struct vm_fault *vmf)
		struct vm_area_struct *vma = vmf->vma;
		struct page page = NULL, swapcache;
		struct mem_cgroup *memcg;
		struct vma_swap_readahead swap_ra;
		swp_entry_t entry;
		pte_t pte;
		int locked;
		int exclusive = 0;
		int ret = 0;
		bool vma_readahead = swap_use_vma_readahead();

		if (vma_readahead)
		page = swap_readahead_detect(vmf, &swap_ra);
		if (!pte_unmap_same(vma->vm_mm, vmf->pmd, vmf->pte, vmf->orig_pte)) {
		if (page)
		put_page(page);
		if (!pte_unmap_same(vma->vm_mm, vmf->pmd, vmf->pte, vmf->orig_pte))
		goto out;
		}

		entry = pte_to_swp_entry(vmf->orig_pte);
		if (unlikely(non_swap_entry(entry))) {
		@@ -2881,17 +2874,36 @@ int do_swap_page(struct vm_fault *vmf)
		}
		goto out;
		}


		delayacct_set_flag(DELAYACCT_PF_SWAPIN);
		if (!page)
		page = lookup_swap_cache(entry, vma_readahead ? vma : NULL,
		vmf->address);
		page = lookup_swap_cache(entry, vma, vmf->address);
		swapcache = page;

		if (!page) {
		if (vma_readahead)
		struct swap_info_struct *si = swp_swap_info(entry);

		if (si->flags & SWP_SYNCHRONOUS_IO &&
		__swap_count(si, entry) == 1) {
		/* skip swapcache */
		page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vmf->address);
		if (page) {
		__SetPageLocked(page);
		__SetPageSwapBacked(page);
		set_page_private(page, entry.val);
		lru_cache_add_anon(page);
		swap_readpage(page, true);
		}
		} else {
		if (swap_use_vma_readahead())
		page = do_swap_page_readahead(entry,
		GFP_HIGHUSER_MOVABLE, vmf, &swap_ra);
		GFP_HIGHUSER_MOVABLE, vmf);
		else
		page = swapin_readahead(entry,
		GFP_HIGHUSER_MOVABLE, vma, vmf->address);
		swapcache = page;
		}

		if (!page) {
		/*
		* Back out if somebody else faulted in this pte
		@@ -2916,11 +2928,9 @@ int do_swap_page(struct vm_fault *vmf)
		*/
		ret = VM_FAULT_HWPOISON;
		delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
		swapcache = page;
		goto out_release;
		}

		swapcache = page;
		locked = lock_page_or_retry(page, vma->vm_mm, vmf->flags);

		delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
		@@ -2935,7 +2945,8 @@ int do_swap_page(struct vm_fault *vmf)
		* test below, are not enough to exclude that. Even if it is still
		* swapcache, we need to check that the page's swap has not changed.
		*/
		if (unlikely(!PageSwapCache(page) \|\| page_private(page) != entry.val))
		if (unlikely((!PageSwapCache(page) \|\|
		page_private(page) != entry.val)) && swapcache)
		goto out_page;

		page = ksm_might_need_to_copy(page, vma, vmf->address);
		@@ -2988,14 +2999,16 @@ int do_swap_page(struct vm_fault *vmf)
		pte = pte_mksoft_dirty(pte);
		set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte);
		vmf->orig_pte = pte;
		if (page == swapcache) {
		do_page_add_anon_rmap(page, vma, vmf->address, exclusive);
		mem_cgroup_commit_charge(page, memcg, true, false);
		activate_page(page);
		} else { /* ksm created a completely new copy */

		/* ksm created a completely new copy */
		if (unlikely(page != swapcache && swapcache)) {
		page_add_new_anon_rmap(page, vma, vmf->address, false);
		mem_cgroup_commit_charge(page, memcg, false, false);
		lru_cache_add_active_or_unevictable(page, vma);
		} else {
		do_page_add_anon_rmap(page, vma, vmf->address, exclusive);
		mem_cgroup_commit_charge(page, memcg, true, false);
		activate_page(page);
		}

		swap_free(entry);
		@@ -3003,7 +3016,7 @@ int do_swap_page(struct vm_fault *vmf)
		(vma->vm_flags & VM_LOCKED) \|\| PageMlocked(page))
		try_to_free_swap(page);
		unlock_page(page);
		if (page != swapcache) {
		if (page != swapcache && swapcache) {
		/*
		* Hold the lock to avoid the swap entry to be reused
		* until we take the PT lock for the pte_same() check
		@@ -3036,7 +3049,7 @@ int do_swap_page(struct vm_fault *vmf)
		unlock_page(page);
		out_release:
		put_page(page);
		if (page != swapcache) {
		if (page != swapcache && swapcache) {
		unlock_page(swapcache);
		put_page(swapcache);
		}

mm/page_io.c

+3 −3

Original line number	Diff line number	Diff line
		@@ -348,7 +348,7 @@ int __swap_writepage(struct page page, struct writeback_control wbc,
		return ret;
		}

		int swap_readpage(struct page *page, bool do_poll)
		int swap_readpage(struct page *page, bool synchronous)
		{
		struct bio *bio;
		int ret = 0;
		@@ -356,7 +356,7 @@ int swap_readpage(struct page *page, bool do_poll)
		blk_qc_t qc;
		struct gendisk *disk;

		VM_BUG_ON_PAGE(!PageSwapCache(page), page);
		VM_BUG_ON_PAGE(!PageSwapCache(page) && !synchronous, page);
		VM_BUG_ON_PAGE(!PageLocked(page), page);
		VM_BUG_ON_PAGE(PageUptodate(page), page);
		if (frontswap_load(page) == 0) {
		@@ -404,7 +404,7 @@ int swap_readpage(struct page *page, bool do_poll)
		count_vm_event(PSWPIN);
		bio_get(bio);
		qc = submit_bio(bio);
		while (do_poll) {
		while (synchronous) {
		set_current_state(TASK_UNINTERRUPTIBLE);
		if (!READ_ONCE(bio->bi_private))
		break;

mm/swap_state.c

+48 −41

Original line number	Diff line number	Diff line
		@@ -332,32 +332,38 @@ struct page lookup_swap_cache(swp_entry_t entry, struct vm_area_struct vma,
		unsigned long addr)
		{
		struct page *page;
		unsigned long ra_info;
		int win, hits, readahead;

		page = find_get_page(swap_address_space(entry), swp_offset(entry));

		INC_CACHE_INFO(find_total);
		if (page) {
		bool vma_ra = swap_use_vma_readahead();
		bool readahead = TestClearPageReadahead(page);

		INC_CACHE_INFO(find_success);
		if (unlikely(PageTransCompound(page)))
		return page;
		readahead = TestClearPageReadahead(page);
		if (vma) {
		ra_info = GET_SWAP_RA_VAL(vma);
		win = SWAP_RA_WIN(ra_info);
		hits = SWAP_RA_HITS(ra_info);

		if (vma && vma_ra) {
		unsigned long ra_val;
		int win, hits;

		ra_val = GET_SWAP_RA_VAL(vma);
		win = SWAP_RA_WIN(ra_val);
		hits = SWAP_RA_HITS(ra_val);
		if (readahead)
		hits = min_t(int, hits + 1, SWAP_RA_HITS_MAX);
		atomic_long_set(&vma->swap_readahead_info,
		SWAP_RA_VAL(addr, win, hits));
		}

		if (readahead) {
		count_vm_event(SWAP_RA_HIT);
		if (!vma)
		if (!vma \|\| !vma_ra)
		atomic_inc(&swapin_readahead_hits);
		}
		}

		return page;
		}

		@@ -646,16 +652,15 @@ static inline void swap_ra_clamp_pfn(struct vm_area_struct *vma,
		PFN_DOWN((faddr & PMD_MASK) + PMD_SIZE));
		}

		struct page swap_readahead_detect(struct vm_fault vmf,
		struct vma_swap_readahead *swap_ra)
		static void swap_ra_info(struct vm_fault *vmf,
		struct vma_swap_readahead *ra_info)
		{
		struct vm_area_struct *vma = vmf->vma;
		unsigned long swap_ra_info;
		struct page *page;
		unsigned long ra_val;
		swp_entry_t entry;
		unsigned long faddr, pfn, fpfn;
		unsigned long start, end;
		pte_t *pte;
		pte_t pte, orig_pte;
		unsigned int max_win, hits, prev_win, win, left;
		#ifndef CONFIG_64BIT
		pte_t *tpte;
		@@ -664,30 +669,32 @@ struct page swap_readahead_detect(struct vm_fault vmf,
		max_win = 1 << min_t(unsigned int, READ_ONCE(page_cluster),
		SWAP_RA_ORDER_CEILING);
		if (max_win == 1) {
		swap_ra->win = 1;
		return NULL;
		ra_info->win = 1;
		return;
		}

		faddr = vmf->address;
		entry = pte_to_swp_entry(vmf->orig_pte);
		if ((unlikely(non_swap_entry(entry))))
		return NULL;
		page = lookup_swap_cache(entry, vma, faddr);
		if (page)
		return page;
		orig_pte = pte = pte_offset_map(vmf->pmd, faddr);
		entry = pte_to_swp_entry(*pte);
		if ((unlikely(non_swap_entry(entry)))) {
		pte_unmap(orig_pte);
		return;
		}

		fpfn = PFN_DOWN(faddr);
		swap_ra_info = GET_SWAP_RA_VAL(vma);
		pfn = PFN_DOWN(SWAP_RA_ADDR(swap_ra_info));
		prev_win = SWAP_RA_WIN(swap_ra_info);
		hits = SWAP_RA_HITS(swap_ra_info);
		swap_ra->win = win = __swapin_nr_pages(pfn, fpfn, hits,
		ra_val = GET_SWAP_RA_VAL(vma);
		pfn = PFN_DOWN(SWAP_RA_ADDR(ra_val));
		prev_win = SWAP_RA_WIN(ra_val);
		hits = SWAP_RA_HITS(ra_val);
		ra_info->win = win = __swapin_nr_pages(pfn, fpfn, hits,
		max_win, prev_win);
		atomic_long_set(&vma->swap_readahead_info,
		SWAP_RA_VAL(faddr, win, 0));

		if (win == 1)
		return NULL;
		if (win == 1) {
		pte_unmap(orig_pte);
		return;
		}

		/* Copy the PTEs because the page table may be unmapped */
		if (fpfn == pfn + 1)
		@@ -700,23 +707,21 @@ struct page swap_readahead_detect(struct vm_fault vmf,
		swap_ra_clamp_pfn(vma, faddr, fpfn - left, fpfn + win - left,
		&start, &end);
		}
		swap_ra->nr_pte = end - start;
		swap_ra->offset = fpfn - start;
		pte = vmf->pte - swap_ra->offset;
		ra_info->nr_pte = end - start;
		ra_info->offset = fpfn - start;
		pte -= ra_info->offset;
		#ifdef CONFIG_64BIT
		swap_ra->ptes = pte;
		ra_info->ptes = pte;
		#else
		tpte = swap_ra->ptes;
		tpte = ra_info->ptes;
		for (pfn = start; pfn != end; pfn++)
		tpte++ = pte++;
		#endif

		return NULL;
		pte_unmap(orig_pte);
		}

		struct page *do_swap_page_readahead(swp_entry_t fentry, gfp_t gfp_mask,
		struct vm_fault *vmf,
		struct vma_swap_readahead *swap_ra)
		struct vm_fault *vmf)
		{
		struct blk_plug plug;
		struct vm_area_struct *vma = vmf->vma;
		@@ -725,12 +730,14 @@ struct page *do_swap_page_readahead(swp_entry_t fentry, gfp_t gfp_mask,
		swp_entry_t entry;
		unsigned int i;
		bool page_allocated;
		struct vma_swap_readahead ra_info = {0,};

		if (swap_ra->win == 1)
		swap_ra_info(vmf, &ra_info);
		if (ra_info.win == 1)
		goto skip;

		blk_start_plug(&plug);
		for (i = 0, pte = swap_ra->ptes; i < swap_ra->nr_pte;
		for (i = 0, pte = ra_info.ptes; i < ra_info.nr_pte;
		i++, pte++) {
		pentry = *pte;
		if (pte_none(pentry))
		@@ -746,7 +753,7 @@ struct page *do_swap_page_readahead(swp_entry_t fentry, gfp_t gfp_mask,
		continue;
		if (page_allocated) {
		swap_readpage(page, false);
		if (i != swap_ra->offset &&
		if (i != ra_info.offset &&
		likely(!PageTransCompound(page))) {
		SetPageReadahead(page);
		count_vm_event(SWAP_RA);
		@@ -758,7 +765,7 @@ struct page *do_swap_page_readahead(swp_entry_t fentry, gfp_t gfp_mask,
		lru_add_drain();
		skip:
		return read_swap_cache_async(fentry, gfp_mask, vma, vmf->address,
		swap_ra->win == 1);
		ra_info.win == 1);
		}

		#ifdef CONFIG_SYSFS

mm/swapfile.c

+14 −4

Original line number	Diff line number	Diff line
		@@ -1328,6 +1328,13 @@ int page_swapcount(struct page *page)
		return count;
		}

		int __swap_count(struct swap_info_struct *si, swp_entry_t entry)
		{
		pgoff_t offset = swp_offset(entry);

		return swap_count(si->swap_map[offset]);
		}

		static int swap_swapcount(struct swap_info_struct *si, swp_entry_t entry)
		{
		int count = 0;
		@@ -3455,10 +3462,15 @@ int swapcache_prepare(swp_entry_t entry)
		return __swap_duplicate(entry, SWAP_HAS_CACHE);
		}

		struct swap_info_struct *swp_swap_info(swp_entry_t entry)
		{
		return swap_info[swp_type(entry)];
		}

		struct swap_info_struct page_swap_info(struct page page)
		{
		swp_entry_t swap = { .val = page_private(page) };
		return swap_info[swp_type(swap)];
		swp_entry_t entry = { .val = page_private(page) };
		return swp_swap_info(entry);
		}

		/*
		@@ -3466,7 +3478,6 @@ struct swap_info_struct page_swap_info(struct page page)
		*/
		struct address_space __page_file_mapping(struct page page)
		{
		VM_BUG_ON_PAGE(!PageSwapCache(page), page);
		return page_swap_info(page)->swap_file->f_mapping;
		}
		EXPORT_SYMBOL_GPL(__page_file_mapping);
		@@ -3474,7 +3485,6 @@ EXPORT_SYMBOL_GPL(__page_file_mapping);
		pgoff_t __page_file_index(struct page *page)
		{
		swp_entry_t swap = { .val = page_private(page) };
		VM_BUG_ON_PAGE(!PageSwapCache(page), page);
		return swp_offset(swap);
		}
		EXPORT_SYMBOL_GPL(__page_file_index);