erofs: force inplace I/O under low memory scenario (33e7e223) · Commits · e / devices / android_kernel_oneplus_sm7250

fs/erofs/compress.h

+3 −0

Original line number	Diff line number	Diff line
		@@ -27,11 +27,13 @@ struct z_erofs_decompress_req {

		/* some special page->private (unsigned long, see below) */
		#define Z_EROFS_SHORTLIVED_PAGE (-1UL << 2)
		#define Z_EROFS_PREALLOCATED_PAGE (-2UL << 2)

		/*
		* For all pages in a pcluster, page->private should be one of
		* Type Last 2bits page->private
		* short-lived page 00 Z_EROFS_SHORTLIVED_PAGE
		* preallocated page (tryalloc) 00 Z_EROFS_PREALLOCATED_PAGE
		* cached/managed page 00 pointer to z_erofs_pcluster
		* online page (file-backed, 01/10/11 sub-index << 2 \| count
		* some pages can be used for inplace I/O)
		@@ -39,6 +41,7 @@ struct z_erofs_decompress_req {
		* page->mapping should be one of
		* Type page->mapping
		* short-lived page NULL
		* preallocated page NULL
		* cached/managed page non-NULL or NULL (invalidated/truncated page)
		* online page non-NULL
		*

fs/erofs/zdata.c

+40 −8

Original line number	Diff line number	Diff line
		@@ -19,6 +19,11 @@
		enum z_erofs_cache_alloctype {
		DONTALLOC, /* don't allocate any cached pages */
		DELAYEDALLOC, /* delayed allocation (at the time of submitting io) */
		/*
		* try to use cached I/O if page allocation succeeds or fallback
		* to in-place I/O instead to avoid any direct reclaim.
		*/
		TRYALLOC,
		};

		/*
		@@ -163,13 +168,16 @@ static DEFINE_MUTEX(z_pagemap_global_lock);

		static void preload_compressed_pages(struct z_erofs_collector *clt,
		struct address_space *mc,
		enum z_erofs_cache_alloctype type)
		enum z_erofs_cache_alloctype type,
		struct list_head *pagepool)
		{
		const struct z_erofs_pcluster *pcl = clt->pcl;
		const unsigned int clusterpages = BIT(pcl->clusterbits);
		struct page **pages = clt->compressedpages;
		pgoff_t index = pcl->obj.index + (pages - pcl->compressed_pages);
		bool standalone = true;
		gfp_t gfp = (mapping_gfp_mask(mc) & ~__GFP_DIRECT_RECLAIM) \|
		__GFP_NOMEMALLOC \| __GFP_NORETRY \| __GFP_NOWARN;

		if (clt->mode < COLLECT_PRIMARY_FOLLOWED)
		return;
		@@ -177,6 +185,7 @@ static void preload_compressed_pages(struct z_erofs_collector *clt,
		for (; pages < pcl->compressed_pages + clusterpages; ++pages) {
		struct page *page;
		compressed_page_t t;
		struct page *newpage = NULL;

		/* the compressed page was loaded before */
		if (READ_ONCE(*pages))
		@@ -188,7 +197,15 @@ static void preload_compressed_pages(struct z_erofs_collector *clt,
		t = tag_compressed_page_justfound(page);
		} else if (type == DELAYEDALLOC) {
		t = tagptr_init(compressed_page_t, PAGE_UNALLOCATED);
		} else if (type == TRYALLOC) {
		newpage = erofs_allocpage(pagepool, gfp);
		if (!newpage)
		goto dontalloc;

		set_page_private(newpage, Z_EROFS_PREALLOCATED_PAGE);
		t = tag_compressed_page_justfound(newpage);
		} else { /* DONTALLOC */
		dontalloc:
		if (standalone)
		clt->compressedpages = pages;
		standalone = false;
		@@ -198,8 +215,12 @@ static void preload_compressed_pages(struct z_erofs_collector *clt,
		if (!cmpxchg_relaxed(pages, NULL, tagptr_cast_ptr(t)))
		continue;

		if (page)
		if (page) {
		put_page(page);
		} else if (newpage) {
		set_page_private(newpage, 0);
		list_add(&newpage->lru, pagepool);
		}
		}

		if (standalone) /* downgrade to PRIMARY_FOLLOWED_NOINPLACE */
		@@ -555,7 +576,7 @@ static bool should_alloc_managed_pages(struct z_erofs_decompress_frontend *fe,
		}

		static int z_erofs_do_read_page(struct z_erofs_decompress_frontend *fe,
		struct page *page)
		struct page page, struct list_head pagepool)
		{
		struct inode *const inode = fe->inode;
		struct erofs_sb_info *const sbi = EROFS_I_SB(inode);
		@@ -608,11 +629,12 @@ static int z_erofs_do_read_page(struct z_erofs_decompress_frontend *fe,

		/* preload all compressed pages (maybe downgrade role if necessary) */
		if (should_alloc_managed_pages(fe, sbi->cache_strategy, map->m_la))
		cache_strategy = DELAYEDALLOC;
		cache_strategy = TRYALLOC;
		else
		cache_strategy = DONTALLOC;

		preload_compressed_pages(clt, MNGD_MAPPING(sbi), cache_strategy);
		preload_compressed_pages(clt, MNGD_MAPPING(sbi),
		cache_strategy, pagepool);

		hitted:
		/*
		@@ -1007,6 +1029,16 @@ static struct page pickup_page_for_submission(struct z_erofs_pcluster pcl,
		justfound = tagptr_unfold_tags(t);
		page = tagptr_unfold_ptr(t);

		/*
		* preallocated cached pages, which is used to avoid direct reclaim
		* otherwise, it will go inplace I/O path instead.
		*/
		if (page->private == Z_EROFS_PREALLOCATED_PAGE) {
		WRITE_ONCE(pcl->compressed_pages[nr], page);
		set_page_private(page, 0);
		tocache = true;
		goto out_tocache;
		}
		mapping = READ_ONCE(page->mapping);

		/*
		@@ -1069,7 +1101,7 @@ static struct page pickup_page_for_submission(struct z_erofs_pcluster pcl,
		cond_resched();
		goto repeat;
		}

		out_tocache:
		if (!tocache \|\| add_to_page_cache_lru(page, mc, index + nr, gfp)) {
		/* turn into temporary page if fails (1 ref) */
		set_page_private(page, Z_EROFS_SHORTLIVED_PAGE);
		@@ -1279,7 +1311,7 @@ static int z_erofs_readpage(struct file file, struct page page)

		f.headoffset = (erofs_off_t)page->index << PAGE_SHIFT;

		err = z_erofs_do_read_page(&f, page);
		err = z_erofs_do_read_page(&f, page, &pagepool);
		(void)z_erofs_collector_end(&f.clt);

		/* if some compressed cluster ready, need submit them anyway */
		@@ -1342,7 +1374,7 @@ static int z_erofs_readpages(struct file filp, struct address_space mapping,
		/* traversal in reverse order */
		head = (void *)page_private(page);

		err = z_erofs_do_read_page(&f, page);
		err = z_erofs_do_read_page(&f, page, &pagepool);
		if (err)
		erofs_err(inode->i_sb,
		"readahead error at page %lu @ nid %llu",