erofs: sync up with kernel 5.10

# SPDX-License-Identifier: GPL-2.0

# SPDX-License-Identifier: GPL-2.0-only

config EROFS_FS

	tristate "EROFS filesystem support"

	depends on BLOCK

	select LIBCRC32C

	help

	  EROFS (Enhanced Read-Only File System) is a lightweight

	  read-only file system with modern designs (eg. page-sized

	  blocks, inline xattrs/data, etc.) for scenarios which need

	  high-performance read-only requirements, eg. firmwares in

	  mobile phone or LIVECDs.

	  high-performance read-only requirements, e.g. Android OS

	  for mobile phones and LIVECDs.

	  It also provides VLE compression support, focusing on

	  random read improvements, keeping relatively lower

	  compression ratios, which is useful for high-performance

	  devices with limited memory and ROM space.

	  It also provides fixed-sized output compression support,

	  which improves storage density, keeps relatively higher

	  compression ratios, which is more useful to achieve high

	  performance for embedded devices with limited memory.

	  If unsure, say N.

	bool "EROFS debugging feature"

	depends on EROFS_FS

	help

	  Print EROFS debugging messages and enable more BUG_ONs

	  which check the filesystem consistency aggressively.

	  Print debugging messages and enable more BUG_ONs which check

	  filesystem consistency and find potential issues aggressively,

	  which can be used for Android eng build, for example.

	  For daily use, say N.

config EROFS_FS_SECURITY

	bool "EROFS Security Labels"

	depends on EROFS_FS_XATTR

	default y

	help

	  Security labels provide an access control facility to support Linux

	  Security Models (LSMs) accepted by AppArmor, SELinux, Smack and TOMOYO

	  If you are not using a security module, say N.

config EROFS_FS_USE_VM_MAP_RAM

	bool "EROFS VM_MAP_RAM Support"

	depends on EROFS_FS

	help

	  use vm_map_ram/vm_unmap_ram instead of vmap/vunmap.

	  If you don't know what these are, say N.

config EROFS_FAULT_INJECTION

	bool "EROFS fault injection facility"

	depends on EROFS_FS

	help

	  Test EROFS to inject faults such as ENOMEM, EIO, and so on.

	  If unsure, say N.

config EROFS_FS_ZIP

	bool "EROFS Data Compresssion Support"

	bool "EROFS Data Compression Support"

	depends on EROFS_FS

	select LZ4_DECOMPRESS

	default y

	help

	  Currently we support VLE Compression only.

	  Play at your own risk.

	  Enable fixed-sized output compression for EROFS.

	  If you don't want to use compression feature, say N.

	  If you don't want to enable compression feature, say N.

config EROFS_FS_CLUSTER_PAGE_LIMIT

	int "EROFS Cluster Pages Hard Limit"

	range 1 256

	default "1"

	help

	  Indicates VLE compressed pages hard limit of a

	  compressed cluster.

	  For example, if files of a image are compressed

	  into 8k-unit, the hard limit should not be less

	  than 2. Otherwise, the image cannot be mounted

	  correctly on this kernel.

choice

	prompt "EROFS VLE Data Decompression mode"

	depends on EROFS_FS_ZIP

	default EROFS_FS_ZIP_CACHE_BIPOLAR

	help

	  EROFS supports three options for VLE decompression.

	  "In-place Decompression Only" consumes the minimum memory

	  with lowest random read.

	  "Bipolar Cached Decompression" consumes the maximum memory

	  with highest random read.

	  If unsure, select "Bipolar Cached Decompression"

config EROFS_FS_ZIP_NO_CACHE

	bool "In-place Decompression Only"

	help

	  Read compressed data into page cache and do in-place

	  decompression directly.

config EROFS_FS_ZIP_CACHE_UNIPOLAR

	bool "Unipolar Cached Decompression"

	help

	  For each request, it caches the last compressed page

	  for further reading.

	  It still decompresses in place for the rest compressed pages.

config EROFS_FS_ZIP_CACHE_BIPOLAR

	bool "Bipolar Cached Decompression"

	help

	  For each request, it caches the both end compressed pages

	  for further reading.

	  It still decompresses in place for the rest compressed pages.

	  Recommended for performance priority.

	  Indicates maximum # of pages of a compressed

	  physical cluster.

endchoice

	  For example, if files in a image were compressed

	  into 8k-unit, hard limit should not be configured

	  less than 2. Otherwise, the image will be refused

	  to mount on this kernel.

# SPDX-License-Identifier: GPL-2.0

# SPDX-License-Identifier: GPL-2.0-only

EROFS_VERSION = "1.0pre1"

EROFS_VERSION = "1.0"

ccflags-y += -Wall -DEROFS_VERSION=\"$(EROFS_VERSION)\"

ccflags-y += -DEROFS_VERSION=\"$(EROFS_VERSION)\"

obj-$(CONFIG_EROFS_FS) += erofs.o

# staging requirement: to be self-contained in its own directory

ccflags-y += -I$(src)/include

erofs-objs := super.o inode.o data.o namei.o dir.o utils.o

erofs-$(CONFIG_EROFS_FS_XATTR) += xattr.o

erofs-$(CONFIG_EROFS_FS_ZIP) += unzip_vle.o unzip_lz4.o unzip_vle_lz4.o

erofs-$(CONFIG_EROFS_FS_ZIP) += decompressor.o zmap.o zdata.o

/* SPDX-License-Identifier: GPL-2.0-only */

/*

 * Copyright (C) 2019 HUAWEI, Inc.

 *             https://www.huawei.com/

 */

#ifndef __EROFS_FS_COMPRESS_H

#define __EROFS_FS_COMPRESS_H

#include "internal.h"

enum {

	Z_EROFS_COMPRESSION_SHIFTED = Z_EROFS_COMPRESSION_MAX,

	Z_EROFS_COMPRESSION_RUNTIME_MAX

};

struct z_erofs_decompress_req {

	struct super_block *sb;

	struct page **in, **out;

	unsigned short pageofs_out;

	unsigned int inputsize, outputsize;

	/* indicate the algorithm will be used for decompression */

	unsigned int alg;

	bool inplace_io, partial_decoding;

};

/*

 * - 0x5A110C8D ('sallocated', Z_EROFS_MAPPING_STAGING) -

 * used to mark temporary allocated pages from other

 * file/cached pages and NULL mapping pages.

 */

#define Z_EROFS_MAPPING_STAGING         ((void *)0x5A110C8D)

/* check if a page is marked as staging */

static inline bool z_erofs_page_is_staging(struct page *page)

{

	return page->mapping == Z_EROFS_MAPPING_STAGING;

}

static inline bool z_erofs_put_stagingpage(struct list_head *pagepool,

					   struct page *page)

{

	if (!z_erofs_page_is_staging(page))

		return false;

	/* staging pages should not be used by others at the same time */

	if (page_ref_count(page) > 1)

		put_page(page);

	else

		list_add(&page->lru, pagepool);

	return true;

}

int z_erofs_decompress(struct z_erofs_decompress_req *rq,

		       struct list_head *pagepool);

#endif

// SPDX-License-Identifier: GPL-2.0

// SPDX-License-Identifier: GPL-2.0-only

/*

 * linux/drivers/staging/erofs/data.c

 *

 * Copyright (C) 2017-2018 HUAWEI, Inc.

 *             http://www.huawei.com/

 * Created by Gao Xiang <gaoxiang25@huawei.com>

 *

 * This file is subject to the terms and conditions of the GNU General Public

 * License.  See the file COPYING in the main directory of the Linux

 * distribution for more details.

 *             https://www.huawei.com/

 */

#include "internal.h"

#include <linux/prefetch.h>

#include <trace/events/erofs.h>

static inline void read_endio(struct bio *bio)

static void erofs_readendio(struct bio *bio)

{

	int i;

	struct bio_vec *bvec;

		/* page is already locked */

		DBG_BUGON(PageUptodate(page));

		if (unlikely(err))

		if (err)

			SetPageError(page);

		else

			SetPageUptodate(page);

	bio_put(bio);

}

/* prio -- true is used for dir */

struct page *erofs_get_meta_page(struct super_block *sb,

	erofs_blk_t blkaddr, bool prio)

struct page *erofs_get_meta_page(struct super_block *sb, erofs_blk_t blkaddr)

{

	struct inode *bd_inode = sb->s_bdev->bd_inode;

	struct address_space *mapping = bd_inode->i_mapping;

	struct address_space *const mapping = sb->s_bdev->bd_inode->i_mapping;

	struct page *page;

repeat:

	page = find_or_create_page(mapping, blkaddr,

	/*

	 * Prefer looping in the allocator rather than here,

	 * at least that code knows what it's doing.

	 */

		mapping_gfp_constraint(mapping, ~__GFP_FS) | __GFP_NOFAIL);

	BUG_ON(!page || !PageLocked(page));

	if (!PageUptodate(page)) {

		struct bio *bio;

		int err;

		bio = prepare_bio(sb, blkaddr, 1, read_endio);

		err = bio_add_page(bio, page, PAGE_SIZE, 0);

		BUG_ON(err != PAGE_SIZE);

		__submit_bio(bio, REQ_OP_READ,

			REQ_META | (prio ? REQ_PRIO : 0));

	page = read_cache_page_gfp(mapping, blkaddr,

				   mapping_gfp_constraint(mapping, ~__GFP_FS));

	/* should already be PageUptodate */

	if (!IS_ERR(page))

		lock_page(page);

		/* the page has been truncated by others? */

		if (unlikely(page->mapping != mapping)) {

			unlock_page(page);

			put_page(page);

			goto repeat;

		}

		/* more likely a read error */

		if (unlikely(!PageUptodate(page))) {

			unlock_page(page);

			put_page(page);

			page = ERR_PTR(-EIO);

		}

	}

	return page;

}

	int err = 0;

	erofs_blk_t nblocks, lastblk;

	u64 offset = map->m_la;

	struct erofs_vnode *vi = EROFS_V(inode);

	struct erofs_inode *vi = EROFS_I(inode);

	bool tailendpacking = (vi->datalayout == EROFS_INODE_FLAT_INLINE);

	trace_erofs_map_blocks_flatmode_enter(inode, map, flags);

	nblocks = DIV_ROUND_UP(inode->i_size, PAGE_SIZE);

	lastblk = nblocks - is_inode_layout_inline(inode);

	lastblk = nblocks - tailendpacking;

	if (unlikely(offset >= inode->i_size)) {

	if (offset >= inode->i_size) {

		/* leave out-of-bound access unmapped */

		map->m_flags = 0;

		map->m_plen = 0;

	if (offset < blknr_to_addr(lastblk)) {

		map->m_pa = blknr_to_addr(vi->raw_blkaddr) + map->m_la;

		map->m_plen = blknr_to_addr(lastblk) - offset;

	} else if (is_inode_layout_inline(inode)) {

	} else if (tailendpacking) {

		/* 2 - inode inline B: inode, [xattrs], inline last blk... */

		struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb);

			vi->xattr_isize + erofs_blkoff(map->m_la);

		map->m_plen = inode->i_size - offset;

		/* inline data should locate in one meta block */

		/* inline data should be located in one meta block */

		if (erofs_blkoff(map->m_pa) + map->m_plen > PAGE_SIZE) {

			erofs_err(inode->i_sb,

				  "inline data cross block boundary @ nid %llu",

				  vi->nid);

			DBG_BUGON(1);

			err = -EIO;

			err = -EFSCORRUPTED;

			goto err_out;

		}

		map->m_flags |= EROFS_MAP_META;

	} else {

		errln("internal error @ nid: %llu (size %llu), m_la 0x%llx",

		erofs_err(inode->i_sb,

			  "internal error @ nid: %llu (size %llu), m_la 0x%llx",

			  vi->nid, inode->i_size, map->m_la);

		DBG_BUGON(1);

		err = -EIO;

	return err;

}

#ifdef CONFIG_EROFS_FS_ZIP

extern int z_erofs_map_blocks_iter(struct inode *,

	struct erofs_map_blocks *, struct page **, int);

#endif

int erofs_map_blocks_iter(struct inode *inode,

	struct erofs_map_blocks *map,

	struct page **mpage_ret, int flags)

{

	/* by default, reading raw data never use erofs_map_blocks_iter */

	if (unlikely(!is_inode_layout_compression(inode))) {

		if (*mpage_ret != NULL)

			put_page(*mpage_ret);

		*mpage_ret = NULL;

		return erofs_map_blocks(inode, map, flags);

	}

#ifdef CONFIG_EROFS_FS_ZIP

	return z_erofs_map_blocks_iter(inode, map, mpage_ret, flags);

#else

	/* data compression is not available */

	return -ENOTSUPP;

#endif

}

int erofs_map_blocks(struct inode *inode,

		     struct erofs_map_blocks *map, int flags)

{

	if (unlikely(is_inode_layout_compression(inode))) {

		struct page *mpage = NULL;

		int err;

	if (erofs_inode_is_data_compressed(EROFS_I(inode)->datalayout)) {

		int err = z_erofs_map_blocks_iter(inode, map, flags);

		err = erofs_map_blocks_iter(inode, map, &mpage, flags);

		if (mpage != NULL)

			put_page(mpage);

		if (map->mpage) {

			put_page(map->mpage);

			map->mpage = NULL;

		}

		return err;

	}

	return erofs_map_blocks_flatmode(inode, map, flags);

}

static inline struct bio *erofs_read_raw_page(

	struct bio *bio,

static inline struct bio *erofs_read_raw_page(struct bio *bio,

					      struct address_space *mapping,

					      struct page *page,

					      erofs_off_t *last_block,

	unsigned nblocks,

					      unsigned int nblocks,

					      bool ra)

{

	struct inode *inode = mapping->host;

	struct inode *const inode = mapping->host;

	struct super_block *const sb = inode->i_sb;

	erofs_off_t current_block = (erofs_off_t)page->index;

	int err;

		goto has_updated;

	}

	if (cleancache_get_page(page) == 0) {

		err = 0;

		SetPageUptodate(page);

		goto has_updated;

	}

	/* note that for readpage case, bio also equals to NULL */

	if (bio != NULL &&

	if (bio &&

	    /* not continuous */

	    *last_block + 1 != current_block) {

submit_bio_retry:

		__submit_bio(bio, REQ_OP_READ, 0);

		submit_bio(bio);

		bio = NULL;

	}

	if (bio == NULL) {

	if (!bio) {

		struct erofs_map_blocks map = {

			.m_la = blknr_to_addr(current_block),

		};

		erofs_blk_t blknr;

		unsigned blkoff;

		unsigned int blkoff;

		err = erofs_map_blocks(inode, &map, EROFS_GET_BLOCKS_RAW);

		if (unlikely(err))

		if (err)

			goto err_out;

		/* zero out the holed page */

		if (unlikely(!(map.m_flags & EROFS_MAP_MAPPED))) {

		if (!(map.m_flags & EROFS_MAP_MAPPED)) {

			zero_user_segment(page, 0, PAGE_SIZE);

			SetPageUptodate(page);

			DBG_BUGON(map.m_plen > PAGE_SIZE);

			ipage = erofs_get_meta_page(inode->i_sb, blknr, 0);

			ipage = erofs_get_meta_page(inode->i_sb, blknr);

			if (IS_ERR(ipage)) {

				err = PTR_ERR(ipage);

		if (nblocks > BIO_MAX_PAGES)

			nblocks = BIO_MAX_PAGES;

		bio = prepare_bio(inode->i_sb, blknr, nblocks, read_endio);

		bio = bio_alloc(GFP_NOIO, nblocks);

		bio->bi_end_io = erofs_readendio;

		bio_set_dev(bio, sb->s_bdev);

		bio->bi_iter.bi_sector = (sector_t)blknr <<

			LOG_SECTORS_PER_BLOCK;

		bio->bi_opf = REQ_OP_READ | (ra ? REQ_RAHEAD : 0);

	}

	err = bio_add_page(bio, page, PAGE_SIZE, 0);

	*last_block = current_block;

	/* shift in advance in case of it followed by too many gaps */

	if (unlikely(bio->bi_vcnt >= bio->bi_max_vecs)) {

	if (bio->bi_iter.bi_size >= bio->bi_max_vecs * PAGE_SIZE) {

		/* err should reassign to 0 after submitting */

		err = 0;

		goto submit_bio_out;

	unlock_page(page);

	/* if updated manually, continuous pages has a gap */

	if (bio != NULL)

	if (bio)

submit_bio_out:

		__submit_bio(bio, REQ_OP_READ, 0);

	return unlikely(err) ? ERR_PTR(err) : NULL;

		submit_bio(bio);

	return err ? ERR_PTR(err) : NULL;

}

/*

static int erofs_raw_access_readpages(struct file *filp,

				      struct address_space *mapping,

	struct list_head *pages, unsigned nr_pages)

				      struct list_head *pages,

				      unsigned int nr_pages)

{

	erofs_off_t last_block;

	struct bio *bio = NULL;

			if (IS_ERR(bio)) {

				pr_err("%s, readahead error at page %lu of nid %llu\n",

				       __func__, page->index,

					EROFS_V(mapping->host)->nid);

				       EROFS_I(mapping->host)->nid);

				bio = NULL;

			}

	DBG_BUGON(!list_empty(pages));

	/* the rare case (end in gaps) */

	if (unlikely(bio != NULL))

		__submit_bio(bio, REQ_OP_READ, 0);

	if (bio)

		submit_bio(bio);

	return 0;

}

static sector_t erofs_bmap(struct address_space *mapping, sector_t block)

{

	struct inode *inode = mapping->host;

	struct erofs_map_blocks map = {

		.m_la = blknr_to_addr(block),

	};

	if (EROFS_I(inode)->datalayout == EROFS_INODE_FLAT_INLINE) {

		erofs_blk_t blks = i_size_read(inode) >> LOG_BLOCK_SIZE;

		if (block >> LOG_SECTORS_PER_BLOCK >= blks)

			return 0;

	}

	if (!erofs_map_blocks(inode, &map, EROFS_GET_BLOCKS_RAW))

		return erofs_blknr(map.m_pa);

	return 0;

}

const struct address_space_operations erofs_raw_access_aops = {

	.readpage = erofs_raw_access_readpage,

	.readpages = erofs_raw_access_readpages,

	.bmap = erofs_bmap,

};

// SPDX-License-Identifier: GPL-2.0-only

/*

 * Copyright (C) 2019 HUAWEI, Inc.

 *             https://www.huawei.com/

 */

#include "compress.h"

#include <linux/module.h>

#include <linux/lz4.h>

#ifndef LZ4_DISTANCE_MAX	/* history window size */

#define LZ4_DISTANCE_MAX 65535	/* set to maximum value by default */

#endif

#define LZ4_MAX_DISTANCE_PAGES	(DIV_ROUND_UP(LZ4_DISTANCE_MAX, PAGE_SIZE) + 1)

#ifndef LZ4_DECOMPRESS_INPLACE_MARGIN

#define LZ4_DECOMPRESS_INPLACE_MARGIN(srcsize)  (((srcsize) >> 8) + 32)

#endif

struct z_erofs_decompressor {

	/*

	 * if destpages have sparsed pages, fill them with bounce pages.

	 * it also check whether destpages indicate continuous physical memory.

	 */

	int (*prepare_destpages)(struct z_erofs_decompress_req *rq,

				 struct list_head *pagepool);

	int (*decompress)(struct z_erofs_decompress_req *rq, u8 *out);

	char *name;

};

static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq,

					 struct list_head *pagepool)

{

	const unsigned int nr =

		PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;

	struct page *availables[LZ4_MAX_DISTANCE_PAGES] = { NULL };

	unsigned long bounced[DIV_ROUND_UP(LZ4_MAX_DISTANCE_PAGES,

					   BITS_PER_LONG)] = { 0 };

	void *kaddr = NULL;

	unsigned int i, j, top;

	top = 0;

	for (i = j = 0; i < nr; ++i, ++j) {

		struct page *const page = rq->out[i];

		struct page *victim;

		if (j >= LZ4_MAX_DISTANCE_PAGES)

			j = 0;

		/* 'valid' bounced can only be tested after a complete round */

		if (test_bit(j, bounced)) {

			DBG_BUGON(i < LZ4_MAX_DISTANCE_PAGES);

			DBG_BUGON(top >= LZ4_MAX_DISTANCE_PAGES);

			availables[top++] = rq->out[i - LZ4_MAX_DISTANCE_PAGES];

		}

		if (page) {

			__clear_bit(j, bounced);

			if (kaddr) {

				if (kaddr + PAGE_SIZE == page_address(page))

					kaddr += PAGE_SIZE;

				else

					kaddr = NULL;

			} else if (!i) {

				kaddr = page_address(page);

			}

			continue;

		}

		kaddr = NULL;

		__set_bit(j, bounced);

		if (top) {

			victim = availables[--top];

			get_page(victim);

		} else {

			victim = erofs_allocpage(pagepool, GFP_KERNEL);

			if (!victim)

				return -ENOMEM;

			victim->mapping = Z_EROFS_MAPPING_STAGING;

		}

		rq->out[i] = victim;

	}

	return kaddr ? 1 : 0;

}

static void *generic_copy_inplace_data(struct z_erofs_decompress_req *rq,

				       u8 *src, unsigned int pageofs_in)

{

	/*

	 * if in-place decompression is ongoing, those decompressed

	 * pages should be copied in order to avoid being overlapped.

	 */

	struct page **in = rq->in;

	u8 *const tmp = erofs_get_pcpubuf(0);

	u8 *tmpp = tmp;

	unsigned int inlen = rq->inputsize - pageofs_in;

	unsigned int count = min_t(uint, inlen, PAGE_SIZE - pageofs_in);

	while (tmpp < tmp + inlen) {

		if (!src)

			src = kmap_atomic(*in);

		memcpy(tmpp, src + pageofs_in, count);

		kunmap_atomic(src);

		src = NULL;

		tmpp += count;

		pageofs_in = 0;

		count = PAGE_SIZE;

		++in;

	}

	return tmp;

}

static int z_erofs_lz4_decompress(struct z_erofs_decompress_req *rq, u8 *out)

{

	unsigned int inputmargin, inlen;

	u8 *src;

	bool copied, support_0padding;

	int ret;

	if (rq->inputsize > PAGE_SIZE)

		return -EOPNOTSUPP;

	src = kmap_atomic(*rq->in);

	inputmargin = 0;

	support_0padding = false;

	/* decompression inplace is only safe when 0padding is enabled */

	if (EROFS_SB(rq->sb)->feature_incompat &

	    EROFS_FEATURE_INCOMPAT_LZ4_0PADDING) {

		support_0padding = true;

		while (!src[inputmargin & ~PAGE_MASK])

			if (!(++inputmargin & ~PAGE_MASK))

				break;

		if (inputmargin >= rq->inputsize) {

			kunmap_atomic(src);

			return -EIO;

		}

	}

	copied = false;

	inlen = rq->inputsize - inputmargin;