Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 87bf54bb authored by Sean Purcell's avatar Sean Purcell Committed by Chris Mason
Browse files

squashfs: Add zstd support 

Add zstd compression and decompression support to SquashFS. zstd is a
great fit for SquashFS because it can compress at ratios approaching xz,
while decompressing twice as fast as zlib. For SquashFS in particular,
it can decompress as fast as lzo and lz4. It also has the flexibility
to turn down the compression ratio for faster compression times.

The compression benchmark is run on the file tree from the SquashFS archive
found in ubuntu-16.10-desktop-amd64.iso [1]. It uses `mksquashfs` with the
default block size (128 KB) and and various compression algorithms/levels.
xz and zstd are also benchmarked with 256 KB blocks. The decompression
benchmark times how long it takes to `tar` the file tree into `/dev/null`.
See the benchmark file in the upstream zstd source repository located under
`contrib/linux-kernel/squashfs-benchmark.sh` [2] for details.

I ran the benchmarks on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM.
The VM is running on a MacBook Pro with a 3.1 GHz Intel Core i7 processor,
16 GB of RAM, and a SSD.

| Method         | Ratio | Compression MB/s | Decompression MB/s |
|----------------|-------|------------------|--------------------|
| gzip           |  2.92 |               15 |                128 |
| lzo            |  2.64 |              9.5 |                217 |
| lz4            |  2.12 |               94 |                218 |
| xz             |  3.43 |              5.5 |                 35 |
| xz 256 KB      |  3.53 |              5.4 |                 40 |
| zstd 1         |  2.71 |               96 |                210 |
| zstd 5         |  2.93 |               69 |                198 |
| zstd 10        |  3.01 |               41 |                225 |
| zstd 15        |  3.13 |             11.4 |                224 |
| zstd 16 256 KB |  3.24 |              8.1 |                210 |

This patch was written by Sean Purcell <me@seanp.xyz>, but I will be
taking over the submission process.

[1] http://releases.ubuntu.com/16.10/
[2] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/squashfs-benchmark.sh

zstd source repository: https://github.com/facebook/zstd



Signed-off-by: default avatarSean Purcell <me@seanp.xyz>
Signed-off-by: default avatarNick Terrell <terrelln@fb.com>
Signed-off-by: default avatarChris Mason <clm@fb.com>
Acked-by: default avatarPhillip Lougher <phillip@squashfs.org.uk>
parent 5c1aab1d

fs/squashfs/Kconfig

+14 −0
Original line number Diff line number Diff line
@@ -165,6 +165,20 @@ config SQUASHFS_XZ 


	  If unsure, say N.



config SQUASHFS_ZSTD

	bool "Include support for ZSTD compressed file systems"

	depends on SQUASHFS

	select ZSTD_DECOMPRESS

	help

	  Saying Y here includes support for reading Squashfs file systems

	  compressed with ZSTD compression.  ZSTD gives better compression than

	  the default ZLIB compression, while using less CPU.



	  ZSTD is not the standard compression used in Squashfs and so most

	  file systems will be readable without selecting this option.



	  If unsure, say N.



config SQUASHFS_4K_DEVBLK_SIZE

	bool "Use 4K device block size?"

	depends on SQUASHFS

fs/squashfs/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -15,3 +15,4 @@ squashfs-$(CONFIG_SQUASHFS_LZ4) += lz4_wrapper.o 
squashfs-$(CONFIG_SQUASHFS_LZO) += lzo_wrapper.o

squashfs-$(CONFIG_SQUASHFS_XZ) += xz_wrapper.o

squashfs-$(CONFIG_SQUASHFS_ZLIB) += zlib_wrapper.o

squashfs-$(CONFIG_SQUASHFS_ZSTD) += zstd_wrapper.o

fs/squashfs/decompressor.c

+7 −0
Original line number Diff line number Diff line
@@ -65,6 +65,12 @@ static const struct squashfs_decompressor squashfs_zlib_comp_ops = { 
};

#endif



#ifndef CONFIG_SQUASHFS_ZSTD

static const struct squashfs_decompressor squashfs_zstd_comp_ops = {

	NULL, NULL, NULL, NULL, ZSTD_COMPRESSION, "zstd", 0

};

#endif



static const struct squashfs_decompressor squashfs_unknown_comp_ops = {

	NULL, NULL, NULL, NULL, 0, "unknown", 0

};
@@ -75,6 +81,7 @@ static const struct squashfs_decompressor *decompressor[] = { 
	&squashfs_lzo_comp_ops,

	&squashfs_xz_comp_ops,

	&squashfs_lzma_unsupported_comp_ops,

	&squashfs_zstd_comp_ops,

	&squashfs_unknown_comp_ops

};

fs/squashfs/decompressor.h

+4 −0
Original line number Diff line number Diff line
@@ -58,4 +58,8 @@ extern const struct squashfs_decompressor squashfs_lzo_comp_ops; 
extern const struct squashfs_decompressor squashfs_zlib_comp_ops;

#endif



#ifdef CONFIG_SQUASHFS_ZSTD

extern const struct squashfs_decompressor squashfs_zstd_comp_ops;

#endif



#endif

fs/squashfs/squashfs_fs.h

+1 −0
Original line number Diff line number Diff line
@@ -241,6 +241,7 @@ struct meta_index { 
#define LZO_COMPRESSION		3

#define XZ_COMPRESSION		4

#define LZ4_COMPRESSION		5

#define ZSTD_COMPRESSION	6



struct squashfs_super_block {

	__le32			s_magic;