Merge "libsparse: Add sparse typed callback" (3014fd00) · Commits · e / os / android_system_core

libsparse/include/sparse/sparse.h

+31 −0

Original line number	Diff line number	Diff line
		@@ -209,6 +209,37 @@ unsigned int sparse_file_block_size(struct sparse_file *s);
		int sparse_file_callback(struct sparse_file *s, bool sparse, bool crc,
		int (write)(void priv, const void data, size_t len), void priv);

		/**
		* sparse_file_callback_typed - call a callback for blocks based on type
		*
		* @s - sparse file cookie
		* @sparse - write in the Android sparse file format
		* @data_write - function to call for data blocks. must not be null
		* @fd_write - function to call for fd blocks
		* @fill_write - function to call for fill blocks
		* @skip_write - function to call for skip blocks
		* @priv - value that will be passed as the first argument to each write
		*
		* Writes a sparse file by calling callback functions. If sparse is true, the
		* file will be written in the Android sparse file format, and fill and skip blocks
		* along with metadata will be written with data_write. If sparse is false, the file
		* will be expanded into normal format and fill and skip blocks will be written with
		* the given callbacks.
		* If a callback function is provided, the library will not unroll data into a buffer,
		* and will instead pass it directly to the caller for custom implementation. If a
		* callback is not provided, that type of block will be converted into a void* and
		* written with data_write. If no callbacks other than data are provided, the behavior
		* is the same as sparse_file_callback(). The callback should return negative on error,
		* 0 on success.
		*
		* Returns 0 on success, negative errno on error.
		*/
		int sparse_file_callback_typed(struct sparse_file* s, bool sparse,
		int (data_write)(void priv, const void* data, size_t len),
		int (fd_write)(void priv, int fd, size_t len),
		int (fill_write)(void priv, uint32_t fill_val, size_t len),
		int (skip_write)(void priv, int64_t len), void* priv);

		/**
		* sparse_file_foreach_chunk - call a callback for data blocks in sparse file
		*

libsparse/output_file.cpp

+178 −86

Original line number	Diff line number	Diff line
		@@ -29,6 +29,8 @@
		#include <unistd.h>
		#include <zlib.h>

		#include <algorithm>

		#include "defs.h"
		#include "output_file.h"
		#include "sparse_crc32.h"
		@@ -48,13 +50,6 @@
		#define off64_t off_t
		#endif

		#define min(a, b) \
		({ \
		typeof(a) _a = (a); \
		typeof(b) _b = (b); \
		(_a < _b) ? _a : _b; \
		})

		#define SPARSE_HEADER_MAJOR_VER 1
		#define SPARSE_HEADER_MINOR_VER 0
		#define SPARSE_HEADER_LEN (sizeof(sparse_header_t))
		@@ -67,11 +62,14 @@ struct output_file_ops {
		int (skip)(struct output_file, int64_t);
		int (pad)(struct output_file, int64_t);
		int (write)(struct output_file, void*, size_t);
		int (write_fd)(struct output_file, int, size_t);
		int (write_fill)(struct output_file, uint32_t, size_t);
		void (close)(struct output_file);
		};

		struct sparse_file_ops {
		int (write_data_chunk)(struct output_file out, unsigned int len, void* data);
		int (write_fd_chunk)(struct output_file out, unsigned int len, int fd);
		int (write_fill_chunk)(struct output_file out, unsigned int len, uint32_t fill_val);
		int (write_skip_chunk)(struct output_file out, int64_t len);
		int (write_end_chunk)(struct output_file out);
		@@ -108,11 +106,96 @@ struct output_file_normal {
		struct output_file_callback {
		struct output_file out;
		void* priv;
		int (write)(void priv, const void* buf, size_t len);
		int (data_write)(void priv, const void* data, size_t len);
		int (fd_write)(void priv, int fd, size_t len);
		int (fill_write)(void priv, uint32_t fill_val, size_t len);
		int (skip_write)(void priv, off64_t len);
		};

		#define to_output_file_callback(_o) container_of((_o), struct output_file_callback, out)

		union handle_data {
		void* data_ptr;
		int fd;
		};

		static int default_file_write_fd(struct output_file* out, int fd, size_t len) {
		int ret;
		int64_t aligned_offset;
		int aligned_diff;
		uint64_t buffer_size;
		char* ptr;
		int64_t offset = lseek64(fd, 0, SEEK_CUR);

		if (offset < 0) {
		return -errno;
		}

		aligned_offset = offset & ~(4096 - 1);
		aligned_diff = offset - aligned_offset;
		buffer_size = (uint64_t)len + (uint64_t)aligned_diff;

		#ifndef _WIN32
		if (buffer_size > SIZE_MAX) {
		return -E2BIG;
		}
		char* data =
		reinterpret_cast<char*>(mmap64(NULL, buffer_size, PROT_READ, MAP_SHARED, fd, aligned_offset));
		if (data == MAP_FAILED) {
		return -errno;
		}
		ptr = data + aligned_diff;
		#else
		char* data = reinterpret_cast<char*>(malloc(len));
		if (!data) {
		return -errno;
		}
		ret = read_all(fd, data, len);
		if (ret < 0) {
		free(data);
		return -errno;
		}
		ptr = data;
		#endif

		ret = out->ops->write(out, ptr, len);

		if (out->use_crc) {
		out->crc32 = sparse_crc32(out->crc32, ptr, len);
		}

		#ifndef _WIN32
		munmap(data, buffer_size);
		#else
		free(data);
		#endif

		return ret;
		}

		static int default_file_write_fill(struct output_file* out, uint32_t fill_val, size_t len) {
		int ret;
		unsigned int i;
		unsigned int write_len;

		/* Initialize fill_buf with the fill_val */
		for (i = 0; i < out->block_size / sizeof(uint32_t); i++) {
		out->fill_buf[i] = fill_val;
		}

		while (len) {
		write_len = std::min(len, static_cast<size_t>(out->block_size));
		ret = out->ops->write(out, out->fill_buf, write_len);
		if (ret < 0) {
		return ret;
		}

		len -= write_len;
		}

		return 0;
		}

		static int file_open(struct output_file* out, int fd) {
		struct output_file_normal* outn = to_output_file_normal(out);

		@@ -176,6 +259,8 @@ static struct output_file_ops file_ops = {
		.skip = file_skip,
		.pad = file_pad,
		.write = file_write,
		.write_fd = default_file_write_fd,
		.write_fill = default_file_write_fill,
		.close = file_close,
		};

		@@ -231,7 +316,7 @@ static int gz_file_write(struct output_file* out, void* data, size_t len) {
		struct output_file_gz* outgz = to_output_file_gz(out);

		while (len > 0) {
		ret = gzwrite(outgz->gz_fd, data, min(len, (unsigned int)INT_MAX));
		ret = gzwrite(outgz->gz_fd, data, std::min(len, static_cast<size_t>(INT_MAX)));
		if (ret == 0) {
		error("gzwrite %s", gzerror(outgz->gz_fd, nullptr));
		return -1;
		@@ -255,6 +340,8 @@ static struct output_file_ops gz_file_ops = {
		.skip = gz_file_skip,
		.pad = gz_file_pad,
		.write = gz_file_write,
		.write_fd = default_file_write_fd,
		.write_fill = default_file_write_fill,
		.close = gz_file_close,
		};

		@@ -267,9 +354,13 @@ static int callback_file_skip(struct output_file* out, int64_t off) {
		int to_write;
		int ret;

		if (outc->skip_write) {
		return outc->skip_write(outc->priv, off);
		}

		while (off > 0) {
		to_write = min(off, (int64_t)INT_MAX);
		ret = outc->write(outc->priv, nullptr, to_write);
		to_write = std::min(off, static_cast<int64_t>(INT_MAX));
		ret = outc->data_write(outc->priv, nullptr, to_write);
		if (ret < 0) {
		return ret;
		}
		@@ -285,8 +376,23 @@ static int callback_file_pad(struct output_file* out __unused, int64_t len __unu

		static int callback_file_write(struct output_file* out, void* data, size_t len) {
		struct output_file_callback* outc = to_output_file_callback(out);
		return outc->data_write(outc->priv, data, len);
		}

		return outc->write(outc->priv, data, len);
		static int callback_file_write_fd(struct output_file* out, int fd, size_t len) {
		struct output_file_callback* outc = to_output_file_callback(out);
		if (outc->fd_write) {
		return outc->fd_write(outc->priv, fd, len);
		}
		return default_file_write_fd(out, fd, len);
		}

		static int callback_file_write_fill(struct output_file* out, uint32_t fill_val, size_t len) {
		struct output_file_callback* outc = to_output_file_callback(out);
		if (outc->fill_write) {
		return outc->fill_write(outc->priv, fill_val, len);
		}
		return default_file_write_fill(out, fill_val, len);
		}

		static void callback_file_close(struct output_file* out) {
		@@ -300,6 +406,8 @@ static struct output_file_ops callback_file_ops = {
		.skip = callback_file_skip,
		.pad = callback_file_pad,
		.write = callback_file_write,
		.write_fd = callback_file_write_fd,
		.write_fill = callback_file_write_fill,
		.close = callback_file_close,
		};

		@@ -376,7 +484,8 @@ static int write_sparse_fill_chunk(struct output_file* out, unsigned int len, ui
		return 0;
		}

		static int write_sparse_data_chunk(struct output_file* out, unsigned int len, void* data) {
		static int write_sparse_data_chunk_variant(struct output_file* out, unsigned int len,
		handle_data data, bool is_fd) {
		chunk_header_t chunk_header;
		int rnd_up_len, zero_len;
		int ret;
		@@ -393,7 +502,16 @@ static int write_sparse_data_chunk(struct output_file* out, unsigned int len, vo
		ret = out->ops->write(out, &chunk_header, sizeof(chunk_header));

		if (ret < 0) return -1;
		ret = out->ops->write(out, data, len);

		if (is_fd) {
		// CRC is handled by write_fd
		ret = out->ops->write_fd(out, data.fd, len);
		} else {
		ret = out->ops->write(out, data.data_ptr, len);
		if (out->use_crc) {
		out->crc32 = sparse_crc32(out->crc32, data.data_ptr, len);
		}
		}
		if (ret < 0) return -1;
		if (zero_len) {
		ret = out->ops->write(out, out->zero_buf, zero_len);
		@@ -401,7 +519,6 @@ static int write_sparse_data_chunk(struct output_file* out, unsigned int len, vo
		}

		if (out->use_crc) {
		out->crc32 = sparse_crc32(out->crc32, data, len);
		if (zero_len) out->crc32 = sparse_crc32(out->crc32, out->zero_buf, zero_len);
		}

		@@ -411,6 +528,16 @@ static int write_sparse_data_chunk(struct output_file* out, unsigned int len, vo
		return 0;
		}

		static int write_sparse_data_chunk(struct output_file* out, unsigned int len, void* data_ptr) {
		handle_data data = {data_ptr};
		return write_sparse_data_chunk_variant(out, len, data, false /* isFd */);
		}

		static int write_sparse_fd_chunk(struct output_file* out, unsigned int len, int fd) {
		handle_data data = {.fd = fd};
		return write_sparse_data_chunk_variant(out, len, data, true /* isFd */);
		}

		int write_sparse_end_chunk(struct output_file* out) {
		chunk_header_t chunk_header;
		int ret;
		@@ -438,16 +565,22 @@ int write_sparse_end_chunk(struct output_file* out) {

		static struct sparse_file_ops sparse_file_ops = {
		.write_data_chunk = write_sparse_data_chunk,
		.write_fd_chunk = write_sparse_fd_chunk,
		.write_fill_chunk = write_sparse_fill_chunk,
		.write_skip_chunk = write_sparse_skip_chunk,
		.write_end_chunk = write_sparse_end_chunk,
		};

		static int write_normal_data_chunk(struct output_file* out, unsigned int len, void* data) {
		static int write_normal_data_chunk_variant(struct output_file* out, unsigned int len,
		handle_data data, bool isFd) {
		int ret;
		unsigned int rnd_up_len = ALIGN(len, out->block_size);

		ret = out->ops->write(out, data, len);
		if (isFd) {
		ret = out->ops->write_fd(out, data.fd, len);
		} else {
		ret = out->ops->write(out, data.data_ptr, len);
		}
		if (ret < 0) {
		return ret;
		}
		@@ -459,27 +592,18 @@ static int write_normal_data_chunk(struct output_file* out, unsigned int len, vo
		return ret;
		}

		static int write_normal_fill_chunk(struct output_file* out, unsigned int len, uint32_t fill_val) {
		int ret;
		unsigned int i;
		unsigned int write_len;

		/* Initialize fill_buf with the fill_val */
		for (i = 0; i < out->block_size / sizeof(uint32_t); i++) {
		out->fill_buf[i] = fill_val;
		}

		while (len) {
		write_len = min(len, out->block_size);
		ret = out->ops->write(out, out->fill_buf, write_len);
		if (ret < 0) {
		return ret;
		static int write_normal_data_chunk(struct output_file* out, unsigned int len, void* data_ptr) {
		handle_data data = {data_ptr};
		return write_normal_data_chunk_variant(out, len, data, false /* isFd */);
		}

		len -= write_len;
		static int write_normal_fd_chunk(struct output_file* out, unsigned int len, int fd) {
		handle_data data = {.fd = fd};
		return write_normal_data_chunk_variant(out, len, data, true /* isFd */);
		}

		return 0;
		static int write_normal_fill_chunk(struct output_file* out, unsigned int len, uint32_t fill_val) {
		return out->ops->write_fill(out, fill_val, len);
		}

		static int write_normal_skip_chunk(struct output_file* out, int64_t len) {
		@@ -492,6 +616,7 @@ int write_normal_end_chunk(struct output_file* out) {

		static struct sparse_file_ops normal_file_ops = {
		.write_data_chunk = write_normal_data_chunk,
		.write_fd_chunk = write_normal_fd_chunk,
		.write_fill_chunk = write_normal_fill_chunk,
		.write_skip_chunk = write_normal_skip_chunk,
		.write_end_chunk = write_normal_end_chunk,
		@@ -589,12 +714,20 @@ static struct output_file* output_file_new_normal(void) {
		return &outn->out;
		}

		struct output_file* output_file_open_callback(int (write)(void, const void, size_t), void priv,
		unsigned int block_size, int64_t len, int gz __unused,
		int sparse, int chunks, int crc) {
		struct output_file* output_file_open_callback(int (data_write)(void, const void*, size_t),
		int (fd_write)(void, int, size_t),
		int (fill_write)(void, uint32_t, size_t),
		int (skip_write)(void, off64_t), void* priv,
		unsigned int block_size, int64_t len, int sparse,
		int chunks, int crc) {
		int ret;
		struct output_file_callback* outc;

		if (!data_write \|\| (crc && (fd_write \|\| fill_write))) {
		errno = EINVAL;
		return nullptr;
		}

		outc =
		reinterpret_cast<struct output_file_callback*>(calloc(1, sizeof(struct output_file_callback)));
		if (!outc) {
		@@ -604,7 +737,10 @@ struct output_file* output_file_open_callback(int (write)(void, const void*, s

		outc->out.ops = &callback_file_ops;
		outc->priv = priv;
		outc->write = write;
		outc->data_write = data_write;
		outc->fd_write = fd_write;
		outc->fill_write = fill_write;
		outc->skip_write = skip_write;

		ret = output_file_init(&outc->out, block_size, len, sparse, chunks, crc);
		if (ret < 0) {
		@@ -651,52 +787,8 @@ int write_fill_chunk(struct output_file* out, unsigned int len, uint32_t fill_va
		}

		int write_fd_chunk(struct output_file* out, unsigned int len, int fd, int64_t offset) {
		int ret;
		int64_t aligned_offset;
		int aligned_diff;
		uint64_t buffer_size;
		char* ptr;

		aligned_offset = offset & ~(4096 - 1);
		aligned_diff = offset - aligned_offset;
		buffer_size = (uint64_t)len + (uint64_t)aligned_diff;

		#ifndef _WIN32
		if (buffer_size > SIZE_MAX) return -E2BIG;
		char* data =
		reinterpret_cast<char*>(mmap64(nullptr, buffer_size, PROT_READ, MAP_SHARED, fd, aligned_offset));
		if (data == MAP_FAILED) {
		return -errno;
		}
		ptr = data + aligned_diff;
		#else
		off64_t pos;
		char* data = reinterpret_cast<char*>(malloc(len));
		if (!data) {
		return -errno;
		}
		pos = lseek64(fd, offset, SEEK_SET);
		if (pos < 0) {
		free(data);
		return -errno;
		}
		ret = read_all(fd, data, len);
		if (ret < 0) {
		free(data);
		return ret;
		}
		ptr = data;
		#endif

		ret = out->sparse_ops->write_data_chunk(out, len, ptr);

		#ifndef _WIN32
		munmap(data, buffer_size);
		#else
		free(data);
		#endif

		return ret;
		lseek64(fd, offset, SEEK_SET);
		return out->sparse_ops->write_fd_chunk(out, len, fd);
		}

		/* Write a contiguous region of data blocks from a file */

libsparse/output_file.h

+7 −3

Original line number	Diff line number	Diff line
		@@ -22,14 +22,18 @@ extern "C" {
		#endif

		#include <sparse/sparse.h>
		#include <sys/types.h>

		struct output_file;

		struct output_file* output_file_open_fd(int fd, unsigned int block_size, int64_t len, int gz,
		int sparse, int chunks, int crc);
		struct output_file* output_file_open_callback(int (write)(void, const void, size_t), void priv,
		unsigned int block_size, int64_t len, int gz,
		int sparse, int chunks, int crc);
		struct output_file* output_file_open_callback(int (data_write)(void, const void*, size_t),
		int (fd_write)(void, int, size_t),
		int (fill_write)(void, uint32_t, size_t),
		int (skip_write)(void, off64_t), void* priv,
		unsigned int block_size, int64_t len, int sparse,
		int chunks, int crc);
		int write_data_chunk(struct output_file* out, unsigned int len, void* data);
		int write_fill_chunk(struct output_file* out, unsigned int len, uint32_t fill_val);
		int write_file_chunk(struct output_file* out, unsigned int len, const char* file, int64_t offset);

libsparse/sparse.cpp

+30 −7

Original line number	Diff line number	Diff line
		@@ -160,7 +160,30 @@ int sparse_file_callback(struct sparse_file* s, bool sparse, bool crc,
		struct output_file* out;

		chunks = sparse_count_chunks(s);
		out = output_file_open_callback(write, priv, s->block_size, s->len, false, sparse, chunks, crc);
		out = output_file_open_callback(write, nullptr, nullptr, nullptr, priv, s->block_size, s->len,
		sparse, chunks, crc);

		if (!out) return -ENOMEM;

		ret = write_all_blocks(s, out);

		output_file_close(out);

		return ret;
		}

		int sparse_file_callback_typed(struct sparse_file* s, bool sparse,
		int (data_write)(void priv, const void* data, size_t len),
		int (fd_write)(void priv, int fd, size_t len),
		int (fill_write)(void priv, uint32_t fill_val, size_t len),
		int (skip_write)(void priv, int64_t len), void* priv) {
		int ret;
		int chunks;
		struct output_file* out;

		chunks = sparse_count_chunks(s);
		out = output_file_open_callback(data_write, fd_write, fill_write, skip_write, priv, s->block_size,
		s->len, sparse, chunks, false);

		if (!out) return -ENOMEM;

		@@ -198,8 +221,8 @@ int sparse_file_foreach_chunk(struct sparse_file* s, bool sparse, bool crc,
		chk.write = write;
		chk.block = chk.nr_blocks = 0;
		chunks = sparse_count_chunks(s);
		out = output_file_open_callback(foreach_chunk_write, &chk, s->block_size, s->len, false, sparse,
		chunks, crc);
		out = output_file_open_callback(foreach_chunk_write, nullptr, nullptr, nullptr, &chk,
		s->block_size, s->len, sparse, chunks, crc);

		if (!out) return -ENOMEM;

		@@ -227,8 +250,8 @@ int64_t sparse_file_len(struct sparse_file* s, bool sparse, bool crc) {
		int64_t count = 0;
		struct output_file* out;

		out = output_file_open_callback(out_counter_write, &count, s->block_size, s->len, false, sparse,
		chunks, crc);
		out = output_file_open_callback(out_counter_write, nullptr, nullptr, nullptr, &count,
		s->block_size, s->len, sparse, chunks, crc);
		if (!out) {
		return -1;
		}
		@@ -267,8 +290,8 @@ static struct backed_block* move_chunks_up_to_len(struct sparse_file* from, stru
		len -= overhead;

		start = backed_block_iter_new(from->backed_block_list);
		out_counter = output_file_open_callback(out_counter_write, &count, to->block_size, to->len, false,
		true, 0, false);
		out_counter = output_file_open_callback(out_counter_write, nullptr, nullptr, nullptr, &count,
		to->block_size, to->len, true, 0, false);
		if (!out_counter) {
		return nullptr;
		}