libsnapshot: Split CowReader into CowParserV2. (d70a174e) · Commits · e / os / android_system_core

fs_mgr/libsnapshot/Android.bp

+1 −0

Original line number	Diff line number	Diff line
		@@ -179,6 +179,7 @@ cc_library_static {
		"libsnapshot_cow/cow_writer.cpp",
		"libsnapshot_cow/cow_format.cpp",
		"libsnapshot_cow/cow_compress.cpp",
		"libsnapshot_cow/parser_v2.cpp",
		],
		host_supported: true,
		recovery_available: true,

fs_mgr/libsnapshot/include/libsnapshot/cow_format.h

+7 −13

Original line number	Diff line number	Diff line
		@@ -52,13 +52,15 @@ static constexpr uint32_t kCowVersionManifest = 2;
		// between writing the last operation/data pair, or the footer itself. In this
		// case, the safest way to proceed is to assume the last operation is faulty.

		struct CowHeader {
		struct CowHeaderPrefix {
		uint64_t magic;
		uint16_t major_version;
		uint16_t minor_version;
		uint16_t header_size; // size of CowHeader.
		} __attribute__((packed));

		// Size of this struct.
		uint16_t header_size;
		struct CowHeader {
		CowHeaderPrefix prefix;

		// Size of footer struct
		uint16_t footer_size;
		@@ -88,7 +90,7 @@ struct CowFooterOperation {
		// the compression type of that data (see constants below).
		uint8_t compression;

		// Length of Footer Data. Currently 64 for both checksums
		// Length of Footer Data. Currently 64.
		uint16_t data_length;

		// The amount of file space used by Cow operations
		@@ -98,14 +100,6 @@ struct CowFooterOperation {
		uint64_t num_ops;
		} __attribute__((packed));

		struct CowFooterData {
		// SHA256 checksums of Footer op
		uint8_t footer_checksum[32];

		// SHA256 of the operation sequence.
		uint8_t ops_checksum[32];
		} __attribute__((packed));

		// Cow operations are currently fixed-size entries, but this may change if
		// needed.
		struct CowOperation {
		@@ -167,7 +161,7 @@ static constexpr uint8_t kCowReadAheadDone = 2;

		struct CowFooter {
		CowFooterOperation op;
		CowFooterData data;
		uint8_t unused[64];
		} __attribute__((packed));

		struct ScratchMetadata {

fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h

+1 −1

Original line number	Diff line number	Diff line
		@@ -171,7 +171,7 @@ class CowWriter : public ICowWriter {

		uint64_t GetCowSize() override;

		uint32_t GetCowVersion() { return header_.major_version; }
		uint32_t GetCowVersion() { return header_.prefix.major_version; }

		protected:
		virtual bool EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) override;

fs_mgr/libsnapshot/libsnapshot_cow/cow_api_test.cpp

+9 −9

Original line number	Diff line number	Diff line
		@@ -65,9 +65,9 @@ TEST_F(CowTest, CopyContiguous) {
		ASSERT_TRUE(reader.Parse(cow_->fd));

		const auto& header = reader.GetHeader();
		ASSERT_EQ(header.magic, kCowMagicNumber);
		ASSERT_EQ(header.major_version, kCowVersionMajor);
		ASSERT_EQ(header.minor_version, kCowVersionMinor);
		ASSERT_EQ(header.prefix.magic, kCowMagicNumber);
		ASSERT_EQ(header.prefix.major_version, kCowVersionMajor);
		ASSERT_EQ(header.prefix.minor_version, kCowVersionMinor);
		ASSERT_EQ(header.block_size, options.block_size);

		CowFooter footer;
		@@ -114,9 +114,9 @@ TEST_F(CowTest, ReadWrite) {
		ASSERT_TRUE(reader.Parse(cow_->fd));

		const auto& header = reader.GetHeader();
		ASSERT_EQ(header.magic, kCowMagicNumber);
		ASSERT_EQ(header.major_version, kCowVersionMajor);
		ASSERT_EQ(header.minor_version, kCowVersionMinor);
		ASSERT_EQ(header.prefix.magic, kCowMagicNumber);
		ASSERT_EQ(header.prefix.major_version, kCowVersionMajor);
		ASSERT_EQ(header.prefix.minor_version, kCowVersionMinor);
		ASSERT_EQ(header.block_size, options.block_size);

		CowFooter footer;
		@@ -193,9 +193,9 @@ TEST_F(CowTest, ReadWriteXor) {
		ASSERT_TRUE(reader.Parse(cow_->fd));

		const auto& header = reader.GetHeader();
		ASSERT_EQ(header.magic, kCowMagicNumber);
		ASSERT_EQ(header.major_version, kCowVersionMajor);
		ASSERT_EQ(header.minor_version, kCowVersionMinor);
		ASSERT_EQ(header.prefix.magic, kCowMagicNumber);
		ASSERT_EQ(header.prefix.major_version, kCowVersionMajor);
		ASSERT_EQ(header.prefix.minor_version, kCowVersionMinor);
		ASSERT_EQ(header.block_size, options.block_size);

		CowFooter footer;

fs_mgr/libsnapshot/libsnapshot_cow/cow_reader.cpp

+13 −213

Original line number	Diff line number	Diff line
		@@ -30,7 +30,7 @@
		#include <zlib.h>

		#include "cow_decompress.h"
		#include "libsnapshot/cow_format.h"
		#include "parser_v2.h"

		namespace android {
		namespace snapshot {
		@@ -43,15 +43,6 @@ CowReader::CowReader(ReaderFlags reader_flag, bool is_merge)
		reader_flag_(reader_flag),
		is_merge_(is_merge) {}

		static void SHA256(const void*, size_t, uint8_t[]) {
		#if 0
		SHA256_CTX c;
		SHA256_Init(&c);
		SHA256_Update(&c, data, length);
		SHA256_Final(out, &c);
		#endif
		}

		std::unique_ptr<CowReader> CowReader::CloneCowReader() {
		auto cow = std::make_unique<CowReader>();
		cow->owned_fd_.reset();
		@@ -63,7 +54,6 @@ std::unique_ptr<CowReader> CowReader::CloneCowReader() {
		cow->merge_op_start_ = merge_op_start_;
		cow->num_total_data_ops_ = num_total_data_ops_;
		cow->num_ordered_ops_to_merge_ = num_ordered_ops_to_merge_;
		cow->has_seq_ops_ = has_seq_ops_;
		cow->data_loc_ = data_loc_;
		cow->block_pos_index_ = block_pos_index_;
		cow->is_merge_ = is_merge_;
		@@ -101,217 +91,26 @@ bool CowReader::Parse(android::base::unique_fd&& fd, std::optional<uint64_t> lab
		bool CowReader::Parse(android::base::borrowed_fd fd, std::optional<uint64_t> label) {
		fd_ = fd;

		auto pos = lseek(fd_.get(), 0, SEEK_END);
		if (pos < 0) {
		PLOG(ERROR) << "lseek end failed";
		if (!ReadCowHeader(fd, &header_)) {
		return false;
		}
		fd_size_ = pos;

		if (lseek(fd_.get(), 0, SEEK_SET) < 0) {
		PLOG(ERROR) << "lseek header failed";
		return false;
		}
		if (!android::base::ReadFully(fd_, &header_, sizeof(header_))) {
		PLOG(ERROR) << "read header failed";
		CowParserV2 parser;
		if (!parser.Parse(fd, header_, label)) {
		return false;
		}

		if (header_.magic != kCowMagicNumber) {
		LOG(ERROR) << "Header Magic corrupted. Magic: " << header_.magic
		<< "Expected: " << kCowMagicNumber;
		return false;
		}
		if (header_.footer_size != sizeof(CowFooter)) {
		LOG(ERROR) << "Footer size unknown, read " << header_.footer_size << ", expected "
		<< sizeof(CowFooter);
		return false;
		}
		if (header_.op_size != sizeof(CowOperation)) {
		LOG(ERROR) << "Operation size unknown, read " << header_.op_size << ", expected "
		<< sizeof(CowOperation);
		return false;
		}
		if (header_.cluster_ops == 1) {
		LOG(ERROR) << "Clusters must contain at least two operations to function.";
		return false;
		}
		if (header_.op_size != sizeof(CowOperation)) {
		LOG(ERROR) << "Operation size unknown, read " << header_.op_size << ", expected "
		<< sizeof(CowOperation);
		return false;
		}
		if (header_.cluster_ops == 1) {
		LOG(ERROR) << "Clusters must contain at least two operations to function.";
		return false;
		}

		if ((header_.major_version > kCowVersionMajor) \|\| (header_.minor_version != kCowVersionMinor)) {
		LOG(ERROR) << "Header version mismatch";
		LOG(ERROR) << "Major version: " << header_.major_version
		<< "Expected: " << kCowVersionMajor;
		LOG(ERROR) << "Minor version: " << header_.minor_version
		<< "Expected: " << kCowVersionMinor;
		return false;
		}
		footer_ = parser.footer();
		fd_size_ = parser.fd_size();
		last_label_ = parser.last_label();
		ops_ = std::move(parser.ops());
		data_loc_ = parser.data_loc();

		if (!ParseOps(label)) {
		return false;
		}
		// If we're resuming a write, we're not ready to merge
		if (label.has_value()) return true;
		return PrepMergeOps();
		}

		bool CowReader::ParseOps(std::optional<uint64_t> label) {
		uint64_t pos;
		auto data_loc = std::make_shared<std::unordered_map<uint64_t, uint64_t>>();

		// Skip the scratch space
		if (header_.major_version >= 2 && (header_.buffer_size > 0)) {
		LOG(DEBUG) << " Scratch space found of size: " << header_.buffer_size;
		size_t init_offset = header_.header_size + header_.buffer_size;
		pos = lseek(fd_.get(), init_offset, SEEK_SET);
		if (pos != init_offset) {
		PLOG(ERROR) << "lseek ops failed";
		return false;
		}
		} else {
		pos = lseek(fd_.get(), header_.header_size, SEEK_SET);
		if (pos != header_.header_size) {
		PLOG(ERROR) << "lseek ops failed";
		return false;
		}
		// Reading a v1 version of COW which doesn't have buffer_size.
		header_.buffer_size = 0;
		}
		uint64_t data_pos = 0;

		if (header_.cluster_ops) {
		data_pos = pos + header_.cluster_ops * sizeof(CowOperation);
		} else {
		data_pos = pos + sizeof(CowOperation);
		}

		auto ops_buffer = std::make_shared<std::vector<CowOperation>>();
		uint64_t current_op_num = 0;
		uint64_t cluster_ops = header_.cluster_ops ?: 1;
		bool done = false;

		// Alternating op clusters and data
		while (!done) {
		uint64_t to_add = std::min(cluster_ops, (fd_size_ - pos) / sizeof(CowOperation));
		if (to_add == 0) break;
		ops_buffer->resize(current_op_num + to_add);
		if (!android::base::ReadFully(fd_, &ops_buffer->data()[current_op_num],
		to_add * sizeof(CowOperation))) {
		PLOG(ERROR) << "read op failed";
		return false;
		}
		// Parse current cluster to find start of next cluster
		while (current_op_num < ops_buffer->size()) {
		auto& current_op = ops_buffer->data()[current_op_num];
		current_op_num++;
		if (current_op.type == kCowXorOp) {
		data_loc->insert({current_op.new_block, data_pos});
		}
		pos += sizeof(CowOperation) + GetNextOpOffset(current_op, header_.cluster_ops);
		data_pos += current_op.data_length + GetNextDataOffset(current_op, header_.cluster_ops);

		if (current_op.type == kCowClusterOp) {
		break;
		} else if (current_op.type == kCowLabelOp) {
		last_label_ = {current_op.source};

		// If we reach the requested label, stop reading.
		if (label && label.value() == current_op.source) {
		done = true;
		break;
		}
		} else if (current_op.type == kCowFooterOp) {
		footer_.emplace();
		CowFooter* footer = &footer_.value();
		memcpy(&footer_->op, &current_op, sizeof(footer->op));
		off_t offs = lseek(fd_.get(), pos, SEEK_SET);
		if (offs < 0 \|\| pos != static_cast<uint64_t>(offs)) {
		PLOG(ERROR) << "lseek next op failed " << offs;
		return false;
		}
		if (!android::base::ReadFully(fd_, &footer->data, sizeof(footer->data))) {
		LOG(ERROR) << "Could not read COW footer";
		return false;
		}

		// Drop the footer from the op stream.
		current_op_num--;
		done = true;
		break;
		} else if (current_op.type == kCowSequenceOp) {
		has_seq_ops_ = true;
		}
		}

		// Position for next cluster read
		off_t offs = lseek(fd_.get(), pos, SEEK_SET);
		if (offs < 0 \|\| pos != static_cast<uint64_t>(offs)) {
		PLOG(ERROR) << "lseek next op failed " << offs;
		return false;
		}
		ops_buffer->resize(current_op_num);
		}

		LOG(DEBUG) << "COW file read complete. Total ops: " << ops_buffer->size();
		// To successfully parse a COW file, we need either:
		// (1) a label to read up to, and for that label to be found, or
		// (2) a valid footer.
		if (label) {
		if (!last_label_) {
		LOG(ERROR) << "Did not find label " << label.value()
		<< " while reading COW (no labels found)";
		return false;
		}
		if (last_label_.value() != label.value()) {
		LOG(ERROR) << "Did not find label " << label.value()
		<< ", last label=" << last_label_.value();
		return false;
		}
		} else if (!footer_) {
		LOG(ERROR) << "No COW footer found";
		return false;
		}

		uint8_t csum[32];
		memset(csum, 0, sizeof(uint8_t) * 32);

		if (footer_) {
		if (ops_buffer->size() != footer_->op.num_ops) {
		LOG(ERROR) << "num ops does not match, expected " << footer_->op.num_ops << ", found "
		<< ops_buffer->size();
		return false;
		}
		if (ops_buffer->size() * sizeof(CowOperation) != footer_->op.ops_size) {
		LOG(ERROR) << "ops size does not match ";
		return false;
		}
		SHA256(&footer_->op, sizeof(footer_->op), footer_->data.footer_checksum);
		if (memcmp(csum, footer_->data.ops_checksum, sizeof(csum)) != 0) {
		LOG(ERROR) << "ops checksum does not match";
		return false;
		}
		SHA256(ops_buffer->data(), footer_->op.ops_size, csum);
		if (memcmp(csum, footer_->data.ops_checksum, sizeof(csum)) != 0) {
		LOG(ERROR) << "ops checksum does not match";
		return false;
		}
		}

		ops_ = ops_buffer;
		ops_->shrink_to_fit();
		data_loc_ = data_loc;

		return true;
		}

		//
		// This sets up the data needed for MergeOpIter. MergeOpIter presents
		// data in the order we intend to merge in.
		@@ -446,7 +245,8 @@ bool CowReader::PrepMergeOps() {
		continue;
		}

		if (!has_seq_ops_ && IsOrderedOp(current_op)) {
		// Sequence ops must be the first ops in the stream.
		if (seq_ops_set.empty() && IsOrderedOp(current_op)) {
		merge_op_blocks->emplace_back(current_op.new_block);
		} else if (seq_ops_set.count(current_op.new_block) == 0) {
		other_ops.push_back(current_op.new_block);
		@@ -718,8 +518,8 @@ std::unique_ptr<ICowOpIter> CowReader::GetMergeOpIter(bool ignore_progress) {

		bool CowReader::GetRawBytes(uint64_t offset, void* buffer, size_t len, size_t* read) {
		// Validate the offset, taking care to acknowledge possible overflow of offset+len.
		if (offset < header_.header_size \|\| offset >= fd_size_ - sizeof(CowFooter) \|\| len >= fd_size_ \|\|
		offset + len > fd_size_ - sizeof(CowFooter)) {
		if (offset < header_.prefix.header_size \|\| offset >= fd_size_ - sizeof(CowFooter) \|\|
		len >= fd_size_ \|\| offset + len > fd_size_ - sizeof(CowFooter)) {
		LOG(ERROR) << "invalid data offset: " << offset << ", " << len << " bytes";
		return false;
		}