Merge changes from topic "CowXorOp" am: c90b727301 am: d8ead66077 (bceac080) · Commits · e / os / android_system_core

fs_mgr/libsnapshot/cow_api_test.cpp

+127 −0

Original line number	Diff line number	Diff line
		@@ -140,6 +140,85 @@ TEST_F(CowTest, ReadWrite) {
		ASSERT_TRUE(iter->Done());
		}

		TEST_F(CowTest, ReadWriteXor) {
		CowOptions options;
		options.cluster_ops = 0;
		CowWriter writer(options);

		ASSERT_TRUE(writer.Initialize(cow_->fd));

		std::string data = "This is some data, believe it";
		data.resize(options.block_size, '\0');

		ASSERT_TRUE(writer.AddCopy(10, 20));
		ASSERT_TRUE(writer.AddXorBlocks(50, data.data(), data.size(), 24, 10));
		ASSERT_TRUE(writer.AddZeroBlocks(51, 2));
		ASSERT_TRUE(writer.Finalize());

		ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);

		CowReader reader;
		CowHeader header;
		CowFooter footer;
		ASSERT_TRUE(reader.Parse(cow_->fd));
		ASSERT_TRUE(reader.GetHeader(&header));
		ASSERT_TRUE(reader.GetFooter(&footer));
		ASSERT_EQ(header.magic, kCowMagicNumber);
		ASSERT_EQ(header.major_version, kCowVersionMajor);
		ASSERT_EQ(header.minor_version, kCowVersionMinor);
		ASSERT_EQ(header.block_size, options.block_size);
		ASSERT_EQ(footer.op.num_ops, 4);

		auto iter = reader.GetOpIter();
		ASSERT_NE(iter, nullptr);
		ASSERT_FALSE(iter->Done());
		auto op = &iter->Get();

		ASSERT_EQ(op->type, kCowCopyOp);
		ASSERT_EQ(op->compression, kCowCompressNone);
		ASSERT_EQ(op->data_length, 0);
		ASSERT_EQ(op->new_block, 10);
		ASSERT_EQ(op->source, 20);

		StringSink sink;

		iter->Next();
		ASSERT_FALSE(iter->Done());
		op = &iter->Get();

		ASSERT_EQ(op->type, kCowXorOp);
		ASSERT_EQ(op->compression, kCowCompressNone);
		ASSERT_EQ(op->data_length, 4096);
		ASSERT_EQ(op->new_block, 50);
		ASSERT_EQ(op->source, 98314); // 4096 * 24 + 10
		ASSERT_TRUE(reader.ReadData(*op, &sink));
		ASSERT_EQ(sink.stream(), data);

		iter->Next();
		ASSERT_FALSE(iter->Done());
		op = &iter->Get();

		// Note: the zero operation gets split into two blocks.
		ASSERT_EQ(op->type, kCowZeroOp);
		ASSERT_EQ(op->compression, kCowCompressNone);
		ASSERT_EQ(op->data_length, 0);
		ASSERT_EQ(op->new_block, 51);
		ASSERT_EQ(op->source, 0);

		iter->Next();
		ASSERT_FALSE(iter->Done());
		op = &iter->Get();

		ASSERT_EQ(op->type, kCowZeroOp);
		ASSERT_EQ(op->compression, kCowCompressNone);
		ASSERT_EQ(op->data_length, 0);
		ASSERT_EQ(op->new_block, 52);
		ASSERT_EQ(op->source, 0);

		iter->Next();
		ASSERT_TRUE(iter->Done());
		}

		TEST_F(CowTest, CompressGz) {
		CowOptions options;
		options.cluster_ops = 0;
		@@ -1034,6 +1113,54 @@ TEST_F(CowTest, MissingSeqOp) {
		ASSERT_FALSE(reader.Parse(cow_->fd));
		}

		TEST_F(CowTest, ResumeSeqOp) {
		CowOptions options;
		auto writer = std::make_unique<CowWriter>(options);
		const int seq_len = 10;
		uint32_t sequence[seq_len];
		for (int i = 0; i < seq_len; i++) {
		sequence[i] = i + 1;
		}

		ASSERT_TRUE(writer->Initialize(cow_->fd));

		ASSERT_TRUE(writer->AddSequenceData(seq_len, sequence));
		ASSERT_TRUE(writer->AddZeroBlocks(1, seq_len / 2));
		ASSERT_TRUE(writer->AddLabel(1));
		ASSERT_TRUE(writer->AddZeroBlocks(1 + seq_len / 2, 1));

		ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
		auto reader = std::make_unique<CowReader>();
		ASSERT_TRUE(reader->Parse(cow_->fd, 1));
		auto itr = reader->GetRevMergeOpIter();
		ASSERT_TRUE(itr->Done());

		writer = std::make_unique<CowWriter>(options);
		ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 1));
		ASSERT_TRUE(writer->AddZeroBlocks(1 + seq_len / 2, seq_len / 2));
		ASSERT_TRUE(writer->Finalize());

		ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);

		reader = std::make_unique<CowReader>();
		ASSERT_TRUE(reader->Parse(cow_->fd));

		auto iter = reader->GetRevMergeOpIter();

		uint64_t expected_block = 10;
		while (!iter->Done() && expected_block > 0) {
		ASSERT_FALSE(iter->Done());
		const auto& op = iter->Get();

		ASSERT_EQ(op.new_block, expected_block);

		iter->Next();
		expected_block--;
		}
		ASSERT_EQ(expected_block, 0);
		ASSERT_TRUE(iter->Done());
		}

		TEST_F(CowTest, RevMergeOpItrTest) {
		CowOptions options;
		options.cluster_ops = 5;

fs_mgr/libsnapshot/cow_format.cpp

+4 −1

Original line number	Diff line number	Diff line
		@@ -37,6 +37,8 @@ std::ostream& operator<<(std::ostream& os, CowOperation const& op) {
		os << "kCowLabelOp, ";
		else if (op.type == kCowClusterOp)
		os << "kCowClusterOp ";
		else if (op.type == kCowXorOp)
		os << "kCowXorOp ";
		else if (op.type == kCowSequenceOp)
		os << "kCowSequenceOp ";
		else if (op.type == kCowFooterOp)
		@@ -61,7 +63,7 @@ std::ostream& operator<<(std::ostream& os, CowOperation const& op) {
		int64_t GetNextOpOffset(const CowOperation& op, uint32_t cluster_ops) {
		if (op.type == kCowClusterOp) {
		return op.source;
		} else if (op.type == kCowReplaceOp && cluster_ops == 0) {
		} else if ((op.type == kCowReplaceOp \|\| op.type == kCowXorOp) && cluster_ops == 0) {
		return op.data_length;
		} else {
		return 0;
		@@ -93,6 +95,7 @@ bool IsMetadataOp(const CowOperation& op) {
		bool IsOrderedOp(const CowOperation& op) {
		switch (op.type) {
		case kCowCopyOp:
		case kCowXorOp:
		return true;
		default:
		return false;

fs_mgr/libsnapshot/cow_reader.cpp

+44 −2

Original line number	Diff line number	Diff line
		@@ -34,7 +34,11 @@
		namespace android {
		namespace snapshot {

		CowReader::CowReader() : fd_(-1), header_(), fd_size_(0) {}
		CowReader::CowReader()
		: fd_(-1),
		header_(),
		fd_size_(0),
		merge_op_blocks_(std::make_shared<std::vector<uint32_t>>()) {}

		static void SHA256(const void*, size_t, uint8_t[]) {
		#if 0
		@@ -45,6 +49,23 @@ static void SHA256(const void*, size_t, uint8_t[]) {
		#endif
		}

		std::unique_ptr<CowReader> CowReader::CloneCowReader() {
		auto cow = std::make_unique<CowReader>();
		cow->owned_fd_.reset();
		cow->header_ = header_;
		cow->footer_ = footer_;
		cow->fd_size_ = fd_size_;
		cow->last_label_ = last_label_;
		cow->ops_ = ops_;
		cow->merge_op_blocks_ = merge_op_blocks_;
		cow->block_map_ = block_map_;
		cow->num_total_data_ops_ = num_total_data_ops_;
		cow->num_ordered_ops_to_merge_ = num_ordered_ops_to_merge_;
		cow->has_seq_ops_ = has_seq_ops_;
		cow->data_loc_ = data_loc_;
		return cow;
		}

		bool CowReader::InitForMerge(android::base::unique_fd&& fd) {
		owned_fd_ = std::move(fd);
		fd_ = owned_fd_.get();
		@@ -133,11 +154,14 @@ bool CowReader::Parse(android::base::borrowed_fd fd, std::optional<uint64_t> lab
		if (!ParseOps(label)) {
		return false;
		}
		// If we're resuming a write, we're not ready to merge
		if (label.has_value()) return true;
		return PrepMergeOps();
		}

		bool CowReader::ParseOps(std::optional<uint64_t> label) {
		uint64_t pos;
		auto data_loc = std::make_shared<std::unordered_map<uint64_t, uint64_t>>();

		// Skip the scratch space
		if (header_.major_version >= 2 && (header_.buffer_size > 0)) {
		@@ -157,6 +181,13 @@ bool CowReader::ParseOps(std::optional<uint64_t> label) {
		// Reading a v1 version of COW which doesn't have buffer_size.
		header_.buffer_size = 0;
		}
		uint64_t data_pos = 0;

		if (header_.cluster_ops) {
		data_pos = pos + header_.cluster_ops * sizeof(CowOperation);
		} else {
		data_pos = pos + sizeof(CowOperation);
		}

		auto ops_buffer = std::make_shared<std::vector<CowOperation>>();
		uint64_t current_op_num = 0;
		@@ -177,7 +208,11 @@ bool CowReader::ParseOps(std::optional<uint64_t> label) {
		while (current_op_num < ops_buffer->size()) {
		auto& current_op = ops_buffer->data()[current_op_num];
		current_op_num++;
		if (current_op.type == kCowXorOp) {
		data_loc->insert({current_op.new_block, data_pos});
		}
		pos += sizeof(CowOperation) + GetNextOpOffset(current_op, header_.cluster_ops);
		data_pos += current_op.data_length + GetNextDataOffset(current_op, header_.cluster_ops);

		if (current_op.type == kCowClusterOp) {
		break;
		@@ -268,6 +303,7 @@ bool CowReader::ParseOps(std::optional<uint64_t> label) {

		ops_ = ops_buffer;
		ops_->shrink_to_fit();
		data_loc_ = data_loc;

		return true;
		}
		@@ -606,7 +642,13 @@ bool CowReader::ReadData(const CowOperation& op, IByteSink* sink) {
		return false;
		}

		CowDataStream stream(this, op.source, op.data_length);
		uint64_t offset;
		if (op.type == kCowXorOp) {
		offset = data_loc_->at(op.new_block);
		} else {
		offset = op.source;
		}
		CowDataStream stream(this, offset, op.data_length);
		decompressor->set_stream(&stream);
		decompressor->set_sink(sink);
		return decompressor->Decompress(header_.block_size);

fs_mgr/libsnapshot/cow_writer.cpp

+34 −6

Original line number	Diff line number	Diff line
		@@ -58,11 +58,25 @@ bool ICowWriter::AddRawBlocks(uint64_t new_block_start, const void* data, size_t
		return EmitRawBlocks(new_block_start, data, size);
		}

		bool ICowWriter::AddXorBlocks(uint32_t /new_block_start/, const void* /data/, size_t /size/,
		uint32_t /old_block/, uint16_t /offset/) {
		LOG(ERROR) << "AddXorBlocks not yet implemented";
		bool ICowWriter::AddXorBlocks(uint32_t new_block_start, const void* data, size_t size,
		uint32_t old_block, uint16_t offset) {
		if (size % options_.block_size != 0) {
		LOG(ERROR) << "AddRawBlocks: size " << size << " is not a multiple of "
		<< options_.block_size;
		return false;
		}

		uint64_t num_blocks = size / options_.block_size;
		uint64_t last_block = new_block_start + num_blocks - 1;
		if (!ValidateNewBlock(last_block)) {
		return false;
		}
		if (offset >= options_.block_size) {
		LOG(ERROR) << "AddXorBlocks: offset " << offset << " is not less than "
		<< options_.block_size;
		}
		return EmitXorBlocks(new_block_start, data, size, old_block, offset);
		}

		bool ICowWriter::AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) {
		uint64_t last_block = new_block_start + num_blocks - 1;
		@@ -278,13 +292,27 @@ bool CowWriter::EmitCopy(uint64_t new_block, uint64_t old_block) {
		}

		bool CowWriter::EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) {
		return EmitBlocks(new_block_start, data, size, 0, 0, kCowReplaceOp);
		}

		bool CowWriter::EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size,
		uint32_t old_block, uint16_t offset) {
		return EmitBlocks(new_block_start, data, size, old_block, offset, kCowXorOp);
		}

		bool CowWriter::EmitBlocks(uint64_t new_block_start, const void* data, size_t size,
		uint64_t old_block, uint16_t offset, uint8_t type) {
		const uint8_t* iter = reinterpret_cast<const uint8_t*>(data);
		CHECK(!merge_in_progress_);
		for (size_t i = 0; i < size / header_.block_size; i++) {
		CowOperation op = {};
		op.type = kCowReplaceOp;
		op.new_block = new_block_start + i;
		op.type = type;
		if (type == kCowXorOp) {
		op.source = (old_block + i) * header_.block_size + offset;
		} else {
		op.source = next_data_pos_;
		}

		if (compression_) {
		auto data = Compress(iter, header_.block_size);

fs_mgr/libsnapshot/include/libsnapshot/cow_format.h

+3 −0

Original line number	Diff line number	Diff line
		@@ -138,6 +138,8 @@ struct CowOperation {
		// For Label operations, this is the value of the applied label.
		//
		// For Cluster operations, this is the length of the following data region
		//
		// For Xor operations, this is the byte location in the source image.
		uint64_t source;
		} __attribute__((packed));

		@@ -148,6 +150,7 @@ static constexpr uint8_t kCowReplaceOp = 2;
		static constexpr uint8_t kCowZeroOp = 3;
		static constexpr uint8_t kCowLabelOp = 4;
		static constexpr uint8_t kCowClusterOp = 5;
		static constexpr uint8_t kCowXorOp = 6;
		static constexpr uint8_t kCowSequenceOp = 7;
		static constexpr uint8_t kCowFooterOp = -1;