Merge "libsnapshot: Add function to verify merge sequence" am: 5125c62cf8 am: 8355803863 (85a39a0c) · Commits · e / os / android_system_core

fs_mgr/libsnapshot/cow_api_test.cpp

+45 −12

Original line number	Original line	Diff line number	Diff line
	@@ -1171,18 +1171,18 @@ TEST_F(CowTest, RevMergeOpItrTest) {
	ASSERT_TRUE(writer.Initialize(cow_->fd));		ASSERT_TRUE(writer.Initialize(cow_->fd));

	ASSERT_TRUE(writer.AddSequenceData(6, sequence));		ASSERT_TRUE(writer.AddSequenceData(6, sequence));
	ASSERT_TRUE(writer.AddCopy(6, 3));		ASSERT_TRUE(writer.AddCopy(6, 13));
	ASSERT_TRUE(writer.AddZeroBlocks(12, 1));		ASSERT_TRUE(writer.AddZeroBlocks(12, 1));
	ASSERT_TRUE(writer.AddZeroBlocks(8, 1));		ASSERT_TRUE(writer.AddZeroBlocks(8, 1));
	ASSERT_TRUE(writer.AddZeroBlocks(11, 1));		ASSERT_TRUE(writer.AddZeroBlocks(11, 1));
	ASSERT_TRUE(writer.AddCopy(3, 5));		ASSERT_TRUE(writer.AddCopy(3, 15));
	ASSERT_TRUE(writer.AddCopy(2, 1));		ASSERT_TRUE(writer.AddCopy(2, 11));
	ASSERT_TRUE(writer.AddZeroBlocks(4, 1));		ASSERT_TRUE(writer.AddZeroBlocks(4, 1));
	ASSERT_TRUE(writer.AddZeroBlocks(9, 1));		ASSERT_TRUE(writer.AddZeroBlocks(9, 1));
	ASSERT_TRUE(writer.AddCopy(5, 6));		ASSERT_TRUE(writer.AddCopy(5, 16));
	ASSERT_TRUE(writer.AddZeroBlocks(1, 1));		ASSERT_TRUE(writer.AddZeroBlocks(1, 1));
	ASSERT_TRUE(writer.AddCopy(10, 2));		ASSERT_TRUE(writer.AddCopy(10, 12));
	ASSERT_TRUE(writer.AddCopy(7, 4));		ASSERT_TRUE(writer.AddCopy(7, 14));
	ASSERT_TRUE(writer.Finalize());		ASSERT_TRUE(writer.Finalize());

	// New block in cow order is 6, 12, 8, 11, 3, 2, 4, 9, 5, 1, 10, 7		// New block in cow order is 6, 12, 8, 11, 3, 2, 4, 9, 5, 1, 10, 7
	@@ -1219,12 +1219,12 @@ TEST_F(CowTest, LegacyRevMergeOpItrTest) {

	ASSERT_TRUE(writer.Initialize(cow_->fd));		ASSERT_TRUE(writer.Initialize(cow_->fd));

	ASSERT_TRUE(writer.AddCopy(2, 1));		ASSERT_TRUE(writer.AddCopy(2, 11));
	ASSERT_TRUE(writer.AddCopy(10, 2));		ASSERT_TRUE(writer.AddCopy(10, 12));
	ASSERT_TRUE(writer.AddCopy(6, 3));		ASSERT_TRUE(writer.AddCopy(6, 13));
	ASSERT_TRUE(writer.AddCopy(7, 4));		ASSERT_TRUE(writer.AddCopy(7, 14));
	ASSERT_TRUE(writer.AddCopy(3, 5));		ASSERT_TRUE(writer.AddCopy(3, 15));
	ASSERT_TRUE(writer.AddCopy(5, 6));		ASSERT_TRUE(writer.AddCopy(5, 16));
	ASSERT_TRUE(writer.AddZeroBlocks(12, 1));		ASSERT_TRUE(writer.AddZeroBlocks(12, 1));
	ASSERT_TRUE(writer.AddZeroBlocks(8, 1));		ASSERT_TRUE(writer.AddZeroBlocks(8, 1));
	ASSERT_TRUE(writer.AddZeroBlocks(11, 1));		ASSERT_TRUE(writer.AddZeroBlocks(11, 1));
	@@ -1260,6 +1260,39 @@ TEST_F(CowTest, LegacyRevMergeOpItrTest) {
	ASSERT_TRUE(iter->Done());		ASSERT_TRUE(iter->Done());
	}		}

			TEST_F(CowTest, InvalidMergeOrderTest) {
			CowOptions options;
			options.cluster_ops = 5;
			options.num_merge_ops = 1;
			std::string data = "This is some data, believe it";
			data.resize(options.block_size, '\0');
			auto writer = std::make_unique<CowWriter>(options);
			CowReader reader;

			ASSERT_TRUE(writer->Initialize(cow_->fd));

			ASSERT_TRUE(writer->AddCopy(3, 2));
			ASSERT_TRUE(writer->AddCopy(2, 1));
			ASSERT_TRUE(writer->AddLabel(1));
			ASSERT_TRUE(writer->Finalize());
			ASSERT_TRUE(reader.Parse(cow_->fd));
			ASSERT_TRUE(reader.VerifyMergeOps());

			ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 1));
			ASSERT_TRUE(writer->AddCopy(4, 2));
			ASSERT_TRUE(writer->Finalize());
			ASSERT_TRUE(reader.Parse(cow_->fd));
			ASSERT_FALSE(reader.VerifyMergeOps());

			writer = std::make_unique<CowWriter>(options);
			ASSERT_TRUE(writer->Initialize(cow_->fd));
			ASSERT_TRUE(writer->AddCopy(2, 1));
			ASSERT_TRUE(writer->AddXorBlocks(3, &data, data.size(), 1, 1));
			ASSERT_TRUE(writer->Finalize());
			ASSERT_TRUE(reader.Parse(cow_->fd));
			ASSERT_FALSE(reader.VerifyMergeOps());
			}

	} // namespace snapshot		} // namespace snapshot
	} // namespace android		} // namespace android

fs_mgr/libsnapshot/cow_format.cpp

+4 −1

Original line number	Original line	Diff line number	Diff line
	@@ -56,7 +56,10 @@ std::ostream& operator<<(std::ostream& os, CowOperation const& op) {
	os << (int)op.compression << "?, ";		os << (int)op.compression << "?, ";
	os << "data_length:" << op.data_length << ",\t";		os << "data_length:" << op.data_length << ",\t";
	os << "new_block:" << op.new_block << ",\t";		os << "new_block:" << op.new_block << ",\t";
	os << "source:" << op.source << ")";		os << "source:" << op.source;
			if (op.type == kCowXorOp)
			os << " (block:" << op.source / BLOCK_SZ << " offset:" << op.source % BLOCK_SZ << ")";
			os << ")";
	return os;		return os;
	}		}

fs_mgr/libsnapshot/cow_reader.cpp

+99 −7

Original line number	Original line	Diff line number	Diff line
	@@ -58,6 +58,7 @@ std::unique_ptr<CowReader> CowReader::CloneCowReader() {
	cow->last_label_ = last_label_;		cow->last_label_ = last_label_;
	cow->ops_ = ops_;		cow->ops_ = ops_;
	cow->merge_op_blocks_ = merge_op_blocks_;		cow->merge_op_blocks_ = merge_op_blocks_;
			cow->merge_op_start_ = merge_op_start_;
	cow->block_map_ = block_map_;		cow->block_map_ = block_map_;
	cow->num_total_data_ops_ = num_total_data_ops_;		cow->num_total_data_ops_ = num_total_data_ops_;
	cow->num_ordered_ops_to_merge_ = num_ordered_ops_to_merge_;		cow->num_ordered_ops_to_merge_ = num_ordered_ops_to_merge_;
	@@ -468,8 +469,7 @@ bool CowReader::PrepMergeOps() {

	num_total_data_ops_ = merge_op_blocks->size();		num_total_data_ops_ = merge_op_blocks->size();
	if (header_.num_merge_ops > 0) {		if (header_.num_merge_ops > 0) {
	merge_op_blocks->erase(merge_op_blocks->begin(),		merge_op_start_ = header_.num_merge_ops;
	merge_op_blocks->begin() + header_.num_merge_ops);
	}		}

	block_map_ = block_map;		block_map_ = block_map;
	@@ -477,6 +477,44 @@ bool CowReader::PrepMergeOps() {
	return true;		return true;
	}		}

			bool CowReader::VerifyMergeOps() {
			auto itr = GetMergeOpIter(true);
			std::unordered_map<uint64_t, CowOperation> overwritten_blocks;
			while (!itr->Done()) {
			CowOperation op = itr->Get();
			uint64_t block;
			bool offset;
			if (op.type == kCowCopyOp) {
			block = op.source;
			offset = false;
			} else if (op.type == kCowXorOp) {
			block = op.source / BLOCK_SZ;
			offset = (op.source % BLOCK_SZ) != 0;
			} else {
			itr->Next();
			continue;
			}

			CowOperation* overwrite = nullptr;
			if (overwritten_blocks.count(block)) {
			overwrite = &overwritten_blocks[block];
			LOG(ERROR) << "Invalid Sequence! Block needed for op:\n"
			<< op << "\noverwritten by previously merged op:\n"
			<< *overwrite;
			}
			if (offset && overwritten_blocks.count(block + 1)) {
			overwrite = &overwritten_blocks[block + 1];
			LOG(ERROR) << "Invalid Sequence! Block needed for op:\n"
			<< op << "\noverwritten by previously merged op:\n"
			<< *overwrite;
			}
			if (overwrite != nullptr) return false;
			overwritten_blocks[op.new_block] = op;
			itr->Next();
			}
			return true;
			}

	bool CowReader::GetHeader(CowHeader* header) {		bool CowReader::GetHeader(CowHeader* header) {
	*header = header_;		*header = header_;
	return true;		return true;
	@@ -530,7 +568,8 @@ class CowRevMergeOpIter final : public ICowOpIter {
	public:		public:
	explicit CowRevMergeOpIter(std::shared_ptr<std::vector<CowOperation>> ops,		explicit CowRevMergeOpIter(std::shared_ptr<std::vector<CowOperation>> ops,
	std::shared_ptr<std::vector<uint32_t>> merge_op_blocks,		std::shared_ptr<std::vector<uint32_t>> merge_op_blocks,
	std::shared_ptr<std::unordered_map<uint32_t, int>> map);		std::shared_ptr<std::unordered_map<uint32_t, int>> map,
			uint64_t start);

	bool Done() override;		bool Done() override;
	const CowOperation& Get() override;		const CowOperation& Get() override;
	@@ -541,20 +580,67 @@ class CowRevMergeOpIter final : public ICowOpIter {
	std::shared_ptr<std::vector<uint32_t>> merge_op_blocks_;		std::shared_ptr<std::vector<uint32_t>> merge_op_blocks_;
	std::shared_ptr<std::unordered_map<uint32_t, int>> map_;		std::shared_ptr<std::unordered_map<uint32_t, int>> map_;
	std::vector<uint32_t>::reverse_iterator block_riter_;		std::vector<uint32_t>::reverse_iterator block_riter_;
			uint64_t start_;
			};

			class CowMergeOpIter final : public ICowOpIter {
			public:
			explicit CowMergeOpIter(std::shared_ptr<std::vector<CowOperation>> ops,
			std::shared_ptr<std::vector<uint32_t>> merge_op_blocks,
			std::shared_ptr<std::unordered_map<uint32_t, int>> map, uint64_t start);

			bool Done() override;
			const CowOperation& Get() override;
			void Next() override;

			private:
			std::shared_ptr<std::vector<CowOperation>> ops_;
			std::shared_ptr<std::vector<uint32_t>> merge_op_blocks_;
			std::shared_ptr<std::unordered_map<uint32_t, int>> map_;
			std::vector<uint32_t>::iterator block_iter_;
			uint64_t start_;
	};		};

			CowMergeOpIter::CowMergeOpIter(std::shared_ptr<std::vector<CowOperation>> ops,
			std::shared_ptr<std::vector<uint32_t>> merge_op_blocks,
			std::shared_ptr<std::unordered_map<uint32_t, int>> map,
			uint64_t start) {
			ops_ = ops;
			merge_op_blocks_ = merge_op_blocks;
			map_ = map;
			start_ = start;

			block_iter_ = merge_op_blocks->begin() + start;
			}

			bool CowMergeOpIter::Done() {
			return block_iter_ == merge_op_blocks_->end();
			}

			void CowMergeOpIter::Next() {
			CHECK(!Done());
			block_iter_++;
			}

			const CowOperation& CowMergeOpIter::Get() {
			CHECK(!Done());
			return ops_->data()[map_->at(*block_iter_)];
			}

	CowRevMergeOpIter::CowRevMergeOpIter(std::shared_ptr<std::vector<CowOperation>> ops,		CowRevMergeOpIter::CowRevMergeOpIter(std::shared_ptr<std::vector<CowOperation>> ops,
	std::shared_ptr<std::vector<uint32_t>> merge_op_blocks,		std::shared_ptr<std::vector<uint32_t>> merge_op_blocks,
	std::shared_ptr<std::unordered_map<uint32_t, int>> map) {		std::shared_ptr<std::unordered_map<uint32_t, int>> map,
			uint64_t start) {
	ops_ = ops;		ops_ = ops;
	merge_op_blocks_ = merge_op_blocks;		merge_op_blocks_ = merge_op_blocks;
	map_ = map;		map_ = map;
			start_ = start;

	block_riter_ = merge_op_blocks->rbegin();		block_riter_ = merge_op_blocks->rbegin();
	}		}

	bool CowRevMergeOpIter::Done() {		bool CowRevMergeOpIter::Done() {
	return block_riter_ == merge_op_blocks_->rend();		return block_riter_ == merge_op_blocks_->rend() - start_;
	}		}

	void CowRevMergeOpIter::Next() {		void CowRevMergeOpIter::Next() {
	@@ -571,8 +657,14 @@ std::unique_ptr<ICowOpIter> CowReader::GetOpIter() {
	return std::make_unique<CowOpIter>(ops_);		return std::make_unique<CowOpIter>(ops_);
	}		}

	std::unique_ptr<ICowOpIter> CowReader::GetRevMergeOpIter() {		std::unique_ptr<ICowOpIter> CowReader::GetRevMergeOpIter(bool ignore_progress) {
	return std::make_unique<CowRevMergeOpIter>(ops_, merge_op_blocks_, block_map_);		return std::make_unique<CowRevMergeOpIter>(ops_, merge_op_blocks_, block_map_,
			ignore_progress ? 0 : merge_op_start_);
			}

			std::unique_ptr<ICowOpIter> CowReader::GetMergeOpIter(bool ignore_progress) {
			return std::make_unique<CowMergeOpIter>(ops_, merge_op_blocks_, block_map_,
			ignore_progress ? 0 : merge_op_start_);
	}		}

	bool CowReader::GetRawBytes(uint64_t offset, void* buffer, size_t len, size_t* read) {		bool CowReader::GetRawBytes(uint64_t offset, void* buffer, size_t len, size_t* read) {

fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h

+8 −2

fs_mgr/libsnapshot/inspect_cow.cpp

+30 −4

Original line number	Original line	Diff line number	Diff line
	@@ -44,10 +44,13 @@ static void usage(void) {
	LOG(ERROR) << "\t -b Show data for failed decompress";		LOG(ERROR) << "\t -b Show data for failed decompress";
	LOG(ERROR) << "\t -l Show ops";		LOG(ERROR) << "\t -l Show ops";
	LOG(ERROR) << "\t -m Show ops in reverse merge order";		LOG(ERROR) << "\t -m Show ops in reverse merge order";
	LOG(ERROR) << "\t -o Shows sequence op block order\n";		LOG(ERROR) << "\t -n Show ops in merge order";
			LOG(ERROR) << "\t -a Include merged ops in any merge order listing";
			LOG(ERROR) << "\t -o Shows sequence op block order";
			LOG(ERROR) << "\t -v Verifies merge order has no conflicts\n";
	}		}

	enum OpIter { Normal, RevMerge };		enum OpIter { Normal, RevMerge, Merge };

	struct Options {		struct Options {
	bool silent;		bool silent;
	@@ -55,7 +58,9 @@ struct Options {
	bool show_ops;		bool show_ops;
	bool show_bad;		bool show_bad;
	bool show_seq;		bool show_seq;
			bool verify_sequence;
	OpIter iter_type;		OpIter iter_type;
			bool include_merged;
	};		};

	// Sink that always appends to the end of a string.		// Sink that always appends to the end of a string.
	@@ -132,11 +137,21 @@ static bool Inspect(const std::string& path, Options opt) {
	}		}
	}		}

			if (opt.verify_sequence) {
			if (reader.VerifyMergeOps()) {
			std::cout << "\nMerge sequence is consistent.\n";
			} else {
			std::cout << "\nMerge sequence is inconsistent!\n";
			}
			}

	std::unique_ptr<ICowOpIter> iter;		std::unique_ptr<ICowOpIter> iter;
	if (opt.iter_type == Normal) {		if (opt.iter_type == Normal) {
	iter = reader.GetOpIter();		iter = reader.GetOpIter();
	} else if (opt.iter_type == RevMerge) {		} else if (opt.iter_type == RevMerge) {
	iter = reader.GetRevMergeOpIter();		iter = reader.GetRevMergeOpIter(opt.include_merged);
			} else if (opt.iter_type == Merge) {
			iter = reader.GetMergeOpIter(opt.include_merged);
	}		}
	StringSink sink;		StringSink sink;
	bool success = true;		bool success = true;
	@@ -188,7 +203,9 @@ int main(int argc, char** argv) {
	opt.decompress = false;		opt.decompress = false;
	opt.show_bad = false;		opt.show_bad = false;
	opt.iter_type = android::snapshot::Normal;		opt.iter_type = android::snapshot::Normal;
	while ((ch = getopt(argc, argv, "sdbmol")) != -1) {		opt.verify_sequence = false;
			opt.include_merged = false;
			while ((ch = getopt(argc, argv, "sdbmnolva")) != -1) {
	switch (ch) {		switch (ch) {
	case 's':		case 's':
	opt.silent = true;		opt.silent = true;
	@@ -202,12 +219,21 @@ int main(int argc, char** argv) {
	case 'm':		case 'm':
	opt.iter_type = android::snapshot::RevMerge;		opt.iter_type = android::snapshot::RevMerge;
	break;		break;
			case 'n':
			opt.iter_type = android::snapshot::Merge;
			break;
	case 'o':		case 'o':
	opt.show_seq = true;		opt.show_seq = true;
	break;		break;
	case 'l':		case 'l':
	opt.show_ops = true;		opt.show_ops = true;
	break;		break;
			case 'v':
			opt.verify_sequence = true;
			break;
			case 'a':
			opt.include_merged = true;
			break;
	default:		default:
	android::snapshot::usage();		android::snapshot::usage();
	}		}

Original line number	Original line	Diff line number	Diff line
	@@ -75,8 +75,11 @@ class ICowReader {
	// Return an iterator for retrieving CowOperation entries.		// Return an iterator for retrieving CowOperation entries.
	virtual std::unique_ptr<ICowOpIter> GetOpIter() = 0;		virtual std::unique_ptr<ICowOpIter> GetOpIter() = 0;

			// Return an iterator for retrieving CowOperation entries in reverse merge order
			virtual std::unique_ptr<ICowOpIter> GetRevMergeOpIter(bool ignore_progress) = 0;

	// Return an iterator for retrieving CowOperation entries in merge order		// Return an iterator for retrieving CowOperation entries in merge order
	virtual std::unique_ptr<ICowOpIter> GetRevMergeOpIter() = 0;		virtual std::unique_ptr<ICowOpIter> GetMergeOpIter(bool ignore_progress) = 0;

	// Get decoded bytes from the data section, handling any decompression.		// Get decoded bytes from the data section, handling any decompression.
	// All retrieved data is passed to the sink.		// All retrieved data is passed to the sink.
	@@ -109,6 +112,7 @@ class CowReader : public ICowReader {
	bool Parse(android::base::borrowed_fd fd, std::optional<uint64_t> label = {});		bool Parse(android::base::borrowed_fd fd, std::optional<uint64_t> label = {});

	bool InitForMerge(android::base::unique_fd&& fd);		bool InitForMerge(android::base::unique_fd&& fd);
			bool VerifyMergeOps();

	bool GetHeader(CowHeader* header) override;		bool GetHeader(CowHeader* header) override;
	bool GetFooter(CowFooter* footer) override;		bool GetFooter(CowFooter* footer) override;
	@@ -120,7 +124,8 @@ class CowReader : public ICowReader {
	// whose lifetime depends on the CowOpIter object; the return		// whose lifetime depends on the CowOpIter object; the return
	// value of these will never be null.		// value of these will never be null.
	std::unique_ptr<ICowOpIter> GetOpIter() override;		std::unique_ptr<ICowOpIter> GetOpIter() override;
	std::unique_ptr<ICowOpIter> GetRevMergeOpIter() override;		std::unique_ptr<ICowOpIter> GetRevMergeOpIter(bool ignore_progress = false) override;
			std::unique_ptr<ICowOpIter> GetMergeOpIter(bool ignore_progress = false) override;

	bool ReadData(const CowOperation& op, IByteSink* sink) override;		bool ReadData(const CowOperation& op, IByteSink* sink) override;

	@@ -152,6 +157,7 @@ class CowReader : public ICowReader {
	std::optional<uint64_t> last_label_;		std::optional<uint64_t> last_label_;
	std::shared_ptr<std::vector<CowOperation>> ops_;		std::shared_ptr<std::vector<CowOperation>> ops_;
	std::shared_ptr<std::vector<uint32_t>> merge_op_blocks_;		std::shared_ptr<std::vector<uint32_t>> merge_op_blocks_;
			uint64_t merge_op_start_;
	std::shared_ptr<std::unordered_map<uint32_t, int>> block_map_;		std::shared_ptr<std::unordered_map<uint32_t, int>> block_map_;
	uint64_t num_total_data_ops_;		uint64_t num_total_data_ops_;
	uint64_t num_ordered_ops_to_merge_;		uint64_t num_ordered_ops_to_merge_;