Merge "libsnapshot: Add function to verify merge sequence" (5125c62c) · Commits · e / os / android_system_core

fs_mgr/libsnapshot/cow_api_test.cpp

+45 −12

Original line number	Diff line number	Diff line
		@@ -1171,18 +1171,18 @@ TEST_F(CowTest, RevMergeOpItrTest) {
		ASSERT_TRUE(writer.Initialize(cow_->fd));

		ASSERT_TRUE(writer.AddSequenceData(6, sequence));
		ASSERT_TRUE(writer.AddCopy(6, 3));
		ASSERT_TRUE(writer.AddCopy(6, 13));
		ASSERT_TRUE(writer.AddZeroBlocks(12, 1));
		ASSERT_TRUE(writer.AddZeroBlocks(8, 1));
		ASSERT_TRUE(writer.AddZeroBlocks(11, 1));
		ASSERT_TRUE(writer.AddCopy(3, 5));
		ASSERT_TRUE(writer.AddCopy(2, 1));
		ASSERT_TRUE(writer.AddCopy(3, 15));
		ASSERT_TRUE(writer.AddCopy(2, 11));
		ASSERT_TRUE(writer.AddZeroBlocks(4, 1));
		ASSERT_TRUE(writer.AddZeroBlocks(9, 1));
		ASSERT_TRUE(writer.AddCopy(5, 6));
		ASSERT_TRUE(writer.AddCopy(5, 16));
		ASSERT_TRUE(writer.AddZeroBlocks(1, 1));
		ASSERT_TRUE(writer.AddCopy(10, 2));
		ASSERT_TRUE(writer.AddCopy(7, 4));
		ASSERT_TRUE(writer.AddCopy(10, 12));
		ASSERT_TRUE(writer.AddCopy(7, 14));
		ASSERT_TRUE(writer.Finalize());

		// New block in cow order is 6, 12, 8, 11, 3, 2, 4, 9, 5, 1, 10, 7
		@@ -1219,12 +1219,12 @@ TEST_F(CowTest, LegacyRevMergeOpItrTest) {

		ASSERT_TRUE(writer.Initialize(cow_->fd));

		ASSERT_TRUE(writer.AddCopy(2, 1));
		ASSERT_TRUE(writer.AddCopy(10, 2));
		ASSERT_TRUE(writer.AddCopy(6, 3));
		ASSERT_TRUE(writer.AddCopy(7, 4));
		ASSERT_TRUE(writer.AddCopy(3, 5));
		ASSERT_TRUE(writer.AddCopy(5, 6));
		ASSERT_TRUE(writer.AddCopy(2, 11));
		ASSERT_TRUE(writer.AddCopy(10, 12));
		ASSERT_TRUE(writer.AddCopy(6, 13));
		ASSERT_TRUE(writer.AddCopy(7, 14));
		ASSERT_TRUE(writer.AddCopy(3, 15));
		ASSERT_TRUE(writer.AddCopy(5, 16));
		ASSERT_TRUE(writer.AddZeroBlocks(12, 1));
		ASSERT_TRUE(writer.AddZeroBlocks(8, 1));
		ASSERT_TRUE(writer.AddZeroBlocks(11, 1));
		@@ -1260,6 +1260,39 @@ TEST_F(CowTest, LegacyRevMergeOpItrTest) {
		ASSERT_TRUE(iter->Done());
		}

		TEST_F(CowTest, InvalidMergeOrderTest) {
		CowOptions options;
		options.cluster_ops = 5;
		options.num_merge_ops = 1;
		std::string data = "This is some data, believe it";
		data.resize(options.block_size, '\0');
		auto writer = std::make_unique<CowWriter>(options);
		CowReader reader;

		ASSERT_TRUE(writer->Initialize(cow_->fd));

		ASSERT_TRUE(writer->AddCopy(3, 2));
		ASSERT_TRUE(writer->AddCopy(2, 1));
		ASSERT_TRUE(writer->AddLabel(1));
		ASSERT_TRUE(writer->Finalize());
		ASSERT_TRUE(reader.Parse(cow_->fd));
		ASSERT_TRUE(reader.VerifyMergeOps());

		ASSERT_TRUE(writer->InitializeAppend(cow_->fd, 1));
		ASSERT_TRUE(writer->AddCopy(4, 2));
		ASSERT_TRUE(writer->Finalize());
		ASSERT_TRUE(reader.Parse(cow_->fd));
		ASSERT_FALSE(reader.VerifyMergeOps());

		writer = std::make_unique<CowWriter>(options);
		ASSERT_TRUE(writer->Initialize(cow_->fd));
		ASSERT_TRUE(writer->AddCopy(2, 1));
		ASSERT_TRUE(writer->AddXorBlocks(3, &data, data.size(), 1, 1));
		ASSERT_TRUE(writer->Finalize());
		ASSERT_TRUE(reader.Parse(cow_->fd));
		ASSERT_FALSE(reader.VerifyMergeOps());
		}

		} // namespace snapshot
		} // namespace android

fs_mgr/libsnapshot/cow_format.cpp

+4 −1

Original line number	Diff line number	Diff line
		@@ -56,7 +56,10 @@ std::ostream& operator<<(std::ostream& os, CowOperation const& op) {
		os << (int)op.compression << "?, ";
		os << "data_length:" << op.data_length << ",\t";
		os << "new_block:" << op.new_block << ",\t";
		os << "source:" << op.source << ")";
		os << "source:" << op.source;
		if (op.type == kCowXorOp)
		os << " (block:" << op.source / BLOCK_SZ << " offset:" << op.source % BLOCK_SZ << ")";
		os << ")";
		return os;
		}

fs_mgr/libsnapshot/cow_reader.cpp

+99 −7

Original line number	Diff line number	Diff line
		@@ -58,6 +58,7 @@ std::unique_ptr<CowReader> CowReader::CloneCowReader() {
		cow->last_label_ = last_label_;
		cow->ops_ = ops_;
		cow->merge_op_blocks_ = merge_op_blocks_;
		cow->merge_op_start_ = merge_op_start_;
		cow->block_map_ = block_map_;
		cow->num_total_data_ops_ = num_total_data_ops_;
		cow->num_ordered_ops_to_merge_ = num_ordered_ops_to_merge_;
		@@ -468,8 +469,7 @@ bool CowReader::PrepMergeOps() {

		num_total_data_ops_ = merge_op_blocks->size();
		if (header_.num_merge_ops > 0) {
		merge_op_blocks->erase(merge_op_blocks->begin(),
		merge_op_blocks->begin() + header_.num_merge_ops);
		merge_op_start_ = header_.num_merge_ops;
		}

		block_map_ = block_map;
		@@ -477,6 +477,44 @@ bool CowReader::PrepMergeOps() {
		return true;
		}

		bool CowReader::VerifyMergeOps() {
		auto itr = GetMergeOpIter(true);
		std::unordered_map<uint64_t, CowOperation> overwritten_blocks;
		while (!itr->Done()) {
		CowOperation op = itr->Get();
		uint64_t block;
		bool offset;
		if (op.type == kCowCopyOp) {
		block = op.source;
		offset = false;
		} else if (op.type == kCowXorOp) {
		block = op.source / BLOCK_SZ;
		offset = (op.source % BLOCK_SZ) != 0;
		} else {
		itr->Next();
		continue;
		}

		CowOperation* overwrite = nullptr;
		if (overwritten_blocks.count(block)) {
		overwrite = &overwritten_blocks[block];
		LOG(ERROR) << "Invalid Sequence! Block needed for op:\n"
		<< op << "\noverwritten by previously merged op:\n"
		<< *overwrite;
		}
		if (offset && overwritten_blocks.count(block + 1)) {
		overwrite = &overwritten_blocks[block + 1];
		LOG(ERROR) << "Invalid Sequence! Block needed for op:\n"
		<< op << "\noverwritten by previously merged op:\n"
		<< *overwrite;
		}
		if (overwrite != nullptr) return false;
		overwritten_blocks[op.new_block] = op;
		itr->Next();
		}
		return true;
		}

		bool CowReader::GetHeader(CowHeader* header) {
		*header = header_;
		return true;
		@@ -530,7 +568,8 @@ class CowRevMergeOpIter final : public ICowOpIter {
		public:
		explicit CowRevMergeOpIter(std::shared_ptr<std::vector<CowOperation>> ops,
		std::shared_ptr<std::vector<uint32_t>> merge_op_blocks,
		std::shared_ptr<std::unordered_map<uint32_t, int>> map);
		std::shared_ptr<std::unordered_map<uint32_t, int>> map,
		uint64_t start);

		bool Done() override;
		const CowOperation& Get() override;
		@@ -541,20 +580,67 @@ class CowRevMergeOpIter final : public ICowOpIter {
		std::shared_ptr<std::vector<uint32_t>> merge_op_blocks_;
		std::shared_ptr<std::unordered_map<uint32_t, int>> map_;
		std::vector<uint32_t>::reverse_iterator block_riter_;
		uint64_t start_;
		};

		class CowMergeOpIter final : public ICowOpIter {
		public:
		explicit CowMergeOpIter(std::shared_ptr<std::vector<CowOperation>> ops,
		std::shared_ptr<std::vector<uint32_t>> merge_op_blocks,
		std::shared_ptr<std::unordered_map<uint32_t, int>> map, uint64_t start);

		bool Done() override;
		const CowOperation& Get() override;
		void Next() override;

		private:
		std::shared_ptr<std::vector<CowOperation>> ops_;
		std::shared_ptr<std::vector<uint32_t>> merge_op_blocks_;
		std::shared_ptr<std::unordered_map<uint32_t, int>> map_;
		std::vector<uint32_t>::iterator block_iter_;
		uint64_t start_;
		};

		CowMergeOpIter::CowMergeOpIter(std::shared_ptr<std::vector<CowOperation>> ops,
		std::shared_ptr<std::vector<uint32_t>> merge_op_blocks,
		std::shared_ptr<std::unordered_map<uint32_t, int>> map,
		uint64_t start) {
		ops_ = ops;
		merge_op_blocks_ = merge_op_blocks;
		map_ = map;
		start_ = start;

		block_iter_ = merge_op_blocks->begin() + start;
		}

		bool CowMergeOpIter::Done() {
		return block_iter_ == merge_op_blocks_->end();
		}

		void CowMergeOpIter::Next() {
		CHECK(!Done());
		block_iter_++;
		}

		const CowOperation& CowMergeOpIter::Get() {
		CHECK(!Done());
		return ops_->data()[map_->at(*block_iter_)];
		}

		CowRevMergeOpIter::CowRevMergeOpIter(std::shared_ptr<std::vector<CowOperation>> ops,
		std::shared_ptr<std::vector<uint32_t>> merge_op_blocks,
		std::shared_ptr<std::unordered_map<uint32_t, int>> map) {
		std::shared_ptr<std::unordered_map<uint32_t, int>> map,
		uint64_t start) {
		ops_ = ops;
		merge_op_blocks_ = merge_op_blocks;
		map_ = map;
		start_ = start;

		block_riter_ = merge_op_blocks->rbegin();
		}

		bool CowRevMergeOpIter::Done() {
		return block_riter_ == merge_op_blocks_->rend();
		return block_riter_ == merge_op_blocks_->rend() - start_;
		}

		void CowRevMergeOpIter::Next() {
		@@ -571,8 +657,14 @@ std::unique_ptr<ICowOpIter> CowReader::GetOpIter() {
		return std::make_unique<CowOpIter>(ops_);
		}

		std::unique_ptr<ICowOpIter> CowReader::GetRevMergeOpIter() {
		return std::make_unique<CowRevMergeOpIter>(ops_, merge_op_blocks_, block_map_);
		std::unique_ptr<ICowOpIter> CowReader::GetRevMergeOpIter(bool ignore_progress) {
		return std::make_unique<CowRevMergeOpIter>(ops_, merge_op_blocks_, block_map_,
		ignore_progress ? 0 : merge_op_start_);
		}

		std::unique_ptr<ICowOpIter> CowReader::GetMergeOpIter(bool ignore_progress) {
		return std::make_unique<CowMergeOpIter>(ops_, merge_op_blocks_, block_map_,
		ignore_progress ? 0 : merge_op_start_);
		}

		bool CowReader::GetRawBytes(uint64_t offset, void* buffer, size_t len, size_t* read) {

fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h

+8 −2

Original line number	Diff line number	Diff line
		@@ -75,8 +75,11 @@ class ICowReader {
		// Return an iterator for retrieving CowOperation entries.
		virtual std::unique_ptr<ICowOpIter> GetOpIter() = 0;

		// Return an iterator for retrieving CowOperation entries in reverse merge order
		virtual std::unique_ptr<ICowOpIter> GetRevMergeOpIter(bool ignore_progress) = 0;

		// Return an iterator for retrieving CowOperation entries in merge order
		virtual std::unique_ptr<ICowOpIter> GetRevMergeOpIter() = 0;
		virtual std::unique_ptr<ICowOpIter> GetMergeOpIter(bool ignore_progress) = 0;

		// Get decoded bytes from the data section, handling any decompression.
		// All retrieved data is passed to the sink.
		@@ -109,6 +112,7 @@ class CowReader : public ICowReader {
		bool Parse(android::base::borrowed_fd fd, std::optional<uint64_t> label = {});

		bool InitForMerge(android::base::unique_fd&& fd);
		bool VerifyMergeOps();

		bool GetHeader(CowHeader* header) override;
		bool GetFooter(CowFooter* footer) override;
		@@ -120,7 +124,8 @@ class CowReader : public ICowReader {
		// whose lifetime depends on the CowOpIter object; the return
		// value of these will never be null.
		std::unique_ptr<ICowOpIter> GetOpIter() override;
		std::unique_ptr<ICowOpIter> GetRevMergeOpIter() override;
		std::unique_ptr<ICowOpIter> GetRevMergeOpIter(bool ignore_progress = false) override;
		std::unique_ptr<ICowOpIter> GetMergeOpIter(bool ignore_progress = false) override;

		bool ReadData(const CowOperation& op, IByteSink* sink) override;

		@@ -152,6 +157,7 @@ class CowReader : public ICowReader {
		std::optional<uint64_t> last_label_;
		std::shared_ptr<std::vector<CowOperation>> ops_;
		std::shared_ptr<std::vector<uint32_t>> merge_op_blocks_;
		uint64_t merge_op_start_;
		std::shared_ptr<std::unordered_map<uint32_t, int>> block_map_;
		uint64_t num_total_data_ops_;
		uint64_t num_ordered_ops_to_merge_;

fs_mgr/libsnapshot/inspect_cow.cpp

+30 −4

Original line number	Diff line number	Diff line
		@@ -44,10 +44,13 @@ static void usage(void) {
		LOG(ERROR) << "\t -b Show data for failed decompress";
		LOG(ERROR) << "\t -l Show ops";
		LOG(ERROR) << "\t -m Show ops in reverse merge order";
		LOG(ERROR) << "\t -o Shows sequence op block order\n";
		LOG(ERROR) << "\t -n Show ops in merge order";
		LOG(ERROR) << "\t -a Include merged ops in any merge order listing";
		LOG(ERROR) << "\t -o Shows sequence op block order";
		LOG(ERROR) << "\t -v Verifies merge order has no conflicts\n";
		}

		enum OpIter { Normal, RevMerge };
		enum OpIter { Normal, RevMerge, Merge };

		struct Options {
		bool silent;
		@@ -55,7 +58,9 @@ struct Options {
		bool show_ops;
		bool show_bad;
		bool show_seq;
		bool verify_sequence;
		OpIter iter_type;
		bool include_merged;
		};

		// Sink that always appends to the end of a string.
		@@ -132,11 +137,21 @@ static bool Inspect(const std::string& path, Options opt) {
		}
		}

		if (opt.verify_sequence) {
		if (reader.VerifyMergeOps()) {
		std::cout << "\nMerge sequence is consistent.\n";
		} else {
		std::cout << "\nMerge sequence is inconsistent!\n";
		}
		}

		std::unique_ptr<ICowOpIter> iter;
		if (opt.iter_type == Normal) {
		iter = reader.GetOpIter();
		} else if (opt.iter_type == RevMerge) {
		iter = reader.GetRevMergeOpIter();
		iter = reader.GetRevMergeOpIter(opt.include_merged);
		} else if (opt.iter_type == Merge) {
		iter = reader.GetMergeOpIter(opt.include_merged);
		}
		StringSink sink;
		bool success = true;
		@@ -188,7 +203,9 @@ int main(int argc, char** argv) {
		opt.decompress = false;
		opt.show_bad = false;
		opt.iter_type = android::snapshot::Normal;
		while ((ch = getopt(argc, argv, "sdbmol")) != -1) {
		opt.verify_sequence = false;
		opt.include_merged = false;
		while ((ch = getopt(argc, argv, "sdbmnolva")) != -1) {
		switch (ch) {
		case 's':
		opt.silent = true;
		@@ -202,12 +219,21 @@ int main(int argc, char** argv) {
		case 'm':
		opt.iter_type = android::snapshot::RevMerge;
		break;
		case 'n':
		opt.iter_type = android::snapshot::Merge;
		break;
		case 'o':
		opt.show_seq = true;
		break;
		case 'l':
		opt.show_ops = true;
		break;
		case 'v':
		opt.verify_sequence = true;
		break;
		case 'a':
		opt.include_merged = true;
		break;
		default:
		android::snapshot::usage();
		}