libsnapshot: Implement OpenReader for CompressedSnapshotWriter. (0544f91a) · Commits · e / os / android_system_core

fs_mgr/libsnapshot/Android.bp

+1 −0

Original line number	Diff line number	Diff line
		@@ -239,6 +239,7 @@ cc_defaults {
		srcs: [
		"partition_cow_creator_test.cpp",
		"snapshot_metadata_updater_test.cpp",
		"snapshot_reader_test.cpp",
		"snapshot_test.cpp",
		],
		shared_libs: [

fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h

+7 −1

Original line number	Diff line number	Diff line
		@@ -39,11 +39,17 @@ class IByteSink {
		// maximum number of bytes that can be written to the returned buffer.
		//
		// The returned buffer is owned by IByteSink, but must remain valid until
		// the ready operation has completed (or the entire buffer has been
		// the read operation has completed (or the entire buffer has been
		// covered by calls to ReturnData).
		//
		// After calling GetBuffer(), all previous buffers returned are no longer
		// valid.
		//
		// GetBuffer() is intended to be sequential. A returned size of N indicates
		// that the output stream will advance by N bytes, and the ReturnData call
		// indicates that those bytes have been fulfilled. Therefore, it is
		// possible to have ReturnBuffer do nothing, if the implementation doesn't
		// care about incremental writes.
		virtual void* GetBuffer(size_t requested, size_t* actual) = 0;

		// Called when a section returned by \|GetBuffer\| has been filled with data.

fs_mgr/libsnapshot/include/libsnapshot/snapshot_writer.h

+3 −2

Original line number	Diff line number	Diff line
		@@ -42,9 +42,10 @@ class ISnapshotWriter : public ICowWriter {
		protected:
		android::base::borrowed_fd GetSourceFd();

		std::optional<std::string> source_device_;

		private:
		android::base::unique_fd source_fd_;
		std::optional<std::string> source_device_;
		};

		// Send writes to a COW or a raw device directly, based on a threshold.
		@@ -52,7 +53,7 @@ class CompressedSnapshotWriter : public ISnapshotWriter {
		public:
		CompressedSnapshotWriter(const CowOptions& options);

		// Sets the COW device, if needed.
		// Sets the COW device; this is required.
		bool SetCowDevice(android::base::unique_fd&& cow_device);

		bool Finalize() override;

fs_mgr/libsnapshot/snapshot_reader.cpp

+253 −2

Original line number	Diff line number	Diff line
		@@ -14,13 +14,17 @@
		// limitations under the License.
		//

		#include <ext4_utils/ext4_utils.h>

		#include "snapshot_reader.h"

		#include <android-base/file.h>
		#include <android-base/logging.h>
		#include <ext4_utils/ext4_utils.h>

		namespace android {
		namespace snapshot {

		using android::base::borrowed_fd;

		// Not supported.
		bool ReadOnlyFileDescriptor::Open(const char*, int, mode_t) {
		errno = EINVAL;
		@@ -73,5 +77,252 @@ bool ReadFdFileDescriptor::Flush() {
		return true;
		}

		bool CompressedSnapshotReader::SetCow(std::unique_ptr<CowReader>&& cow) {
		cow_ = std::move(cow);

		CowHeader header;
		if (!cow_->GetHeader(&header)) {
		return false;
		}
		block_size_ = header.block_size;

		// Populate the operation map.
		op_iter_ = cow_->GetOpIter();
		while (!op_iter_->Done()) {
		const CowOperation* op = &op_iter_->Get();
		if (op->new_block >= ops_.size()) {
		ops_.resize(op->new_block + 1, nullptr);
		}
		ops_[op->new_block] = op;
		op_iter_->Next();
		}

		return true;
		}

		void CompressedSnapshotReader::SetSourceDevice(const std::string& source_device) {
		source_device_ = {source_device};
		}

		void CompressedSnapshotReader::SetBlockDeviceSize(uint64_t block_device_size) {
		block_device_size_ = block_device_size;
		}

		borrowed_fd CompressedSnapshotReader::GetSourceFd() {
		if (source_fd_ < 0) {
		if (!source_device_) {
		LOG(ERROR) << "CompressedSnapshotReader needs source device, but none was set";
		errno = EINVAL;
		return {-1};
		}
		source_fd_.reset(open(source_device_->c_str(), O_RDONLY \| O_CLOEXEC));
		if (source_fd_ < 0) {
		PLOG(ERROR) << "open " << *source_device_;
		return {-1};
		}
		}
		return source_fd_;
		}

		class MemoryByteSink : public IByteSink {
		public:
		MemoryByteSink(void* buf, size_t count) {
		buf_ = reinterpret_cast<uint8_t*>(buf);
		pos_ = buf_;
		end_ = buf_ + count;
		}

		void* GetBuffer(size_t requested, size_t* actual) override {
		*actual = std::min(remaining(), requested);
		if (!*actual) {
		return nullptr;
		}

		uint8_t* start = pos_;
		pos_ += *actual;
		return start;
		}

		bool ReturnData(void*, size_t) override { return true; }

		uint8_t* buf() const { return buf_; }
		uint8_t* pos() const { return pos_; }
		size_t remaining() const { return end_ - pos_; }

		private:
		uint8_t* buf_;
		uint8_t* pos_;
		uint8_t* end_;
		};

		ssize_t CompressedSnapshotReader::Read(void* buf, size_t count) {
		// Find the start and end chunks, inclusive.
		uint64_t start_chunk = offset_ / block_size_;
		uint64_t end_chunk = (offset_ + count - 1) / block_size_;

		// Chop off the first N bytes if the position is not block-aligned.
		size_t start_offset = offset_ % block_size_;

		MemoryByteSink sink(buf, count);

		size_t initial_bytes = std::min(block_size_ - start_offset, sink.remaining());
		ssize_t rv = ReadBlock(start_chunk, &sink, start_offset, initial_bytes);
		if (rv < 0) {
		return -1;
		}
		offset_ += rv;

		for (uint64_t chunk = start_chunk + 1; chunk < end_chunk; chunk++) {
		ssize_t rv = ReadBlock(chunk, &sink, 0);
		if (rv < 0) {
		return -1;
		}
		offset_ += rv;
		}

		if (sink.remaining()) {
		ssize_t rv = ReadBlock(end_chunk, &sink, 0, {sink.remaining()});
		if (rv < 0) {
		return -1;
		}
		offset_ += rv;
		}

		errno = 0;

		DCHECK(sink.pos() - sink.buf() == count);
		return count;
		}

		// Discard the first N bytes of a sink request, or any excess bytes.
		class PartialSink : public MemoryByteSink {
		public:
		PartialSink(void* buffer, size_t size, size_t ignore_start)
		: MemoryByteSink(buffer, size), ignore_start_(ignore_start) {}

		void* GetBuffer(size_t requested, size_t* actual) override {
		// Throw away the first N bytes if needed.
		if (ignore_start_) {
		*actual = std::min({requested, ignore_start_, sizeof(discard_)});
		ignore_start_ -= *actual;
		return discard_;
		}
		// Throw away any excess bytes if needed.
		if (remaining() == 0) {
		*actual = std::min(requested, sizeof(discard_));
		return discard_;
		}
		return MemoryByteSink::GetBuffer(requested, actual);
		}

		private:
		size_t ignore_start_;
		char discard_[4096];
		};

		ssize_t CompressedSnapshotReader::ReadBlock(uint64_t chunk, IByteSink* sink, size_t start_offset,
		const std::optional<uint64_t>& max_bytes) {
		size_t bytes_to_read = block_size_;
		if (max_bytes) {
		bytes_to_read = *max_bytes;
		}

		// The offset is relative to the chunk; we should be reading no more than
		// one chunk.
		CHECK(start_offset + bytes_to_read <= block_size_);

		const CowOperation* op = nullptr;
		if (chunk < ops_.size()) {
		op = ops_[chunk];
		}

		size_t actual;
		void* buffer = sink->GetBuffer(bytes_to_read, &actual);
		if (!buffer \|\| actual < bytes_to_read) {
		// This should never happen unless we calculated the read size wrong
		// somewhere. MemoryByteSink always fulfills the entire requested
		// region unless there's not enough buffer remaining.
		LOG(ERROR) << "Asked for buffer of size " << bytes_to_read << ", got " << actual;
		errno = EINVAL;
		return -1;
		}

		if (!op \|\| op->type == kCowCopyOp) {
		borrowed_fd fd = GetSourceFd();
		if (fd < 0) {
		// GetSourceFd sets errno.
		return -1;
		}

		if (op) {
		chunk = op->source;
		}

		off64_t offset = (chunk * block_size_) + start_offset;
		if (!android::base::ReadFullyAtOffset(fd, buffer, bytes_to_read, offset)) {
		PLOG(ERROR) << "read " << *source_device_;
		// ReadFullyAtOffset sets errno.
		return -1;
		}
		} else if (op->type == kCowZeroOp) {
		memset(buffer, 0, bytes_to_read);
		} else if (op->type == kCowReplaceOp) {
		PartialSink partial_sink(buffer, bytes_to_read, start_offset);
		if (!cow_->ReadData(*op, &partial_sink)) {
		LOG(ERROR) << "CompressedSnapshotReader failed to read replace op";
		errno = EIO;
		return -1;
		}
		} else {
		LOG(ERROR) << "CompressedSnapshotReader unknown op type: " << op->type;
		errno = EINVAL;
		return -1;
		}

		// MemoryByteSink doesn't do anything in ReturnBuffer, so don't bother calling it.
		return bytes_to_read;
		}

		off64_t CompressedSnapshotReader::Seek(off64_t offset, int whence) {
		switch (whence) {
		case SEEK_SET:
		offset_ = offset;
		break;
		case SEEK_END:
		offset_ = static_cast<off64_t>(block_device_size_) + offset;
		break;
		case SEEK_CUR:
		offset_ += offset;
		break;
		default:
		LOG(ERROR) << "Unrecognized seek whence: " << whence;
		errno = EINVAL;
		return -1;
		}
		return offset_;
		}

		uint64_t CompressedSnapshotReader::BlockDevSize() {
		return block_device_size_;
		}

		bool CompressedSnapshotReader::Close() {
		cow_ = nullptr;
		source_fd_ = {};
		return true;
		}

		bool CompressedSnapshotReader::IsSettingErrno() {
		return true;
		}

		bool CompressedSnapshotReader::IsOpen() {
		return cow_ != nullptr;
		}

		bool CompressedSnapshotReader::Flush() {
		return true;
		}

		} // namespace snapshot
		} // namespace android

fs_mgr/libsnapshot/snapshot_reader.h

+35 −0

Original line number	Diff line number	Diff line
		@@ -16,7 +16,11 @@

		#pragma once

		#include <optional>
		#include <vector>

		#include <android-base/file.h>
		#include <libsnapshot/cow_reader.h>
		#include <payload_consumer/file_descriptor.h>

		namespace android {
		@@ -46,5 +50,36 @@ class ReadFdFileDescriptor : public ReadOnlyFileDescriptor {
		android::base::unique_fd fd_;
		};

		class CompressedSnapshotReader : public ReadOnlyFileDescriptor {
		public:
		bool SetCow(std::unique_ptr<CowReader>&& cow);
		void SetSourceDevice(const std::string& source_device);
		void SetBlockDeviceSize(uint64_t block_device_size);

		ssize_t Read(void* buf, size_t count) override;
		off64_t Seek(off64_t offset, int whence) override;
		uint64_t BlockDevSize() override;
		bool Close() override;
		bool IsSettingErrno() override;
		bool IsOpen() override;
		bool Flush() override;

		private:
		ssize_t ReadBlock(uint64_t chunk, IByteSink* sink, size_t start_offset,
		const std::optional<uint64_t>& max_bytes = {});
		android::base::borrowed_fd GetSourceFd();

		std::unique_ptr<CowReader> cow_;
		std::unique_ptr<ICowOpIter> op_iter_;
		uint32_t block_size_ = 0;

		std::optional<std::string> source_device_;
		android::base::unique_fd source_fd_;
		uint64_t block_device_size_ = 0;
		off64_t offset_ = 0;

		std::vector<const CowOperation*> ops_;
		};

		} // namespace snapshot
		} // namespace android