liblp: Add a helper class for building sparse-compatible super image layouts. (e6cef616) · Commits · e / os / android_system_core

fs_mgr/liblp/Android.bp

+21 −14

Original line number	Original line	Diff line number	Diff line
	@@ -39,6 +39,7 @@ cc_library {
	],		],
	srcs: [		srcs: [
	"builder.cpp",		"builder.cpp",
			"super_layout_builder.cpp",
	"images.cpp",		"images.cpp",
	"partition_opener.cpp",		"partition_opener.cpp",
	"property_fetcher.cpp",		"property_fetcher.cpp",
	@@ -62,17 +63,6 @@ cc_library {
	export_include_dirs: ["include"],		export_include_dirs: ["include"],
	}		}

	filegroup {
	name: "liblp_test_srcs",
	srcs: [
	"builder_test.cpp",
	"device_test.cpp",
	"io_test.cpp",
	"test_partition_opener.cpp",
	"utility_test.cpp",
	],
	}

	cc_defaults {		cc_defaults {
	name: "liblp_test_defaults",		name: "liblp_test_defaults",
	defaults: ["fs_mgr_defaults"],		defaults: ["fs_mgr_defaults"],
	@@ -82,22 +72,39 @@ cc_defaults {
	static_libs: [		static_libs: [
	"libcutils",		"libcutils",
	"libgmock",		"libgmock",
	"libfs_mgr",
	"liblp",		"liblp",
	"libcrypto_static",		"libcrypto_static",
	] + liblp_lib_deps,		] + liblp_lib_deps,
	header_libs: [		header_libs: [
	"libstorage_literals_headers",		"libstorage_literals_headers",
	],		],
			target: {
			android: {
			srcs: [
			"device_test.cpp",
			"io_test.cpp",
			],
			static_libs: [
			"libfs_mgr",
			],
			}
			},
	stl: "libc++_static",		stl: "libc++_static",
	srcs: [":liblp_test_srcs"],		srcs: [
			"builder_test.cpp",
			"super_layout_builder_test.cpp",
			"test_partition_opener.cpp",
			"utility_test.cpp",
			],
	}		}

	cc_test {		cc_test {
	name: "liblp_test",		name: "liblp_test",
	defaults: ["liblp_test_defaults"],		defaults: ["liblp_test_defaults"],
	test_config: "liblp_test.xml",
	test_suites: ["device-tests"],		test_suites: ["device-tests"],
			auto_gen_config: true,
			require_root: true,
			host_supported: true
	}		}

	cc_test {		cc_test {

fs_mgr/liblp/include/liblp/super_layout_builder.h

0 → 100644

+104 −0

Original line number	Original line	Diff line number	Diff line
			//
			// Copyright (C) 2023 The Android Open Source Project
			//
			// Licensed under the Apache License, Version 2.0 (the "License");
			// you may not use this file except in compliance with the License.
			// You may obtain a copy of the License at
			//
			// http://www.apache.org/licenses/LICENSE-2.0
			//
			// Unless required by applicable law or agreed to in writing, software
			// distributed under the License is distributed on an "AS IS" BASIS,
			// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
			// See the License for the specific language governing permissions and
			// limitations under the License.
			//
			#pragma once

			#include <stdint.h>

			#include <memory>
			#include <ostream>
			#include <string>
			#include <unordered_map>
			#include <utility>

			#include <android-base/unique_fd.h>
			#include <liblp/builder.h>

			namespace android {
			namespace fs_mgr {

			struct SuperImageExtent {
			enum class Type { INVALID, DATA, PARTITION, ZERO, DONTCARE };

			SuperImageExtent(const SuperImageExtent& other) = default;
			SuperImageExtent(SuperImageExtent&& other) = default;
			SuperImageExtent(uint64_t offset, uint64_t size, Type type)
			: offset(offset), size(size), type(type) {}

			SuperImageExtent(uint64_t offset, std::shared_ptr<std::string> blob)
			: SuperImageExtent(offset, blob->size(), Type::DATA) {
			this->blob = blob;
			}

			SuperImageExtent(uint64_t offset, uint64_t size, const std::string& image_name,
			uint64_t image_offset)
			: SuperImageExtent(offset, size, Type::PARTITION) {
			this->image_name = image_name;
			this->image_offset = image_offset;
			}

			SuperImageExtent& operator=(const SuperImageExtent& other) = default;
			SuperImageExtent& operator=(SuperImageExtent&& other) = default;

			bool operator<(const SuperImageExtent& other) const { return offset < other.offset; }
			bool operator==(const SuperImageExtent& other) const;

			// Location, size, and type of the extent.
			uint64_t offset = 0;
			uint64_t size = 0;
			Type type = Type::INVALID;

			// If type == DATA, this contains the bytes to write.
			std::shared_ptr<std::string> blob;
			// If type == PARTITION, this contains the partition image name and
			// offset within that file.
			std::string image_name;
			uint64_t image_offset = 0;
			};

			// The SuperLayoutBuilder allows building a sparse view of a super image. This
			// is useful for efficient flashing, eg to bypass fastbootd and directly flash
			// super without physically building and storing the image.
			class SuperLayoutBuilder final {
			public:
			// Open a super_empty.img, return false on failure. This must be called to
			// initialize the tool. If it returns false, either the image failed to
			// parse, or the tool is not compatible with how the device is configured
			// (in which case fastbootd should be preferred).
			[[nodiscard]] bool Open(android::base::borrowed_fd fd);
			[[nodiscard]] bool Open(const void* data, size_t bytes);
			[[nodiscard]] bool Open(const LpMetadata& metadata);

			// Add a partition's image and size to the work list. If false is returned,
			// there was either a duplicate partition or not enough space in super.
			bool AddPartition(const std::string& partition_name, const std::string& image_name,
			uint64_t partition_size);

			// Return the list of extents describing the super image. If this list is
			// empty, then there was an unrecoverable error in building the list.
			std::vector<SuperImageExtent> GetImageLayout();

			// Return the current metadata.
			std::unique_ptr<LpMetadata> Export() const { return builder_->Export(); }

			private:
			std::unique_ptr<MetadataBuilder> builder_;
			std::unordered_map<std::string, std::string> image_map_;
			};

			std::ostream& operator<<(std::ostream& stream, const SuperImageExtent& extent);

			} // namespace fs_mgr
			} // namespace android

fs_mgr/liblp/liblp_test.xml

deleted100644 → 0

+0 −26

Original line number	Original line	Diff line number	Diff line
	<?xml version="1.0" encoding="utf-8"?>
	<!-- Copyright (C) 2019 The Android Open Source Project

	Licensed under the Apache License, Version 2.0 (the "License");
	you may not use this file except in compliance with the License.
	You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

	Unless required by applicable law or agreed to in writing, software
	distributed under the License is distributed on an "AS IS" BASIS,
	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	See the License for the specific language governing permissions and
	limitations under the License.
	-->
	<configuration description="Config for liblp_test">
	<target_preparer class="com.android.tradefed.targetprep.RootTargetPreparer"/>
	<target_preparer class="com.android.tradefed.targetprep.PushFilePreparer">
	<option name="cleanup" value="true" />
	<option name="push" value="liblp_test->/data/local/tmp/liblp_test" />
	</target_preparer>
	<test class="com.android.tradefed.testtype.GTest" >
	<option name="native-test-device-path" value="/data/local/tmp" />
	<option name="module-name" value="liblp_test" />
	</test>
	</configuration>

fs_mgr/liblp/super_layout_builder.cpp

0 → 100644

+241 −0

Original line number	Original line	Diff line number	Diff line
			//
			// Copyright (C) 2023 The Android Open Source Project
			//
			// Licensed under the Apache License, Version 2.0 (the "License");
			// you may not use this file except in compliance with the License.
			// You may obtain a copy of the License at
			//
			// http://www.apache.org/licenses/LICENSE-2.0
			//
			// Unless required by applicable law or agreed to in writing, software
			// distributed under the License is distributed on an "AS IS" BASIS,
			// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
			// See the License for the specific language governing permissions and
			// limitations under the License.
			//
			#include <liblp/super_layout_builder.h>

			#include <liblp/liblp.h>

			#include "images.h"
			#include "utility.h"
			#include "writer.h"

			using android::base::borrowed_fd;
			using android::base::unique_fd;

			namespace android {
			namespace fs_mgr {

			bool SuperLayoutBuilder::Open(borrowed_fd fd) {
			auto metadata = ReadFromImageFile(fd.get());
			if (!metadata) {
			return false;
			}
			return Open(*metadata.get());
			}

			bool SuperLayoutBuilder::Open(const void* data, size_t size) {
			auto metadata = ReadFromImageBlob(data, size);
			if (!metadata) {
			return false;
			}
			return Open(*metadata.get());
			}

			bool SuperLayoutBuilder::Open(const LpMetadata& metadata) {
			for (const auto& partition : metadata.partitions) {
			if (partition.attributes & LP_PARTITION_ATTR_SLOT_SUFFIXED) {
			// Retrofit devices are not supported.
			return false;
			}
			if (!(partition.attributes & LP_PARTITION_ATTR_READONLY)) {
			// Writable partitions are not supported.
			return false;
			}
			}
			if (!metadata.extents.empty()) {
			// Partitions that already have extents are not supported (should
			// never be true of super_empty.img).
			return false;
			}
			if (metadata.block_devices.size() != 1) {
			// Only one "super" is supported.
			return false;
			}

			builder_ = MetadataBuilder::New(metadata);
			return !!builder_;
			}

			bool SuperLayoutBuilder::AddPartition(const std::string& partition_name,
			const std::string& image_name, uint64_t partition_size) {
			auto p = builder_->FindPartition(partition_name);
			if (!p) {
			return false;
			}
			if (!builder_->ResizePartition(p, partition_size)) {
			return false;
			}
			image_map_.emplace(partition_name, image_name);
			return true;
			}

			// Fill the space between each extent, if any, with either a fill or dontcare
			// extent. The caller constructs a sample extent to re-use.
			static bool AddGapExtents(std::vector<SuperImageExtent>* extents, SuperImageExtent::Type gap_type) {
			std::vector<SuperImageExtent> old = std::move(*extents);
			std::sort(old.begin(), old.end());

			*extents = {};

			uint64_t current_offset = 0;
			for (const auto& extent : old) {
			// Check for overlapping extents - this would be a serious error.
			if (current_offset > extent.offset) {
			LOG(INFO) << "Overlapping extents detected; cannot layout temporary super image";
			return false;
			}

			if (extent.offset != current_offset) {
			uint64_t gap_size = extent.offset - current_offset;
			extents->emplace_back(current_offset, gap_size, gap_type);
			current_offset = extent.offset;
			}

			extents->emplace_back(extent);
			current_offset += extent.size;
			}
			return true;
			}

			std::vector<SuperImageExtent> SuperLayoutBuilder::GetImageLayout() {
			auto metadata = builder_->Export();
			if (!metadata) {
			return {};
			}

			std::vector<SuperImageExtent> extents;

			// Write the primary and backup copies of geometry.
			std::string geometry_bytes = SerializeGeometry(metadata->geometry);
			auto blob = std::make_shared<std::string>(std::move(geometry_bytes));

			extents.emplace_back(0, GetPrimaryGeometryOffset(), SuperImageExtent::Type::ZERO);
			extents.emplace_back(GetPrimaryGeometryOffset(), blob);
			extents.emplace_back(GetBackupGeometryOffset(), blob);

			// Write the primary and backup copies of each metadata slot. When flashing,
			// all metadata copies are the same, even for different slots.
			std::string metadata_bytes = SerializeMetadata(*metadata.get());

			// Align metadata size to 4KB. This makes the layout easily compatible with
			// libsparse.
			static constexpr size_t kSparseAlignment = 4096;
			size_t metadata_aligned_bytes;
			if (!AlignTo(metadata_bytes.size(), kSparseAlignment, &metadata_aligned_bytes)) {
			LOG(ERROR) << "Unable to align metadata size " << metadata_bytes.size() << " to "
			<< kSparseAlignment;
			return {};
			}
			metadata_bytes.resize(metadata_aligned_bytes, '\0');

			// However, alignment can cause larger-than-supported metadata blocks. Fall
			// back to fastbootd/update-super.
			if (metadata_bytes.size() > metadata->geometry.metadata_max_size) {
			LOG(VERBOSE) << "Aligned metadata size " << metadata_bytes.size()
			<< " is larger than maximum metadata size "
			<< metadata->geometry.metadata_max_size;
			return {};
			}

			blob = std::make_shared<std::string>(std::move(metadata_bytes));
			for (uint32_t i = 0; i < metadata->geometry.metadata_slot_count; i++) {
			int64_t metadata_primary = GetPrimaryMetadataOffset(metadata->geometry, i);
			int64_t metadata_backup = GetBackupMetadataOffset(metadata->geometry, i);
			extents.emplace_back(metadata_primary, blob);
			extents.emplace_back(metadata_backup, blob);
			}

			// Add extents for each partition.
			for (const auto& partition : metadata->partitions) {
			auto partition_name = GetPartitionName(partition);
			auto image_name_iter = image_map_.find(partition_name);
			if (image_name_iter == image_map_.end()) {
			if (partition.num_extents != 0) {
			LOG(ERROR) << "Partition " << partition_name
			<< " has extents but no image filename";
			return {};
			}
			continue;
			}
			const auto& image_name = image_name_iter->second;

			uint64_t image_offset = 0;
			for (uint32_t i = 0; i < partition.num_extents; i++) {
			const auto& e = metadata->extents[partition.first_extent_index + i];

			if (e.target_type != LP_TARGET_TYPE_LINEAR) {
			// Any type other than LINEAR isn't understood here. We don't even
			// bother with ZERO, which is never generated.
			LOG(INFO) << "Unknown extent type from liblp: " << e.target_type;
			return {};
			}

			size_t size = e.num_sectors * LP_SECTOR_SIZE;
			uint64_t super_offset = e.target_data * LP_SECTOR_SIZE;
			extents.emplace_back(super_offset, size, image_name, image_offset);

			image_offset += size;
			}
			}

			if (!AddGapExtents(&extents, SuperImageExtent::Type::DONTCARE)) {
			return {};
			}
			return extents;
			}

			bool SuperImageExtent::operator==(const SuperImageExtent& other) const {
			if (offset != other.offset) {
			return false;
			}
			if (size != other.size) {
			return false;
			}
			if (type != other.type) {
			return false;
			}
			switch (type) {
			case Type::DATA:
			return blob == other.blob;
			case Type::PARTITION:
			return image_name == other.image_name && image_offset == other.image_offset;
			default:
			return true;
			}
			}

			std::ostream& operator<<(std::ostream& stream, const SuperImageExtent& extent) {
			stream << "extent:" << extent.offset << ":" << extent.size << ":";
			switch (extent.type) {
			case SuperImageExtent::Type::DATA:
			stream << "data";
			break;
			case SuperImageExtent::Type::PARTITION:
			stream << "partition:" << extent.image_name << ":" << extent.image_offset;
			break;
			case SuperImageExtent::Type::ZERO:
			stream << "zero";
			break;
			case SuperImageExtent::Type::DONTCARE:
			stream << "dontcare";
			break;
			default:
			stream << "invalid";
			}
			return stream;
			}

			} // namespace fs_mgr
			} // namespace android

fs_mgr/liblp/super_layout_builder_test.cpp

0 → 100644

+141 −0

Original line number	Original line	Diff line number	Diff line
			//
			// Copyright (C) 2023 The Android Open Source Project
			//
			// Licensed under the Apache License, Version 2.0 (the "License");
			// you may not use this file except in compliance with the License.
			// You may obtain a copy of the License at
			//
			// http://www.apache.org/licenses/LICENSE-2.0
			//
			// Unless required by applicable law or agreed to in writing, software
			// distributed under the License is distributed on an "AS IS" BASIS,
			// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
			// See the License for the specific language governing permissions and
			// limitations under the License.
			//

			#include <gmock/gmock.h>
			#include <gtest/gtest.h>
			#include <liblp/builder.h>
			#include <liblp/super_layout_builder.h>
			#include <storage_literals/storage_literals.h>

			#include "images.h"
			#include "writer.h"

			using namespace android::fs_mgr;
			using namespace android::storage_literals;

			TEST(SuperImageTool, Layout) {
			auto builder = MetadataBuilder::New(4_MiB, 8_KiB, 2);
			ASSERT_NE(builder, nullptr);

			Partition* p = builder->AddPartition("system_a", LP_PARTITION_ATTR_READONLY);
			ASSERT_NE(p, nullptr);

			auto metadata = builder->Export();
			ASSERT_NE(metadata, nullptr);

			SuperLayoutBuilder tool;
			ASSERT_TRUE(tool.Open(*metadata.get()));
			ASSERT_TRUE(tool.AddPartition("system_a", "system.img", 16_KiB));

			// Get a copy of the metadata we'd expect if flashing.
			ASSERT_TRUE(builder->ResizePartition(p, 16_KiB));
			metadata = builder->Export();
			ASSERT_NE(metadata, nullptr);

			auto geometry_blob = std::make_shared<std::string>(SerializeGeometry(metadata->geometry));
			auto metadata_blob = std::make_shared<std::string>(SerializeMetadata(*metadata.get()));
			metadata_blob->resize(4_KiB, '\0');

			auto extents = tool.GetImageLayout();
			ASSERT_EQ(extents.size(), 12);
			EXPECT_EQ(extents[0], SuperImageExtent(0, 4096, SuperImageExtent::Type::ZERO));
			EXPECT_EQ(extents[1], SuperImageExtent(4096, geometry_blob));
			EXPECT_EQ(extents[2], SuperImageExtent(8192, geometry_blob));
			EXPECT_EQ(extents[3], SuperImageExtent(12288, metadata_blob));
			EXPECT_EQ(extents[4], SuperImageExtent(16384, 4096, SuperImageExtent::Type::DONTCARE));
			EXPECT_EQ(extents[5], SuperImageExtent(20480, metadata_blob));
			EXPECT_EQ(extents[6], SuperImageExtent(24576, 4096, SuperImageExtent::Type::DONTCARE));
			EXPECT_EQ(extents[7], SuperImageExtent(28672, metadata_blob));
			EXPECT_EQ(extents[8], SuperImageExtent(32768, 4096, SuperImageExtent::Type::DONTCARE));
			EXPECT_EQ(extents[9], SuperImageExtent(36864, metadata_blob));
			EXPECT_EQ(extents[10], SuperImageExtent(40960, 4096, SuperImageExtent::Type::DONTCARE));
			EXPECT_EQ(extents[11], SuperImageExtent(45056, 16384, "system.img", 0));
			}

			TEST(SuperImageTool, NoWritablePartitions) {
			auto builder = MetadataBuilder::New(4_MiB, 8_KiB, 2);
			ASSERT_NE(builder, nullptr);

			Partition* p = builder->AddPartition("system_a", 0);
			ASSERT_NE(p, nullptr);

			auto metadata = builder->Export();
			ASSERT_NE(metadata, nullptr);

			SuperLayoutBuilder tool;
			ASSERT_FALSE(tool.Open(*metadata.get()));
			}

			TEST(SuperImageTool, NoRetrofit) {
			auto builder = MetadataBuilder::New(4_MiB, 8_KiB, 2);
			ASSERT_NE(builder, nullptr);

			Partition* p = builder->AddPartition("system_a", LP_PARTITION_ATTR_READONLY);
			ASSERT_NE(p, nullptr);

			auto metadata = builder->Export();
			ASSERT_NE(metadata, nullptr);

			// Add an extra block device.
			metadata->block_devices.emplace_back(metadata->block_devices[0]);

			SuperLayoutBuilder tool;
			ASSERT_FALSE(tool.Open(*metadata.get()));
			}

			TEST(SuperImageTool, NoRetrofit2) {
			auto builder = MetadataBuilder::New(4_MiB, 8_KiB, 2);
			ASSERT_NE(builder, nullptr);

			Partition* p = builder->AddPartition(
			"system_a", LP_PARTITION_ATTR_READONLY \| LP_PARTITION_ATTR_SLOT_SUFFIXED);
			ASSERT_NE(p, nullptr);

			auto metadata = builder->Export();
			ASSERT_NE(metadata, nullptr);

			SuperLayoutBuilder tool;
			ASSERT_FALSE(tool.Open(*metadata.get()));
			}

			TEST(SuperImageTool, NoFixedPartitions) {
			auto builder = MetadataBuilder::New(4_MiB, 8_KiB, 2);
			ASSERT_NE(builder, nullptr);

			Partition* p = builder->AddPartition("system_a", LP_PARTITION_ATTR_READONLY);
			ASSERT_NE(p, nullptr);
			ASSERT_TRUE(builder->ResizePartition(p, 4_KiB));

			auto metadata = builder->Export();
			ASSERT_NE(metadata, nullptr);

			SuperLayoutBuilder tool;
			ASSERT_FALSE(tool.Open(*metadata.get()));
			}

			TEST(SuperImageTool, LargeAlignedMetadata) {
			auto builder = MetadataBuilder::New(4_MiB, 512, 2);
			ASSERT_NE(builder, nullptr);

			auto metadata = builder->Export();
			ASSERT_NE(metadata, nullptr);

			SuperLayoutBuilder tool;
			ASSERT_TRUE(tool.Open(*metadata.get()));

			auto extents = tool.GetImageLayout();
			ASSERT_TRUE(extents.empty());
			}