Merge changes from topic "read_api" into main (7c30411d) · Commits · e / os / android_build

tools/aconfig/aconfig_storage_read_api/Android.bp

+5 −0

Original line number	Diff line number	Diff line
		@@ -73,4 +73,9 @@ cc_library_static {
		generated_sources: ["libcxx_aconfig_storage_read_api_bridge_code"],
		whole_static_libs: ["libaconfig_storage_read_api_cxx_bridge"],
		export_include_dirs: ["include"],
		static_libs: [
		"libaconfig_storage_protos_cc",
		"libprotobuf-cpp-lite",
		"libbase",
		],
		}

tools/aconfig/aconfig_storage_read_api/aconfig_storage_read_api.cpp

+147 −66

Original line number	Diff line number	Diff line
		#include "aconfig_storage/aconfig_storage_read_api.hpp"
		#include <android-base/file.h>
		#include <android-base/logging.h>
		#include <android-base/result.h>
		#include <protos/aconfig_storage_metadata.pb.h>

		#include <sys/mman.h>
		#include <sys/stat.h>
		#include <fcntl.h>

		#include "rust/cxx.h"
		#include "aconfig_storage/lib.rs.h"
		#include "aconfig_storage/aconfig_storage_read_api.hpp"

		using storage_records_pb = android::aconfig_storage_metadata::storage_files;
		using storage_record_pb = android::aconfig_storage_metadata::storage_file_info;
		using namespace android::base;

		namespace aconfig_storage {

		/// Storage location pb file
		static constexpr char kAvailableStorageRecordsPb[] =
		"/metadata/aconfig/available_storage_file_records.pb";

		/// Read aconfig storage records pb file
		static Result<storage_records_pb> read_storage_records_pb(std::string const& pb_file) {
		auto records = storage_records_pb();
		auto content = std::string();
		if (!ReadFileToString(pb_file, &content)) {
		return ErrnoError() << "ReadFileToString failed";
		}

		if (!records.ParseFromString(content)) {
		return ErrnoError() << "Unable to parse persistent storage records protobuf";
		}
		return records;
		}

		/// Get storage file path
		static Result<std::string> find_storage_file(
		std::string const& pb_file,
		std::string const& container,
		StorageFileType file_type) {
		auto records_pb = read_storage_records_pb(pb_file);
		if (!records_pb.ok()) {
		return Error() << "Unable to read storage records from " << pb_file
		<< " : " << records_pb.error();
		}

		for (auto& entry : records_pb->files()) {
		if (entry.container() == container) {
		switch(file_type) {
		case StorageFileType::package_map:
		return entry.package_map();
		case StorageFileType::flag_map:
		return entry.flag_map();
		case StorageFileType::flag_val:
		return entry.flag_val();
		default:
		return Error() << "Invalid file type " << file_type;
		}
		}
		}

		return Error() << "Unable to find storage files for container " << container;;
		}

		/// Map a storage file
		static Result<MappedStorageFile> map_storage_file(std::string const& file) {
		int fd = open(file.c_str(), O_CLOEXEC \| O_NOFOLLOW \| O_RDONLY);
		if (fd == -1) {
		return Error() << "failed to open " << file;
		};

		struct stat fd_stat;
		if (fstat(fd, &fd_stat) < 0) {
		return Error() << "fstat failed";
		}

		if ((fd_stat.st_mode & (S_IWUSR \| S_IWGRP \| S_IWOTH)) != 0) {
		return Error() << "cannot map writeable file";
		}

		size_t file_size = fd_stat.st_size;

		void* const map_result = mmap(nullptr, file_size, PROT_READ, MAP_SHARED, fd, 0);
		if (map_result == MAP_FAILED) {
		return Error() << "mmap failed";
		}

		auto mapped_file = MappedStorageFile();
		mapped_file.file_ptr = map_result;
		mapped_file.file_size = file_size;

		return mapped_file;
		}

		namespace private_internal_api {

		/// Get mapped file implementation.
		MappedStorageFileQuery get_mapped_file_impl(
		std::string const& pb_file,
		std::string const& container,
		StorageFileType file_type) {
		auto query = MappedStorageFileQuery();

		auto file_result = find_storage_file(pb_file, container, file_type);
		if (!file_result.ok()) {
		query.query_success = false;
		query.error_message = file_result.error().message();
		query.mapped_file.file_ptr = nullptr;
		query.mapped_file.file_size = 0;
		return query;
		}

		auto mapped_file_result = map_storage_file(*file_result);
		if (!mapped_file_result.ok()) {
		query.query_success = false;
		query.error_message = mapped_file_result.error().message();
		query.mapped_file.file_ptr = nullptr;
		query.mapped_file.file_size = 0;
		} else {
		query.query_success = true;
		query.error_message = "";
		query.mapped_file = *mapped_file_result;
		}

		return query;
		}

		} // namespace private internal api

		/// Get mapped storage file
		MappedStorageFileQuery get_mapped_file(
		std::string const& container,
		StorageFileType file_type) {
		return private_internal_api::get_mapped_file_impl(
		kAvailableStorageRecordsPb, container, file_type);
		}

		/// Get storage file version number
		VersionNumberQuery get_storage_file_version(
		std::string const& file_path) {
		@@ -18,11 +151,11 @@ VersionNumberQuery get_storage_file_version(

		/// Get package offset
		PackageOffsetQuery get_package_offset(
		std::string const& container,
		MappedStorageFile const& file,
		std::string const& package) {
		auto offset_cxx = get_package_offset_cxx(
		rust::Str(container.c_str()),
		rust::Str(package.c_str()));
		auto content = rust::Slice<const uint8_t>(
		static_cast<uint8_t*>(file.file_ptr), file.file_size);
		auto offset_cxx = get_package_offset_cxx(content, rust::Str(package.c_str()));
		auto offset = PackageOffsetQuery();
		offset.query_success = offset_cxx.query_success;
		offset.error_message = std::string(offset_cxx.error_message.c_str());
		@@ -34,13 +167,12 @@ PackageOffsetQuery get_package_offset(

		/// Get flag offset
		FlagOffsetQuery get_flag_offset(
		std::string const& container,
		MappedStorageFile const& file,
		uint32_t package_id,
		std::string const& flag_name){
		auto offset_cxx = get_flag_offset_cxx(
		rust::Str(container.c_str()),
		package_id,
		rust::Str(flag_name.c_str()));
		auto content = rust::Slice<const uint8_t>(
		static_cast<uint8_t*>(file.file_ptr), file.file_size);
		auto offset_cxx = get_flag_offset_cxx(content, package_id, rust::Str(flag_name.c_str()));
		auto offset = FlagOffsetQuery();
		offset.query_success = offset_cxx.query_success;
		offset.error_message = std::string(offset_cxx.error_message.c_str());
		@@ -51,11 +183,11 @@ FlagOffsetQuery get_flag_offset(

		/// Get boolean flag value
		BooleanFlagValueQuery get_boolean_flag_value(
		std::string const& container,
		MappedStorageFile const& file,
		uint32_t offset) {
		auto value_cxx = get_boolean_flag_value_cxx(
		rust::Str(container.c_str()),
		offset);
		auto content = rust::Slice<const uint8_t>(
		static_cast<uint8_t*>(file.file_ptr), file.file_size);
		auto value_cxx = get_boolean_flag_value_cxx(content, offset);
		auto value = BooleanFlagValueQuery();
		value.query_success = value_cxx.query_success;
		value.error_message = std::string(value_cxx.error_message.c_str());
		@@ -63,55 +195,4 @@ BooleanFlagValueQuery get_boolean_flag_value(
		return value;
		}

		namespace test_only_api {
		PackageOffsetQuery get_package_offset_impl(
		std::string const& pb_file,
		std::string const& container,
		std::string const& package) {
		auto offset_cxx = get_package_offset_cxx_impl(
		rust::Str(pb_file.c_str()),
		rust::Str(container.c_str()),
		rust::Str(package.c_str()));
		auto offset = PackageOffsetQuery();
		offset.query_success = offset_cxx.query_success;
		offset.error_message = std::string(offset_cxx.error_message.c_str());
		offset.package_exists = offset_cxx.package_exists;
		offset.package_id = offset_cxx.package_id;
		offset.boolean_offset = offset_cxx.boolean_offset;
		return offset;
		}

		FlagOffsetQuery get_flag_offset_impl(
		std::string const& pb_file,
		std::string const& container,
		uint32_t package_id,
		std::string const& flag_name) {
		auto offset_cxx = get_flag_offset_cxx_impl(
		rust::Str(pb_file.c_str()),
		rust::Str(container.c_str()),
		package_id,
		rust::Str(flag_name.c_str()));
		auto offset = FlagOffsetQuery();
		offset.query_success = offset_cxx.query_success;
		offset.error_message = std::string(offset_cxx.error_message.c_str());
		offset.flag_exists = offset_cxx.flag_exists;
		offset.flag_offset = offset_cxx.flag_offset;
		return offset;
		}

		BooleanFlagValueQuery get_boolean_flag_value_impl(
		std::string const& pb_file,
		std::string const& container,
		uint32_t offset) {
		auto value_cxx = get_boolean_flag_value_cxx_impl(
		rust::Str(pb_file.c_str()),
		rust::Str(container.c_str()),
		offset);
		auto value = BooleanFlagValueQuery();
		value.query_success = value_cxx.query_success;
		value.error_message = std::string(value_cxx.error_message.c_str());
		value.flag_value = value_cxx.flag_value;
		return value;
		}
		} // namespace test_only_api
		} // namespace aconfig_storage

tools/aconfig/aconfig_storage_read_api/include/aconfig_storage/aconfig_storage_read_api.hpp

+44 −24

Original line number	Diff line number	Diff line
		@@ -5,6 +5,36 @@

		namespace aconfig_storage {

		/// Storage file type enum
		enum StorageFileType {
		package_map,
		flag_map,
		flag_val
		};

		/// Mapped storage file
		struct MappedStorageFile {
		void* file_ptr;
		size_t file_size;
		};

		/// Mapped storage file query
		struct MappedStorageFileQuery {
		bool query_success;
		std::string error_message;
		MappedStorageFile mapped_file;
		};

		/// DO NOT USE APIS IN THE FOLLOWING NAMESPACE DIRECTLY
		namespace private_internal_api {

		MappedStorageFileQuery get_mapped_file_impl(
		std::string const& pb_file,
		std::string const& container,
		StorageFileType file_type);

		} // namespace private_internal_api

		/// Storage version number query result
		struct VersionNumberQuery {
		bool query_success;
		@@ -36,6 +66,14 @@ struct BooleanFlagValueQuery {
		bool flag_value;
		};

		/// Get mapped storage file
		/// \input container: stoarge container name
		/// \input file_type: storage file type enum
		/// \returns a MappedStorageFileQuery
		MappedStorageFileQuery get_mapped_file(
		std::string const& container,
		StorageFileType file_type);

		/// Get storage file version number
		/// \input file_path: the path to the storage file
		/// \returns a VersionNumberQuery
		@@ -43,47 +81,29 @@ VersionNumberQuery get_storage_file_version(
		std::string const& file_path);

		/// Get package offset
		/// \input container: the flag container name
		/// \input file: mapped storage file
		/// \input package: the flag package name
		/// \returns a PackageOffsetQuery
		PackageOffsetQuery get_package_offset(
		std::string const& container,
		MappedStorageFile const& file,
		std::string const& package);

		/// Get flag offset
		/// \input container: the flag container name
		/// \input file: mapped storage file
		/// \input package_id: the flag package id obtained from package offset query
		/// \input flag_name: flag name
		/// \returns a FlagOffsetQuery
		FlagOffsetQuery get_flag_offset(
		std::string const& container,
		MappedStorageFile const& file,
		uint32_t package_id,
		std::string const& flag_name);

		/// Get boolean flag value
		/// \input container: the flag container name
		/// \input file: mapped storage file
		/// \input offset: the boolean flag value byte offset in the file
		/// \returns a BooleanFlagValueQuery
		BooleanFlagValueQuery get_boolean_flag_value(
		std::string const& container,
		MappedStorageFile const& file,
		uint32_t offset);

		/// DO NOT USE APIS IN THE FOLLOWING NAMESPACE, TEST ONLY
		namespace test_only_api {
		PackageOffsetQuery get_package_offset_impl(
		std::string const& pb_file,
		std::string const& container,
		std::string const& package);

		FlagOffsetQuery get_flag_offset_impl(
		std::string const& pb_file,
		std::string const& container,
		uint32_t package_id,
		std::string const& flag_name);

		BooleanFlagValueQuery get_boolean_flag_value_impl(
		std::string const& pb_file,
		std::string const& container,
		uint32_t offset);
		} // namespace test_only_api
		} // namespace aconfig_storage

tools/aconfig/aconfig_storage_read_api/src/lib.rs

+105 −202

File changed.

Preview size limit exceeded, changes collapsed.

tools/aconfig/aconfig_storage_read_api/src/mapped_file.rs

+50 −128

Original line number	Diff line number	Diff line
		@@ -14,14 +14,11 @@
		* limitations under the License.
		*/

		use std::collections::HashMap;
		use std::fs::{self, File};
		use std::io::{BufReader, Read};
		use std::sync::{Arc, Mutex};

		use anyhow::anyhow;
		use memmap2::Mmap;
		use once_cell::sync::Lazy;

		use crate::AconfigStorageError::{
		self, FileReadFail, MapFileFail, ProtobufParseFail, StorageFileNotFound,
		@@ -31,20 +28,6 @@ use aconfig_storage_file::protos::{
		storage_record_pb::try_from_binary_proto, ProtoStorageFileInfo, ProtoStorageFiles,
		};

		/// Cache for already mapped files
		static ALL_MAPPED_FILES: Lazy<Mutex<HashMap<String, MappedStorageFileSet>>> = Lazy::new(\|\| {
		let mapped_files = HashMap::new();
		Mutex::new(mapped_files)
		});

		/// Mapped storage files for a particular container
		#[derive(Debug)]
		struct MappedStorageFileSet {
		package_map: Arc<Mmap>,
		flag_map: Arc<Mmap>,
		flag_val: Arc<Mmap>,
		}

		/// Find where storage files are stored for a particular container
		fn find_container_storage_location(
		location_pb_file: &str,
		@@ -101,49 +84,26 @@ fn verify_read_only_and_map(file_path: &str) -> Result<Mmap, AconfigStorageError
		}
		}

		/// Map all storage files for a particular container
		fn map_container_storage_files(
		location_pb_file: &str,
		container: &str,
		) -> Result<MappedStorageFileSet, AconfigStorageError> {
		let files_location = find_container_storage_location(location_pb_file, container)?;
		let package_map = Arc::new(verify_read_only_and_map(files_location.package_map())?);
		let flag_map = Arc::new(verify_read_only_and_map(files_location.flag_map())?);
		let flag_val = Arc::new(verify_read_only_and_map(files_location.flag_val())?);
		Ok(MappedStorageFileSet { package_map, flag_map, flag_val })
		}

		/// Get a mapped storage file given the container and file type
		pub(crate) fn get_mapped_file(
		pub fn get_mapped_file(
		location_pb_file: &str,
		container: &str,
		file_selection: StorageFileType,
		) -> Result<Arc<Mmap>, AconfigStorageError> {
		let mut all_mapped_files = ALL_MAPPED_FILES.lock().unwrap();
		match all_mapped_files.get(container) {
		Some(mapped_files) => Ok(match file_selection {
		StorageFileType::PackageMap => Arc::clone(&mapped_files.package_map),
		StorageFileType::FlagMap => Arc::clone(&mapped_files.flag_map),
		StorageFileType::FlagVal => Arc::clone(&mapped_files.flag_val),
		}),
		None => {
		let mapped_files = map_container_storage_files(location_pb_file, container)?;
		let file_ptr = match file_selection {
		StorageFileType::PackageMap => Arc::clone(&mapped_files.package_map),
		StorageFileType::FlagMap => Arc::clone(&mapped_files.flag_map),
		StorageFileType::FlagVal => Arc::clone(&mapped_files.flag_val),
		};
		all_mapped_files.insert(container.to_string(), mapped_files);
		Ok(file_ptr)
		}
		file_type: StorageFileType,
		) -> Result<Mmap, AconfigStorageError> {
		let files_location = find_container_storage_location(location_pb_file, container)?;
		match file_type {
		StorageFileType::PackageMap => verify_read_only_and_map(files_location.package_map()),
		StorageFileType::FlagMap => verify_read_only_and_map(files_location.flag_map()),
		StorageFileType::FlagVal => verify_read_only_and_map(files_location.flag_val()),
		}
		}

		#[cfg(test)]
		mod tests {
		use super::*;
		use crate::test_utils::TestStorageFileSet;
		use crate::test_utils::copy_to_temp_file;
		use aconfig_storage_file::protos::storage_record_pb::write_proto_to_temp_file;
		use tempfile::NamedTempFile;

		#[test]
		fn test_find_storage_file_location() {
		@@ -190,28 +150,20 @@ files {
		);
		}

		fn map_and_verify(location_pb_file: &str, file_selection: StorageFileType, actual_file: &str) {
		fn map_and_verify(location_pb_file: &str, file_type: StorageFileType, actual_file: &str) {
		let mut opened_file = File::open(actual_file).unwrap();
		let mut content = Vec::new();
		opened_file.read_to_end(&mut content).unwrap();

		let mmaped_file = get_mapped_file(location_pb_file, "system", file_selection).unwrap();
		let mmaped_file = get_mapped_file(location_pb_file, "system", file_type).unwrap();
		assert_eq!(mmaped_file[..], content[..]);
		}

		fn create_test_storage_files(read_only: bool) -> TestStorageFileSet {
		TestStorageFileSet::new(
		"./tests/package.map",
		"./tests/flag.map",
		"./tests/flag.val",
		read_only,
		)
		.unwrap()
		}
		fn create_test_storage_files(read_only: bool) -> [NamedTempFile; 4] {
		let package_map = copy_to_temp_file("./tests/package.map", read_only).unwrap();
		let flag_map = copy_to_temp_file("./tests/flag.map", read_only).unwrap();
		let flag_val = copy_to_temp_file("./tests/package.map", read_only).unwrap();

		#[test]
		fn test_mapped_file_contents() {
		let ro_files = create_test_storage_files(true);
		let text_proto = format!(
		r#"
		files {{
		@@ -223,92 +175,62 @@ files {{
		timestamp: 12345
		}}
		"#,
		ro_files.package_map.name, ro_files.flag_map.name, ro_files.flag_val.name
		package_map.path().display(),
		flag_map.path().display(),
		flag_val.path().display()
		);

		let file = write_proto_to_temp_file(&text_proto).unwrap();
		let file_full_path = file.path().display().to_string();
		map_and_verify(&file_full_path, StorageFileType::PackageMap, &ro_files.package_map.name);
		map_and_verify(&file_full_path, StorageFileType::FlagMap, &ro_files.flag_map.name);
		map_and_verify(&file_full_path, StorageFileType::FlagVal, &ro_files.flag_val.name);
		let pb_file = write_proto_to_temp_file(&text_proto).unwrap();
		[package_map, flag_map, flag_val, pb_file]
		}

		#[test]
		fn test_map_non_read_only_file() {
		let ro_files = create_test_storage_files(true);
		let rw_files = create_test_storage_files(false);
		let text_proto = format!(
		r#"
		files {{
		version: 0
		container: "system"
		package_map: "{}"
		flag_map: "{}"
		flag_val: "{}"
		timestamp: 12345
		}}
		"#,
		rw_files.package_map.name, ro_files.flag_map.name, ro_files.flag_val.name
		fn test_mapped_file_contents() {
		let [package_map, flag_map, flag_val, pb_file] = create_test_storage_files(true);
		let pb_file_path = pb_file.path().display().to_string();
		map_and_verify(
		&pb_file_path,
		StorageFileType::PackageMap,
		&package_map.path().display().to_string(),
		);
		map_and_verify(
		&pb_file_path,
		StorageFileType::FlagMap,
		&flag_map.path().display().to_string(),
		);
		map_and_verify(
		&pb_file_path,
		StorageFileType::FlagVal,
		&flag_val.path().display().to_string(),
		);
		}

		let file = write_proto_to_temp_file(&text_proto).unwrap();
		let file_full_path = file.path().display().to_string();
		let error = map_container_storage_files(&file_full_path, "system").unwrap_err();
		#[test]
		fn test_map_non_read_only_file() {
		let [package_map, flag_map, flag_val, pb_file] = create_test_storage_files(false);
		let pb_file_path = pb_file.path().display().to_string();
		let error =
		get_mapped_file(&pb_file_path, "system", StorageFileType::PackageMap).unwrap_err();
		assert_eq!(
		format!("{:?}", error),
		format!(
		"MapFileFail(fail to map non read only storage file {})",
		rw_files.package_map.name
		package_map.path().display()
		)
		);

		let text_proto = format!(
		r#"
		files {{
		version: 0
		container: "system"
		package_map: "{}"
		flag_map: "{}"
		flag_val: "{}"
		timestamp: 12345
		}}
		"#,
		ro_files.package_map.name, rw_files.flag_map.name, ro_files.flag_val.name
		);

		let file = write_proto_to_temp_file(&text_proto).unwrap();
		let file_full_path = file.path().display().to_string();
		let error = map_container_storage_files(&file_full_path, "system").unwrap_err();
		let error = get_mapped_file(&pb_file_path, "system", StorageFileType::FlagMap).unwrap_err();
		assert_eq!(
		format!("{:?}", error),
		format!(
		"MapFileFail(fail to map non read only storage file {})",
		rw_files.flag_map.name
		flag_map.path().display()
		)
		);

		let text_proto = format!(
		r#"
		files {{
		version: 0
		container: "system"
		package_map: "{}"
		flag_map: "{}"
		flag_val: "{}"
		timestamp: 12345
		}}
		"#,
		ro_files.package_map.name, ro_files.flag_map.name, rw_files.flag_val.name
		);

		let file = write_proto_to_temp_file(&text_proto).unwrap();
		let file_full_path = file.path().display().to_string();
		let error = map_container_storage_files(&file_full_path, "system").unwrap_err();
		let error = get_mapped_file(&pb_file_path, "system", StorageFileType::FlagVal).unwrap_err();
		assert_eq!(
		format!("{:?}", error),
		format!(
		"MapFileFail(fail to map non read only storage file {})",
		rw_files.flag_val.name
		flag_val.path().display()
		)
		);
		}