Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 55904785 authored by Dennis Shen's avatar Dennis Shen
Browse files

aconfig: update storage read api 

Update storage read api to not find storage file location from a pb
file, instead directly read from /metadata copy. Previously for
package.map and flag.map, we are reading from the respective RO
partition. Now we are reading from /metadata/maps dir. This has a few
advantages:

1, early flag availability, since /metadata can be mounted much earlier
than mainline modules, so it would make mainline flags availabile even
before mainline modules are mounted.

2, we no longer need to read from a pb file to find where package.map
and flag.map are. Thus the read api can be further simplified and
downsized. With this change, we are able to shrink the cc flag read api
lib size from 171k to 120k.

Bug: b/321077378
Test atest -c

Change-Id: Ic9086fe4c49c139a4d1c66a8d472023a88c9dd17
parent 22135a0b

tools/aconfig/aconfig_storage_read_api/Android.bp

+3 −2
Original line number Diff line number Diff line
@@ -9,8 +9,6 @@ rust_defaults { 
    srcs: ["src/lib.rs"],

    rustlibs: [

        "libanyhow",

        "libonce_cell",

        "libtempfile",

        "libmemmap2",

        "libcxx",

        "libthiserror",
@@ -34,6 +32,9 @@ rust_test_host { 
    name: "aconfig_storage_read_api.test",

    test_suites: ["general-tests"],

    defaults: ["aconfig_storage_read_api.defaults"],

    rustlibs: [

        "librand",

    ],

    data: [

        "tests/package.map",

        "tests/flag.map",

tools/aconfig/aconfig_storage_read_api/Cargo.toml

+1 −2
Original line number Diff line number Diff line
@@ -8,10 +8,9 @@ default = ["cargo"] 
cargo = []



[dependencies]

rand = "0.8.5"

anyhow = "1.0.69"

memmap2 = "0.8.0"

once_cell = "1.19.0"

tempfile = "3.9.0"

cxx = "1.0"

thiserror = "1.0.56"

aconfig_storage_file = { path = "../aconfig_storage_file" }

tools/aconfig/aconfig_storage_read_api/src/lib.rs

+37 −47
Original line number Diff line number Diff line
@@ -42,9 +42,6 @@ pub mod flag_value_query; 
pub mod mapped_file;

pub mod package_table_query;



#[cfg(test)]

mod test_utils;



pub use aconfig_storage_file::{AconfigStorageError, FlagValueType, StorageFileType};

pub use flag_table_query::FlagReadContext;

pub use package_table_query::PackageReadContext;
@@ -60,8 +57,8 @@ use memmap2::Mmap; 
use std::fs::File;

use std::io::Read;



/// Storage file location pb file

pub const STORAGE_LOCATION_FILE: &str = "/metadata/aconfig/boot/available_storage_file_records.pb";

/// Storage file location

pub const STORAGE_LOCATION: &str = "/metadata/aconfig";



/// Get read only mapped storage files.

///
@@ -78,7 +75,7 @@ pub unsafe fn get_mapped_storage_file( 
    container: &str,

    file_type: StorageFileType,

) -> Result<Mmap, AconfigStorageError> {

    unsafe { crate::mapped_file::get_mapped_file(STORAGE_LOCATION_FILE, container, file_type) }

    unsafe { crate::mapped_file::get_mapped_file(STORAGE_LOCATION, container, file_type) }

}



/// Get package read context for a specific package.
@@ -394,45 +391,41 @@ pub fn get_storage_file_version_cxx(file_path: &str) -> ffi::VersionNumberQueryC 
mod tests {

    use super::*;

    use crate::mapped_file::get_mapped_file;

    use crate::test_utils::copy_to_temp_file;

    use aconfig_storage_file::protos::storage_record_pb::write_proto_to_temp_file;

    use aconfig_storage_file::{FlagInfoBit, StoredFlagType};

    use tempfile::NamedTempFile;



    fn create_test_storage_files() -> [NamedTempFile; 5] {

        let package_map = copy_to_temp_file("./tests/package.map").unwrap();

        let flag_map = copy_to_temp_file("./tests/flag.map").unwrap();

        let flag_val = copy_to_temp_file("./tests/flag.val").unwrap();

        let flag_info = copy_to_temp_file("./tests/flag.info").unwrap();



        let text_proto = format!(

            r#"

files {{

    version: 0

    container: "mockup"

    package_map: "{}"

    flag_map: "{}"

    flag_val: "{}"

    flag_info: "{}"

    timestamp: 12345

}}

"#,

            package_map.path().display(),

            flag_map.path().display(),

            flag_val.path().display(),

            flag_info.path().display()

        );

        let pb_file = write_proto_to_temp_file(&text_proto).unwrap();

        [package_map, flag_map, flag_val, flag_info, pb_file]

    use rand::Rng;

    use std::fs;



    fn create_test_storage_files() -> String {

        let mut rng = rand::thread_rng();

        let number: u32 = rng.gen();

        let storage_dir = String::from("/tmp/") + &number.to_string();

        if std::fs::metadata(&storage_dir).is_ok() {

            fs::remove_dir_all(&storage_dir).unwrap();

        }

        let maps_dir = storage_dir.clone() + "/maps";

        let boot_dir = storage_dir.clone() + "/boot";

        fs::create_dir(&storage_dir).unwrap();

        fs::create_dir(&maps_dir).unwrap();

        fs::create_dir(&boot_dir).unwrap();



        let package_map = storage_dir.clone() + "/maps/mockup.package.map";

        let flag_map = storage_dir.clone() + "/maps/mockup.flag.map";

        let flag_val = storage_dir.clone() + "/boot/mockup.val";

        let flag_info = storage_dir.clone() + "/boot/mockup.info";

        fs::copy("./tests/package.map", &package_map).unwrap();

        fs::copy("./tests/flag.map", &flag_map).unwrap();

        fs::copy("./tests/flag.val", &flag_val).unwrap();

        fs::copy("./tests/flag.info", &flag_info).unwrap();



        return storage_dir;

    }



    #[test]

    // this test point locks down flag package read context query

    fn test_package_context_query() {

        let [_package_map, _flag_map, _flag_val, _flag_info, pb_file] = create_test_storage_files();

        let pb_file_path = pb_file.path().display().to_string();

        let storage_dir = create_test_storage_files();

        let package_mapped_file = unsafe {

            get_mapped_file(&pb_file_path, "mockup", StorageFileType::PackageMap).unwrap()

            get_mapped_file(&storage_dir, "mockup", StorageFileType::PackageMap).unwrap()

        };



        let package_context =
@@ -460,10 +453,9 @@ files {{ 
    #[test]

    // this test point locks down flag read context query

    fn test_flag_context_query() {

        let [_package_map, _flag_map, _flag_val, _flag_info, pb_file] = create_test_storage_files();

        let pb_file_path = pb_file.path().display().to_string();

        let storage_dir = create_test_storage_files();

        let flag_mapped_file =

            unsafe { get_mapped_file(&pb_file_path, "mockup", StorageFileType::FlagMap).unwrap() };

            unsafe { get_mapped_file(&storage_dir, "mockup", StorageFileType::FlagMap).unwrap() };



        let baseline = vec![

            (0, "enabled_ro", StoredFlagType::ReadOnlyBoolean, 1u16),
@@ -486,10 +478,9 @@ files {{ 
    #[test]

    // this test point locks down flag value query

    fn test_flag_value_query() {

        let [_package_map, _flag_map, _flag_val, _flag_info, pb_file] = create_test_storage_files();

        let pb_file_path = pb_file.path().display().to_string();

        let storage_dir = create_test_storage_files();

        let flag_value_file =

            unsafe { get_mapped_file(&pb_file_path, "mockup", StorageFileType::FlagVal).unwrap() };

            unsafe { get_mapped_file(&storage_dir, "mockup", StorageFileType::FlagVal).unwrap() };

        let baseline: Vec<bool> = vec![false, true, true, false, true, true, true, true];

        for (offset, expected_value) in baseline.into_iter().enumerate() {

            let flag_value = get_boolean_flag_value(&flag_value_file, offset as u32).unwrap();
@@ -500,10 +491,9 @@ files {{ 
    #[test]

    // this test point locks donw flag info query

    fn test_flag_info_query() {

        let [_package_map, _flag_map, _flag_val, _flag_info, pb_file] = create_test_storage_files();

        let pb_file_path = pb_file.path().display().to_string();

        let storage_dir = create_test_storage_files();

        let flag_info_file =

            unsafe { get_mapped_file(&pb_file_path, "mockup", StorageFileType::FlagInfo).unwrap() };

            unsafe { get_mapped_file(&storage_dir, "mockup", StorageFileType::FlagInfo).unwrap() };

        let is_rw: Vec<bool> = vec![true, false, true, true, false, false, false, true];

        for (offset, expected_value) in is_rw.into_iter().enumerate() {

            let attribute =

tools/aconfig/aconfig_storage_read_api/src/mapped_file.rs

+46 −134
Original line number Diff line number Diff line
@@ -14,47 +14,12 @@ 
 * limitations under the License.

 */



use std::fs::File;

use std::io::{BufReader, Read};



use anyhow::anyhow;

use memmap2::Mmap;

use std::fs::File;



use crate::AconfigStorageError::{

    self, FileReadFail, MapFileFail, ProtobufParseFail, StorageFileNotFound,

};

use crate::AconfigStorageError::{self, FileReadFail, MapFileFail, StorageFileNotFound};

use crate::StorageFileType;

use aconfig_storage_file::protos::{

    storage_record_pb::try_from_binary_proto, ProtoStorageFileInfo, ProtoStorageFiles,

};



/// Find where storage files are stored for a particular container

pub fn find_container_storage_location(

    location_pb_file: &str,

    container: &str,

) -> Result<ProtoStorageFileInfo, AconfigStorageError> {

    let file = File::open(location_pb_file).map_err(|errmsg| {

        FileReadFail(anyhow!("Failed to open file {}: {}", location_pb_file, errmsg))

    })?;

    let mut reader = BufReader::new(file);

    let mut bytes = Vec::new();

    reader.read_to_end(&mut bytes).map_err(|errmsg| {

        FileReadFail(anyhow!("Failed to read file {}: {}", location_pb_file, errmsg))

    })?;

    let storage_locations: ProtoStorageFiles = try_from_binary_proto(&bytes).map_err(|errmsg| {

        ProtobufParseFail(anyhow!(

            "Failed to parse storage location pb file {}: {}",

            location_pb_file,

            errmsg

        ))

    })?;

    for location_info in storage_locations.files.iter() {

        if location_info.container() == container {

            return Ok(location_info.clone());

        }

    }

    Err(StorageFileNotFound(anyhow!("Storage file does not exist for {}", container)))

}



/// Get the read only memory mapping of a storage file

///
@@ -82,123 +47,70 @@ unsafe fn map_file(file_path: &str) -> Result<Mmap, AconfigStorageError> { 
/// file after being mapped. Ensure no writes can happen to this file while this

/// mapping stays alive.

pub unsafe fn get_mapped_file(

    location_pb_file: &str,

    storage_dir: &str,

    container: &str,

    file_type: StorageFileType,

) -> Result<Mmap, AconfigStorageError> {

    let files_location = find_container_storage_location(location_pb_file, container)?;

    match file_type {

        StorageFileType::PackageMap => unsafe { map_file(files_location.package_map()) },

        StorageFileType::FlagMap => unsafe { map_file(files_location.flag_map()) },

        StorageFileType::FlagVal => unsafe { map_file(files_location.flag_val()) },

        StorageFileType::FlagInfo => unsafe { map_file(files_location.flag_info()) },

    let storage_file = match file_type {

        StorageFileType::PackageMap => {

            String::from(storage_dir) + "/maps/" + container + ".package.map"

        }

        StorageFileType::FlagMap => String::from(storage_dir) + "/maps/" + container + ".flag.map",

        StorageFileType::FlagVal => String::from(storage_dir) + "/boot/" + container + ".val",

        StorageFileType::FlagInfo => String::from(storage_dir) + "/boot/" + container + ".info",

    };

    if std::fs::metadata(&storage_file).is_err() {

        return Err(StorageFileNotFound(anyhow!("storage file {} does not exist", storage_file)));

    }

    unsafe { map_file(&storage_file) }

}



#[cfg(test)]

mod tests {

    use super::*;

    use crate::test_utils::copy_to_temp_file;

    use aconfig_storage_file::protos::storage_record_pb::write_proto_to_temp_file;

    use tempfile::NamedTempFile;



    #[test]

    fn test_find_storage_file_location() {

        let text_proto = r#"

files {

    version: 0

    container: "system"

    package_map: "/system/etc/package.map"

    flag_map: "/system/etc/flag.map"

    flag_val: "/metadata/aconfig/system.val"

    timestamp: 12345

}

files {

    version: 1

    container: "product"

    package_map: "/product/etc/package.map"

    flag_map: "/product/etc/flag.map"

    flag_val: "/metadata/aconfig/product.val"

    timestamp: 54321

}

"#;

        let file = write_proto_to_temp_file(&text_proto).unwrap();

        let file_full_path = file.path().display().to_string();

        let file_info = find_container_storage_location(&file_full_path, "system").unwrap();

        assert_eq!(file_info.version(), 0);

        assert_eq!(file_info.container(), "system");

        assert_eq!(file_info.package_map(), "/system/etc/package.map");

        assert_eq!(file_info.flag_map(), "/system/etc/flag.map");

        assert_eq!(file_info.flag_val(), "/metadata/aconfig/system.val");

        assert_eq!(file_info.timestamp(), 12345);

    use rand::Rng;

    use std::fs;

    use std::io::Read;



        let file_info = find_container_storage_location(&file_full_path, "product").unwrap();

        assert_eq!(file_info.version(), 1);

        assert_eq!(file_info.container(), "product");

        assert_eq!(file_info.package_map(), "/product/etc/package.map");

        assert_eq!(file_info.flag_map(), "/product/etc/flag.map");

        assert_eq!(file_info.flag_val(), "/metadata/aconfig/product.val");

        assert_eq!(file_info.timestamp(), 54321);



        let err = find_container_storage_location(&file_full_path, "vendor").unwrap_err();

        assert_eq!(

            format!("{:?}", err),

            "StorageFileNotFound(Storage file does not exist for vendor)"

        );

    }



    fn map_and_verify(location_pb_file: &str, file_type: StorageFileType, actual_file: &str) {

    fn map_and_verify(storage_dir: &str, file_type: StorageFileType, actual_file: &str) {

        let mut opened_file = File::open(actual_file).unwrap();

        let mut content = Vec::new();

        opened_file.read_to_end(&mut content).unwrap();



        let mmaped_file =

            unsafe { get_mapped_file(location_pb_file, "system", file_type).unwrap() };

        let mmaped_file = unsafe { get_mapped_file(storage_dir, "mockup", file_type).unwrap() };

        assert_eq!(mmaped_file[..], content[..]);

    }



    fn create_test_storage_files() -> [NamedTempFile; 4] {

        let package_map = copy_to_temp_file("./tests/package.map").unwrap();

        let flag_map = copy_to_temp_file("./tests/flag.map").unwrap();

        let flag_val = copy_to_temp_file("./tests/package.map").unwrap();

    fn create_test_storage_files() -> String {

        let mut rng = rand::thread_rng();

        let number: u32 = rng.gen();

        let storage_dir = String::from("/tmp/") + &number.to_string();

        if std::fs::metadata(&storage_dir).is_ok() {

            fs::remove_dir_all(&storage_dir).unwrap();

        }

        let maps_dir = storage_dir.clone() + "/maps";

        let boot_dir = storage_dir.clone() + "/boot";

        fs::create_dir(&storage_dir).unwrap();

        fs::create_dir(&maps_dir).unwrap();

        fs::create_dir(&boot_dir).unwrap();



        let package_map = storage_dir.clone() + "/maps/mockup.package.map";

        let flag_map = storage_dir.clone() + "/maps/mockup.flag.map";

        let flag_val = storage_dir.clone() + "/boot/mockup.val";

        let flag_info = storage_dir.clone() + "/boot/mockup.info";

        fs::copy("./tests/package.map", &package_map).unwrap();

        fs::copy("./tests/flag.map", &flag_map).unwrap();

        fs::copy("./tests/flag.val", &flag_val).unwrap();

        fs::copy("./tests/flag.info", &flag_info).unwrap();



        let text_proto = format!(

            r#"

files {{

    version: 0

    container: "system"

    package_map: "{}"

    flag_map: "{}"

    flag_val: "{}"

    timestamp: 12345

}}

"#,

            package_map.path().display(),

            flag_map.path().display(),

            flag_val.path().display()

        );

        let pb_file = write_proto_to_temp_file(&text_proto).unwrap();

        [package_map, flag_map, flag_val, pb_file]

        return storage_dir;

    }



    #[test]

    fn test_mapped_file_contents() {

        let [package_map, flag_map, flag_val, pb_file] = create_test_storage_files();

        let pb_file_path = pb_file.path().display().to_string();

        map_and_verify(

            &pb_file_path,

            StorageFileType::PackageMap,

            &package_map.path().display().to_string(),

        );

        map_and_verify(

            &pb_file_path,

            StorageFileType::FlagMap,

            &flag_map.path().display().to_string(),

        );

        map_and_verify(

            &pb_file_path,

            StorageFileType::FlagVal,

            &flag_val.path().display().to_string(),

        );

        let storage_dir = create_test_storage_files();

        map_and_verify(&storage_dir, StorageFileType::PackageMap, "./tests/package.map");

        map_and_verify(&storage_dir, StorageFileType::FlagMap, "./tests/flag.map");

        map_and_verify(&storage_dir, StorageFileType::FlagVal, "./tests/flag.val");

        map_and_verify(&storage_dir, StorageFileType::FlagInfo, "./tests/flag.info");

    }

}

tools/aconfig/aconfig_storage_read_api/src/test_utils.rs

deleted100644 → 0
+0 −26
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2023 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



use anyhow::Result;

use std::fs;

use tempfile::NamedTempFile;



/// Create temp file copy

pub(crate) fn copy_to_temp_file(source_file: &str) -> Result<NamedTempFile> {

    let file = NamedTempFile::new()?;

    fs::copy(source_file, file.path())?;

    Ok(file)

}