Merge "aconfig: package.map file serialization" into main (4575b273) · Commits · e / os / android_build

tools/aconfig/src/storage/mod.rs

+58 −6

Original line number	Diff line number	Diff line
		@@ -14,11 +14,41 @@
		* limitations under the License.
		*/

		use anyhow::Result;
		use std::collections::{HashMap, HashSet};
		pub mod package_table;

		use anyhow::{anyhow, Result};
		use std::collections::{hash_map::DefaultHasher, HashMap, HashSet};
		use std::hash::{Hash, Hasher};
		use std::path::PathBuf;

		use crate::commands::OutputFile;
		use crate::protos::{ProtoParsedFlag, ProtoParsedFlags};
		use crate::storage::package_table::PackageTable;

		pub const FILE_VERSION: u32 = 1;

		pub const HASH_PRIMES: [u32; 29] = [
		7, 13, 29, 53, 97, 193, 389, 769, 1543, 3079, 6151, 12289, 24593, 49157, 98317, 196613, 393241,
		786433, 1572869, 3145739, 6291469, 12582917, 25165843, 50331653, 100663319, 201326611,
		402653189, 805306457, 1610612741,
		];

		/// Get the right hash table size given number of entries in the table. Use a
		/// load factor of 0.5 for performance.
		pub fn get_table_size(entries: u32) -> Result<u32> {
		HASH_PRIMES
		.iter()
		.find(\|&&num\| num >= 2 * entries)
		.copied()
		.ok_or(anyhow!("Number of packages is too large"))
		}

		/// Get the corresponding bucket index given the key and number of buckets
		pub fn get_bucket_index<T: Hash>(val: &T, num_buckets: u32) -> u32 {
		let mut s = DefaultHasher::new();
		val.hash(&mut s);
		(s.finish() % num_buckets as u64) as u32
		}

		pub struct FlagPackage<'a> {
		pub package_name: &'a str,
		@@ -52,7 +82,7 @@ where
		{
		// group flags by package
		let mut packages: Vec<FlagPackage<'a>> = Vec::new();
		let mut package_index: HashMap<&'a str, usize> = HashMap::new();
		let mut package_index: HashMap<&str, usize> = HashMap::new();
		for parsed_flags in parsed_flags_vec_iter {
		for parsed_flag in parsed_flags.parsed_flag.iter() {
		let index = *(package_index.entry(parsed_flag.package()).or_insert(packages.len()));
		@@ -76,14 +106,21 @@ where
		}

		pub fn generate_storage_files<'a, I>(
		_containser: &str,
		container: &str,
		parsed_flags_vec_iter: I,
		) -> Result<Vec<OutputFile>>
		where
		I: Iterator<Item = &'a ProtoParsedFlags>,
		{
		let _packages = group_flags_by_package(parsed_flags_vec_iter);
		Ok(vec![])
		let packages = group_flags_by_package(parsed_flags_vec_iter);

		// create and serialize package map
		let package_table = PackageTable::new(container, &packages)?;
		let package_table_file_path = PathBuf::from("package.map");
		let package_table_file =
		OutputFile { contents: package_table.as_bytes(), path: package_table_file_path };

		Ok(vec![package_table_file])
		}

		#[cfg(test)]
		@@ -91,6 +128,21 @@ mod tests {
		use super::*;
		use crate::Input;

		/// Read and parse bytes as u32
		pub fn read_u32_from_bytes(buf: &[u8], head: &mut usize) -> Result<u32> {
		let val = u32::from_le_bytes(buf[head..head + 4].try_into()?);
		*head += 4;
		Ok(val)
		}

		/// Read and parse bytes as string
		pub fn read_str_from_bytes(buf: &[u8], head: &mut usize) -> Result<String> {
		let num_bytes = read_u32_from_bytes(buf, head)? as usize;
		let val = String::from_utf8(buf[head..head + num_bytes].to_vec())?;
		*head += num_bytes;
		Ok(val)
		}

		pub fn parse_all_test_flags() -> Vec<ProtoParsedFlags> {
		let aconfig_files = [
		(

tools/aconfig/src/storage/package_table.rs

0 → 100644

+259 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2023 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		use crate::storage::{self, FlagPackage};
		use anyhow::Result;

		#[derive(PartialEq, Debug)]
		pub struct PackageTableHeader {
		pub version: u32,
		pub container: String,
		pub file_size: u32,
		pub num_packages: u32,
		pub bucket_offset: u32,
		pub node_offset: u32,
		}

		impl PackageTableHeader {
		fn new(container: &str, num_packages: u32) -> Self {
		Self {
		version: storage::FILE_VERSION,
		container: String::from(container),
		file_size: 0,
		num_packages,
		bucket_offset: 0,
		node_offset: 0,
		}
		}

		fn as_bytes(&self) -> Vec<u8> {
		let mut result = Vec::new();
		result.extend_from_slice(&self.version.to_le_bytes());
		let container_bytes = self.container.as_bytes();
		result.extend_from_slice(&(container_bytes.len() as u32).to_le_bytes());
		result.extend_from_slice(container_bytes);
		result.extend_from_slice(&self.file_size.to_le_bytes());
		result.extend_from_slice(&self.num_packages.to_le_bytes());
		result.extend_from_slice(&self.bucket_offset.to_le_bytes());
		result.extend_from_slice(&self.node_offset.to_le_bytes());
		result
		}
		}

		#[derive(PartialEq, Debug)]
		pub struct PackageTableNode {
		pub package_name: String,
		pub package_id: u32,
		pub boolean_offset: u32,
		pub next_offset: Option<u32>,
		pub bucket_index: u32,
		}

		impl PackageTableNode {
		fn new(package: &FlagPackage, num_buckets: u32) -> Self {
		let bucket_index =
		storage::get_bucket_index(&package.package_name.to_string(), num_buckets);
		Self {
		package_name: String::from(package.package_name),
		package_id: package.package_id,
		boolean_offset: package.boolean_offset,
		next_offset: None,
		bucket_index,
		}
		}

		fn as_bytes(&self) -> Vec<u8> {
		let mut result = Vec::new();
		let name_bytes = self.package_name.as_bytes();
		result.extend_from_slice(&(name_bytes.len() as u32).to_le_bytes());
		result.extend_from_slice(name_bytes);
		result.extend_from_slice(&self.package_id.to_le_bytes());
		result.extend_from_slice(&self.boolean_offset.to_le_bytes());
		result.extend_from_slice(&self.next_offset.unwrap_or(0).to_le_bytes());
		result
		}
		}

		pub struct PackageTable {
		pub header: PackageTableHeader,
		pub buckets: Vec<Option<u32>>,
		pub nodes: Vec<PackageTableNode>,
		}

		impl PackageTable {
		pub fn new(container: &str, packages: &[FlagPackage]) -> Result<Self> {
		// create table
		let num_packages = packages.len() as u32;
		let num_buckets = storage::get_table_size(num_packages)?;
		let mut table = Self {
		header: PackageTableHeader::new(container, num_packages),
		buckets: vec![None; num_buckets as usize],
		nodes: packages.iter().map(\|pkg\| PackageTableNode::new(pkg, num_buckets)).collect(),
		};

		// sort nodes by bucket index for efficiency
		table.nodes.sort_by(\|a, b\| a.bucket_index.cmp(&b.bucket_index));

		// fill all node offset
		let mut offset = 0;
		for i in 0..table.nodes.len() {
		let node_bucket_idx = table.nodes[i].bucket_index;
		let next_node_bucket_idx = if i + 1 < table.nodes.len() {
		Some(table.nodes[i + 1].bucket_index)
		} else {
		None
		};

		if table.buckets[node_bucket_idx as usize].is_none() {
		table.buckets[node_bucket_idx as usize] = Some(offset);
		}
		offset += table.nodes[i].as_bytes().len() as u32;

		if let Some(index) = next_node_bucket_idx {
		if index == node_bucket_idx {
		table.nodes[i].next_offset = Some(offset);
		}
		}
		}

		// fill table region offset
		table.header.bucket_offset = table.header.as_bytes().len() as u32;
		table.header.node_offset = table.header.bucket_offset + num_buckets * 4;
		table.header.file_size = table.header.node_offset
		+ table.nodes.iter().map(\|x\| x.as_bytes().len()).sum::<usize>() as u32;

		Ok(table)
		}

		pub fn as_bytes(&self) -> Vec<u8> {
		[
		self.header.as_bytes(),
		self.buckets.iter().map(\|v\| v.unwrap_or(0).to_le_bytes()).collect::<Vec<_>>().concat(),
		self.nodes.iter().map(\|v\| v.as_bytes()).collect::<Vec<_>>().concat(),
		]
		.concat()
		}
		}

		#[cfg(test)]
		mod tests {
		use super::*;
		use crate::storage::{
		group_flags_by_package, tests::parse_all_test_flags, tests::read_str_from_bytes,
		tests::read_u32_from_bytes,
		};

		impl PackageTableHeader {
		// test only method to deserialize back into the header struct
		fn from_bytes(bytes: &[u8]) -> Result<Self> {
		let mut head = 0;
		Ok(Self {
		version: read_u32_from_bytes(bytes, &mut head)?,
		container: read_str_from_bytes(bytes, &mut head)?,
		file_size: read_u32_from_bytes(bytes, &mut head)?,
		num_packages: read_u32_from_bytes(bytes, &mut head)?,
		bucket_offset: read_u32_from_bytes(bytes, &mut head)?,
		node_offset: read_u32_from_bytes(bytes, &mut head)?,
		})
		}
		}

		impl PackageTableNode {
		// test only method to deserialize back into the node struct
		fn from_bytes(bytes: &[u8], num_buckets: u32) -> Result<Self> {
		let mut head = 0;
		let mut node = Self {
		package_name: read_str_from_bytes(bytes, &mut head)?,
		package_id: read_u32_from_bytes(bytes, &mut head)?,
		boolean_offset: read_u32_from_bytes(bytes, &mut head)?,
		next_offset: match read_u32_from_bytes(bytes, &mut head)? {
		0 => None,
		val => Some(val),
		},
		bucket_index: 0,
		};
		node.bucket_index = storage::get_bucket_index(&node.package_name, num_buckets);
		Ok(node)
		}
		}

		pub fn create_test_package_table() -> Result<PackageTable> {
		let caches = parse_all_test_flags();
		let packages = group_flags_by_package(caches.iter());
		PackageTable::new("system", &packages)
		}

		#[test]
		// this test point locks down the table creation and each field
		fn test_table_contents() {
		let package_table = create_test_package_table();
		assert!(package_table.is_ok());

		let header: &PackageTableHeader = &package_table.as_ref().unwrap().header;
		let expected_header = PackageTableHeader {
		version: storage::FILE_VERSION,
		container: String::from("system"),
		file_size: 158,
		num_packages: 2,
		bucket_offset: 30,
		node_offset: 58,
		};
		assert_eq!(header, &expected_header);

		let buckets: &Vec<Option<u32>> = &package_table.as_ref().unwrap().buckets;
		let expected: Vec<Option<u32>> = vec![Some(0), None, None, Some(50), None, None, None];
		assert_eq!(buckets, &expected);

		let nodes: &Vec<PackageTableNode> = &package_table.as_ref().unwrap().nodes;
		assert_eq!(nodes.len(), 2);
		let first_node_expected = PackageTableNode {
		package_name: String::from("com.android.aconfig.storage.test_2"),
		package_id: 1,
		boolean_offset: 10,
		next_offset: None,
		bucket_index: 0,
		};
		assert_eq!(nodes[0], first_node_expected);
		let second_node_expected = PackageTableNode {
		package_name: String::from("com.android.aconfig.storage.test_1"),
		package_id: 0,
		boolean_offset: 0,
		next_offset: None,
		bucket_index: 3,
		};
		assert_eq!(nodes[1], second_node_expected);
		}

		#[test]
		// this test point locks down the table serialization
		fn test_serialization() {
		let package_table = create_test_package_table();
		assert!(package_table.is_ok());

		let header: &PackageTableHeader = &package_table.as_ref().unwrap().header;
		let reinterpreted_header = PackageTableHeader::from_bytes(&header.as_bytes());
		assert!(reinterpreted_header.is_ok());
		assert_eq!(header, &reinterpreted_header.unwrap());

		let nodes: &Vec<PackageTableNode> = &package_table.as_ref().unwrap().nodes;
		let num_buckets = storage::get_table_size(header.num_packages).unwrap();
		for node in nodes.iter() {
		let reinterpreted_node = PackageTableNode::from_bytes(&node.as_bytes(), num_buckets);
		assert!(reinterpreted_node.is_ok());
		assert_eq!(node, &reinterpreted_node.unwrap());
		}
		}
		}