Merge "Reland "Calculate the runtime fingerprint prefixes from build prop"" (e9ab8595) · Commits · e / os / android_build

tools/releasetools/common.py

+66 −11

Original line number	Diff line number	Diff line
		@@ -738,18 +738,22 @@ class PartitionBuildProps(object):
		partition: name of the partition.
		props_allow_override: a list of build properties to search for the
		alternative values during runtime.
		build_props: a dictionary of build properties for the given partition.
		prop_overrides: a dict of list. And each list holds the overridden values
		for props_allow_override.
		build_props: a dict of build properties for the given partition.
		prop_overrides: a set of props that are overridden by import.
		placeholder_values: A dict of runtime variables' values to replace the
		placeholders in the build.prop file. We expect exactly one value for
		each of the variables.
		"""

		def __init__(self, input_file, name):
		def __init__(self, input_file, name, placeholder_values=None):
		self.input_file = input_file
		self.partition = name
		self.props_allow_override = [props.format(name) for props in [
		'ro.product.{}.name', 'ro.product.{}.device']]
		'ro.product.{}.brand', 'ro.product.{}.name', 'ro.product.{}.device']]
		self.build_props = {}
		self.prop_overrides = {}
		self.prop_overrides = set()
		self.placeholder_values = {}
		if placeholder_values:
		self.placeholder_values = copy.deepcopy(placeholder_values)

		@staticmethod
		def FromDictionary(name, build_props):
		@@ -760,9 +764,8 @@ class PartitionBuildProps(object):
		return props

		@staticmethod
		def FromInputFile(input_file, name):
		def FromInputFile(input_file, name, placeholder_values=None):
		"""Loads the build.prop file and builds the attributes."""

		data = ''
		for prop_file in ['{}/etc/build.prop'.format(name.upper()),
		'{}/build.prop'.format(name.upper())]:
		@@ -772,10 +775,62 @@ class PartitionBuildProps(object):
		except KeyError:
		logger.warning('Failed to read %s', prop_file)

		props = PartitionBuildProps(input_file, name)
		props.build_props = LoadDictionaryFromLines(data.split('\n'))
		props = PartitionBuildProps(input_file, name, placeholder_values)
		props._LoadBuildProp(data)
		return props

		def _LoadBuildProp(self, data):
		for line in data.split('\n'):
		line = line.strip()
		if not line or line.startswith("#"):
		continue
		if line.startswith("import"):
		overrides = self._ImportParser(line)
		duplicates = self.prop_overrides.intersection(overrides.keys())
		if duplicates:
		raise ValueError('prop {} is overridden multiple times'.format(
		','.join(duplicates)))
		self.prop_overrides = self.prop_overrides.union(overrides.keys())
		self.build_props.update(overrides)
		elif "=" in line:
		name, value = line.split("=", 1)
		if name in self.prop_overrides:
		raise ValueError('prop {} is set again after overridden by import '
		'statement'.format(name))
		self.build_props[name] = value

		def _ImportParser(self, line):
		"""Parses the build prop in a given import statement."""

		tokens = line.split()
		if len(tokens) != 2 or tokens[0] != 'import':
		raise ValueError('Unrecognized import statement {}'.format(line))
		import_path = tokens[1]
		if not re.match(r'^/{}/.*\.prop$'.format(self.partition), import_path):
		raise ValueError('Unrecognized import path {}'.format(line))

		# We only recognize a subset of import statement that the init process
		# supports. And we can loose the restriction based on how the dynamic
		# fingerprint is used in practice. The placeholder format should be
		# ${placeholder}, and its value should be provided by the caller through
		# the placeholder_values.
		for prop, value in self.placeholder_values.items():
		prop_place_holder = '${{{}}}'.format(prop)
		if prop_place_holder in import_path:
		import_path = import_path.replace(prop_place_holder, value)
		if '$' in import_path:
		logger.info('Unresolved place holder in import path %s', import_path)
		return {}

		import_path = import_path.replace('/{}'.format(self.partition),
		self.partition.upper())
		logger.info('Parsing build props override from %s', import_path)

		lines = ReadFromInputFile(self.input_file, import_path).split('\n')
		d = LoadDictionaryFromLines(lines)
		return {key: val for key, val in d.items()
		if key in self.props_allow_override}

		def GetProp(self, prop):
		return self.build_props.get(prop)

tools/releasetools/ota_from_target_files.py

+34 −0

Original line number	Diff line number	Diff line
		@@ -193,6 +193,8 @@ A/B OTA specific options
		from __future__ import print_function

		import collections
		import copy
		import itertools
		import logging
		import multiprocessing
		import os.path
		@@ -229,6 +231,7 @@ OPTIONS.include_secondary = False
		OPTIONS.no_signing = False
		OPTIONS.block_based = True
		OPTIONS.updater_binary = None
		OPTIONS.oem_dicts = None
		OPTIONS.oem_source = None
		OPTIONS.oem_no_mount = False
		OPTIONS.full_radio = False
		@@ -247,6 +250,7 @@ OPTIONS.retrofit_dynamic_partitions = False
		OPTIONS.skip_compatibility_check = False
		OPTIONS.output_metadata_path = None
		OPTIONS.disable_fec_computation = False
		OPTIONS.boot_variable_values = None


		METADATA_NAME = 'META-INF/com/android/metadata'
		@@ -1959,6 +1963,36 @@ def GenerateNonAbOtaPackage(target_file, output_file, source_file=None):
		output_file)


		def CalculateRuntimeFingerprints():
		"""Returns a set of runtime fingerprints based on the boot variables."""

		build_info = common.BuildInfo(OPTIONS.info_dict, OPTIONS.oem_dicts)
		fingerprints = {build_info.fingerprint}

		if not OPTIONS.boot_variable_values:
		return fingerprints

		# Calculate all possible combinations of the values for the boot variables.
		keys = OPTIONS.boot_variable_values.keys()
		value_list = OPTIONS.boot_variable_values.values()
		combinations = [dict(zip(keys, values))
		for values in itertools.product(*value_list)]
		for placeholder_values in combinations:
		# Reload the info_dict as some build properties may change their values
		# based on the value of ro.boot* properties.
		info_dict = copy.deepcopy(OPTIONS.info_dict)
		for partition in common.PARTITIONS_WITH_CARE_MAP:
		partition_prop_key = "{}.build.prop".format(partition)
		old_props = info_dict[partition_prop_key]
		info_dict[partition_prop_key] = common.PartitionBuildProps.FromInputFile(
		old_props.input_file, partition, placeholder_values)
		info_dict["build.prop"] = info_dict["system.build.prop"]

		build_info = common.BuildInfo(info_dict, OPTIONS.oem_dicts)
		fingerprints.add(build_info.fingerprint)
		return fingerprints


		def main(argv):

		def option_handler(o, a):

tools/releasetools/test_common.py

+202 −8

Original line number	Diff line number	Diff line
		@@ -1899,7 +1899,7 @@ super_group_foo_group_size={group_foo_size}

		class PartitionBuildPropsTest(test_utils.ReleaseToolsTestCase):
		def setUp(self):
		self.build_prop = [
		self.odm_build_prop = [
		'ro.odm.build.date.utc=1578430045',
		'ro.odm.build.fingerprint='
		'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys',
		@@ -1927,6 +1927,74 @@ class PartitionBuildPropsTest(test_utils.ReleaseToolsTestCase):
		'ODM/etc/build.prop': '\n'.join(build_prop),
		})

		with zipfile.ZipFile(input_file, 'r') as input_zip:
		placeholder_values = {
		'ro.boot.product.device_name': ['std', 'pro']
		}
		partition_props = common.PartitionBuildProps.FromInputFile(
		input_zip, 'odm', placeholder_values)

		self.assertEqual({
		'ro.odm.build.date.utc': '1578430045',
		'ro.odm.build.fingerprint':
		'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys',
		'ro.product.odm.device': 'coral',
		}, partition_props.build_props)

		self.assertEqual(set(), partition_props.prop_overrides)

		def test_parseBuildProps_singleImportStatement(self):
		build_std_prop = [
		'ro.product.odm.device=coral',
		'ro.product.odm.name=product1',
		]
		build_pro_prop = [
		'ro.product.odm.device=coralpro',
		'ro.product.odm.name=product2',
		]

		input_file = self._BuildZipFile({
		'ODM/etc/build.prop': '\n'.join(self.odm_build_prop),
		'ODM/etc/build_std.prop': '\n'.join(build_std_prop),
		'ODM/etc/build_pro.prop': '\n'.join(build_pro_prop),
		})

		with zipfile.ZipFile(input_file, 'r') as input_zip:
		placeholder_values = {
		'ro.boot.product.device_name': 'std'
		}
		partition_props = common.PartitionBuildProps.FromInputFile(
		input_zip, 'odm', placeholder_values)

		self.assertEqual({
		'ro.odm.build.date.utc': '1578430045',
		'ro.odm.build.fingerprint':
		'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys',
		'ro.product.odm.device': 'coral',
		'ro.product.odm.name': 'product1',
		}, partition_props.build_props)

		with zipfile.ZipFile(input_file, 'r') as input_zip:
		placeholder_values = {
		'ro.boot.product.device_name': 'pro'
		}
		partition_props = common.PartitionBuildProps.FromInputFile(
		input_zip, 'odm', placeholder_values)

		self.assertEqual({
		'ro.odm.build.date.utc': '1578430045',
		'ro.odm.build.fingerprint':
		'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys',
		'ro.product.odm.device': 'coralpro',
		'ro.product.odm.name': 'product2',
		}, partition_props.build_props)

		def test_parseBuildProps_noPlaceHolders(self):
		build_prop = copy.copy(self.odm_build_prop)
		input_file = self._BuildZipFile({
		'ODM/etc/build.prop': '\n'.join(build_prop),
		})

		with zipfile.ZipFile(input_file, 'r') as input_zip:
		partition_props = common.PartitionBuildProps.FromInputFile(
		input_zip, 'odm')
		@@ -1938,4 +2006,130 @@ class PartitionBuildPropsTest(test_utils.ReleaseToolsTestCase):
		'ro.product.odm.device': 'coral',
		}, partition_props.build_props)

		self.assertEqual({}, partition_props.prop_overrides)
		self.assertEqual(set(), partition_props.prop_overrides)

		def test_parseBuildProps_multipleImportStatements(self):
		build_prop = copy.deepcopy(self.odm_build_prop)
		build_prop.append(
		'import /odm/etc/build_${ro.boot.product.product_name}.prop')

		build_std_prop = [
		'ro.product.odm.device=coral',
		]
		build_pro_prop = [
		'ro.product.odm.device=coralpro',
		]

		product1_prop = [
		'ro.product.odm.name=product1',
		'ro.product.not_care=not_care',
		]

		product2_prop = [
		'ro.product.odm.name=product2',
		'ro.product.not_care=not_care',
		]

		input_file = self._BuildZipFile({
		'ODM/etc/build.prop': '\n'.join(build_prop),
		'ODM/etc/build_std.prop': '\n'.join(build_std_prop),
		'ODM/etc/build_pro.prop': '\n'.join(build_pro_prop),
		'ODM/etc/build_product1.prop': '\n'.join(product1_prop),
		'ODM/etc/build_product2.prop': '\n'.join(product2_prop),
		})

		with zipfile.ZipFile(input_file, 'r') as input_zip:
		placeholder_values = {
		'ro.boot.product.device_name': 'std',
		'ro.boot.product.product_name': 'product1',
		'ro.boot.product.not_care': 'not_care',
		}
		partition_props = common.PartitionBuildProps.FromInputFile(
		input_zip, 'odm', placeholder_values)

		self.assertEqual({
		'ro.odm.build.date.utc': '1578430045',
		'ro.odm.build.fingerprint':
		'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys',
		'ro.product.odm.device': 'coral',
		'ro.product.odm.name': 'product1'
		}, partition_props.build_props)

		with zipfile.ZipFile(input_file, 'r') as input_zip:
		placeholder_values = {
		'ro.boot.product.device_name': 'pro',
		'ro.boot.product.product_name': 'product2',
		'ro.boot.product.not_care': 'not_care',
		}
		partition_props = common.PartitionBuildProps.FromInputFile(
		input_zip, 'odm', placeholder_values)

		self.assertEqual({
		'ro.odm.build.date.utc': '1578430045',
		'ro.odm.build.fingerprint':
		'google/coral/coral:10/RP1A.200325.001/6337676:user/dev-keys',
		'ro.product.odm.device': 'coralpro',
		'ro.product.odm.name': 'product2'
		}, partition_props.build_props)

		def test_parseBuildProps_defineAfterOverride(self):
		build_prop = copy.deepcopy(self.odm_build_prop)
		build_prop.append('ro.product.odm.device=coral')

		build_std_prop = [
		'ro.product.odm.device=coral',
		]
		build_pro_prop = [
		'ro.product.odm.device=coralpro',
		]

		input_file = self._BuildZipFile({
		'ODM/etc/build.prop': '\n'.join(build_prop),
		'ODM/etc/build_std.prop': '\n'.join(build_std_prop),
		'ODM/etc/build_pro.prop': '\n'.join(build_pro_prop),
		})

		with zipfile.ZipFile(input_file, 'r') as input_zip:
		placeholder_values = {
		'ro.boot.product.device_name': 'std',
		}

		self.assertRaises(ValueError, common.PartitionBuildProps.FromInputFile,
		input_zip, 'odm', placeholder_values)

		def test_parseBuildProps_duplicateOverride(self):
		build_prop = copy.deepcopy(self.odm_build_prop)
		build_prop.append(
		'import /odm/etc/build_${ro.boot.product.product_name}.prop')

		build_std_prop = [
		'ro.product.odm.device=coral',
		'ro.product.odm.name=product1',
		]
		build_pro_prop = [
		'ro.product.odm.device=coralpro',
		]

		product1_prop = [
		'ro.product.odm.name=product1',
		]

		product2_prop = [
		'ro.product.odm.name=product2',
		]

		input_file = self._BuildZipFile({
		'ODM/etc/build.prop': '\n'.join(build_prop),
		'ODM/etc/build_std.prop': '\n'.join(build_std_prop),
		'ODM/etc/build_pro.prop': '\n'.join(build_pro_prop),
		'ODM/etc/build_product1.prop': '\n'.join(product1_prop),
		'ODM/etc/build_product2.prop': '\n'.join(product2_prop),
		})

		with zipfile.ZipFile(input_file, 'r') as input_zip:
		placeholder_values = {
		'ro.boot.product.device_name': 'std',
		'ro.boot.product.product_name': 'product1',
		}
		self.assertRaises(ValueError, common.PartitionBuildProps.FromInputFile,
		input_zip, 'odm', placeholder_values)

tools/releasetools/test_ota_from_target_files.py

+124 −1

Original line number	Diff line number	Diff line
		@@ -26,7 +26,8 @@ from ota_from_target_files import (
		GetPackageMetadata, GetTargetFilesZipForSecondaryImages,
		GetTargetFilesZipWithoutPostinstallConfig, NonAbOtaPropertyFiles,
		Payload, PayloadSigner, POSTINSTALL_CONFIG, PropertyFiles,
		StreamingPropertyFiles, WriteFingerprintAssertion)
		StreamingPropertyFiles, WriteFingerprintAssertion,
		CalculateRuntimeFingerprints)


		def construct_target_files(secondary=False):
		@@ -1318,3 +1319,125 @@ class PayloadTest(test_utils.ReleaseToolsTestCase):
		Payload.SECONDARY_PAYLOAD_PROPERTIES_TXT):
		continue
		self.assertEqual(zipfile.ZIP_STORED, entry_info.compress_type)


		class RuntimeFingerprintTest(test_utils.ReleaseToolsTestCase):
		MISC_INFO = [
		'recovery_api_version=3',
		'fstab_version=2',
		'recovery_as_boot=true',
		]

		BUILD_PROP = [
		'ro.build.version.release=version-release',
		'ro.build.id=build-id',
		'ro.build.version.incremental=version-incremental',
		'ro.build.type=build-type',
		'ro.build.tags=build-tags',
		]

		VENDOR_BUILD_PROP = [
		'ro.product.vendor.brand=vendor-product-brand',
		'ro.product.vendor.name=vendor-product-name',
		'ro.product.vendor.device=vendor-product-device'
		]

		def setUp(self):
		common.OPTIONS.oem_dicts = None
		self.test_dir = common.MakeTempDir()
		self.writeFiles({'META/misc_info.txt': '\n'.join(self.MISC_INFO)})

		def writeFiles(self, contents_dict):
		for path, content in contents_dict.items():
		abs_path = os.path.join(self.test_dir, path)
		dir_name = os.path.dirname(abs_path)
		if not os.path.exists(dir_name):
		os.makedirs(dir_name)
		with open(abs_path, 'w') as f:
		f.write(content)

		@staticmethod
		def constructFingerprint(prefix):
		return '{}:version-release/build-id/version-incremental:' \
		'build-type/build-tags'.format(prefix)

		def test_CalculatePossibleFingerprints_no_dynamic_fingerprint(self):
		build_prop = copy.deepcopy(self.BUILD_PROP)
		build_prop.extend([
		'ro.product.brand=product-brand',
		'ro.product.name=product-name',
		'ro.product.device=product-device',
		])
		self.writeFiles({
		'SYSTEM/build.prop': '\n'.join(build_prop),
		'VENDOR/build.prop': '\n'.join(self.VENDOR_BUILD_PROP),
		})
		common.OPTIONS.info_dict = common.LoadInfoDict(self.test_dir)

		self.assertEqual({
		self.constructFingerprint('product-brand/product-name/product-device')
		}, CalculateRuntimeFingerprints())

		def test_CalculatePossibleFingerprints_single_override(self):
		vendor_build_prop = copy.deepcopy(self.VENDOR_BUILD_PROP)
		vendor_build_prop.extend([
		'import /vendor/etc/build_${ro.boot.sku_name}.prop',
		])
		self.writeFiles({
		'SYSTEM/build.prop': '\n'.join(self.BUILD_PROP),
		'VENDOR/build.prop': '\n'.join(vendor_build_prop),
		'VENDOR/etc/build_std.prop':
		'ro.product.vendor.name=vendor-product-std',
		'VENDOR/etc/build_pro.prop':
		'ro.product.vendor.name=vendor-product-pro',
		})
		common.OPTIONS.info_dict = common.LoadInfoDict(self.test_dir)
		common.OPTIONS.boot_variable_values = {
		'ro.boot.sku_name': ['std', 'pro']
		}

		self.assertEqual({
		self.constructFingerprint(
		'vendor-product-brand/vendor-product-name/vendor-product-device'),
		self.constructFingerprint(
		'vendor-product-brand/vendor-product-std/vendor-product-device'),
		self.constructFingerprint(
		'vendor-product-brand/vendor-product-pro/vendor-product-device'),
		}, CalculateRuntimeFingerprints())

		def test_CalculatePossibleFingerprints_multiple_overrides(self):
		vendor_build_prop = copy.deepcopy(self.VENDOR_BUILD_PROP)
		vendor_build_prop.extend([
		'import /vendor/etc/build_${ro.boot.sku_name}.prop',
		'import /vendor/etc/build_${ro.boot.device_name}.prop',
		])
		self.writeFiles({
		'SYSTEM/build.prop': '\n'.join(self.BUILD_PROP),
		'VENDOR/build.prop': '\n'.join(vendor_build_prop),
		'VENDOR/etc/build_std.prop':
		'ro.product.vendor.name=vendor-product-std',
		'VENDOR/etc/build_product1.prop':
		'ro.product.vendor.device=vendor-device-product1',
		'VENDOR/etc/build_pro.prop':
		'ro.product.vendor.name=vendor-product-pro',
		'VENDOR/etc/build_product2.prop':
		'ro.product.vendor.device=vendor-device-product2',
		})
		common.OPTIONS.info_dict = common.LoadInfoDict(self.test_dir)
		common.OPTIONS.boot_variable_values = {
		'ro.boot.sku_name': ['std', 'pro'],
		'ro.boot.device_name': ['product1', 'product2'],
		}

		self.assertEqual({
		self.constructFingerprint(
		'vendor-product-brand/vendor-product-name/vendor-product-device'),
		self.constructFingerprint(
		'vendor-product-brand/vendor-product-std/vendor-device-product1'),
		self.constructFingerprint(
		'vendor-product-brand/vendor-product-pro/vendor-device-product1'),
		self.constructFingerprint(
		'vendor-product-brand/vendor-product-std/vendor-device-product2'),
		self.constructFingerprint(
		'vendor-product-brand/vendor-product-pro/vendor-device-product2'),
		}, CalculateRuntimeFingerprints())