Merge "Define the protobuf for OTA metadata" am: 0068973f73 am: 9e302c8c78 am: dd3f52831c (02e47379) · Commits · e / os / android_build

tools/releasetools/Android.bp

+21 −2

Original line number	Diff line number	Diff line
		@@ -89,16 +89,35 @@ python_defaults {
		],
		}

		python_library_host {
		name: "ota_metadata_proto",
		version: {
		py2: {
		enabled: true,
		},
		py3: {
		enabled: true,
		},
		},
		srcs: [
		"ota_metadata.proto",
		],
		proto: {
		canonical_path_from_root: false,
		},
		}

		python_defaults {
		name: "releasetools_ota_from_target_files_defaults",
		srcs: [
		"edify_generator.py",
		"ota_from_target_files.py",
		"non_ab_ota.py",
		"target_files_diff.py",
		"ota_from_target_files.py",
		"ota_utils.py",
		"target_files_diff.py",
		],
		libs: [
		"ota_metadata_proto",
		"releasetools_check_target_files_vintf",
		"releasetools_common",
		"releasetools_verity_utils",

tools/releasetools/non_ab_ota.py

+2 −2

Original line number	Diff line number	Diff line
		@@ -276,7 +276,7 @@ endif;

		script.SetProgress(1)
		script.AddToZip(input_zip, output_zip, input_path=OPTIONS.updater_binary)
		metadata["ota-required-cache"] = str(script.required_cache)
		metadata.required_cache = script.required_cache

		# We haven't written the metadata entry, which will be done in
		# FinalizeMetadata.
		@@ -530,7 +530,7 @@ endif;
		script.AddToZip(source_zip, output_zip, input_path=OPTIONS.updater_binary)
		else:
		script.AddToZip(target_zip, output_zip, input_path=OPTIONS.updater_binary)
		metadata["ota-required-cache"] = str(script.required_cache)
		metadata.required_cache = script.required_cache

		# We haven't written the metadata entry yet, which will be handled in
		# FinalizeMetadata().

tools/releasetools/ota_from_target_files.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -848,7 +848,7 @@ def GenerateAbOtaPackage(target_file, output_file, source_file=None):
		if OPTIONS.downgrade:
		max_timestamp = source_info.GetBuildProp("ro.build.date.utc")
		else:
		max_timestamp = metadata["post-timestamp"]
		max_timestamp = str(metadata.postcondition.timestamp)
		additional_args = ["--max_timestamp", max_timestamp]

		payload.Generate(target_file, source_file, additional_args)

tools/releasetools/ota_metadata.proto

0 → 100644

+88 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2020 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		syntax = "proto3";

		package build.tools.releasetools;
		option optimize_for = LITE_RUNTIME;

		// The build information of a particular partition on the device.
		message PartitionState {
		string partition_name = 1;
		repeated string device = 2;
		repeated string build = 3;
		// The version string of the partition. It's usually timestamp if present.
		// One known exception is the boot image, who uses the kmi version, e.g.
		// 5.4.42-android12-0
		string version = 4;

		// TODO(xunchang), revisit other necessary fields, e.g. security_patch_level.
		}

		// The build information on the device. The bytes of the running images are thus
		// inferred from the device state. For more information of the meaning of each
		// subfield, check
		// https://source.android.com/compatibility/android-cdd#3_2_2_build_parameters
		message DeviceState {
		// device name. i.e. ro.product.device; if the field has multiple values, it
		// means the ota package supports multiple devices. This usually happens when
		// we use the same image to support multiple skus.
		repeated string device = 1;
		// device fingerprint. Up to R build, the value reads from
		// ro.build.fingerprint.
		repeated string build = 2;
		// A value that specify a version of the android build.
		string build_incremental = 3;
		// The timestamp when the build is generated.
		int64 timestamp = 4;
		// The version of the currently-executing Android system.
		string sdk_level = 5;
		// A value indicating the security patch level of a build.
		string security_patch_level = 6;

		// The detailed state of each partition. For partial updates or devices with
		// mixed build of partitions, some of the above fields may left empty. And the
		// client will rely on the information of specific partitions to target the
		// update.
		repeated PartitionState partition_state = 7;
		}

		// The metadata of an OTA package. It contains the information of the package
		// and prerequisite to install the update correctly.
		message OtaMetadata {
		enum OtaType {
		AB = 0;
		BLOCK = 1;
		};
		OtaType type = 1;
		// True if we need to wipe after the update.
		bool wipe = 2;
		// True if the timestamp of the post build is older than the pre build.
		bool downgrade = 3;
		// A map of name:content of property files, e.g. ota-property-files.
		map<string, string> property_files = 4;

		// The required device state in order to install the package.
		DeviceState precondition = 5;
		// The expected device state after the update.
		DeviceState postcondition = 6;

		// True if the ota that updates a device to support dynamic partitions, where
		// the source build doesn't support it.
		bool retrofit_dynamic_partitions = 7;
		// The required size of the cache partition, only valid for non-A/B update.
		int64 required_cache = 8;
		}

tools/releasetools/ota_utils.py

+118 −51

Original line number	Diff line number	Diff line
		@@ -17,6 +17,7 @@ import itertools
		import os
		import zipfile

		import ota_metadata_pb2
		from common import (ZipDelete, ZipClose, OPTIONS, MakeTempFile,
		ZipWriteStr, BuildInfo, LoadDictionaryFromFile,
		SignFile, PARTITIONS_WITH_CARE_MAP, PartitionBuildProps)
		@@ -34,6 +35,7 @@ OPTIONS.output_metadata_path = None
		OPTIONS.boot_variable_file = None

		METADATA_NAME = 'META-INF/com/android/metadata'
		METADATA_PROTO_NAME = 'META-INF/com/android/metadata.pb'
		UNZIP_PATTERN = ['IMAGES/', 'META/', 'OTA/', 'RADIO/']


		@@ -62,11 +64,12 @@ def FinalizeMetadata(metadata, input_file, output_file, needed_property_files):
		# Write the current metadata entry with placeholders.
		with zipfile.ZipFile(input_file) as input_zip:
		for property_files in needed_property_files:
		metadata[property_files.name] = property_files.Compute(input_zip)
		metadata.property_files[property_files.name] = property_files.Compute(
		input_zip)
		namelist = input_zip.namelist()

		if METADATA_NAME in namelist:
		ZipDelete(input_file, METADATA_NAME)
		if METADATA_NAME in namelist or METADATA_PROTO_NAME in namelist:
		ZipDelete(input_file, [METADATA_NAME, METADATA_PROTO_NAME])
		output_zip = zipfile.ZipFile(input_file, 'a')
		WriteMetadata(metadata, output_zip)
		ZipClose(output_zip)
		@@ -81,8 +84,9 @@ def FinalizeMetadata(metadata, input_file, output_file, needed_property_files):
		def FinalizeAllPropertyFiles(prelim_signing, needed_property_files):
		with zipfile.ZipFile(prelim_signing) as prelim_signing_zip:
		for property_files in needed_property_files:
		metadata[property_files.name] = property_files.Finalize(
		prelim_signing_zip, len(metadata[property_files.name]))
		metadata.property_files[property_files.name] = property_files.Finalize(
		prelim_signing_zip,
		len(metadata.property_files[property_files.name]))

		# SignOutput(), which in turn calls signapk.jar, will possibly reorder the ZIP
		# entries, as well as padding the entry headers. We do a preliminary signing
		@@ -103,7 +107,7 @@ def FinalizeMetadata(metadata, input_file, output_file, needed_property_files):
		FinalizeAllPropertyFiles(prelim_signing, needed_property_files)

		# Replace the METADATA entry.
		ZipDelete(prelim_signing, METADATA_NAME)
		ZipDelete(prelim_signing, [METADATA_NAME, METADATA_PROTO_NAME])
		output_zip = zipfile.ZipFile(prelim_signing, 'a')
		WriteMetadata(metadata, output_zip)
		ZipClose(output_zip)
		@@ -117,7 +121,8 @@ def FinalizeMetadata(metadata, input_file, output_file, needed_property_files):
		# Reopen the final signed zip to double check the streaming metadata.
		with zipfile.ZipFile(output_file) as output_zip:
		for property_files in needed_property_files:
		property_files.Verify(output_zip, metadata[property_files.name].strip())
		property_files.Verify(
		output_zip, metadata.property_files[property_files.name].strip())

		# If requested, dump the metadata to a separate file.
		output_metadata_path = OPTIONS.output_metadata_path
		@@ -125,30 +130,60 @@ def FinalizeMetadata(metadata, input_file, output_file, needed_property_files):
		WriteMetadata(metadata, output_metadata_path)


		def WriteMetadata(metadata, output):
		def WriteMetadata(metadata_proto, output):
		"""Writes the metadata to the zip archive or a file.

		Args:
		metadata: The metadata dict for the package.
		output: A ZipFile object or a string of the output file path.
		metadata_proto: The metadata protobuf for the package.
		output: A ZipFile object or a string of the output file path. If a string
		path is given, the metadata in the protobuf format will be written to
		{output}.pb, e.g. ota_metadata.pb
		"""

		value = "".join(["%s=%s\n" % kv for kv in sorted(metadata.items())])
		metadata_dict = BuildLegacyOtaMetadata(metadata_proto)
		legacy_metadata = "".join(["%s=%s\n" % kv for kv in
		sorted(metadata_dict.items())])
		if isinstance(output, zipfile.ZipFile):
		ZipWriteStr(output, METADATA_NAME, value,
		ZipWriteStr(output, METADATA_PROTO_NAME, metadata_proto.SerializeToString(),
		compress_type=zipfile.ZIP_STORED)
		ZipWriteStr(output, METADATA_NAME, legacy_metadata,
		compress_type=zipfile.ZIP_STORED)
		return

		with open('{}.pb'.format(output), 'w') as f:
		f.write(metadata_proto.SerializeToString())
		with open(output, 'w') as f:
		f.write(value)
		f.write(legacy_metadata)


		def UpdateDeviceState(device_state, build_info, boot_variable_values,
		is_post_build):
		"""Update the fields of the DeviceState proto with build info."""

		build_devices, build_fingerprints = \
		CalculateRuntimeDevicesAndFingerprints(build_info, boot_variable_values)
		device_state.device.extend(sorted(build_devices))
		device_state.build.extend(sorted(build_fingerprints))
		device_state.build_incremental = build_info.GetBuildProp(
		'ro.build.version.incremental')

		# TODO(xunchang) update the partition state

		if is_post_build:
		device_state.sdk_level = build_info.GetBuildProp(
		'ro.build.version.sdk')
		device_state.security_patch_level = build_info.GetBuildProp(
		'ro.build.version.security_patch')
		# Use the actual post-timestamp, even for a downgrade case.
		device_state.timestamp = int(build_info.GetBuildProp('ro.build.date.utc'))


		def GetPackageMetadata(target_info, source_info=None):
		"""Generates and returns the metadata dict.
		"""Generates and returns the metadata proto.

		It generates a dict() that contains the info to be written into an OTA
		package (META-INF/com/android/metadata). It also handles the detection of
		downgrade / data wipe based on the global options.
		It generates a ota_metadata protobuf that contains the info to be written
		into an OTA package (META-INF/com/android/metadata.pb). It also handles the
		detection of downgrade / data wipe based on the global options.

		Args:
		target_info: The BuildInfo instance that holds the target build info.
		@@ -156,66 +191,96 @@ def GetPackageMetadata(target_info, source_info=None):
		None if generating full OTA.

		Returns:
		A dict to be written into package metadata entry.
		A protobuf to be written into package metadata entry.
		"""
		assert isinstance(target_info, BuildInfo)
		assert source_info is None or isinstance(source_info, BuildInfo)

		separator = '\|'

		boot_variable_values = {}
		if OPTIONS.boot_variable_file:
		d = LoadDictionaryFromFile(OPTIONS.boot_variable_file)
		for key, values in d.items():
		boot_variable_values[key] = [val.strip() for val in values.split(',')]

		post_build_devices, post_build_fingerprints = \
		CalculateRuntimeDevicesAndFingerprints(target_info, boot_variable_values)
		metadata = {
		'post-build': separator.join(sorted(post_build_fingerprints)),
		'post-build-incremental': target_info.GetBuildProp(
		'ro.build.version.incremental'),
		'post-sdk-level': target_info.GetBuildProp(
		'ro.build.version.sdk'),
		'post-security-patch-level': target_info.GetBuildProp(
		'ro.build.version.security_patch'),
		}
		metadata_proto = ota_metadata_pb2.OtaMetadata()
		# TODO(xunchang) some fields, e.g. post-device isn't necessary. We can
		# consider skipping them if they aren't used by clients.
		UpdateDeviceState(metadata_proto.postcondition, target_info,
		boot_variable_values, True)

		if target_info.is_ab and not OPTIONS.force_non_ab:
		metadata['ota-type'] = 'AB'
		metadata['ota-required-cache'] = '0'
		metadata_proto.type = ota_metadata_pb2.OtaMetadata.AB
		metadata_proto.required_cache = 0
		else:
		metadata['ota-type'] = 'BLOCK'
		metadata_proto.type = ota_metadata_pb2.OtaMetadata.BLOCK
		# cache requirement will be updated by the non-A/B codes.

		if OPTIONS.wipe_user_data:
		metadata['ota-wipe'] = 'yes'
		metadata_proto.wipe = True

		if OPTIONS.retrofit_dynamic_partitions:
		metadata['ota-retrofit-dynamic-partitions'] = 'yes'
		metadata_proto.retrofit_dynamic_partitions = True

		is_incremental = source_info is not None
		if is_incremental:
		pre_build_devices, pre_build_fingerprints = \
		CalculateRuntimeDevicesAndFingerprints(source_info,
		boot_variable_values)
		metadata['pre-build'] = separator.join(sorted(pre_build_fingerprints))
		metadata['pre-build-incremental'] = source_info.GetBuildProp(
		'ro.build.version.incremental')
		metadata['pre-device'] = separator.join(sorted(pre_build_devices))
		UpdateDeviceState(metadata_proto.precondition, source_info,
		boot_variable_values, False)
		else:
		metadata['pre-device'] = separator.join(sorted(post_build_devices))

		# Use the actual post-timestamp, even for a downgrade case.
		metadata['post-timestamp'] = target_info.GetBuildProp('ro.build.date.utc')
		metadata_proto.precondition.device.extend(
		metadata_proto.postcondition.device)

		# Detect downgrades and set up downgrade flags accordingly.
		if is_incremental:
		HandleDowngradeMetadata(metadata, target_info, source_info)
		HandleDowngradeMetadata(metadata_proto, target_info, source_info)

		return metadata_proto


		def BuildLegacyOtaMetadata(metadata_proto):
		"""Converts the metadata proto to a legacy metadata dict.

		This metadata dict is used to build the legacy metadata text file for
		backward compatibility. We won't add new keys to the legacy metadata format.
		If new information is needed, we should add it as a new field in OtaMetadata
		proto definition.
		"""

		separator = '\|'

		metadata_dict = {}
		if metadata_proto.type == ota_metadata_pb2.OtaMetadata.AB:
		metadata_dict['ota-type'] = 'AB'
		elif metadata_proto.type == ota_metadata_pb2.OtaMetadata.BLOCK:
		metadata_dict['ota-type'] = 'BLOCK'
		if metadata_proto.wipe:
		metadata_dict['ota-wipe'] = 'yes'
		if metadata_proto.retrofit_dynamic_partitions:
		metadata_dict['ota-retrofit-dynamic-partitions'] = 'yes'
		if metadata_proto.downgrade:
		metadata_dict['ota-downgrade'] = 'yes'

		metadata_dict['ota-required-cache'] = str(metadata_proto.required_cache)

		post_build = metadata_proto.postcondition
		metadata_dict['post-build'] = separator.join(post_build.build)
		metadata_dict['post-build-incremental'] = post_build.build_incremental
		metadata_dict['post-sdk-level'] = post_build.sdk_level
		metadata_dict['post-security-patch-level'] = post_build.security_patch_level
		metadata_dict['post-timestamp'] = str(post_build.timestamp)

		pre_build = metadata_proto.precondition
		metadata_dict['pre-device'] = separator.join(pre_build.device)
		# incremental updates
		if len(pre_build.build) != 0:
		metadata_dict['pre-build'] = separator.join(pre_build.build)
		metadata_dict['pre-build-incremental'] = pre_build.build_incremental

		metadata_dict.update(metadata_proto.property_files)

		return metadata
		return metadata_dict


		def HandleDowngradeMetadata(metadata, target_info, source_info):
		def HandleDowngradeMetadata(metadata_proto, target_info, source_info):
		# Only incremental OTAs are allowed to reach here.
		assert OPTIONS.incremental_source is not None

		@@ -228,7 +293,7 @@ def HandleDowngradeMetadata(metadata, target_info, source_info):
		raise RuntimeError(
		"--downgrade or --override_timestamp specified but no downgrade "
		"detected: pre: %s, post: %s" % (pre_timestamp, post_timestamp))
		metadata["ota-downgrade"] = "yes"
		metadata_proto.downgrade = True
		else:
		if is_downgrade:
		raise RuntimeError(
		@@ -415,8 +480,10 @@ class PropertyFiles(object):
		# reserved space serves the metadata entry only.
		if reserve_space:
		tokens.append('metadata:' + ' ' * 15)
		tokens.append('metadata.pb:' + ' ' * 15)
		else:
		tokens.append(ComputeEntryOffsetSize(METADATA_NAME))
		tokens.append(ComputeEntryOffsetSize(METADATA_PROTO_NAME))

		return ','.join(tokens)