Merge "Move GetCareMap from common.py to add_img_to_target_files.py" (ed024f1b) · Commits · e / os / android_build

tools/releasetools/add_img_to_target_files.py

+135 −4

Original line number	Diff line number	Diff line
		@@ -64,7 +64,9 @@ import verity_utils
		import ota_metadata_pb2

		from apex_utils import GetApexInfoFromTargetFiles
		from common import AddCareMapForAbOta, ZipDelete
		from common import ZipDelete, PARTITIONS_WITH_CARE_MAP, ExternalError, RunAndCheckOutput, MakeTempFile, ZipWrite
		import rangelib
		import sparse_img

		if sys.hexversion < 0x02070000:
		print("Python 2.7 or newer is required.", file=sys.stderr)
		@@ -85,6 +87,132 @@ FIXED_FILE_TIMESTAMP = int((
		datetime.datetime.utcfromtimestamp(0)).total_seconds())


		def GetCareMap(which, imgname):
		"""Returns the care_map string for the given partition.

		Args:
		which: The partition name, must be listed in PARTITIONS_WITH_CARE_MAP.
		imgname: The filename of the image.

		Returns:
		(which, care_map_ranges): care_map_ranges is the raw string of the care_map
		RangeSet; or None.
		"""
		assert which in PARTITIONS_WITH_CARE_MAP

		# which + "_image_size" contains the size that the actual filesystem image
		# resides in, which is all that needs to be verified. The additional blocks in
		# the image file contain verity metadata, by reading which would trigger
		# invalid reads.
		image_size = OPTIONS.info_dict.get(which + "_image_size")
		if not image_size:
		return None

		disable_sparse = OPTIONS.info_dict.get(which + "_disable_sparse")

		image_blocks = int(image_size) // 4096 - 1
		# It's OK for image_blocks to be 0, because care map ranges are inclusive.
		# So 0-0 means "just block 0", which is valid.
		assert image_blocks >= 0, "blocks for {} must be non-negative, image size: {}".format(
		which, image_size)

		# For sparse images, we will only check the blocks that are listed in the care
		# map, i.e. the ones with meaningful data.
		if "extfs_sparse_flag" in OPTIONS.info_dict and not disable_sparse:
		simg = sparse_img.SparseImage(imgname)
		care_map_ranges = simg.care_map.intersect(
		rangelib.RangeSet("0-{}".format(image_blocks)))

		# Otherwise for non-sparse images, we read all the blocks in the filesystem
		# image.
		else:
		care_map_ranges = rangelib.RangeSet("0-{}".format(image_blocks))

		return [which, care_map_ranges.to_string_raw()]


		def AddCareMapForAbOta(output_file, ab_partitions, image_paths):
		"""Generates and adds care_map.pb for a/b partition that has care_map.

		Args:
		output_file: The output zip file (needs to be already open),
		or file path to write care_map.pb.
		ab_partitions: The list of A/B partitions.
		image_paths: A map from the partition name to the image path.
		"""
		if not output_file:
		raise ExternalError('Expected output_file for AddCareMapForAbOta')

		care_map_list = []
		for partition in ab_partitions:
		partition = partition.strip()
		if partition not in PARTITIONS_WITH_CARE_MAP:
		continue

		verity_block_device = "{}_verity_block_device".format(partition)
		avb_hashtree_enable = "avb_{}_hashtree_enable".format(partition)
		if (verity_block_device in OPTIONS.info_dict or
		OPTIONS.info_dict.get(avb_hashtree_enable) == "true"):
		if partition not in image_paths:
		logger.warning('Potential partition with care_map missing from images: %s',
		partition)
		continue
		image_path = image_paths[partition]
		if not os.path.exists(image_path):
		raise ExternalError('Expected image at path {}'.format(image_path))

		care_map = GetCareMap(partition, image_path)
		if not care_map:
		continue
		care_map_list += care_map

		# adds fingerprint field to the care_map
		# TODO(xunchang) revisit the fingerprint calculation for care_map.
		partition_props = OPTIONS.info_dict.get(partition + ".build.prop")
		prop_name_list = ["ro.{}.build.fingerprint".format(partition),
		"ro.{}.build.thumbprint".format(partition)]

		present_props = [x for x in prop_name_list if
		partition_props and partition_props.GetProp(x)]
		if not present_props:
		logger.warning(
		"fingerprint is not present for partition %s", partition)
		property_id, fingerprint = "unknown", "unknown"
		else:
		property_id = present_props[0]
		fingerprint = partition_props.GetProp(property_id)
		care_map_list += [property_id, fingerprint]

		if not care_map_list:
		return

		# Converts the list into proto buf message by calling care_map_generator; and
		# writes the result to a temp file.
		temp_care_map_text = MakeTempFile(prefix="caremap_text-",
		suffix=".txt")
		with open(temp_care_map_text, 'w') as text_file:
		text_file.write('\n'.join(care_map_list))

		temp_care_map = MakeTempFile(prefix="caremap-", suffix=".pb")
		care_map_gen_cmd = ["care_map_generator", temp_care_map_text, temp_care_map]
		RunAndCheckOutput(care_map_gen_cmd)

		if not isinstance(output_file, zipfile.ZipFile):
		shutil.copy(temp_care_map, output_file)
		return
		# output_file is a zip file
		care_map_path = "META/care_map.pb"
		if care_map_path in output_file.namelist():
		# Copy the temp file into the OPTIONS.input_tmp dir and update the
		# replace_updated_files_list used by add_img_to_target_files
		if not OPTIONS.replace_updated_files_list:
		OPTIONS.replace_updated_files_list = []
		shutil.copy(temp_care_map, os.path.join(OPTIONS.input_tmp, care_map_path))
		OPTIONS.replace_updated_files_list.append(care_map_path)
		else:
		ZipWrite(output_file, temp_care_map, arcname=care_map_path)


		class OutputFile(object):
		"""A helper class to write a generated file to the given dir or zip.

		@@ -277,6 +405,7 @@ def AddOdmDlkm(output_zip):
		block_list=block_list)
		return img.name


		def AddSystemDlkm(output_zip):
		"""Turn the contents of SystemDlkm into an system_dlkm image and store it in output_zip."""

		@@ -780,7 +909,8 @@ def AddImagesToTargetFiles(filename):
		has_boot = OPTIONS.info_dict.get("no_boot") != "true"
		has_init_boot = OPTIONS.info_dict.get("init_boot") == "true"
		has_vendor_boot = OPTIONS.info_dict.get("vendor_boot") == "true"
		has_vendor_kernel_boot = OPTIONS.info_dict.get("vendor_kernel_boot") == "true"
		has_vendor_kernel_boot = OPTIONS.info_dict.get(
		"vendor_kernel_boot") == "true"

		# {vendor,odm,product,system_ext,vendor_dlkm,odm_dlkm, system_dlkm, system, system_other}.img
		# can be built from source, or dropped into target_files.zip as a prebuilt blob.
		@@ -1051,7 +1181,8 @@ def OptimizeCompressedEntries(zipfile_path):
		ZipDelete(zipfile_path, [entry.filename for entry in entries_to_store])
		with zipfile.ZipFile(zipfile_path, "a", allowZip64=True) as zfp:
		for entry in entries_to_store:
		zfp.write(os.path.join(tmpdir, entry.filename), entry.filename, compress_type=zipfile.ZIP_STORED)
		zfp.write(os.path.join(tmpdir, entry.filename),
		entry.filename, compress_type=zipfile.ZIP_STORED)


		def main(argv):

tools/releasetools/common.py

+17 −137

Original line number	Diff line number	Diff line
		@@ -1682,7 +1682,6 @@ def _BuildBootableImage(image_name, sourcedir, fs_config_file, info_dict=None,
		with open(img.name, 'ab') as f:
		f.write(boot_signature_bytes)


		# Sign the image if vboot is non-empty.
		if info_dict.get("vboot"):
		path = "/" + partition_name
		@@ -1863,7 +1862,8 @@ def _BuildVendorBootImage(sourcedir, partition_name, info_dict=None):

		fn = os.path.join(sourcedir, "dtb")
		if os.access(fn, os.F_OK):
		has_vendor_kernel_boot = (info_dict.get("vendor_kernel_boot", "").lower() == "true")
		has_vendor_kernel_boot = (info_dict.get(
		"vendor_kernel_boot", "").lower() == "true")

		# Pack dtb into vendor_kernel_boot if building vendor_kernel_boot.
		# Otherwise pack dtb into vendor_boot.
		@@ -2360,6 +2360,7 @@ def SignFile(input_name, output_name, key, password, min_api_level=None,
		"Failed to run {}: return code {}:\n{}".format(cmd,
		proc.returncode, stdoutdata))


		def SignSePolicy(sepolicy, key, password):
		"""Sign the sepolicy zip, producing an fsverity .fsv_sig and
		an RSA .sig signature files.
		@@ -2391,6 +2392,7 @@ def SignSePolicy(sepolicy, key, password):
		proc.returncode, stdoutdata))
		return True


		def CheckSize(data, target, info_dict):
		"""Checks the data string passed against the max size limit.

		@@ -2608,13 +2610,17 @@ def ParseOptions(argv,
		elif o in ("--private_key_suffix",):
		OPTIONS.private_key_suffix = a
		elif o in ("--boot_signer_path",):
		raise ValueError("--boot_signer_path is no longer supported, please switch to AVB")
		raise ValueError(
		"--boot_signer_path is no longer supported, please switch to AVB")
		elif o in ("--boot_signer_args",):
		raise ValueError("--boot_signer_args is no longer supported, please switch to AVB")
		raise ValueError(
		"--boot_signer_args is no longer supported, please switch to AVB")
		elif o in ("--verity_signer_path",):
		raise ValueError("--verity_signer_path is no longer supported, please switch to AVB")
		raise ValueError(
		"--verity_signer_path is no longer supported, please switch to AVB")
		elif o in ("--verity_signer_args",):
		raise ValueError("--verity_signer_args is no longer supported, please switch to AVB")
		raise ValueError(
		"--verity_signer_args is no longer supported, please switch to AVB")
		elif o in ("-s", "--device_specific"):
		OPTIONS.device_specific = a
		elif o in ("-x", "--extra"):
		@@ -3972,132 +3978,6 @@ def GetBootImageTimestamp(boot_img):
		return None


		def GetCareMap(which, imgname):
		"""Returns the care_map string for the given partition.

		Args:
		which: The partition name, must be listed in PARTITIONS_WITH_CARE_MAP.
		imgname: The filename of the image.

		Returns:
		(which, care_map_ranges): care_map_ranges is the raw string of the care_map
		RangeSet; or None.
		"""
		assert which in PARTITIONS_WITH_CARE_MAP

		# which + "_image_size" contains the size that the actual filesystem image
		# resides in, which is all that needs to be verified. The additional blocks in
		# the image file contain verity metadata, by reading which would trigger
		# invalid reads.
		image_size = OPTIONS.info_dict.get(which + "_image_size")
		if not image_size:
		return None

		disable_sparse = OPTIONS.info_dict.get(which + "_disable_sparse")

		image_blocks = int(image_size) // 4096 - 1
		# It's OK for image_blocks to be 0, because care map ranges are inclusive.
		# So 0-0 means "just block 0", which is valid.
		assert image_blocks >= 0, "blocks for {} must be non-negative, image size: {}".format(
		which, image_size)

		# For sparse images, we will only check the blocks that are listed in the care
		# map, i.e. the ones with meaningful data.
		if "extfs_sparse_flag" in OPTIONS.info_dict and not disable_sparse:
		simg = sparse_img.SparseImage(imgname)
		care_map_ranges = simg.care_map.intersect(
		rangelib.RangeSet("0-{}".format(image_blocks)))

		# Otherwise for non-sparse images, we read all the blocks in the filesystem
		# image.
		else:
		care_map_ranges = rangelib.RangeSet("0-{}".format(image_blocks))

		return [which, care_map_ranges.to_string_raw()]


		def AddCareMapForAbOta(output_file, ab_partitions, image_paths):
		"""Generates and adds care_map.pb for a/b partition that has care_map.

		Args:
		output_file: The output zip file (needs to be already open),
		or file path to write care_map.pb.
		ab_partitions: The list of A/B partitions.
		image_paths: A map from the partition name to the image path.
		"""
		if not output_file:
		raise ExternalError('Expected output_file for AddCareMapForAbOta')

		care_map_list = []
		for partition in ab_partitions:
		partition = partition.strip()
		if partition not in PARTITIONS_WITH_CARE_MAP:
		continue

		verity_block_device = "{}_verity_block_device".format(partition)
		avb_hashtree_enable = "avb_{}_hashtree_enable".format(partition)
		if (verity_block_device in OPTIONS.info_dict or
		OPTIONS.info_dict.get(avb_hashtree_enable) == "true"):
		if partition not in image_paths:
		logger.warning('Potential partition with care_map missing from images: %s',
		partition)
		continue
		image_path = image_paths[partition]
		if not os.path.exists(image_path):
		raise ExternalError('Expected image at path {}'.format(image_path))

		care_map = GetCareMap(partition, image_path)
		if not care_map:
		continue
		care_map_list += care_map

		# adds fingerprint field to the care_map
		# TODO(xunchang) revisit the fingerprint calculation for care_map.
		partition_props = OPTIONS.info_dict.get(partition + ".build.prop")
		prop_name_list = ["ro.{}.build.fingerprint".format(partition),
		"ro.{}.build.thumbprint".format(partition)]

		present_props = [x for x in prop_name_list if
		partition_props and partition_props.GetProp(x)]
		if not present_props:
		logger.warning(
		"fingerprint is not present for partition %s", partition)
		property_id, fingerprint = "unknown", "unknown"
		else:
		property_id = present_props[0]
		fingerprint = partition_props.GetProp(property_id)
		care_map_list += [property_id, fingerprint]

		if not care_map_list:
		return

		# Converts the list into proto buf message by calling care_map_generator; and
		# writes the result to a temp file.
		temp_care_map_text = MakeTempFile(prefix="caremap_text-",
		suffix=".txt")
		with open(temp_care_map_text, 'w') as text_file:
		text_file.write('\n'.join(care_map_list))

		temp_care_map = MakeTempFile(prefix="caremap-", suffix=".pb")
		care_map_gen_cmd = ["care_map_generator", temp_care_map_text, temp_care_map]
		RunAndCheckOutput(care_map_gen_cmd)

		if not isinstance(output_file, zipfile.ZipFile):
		shutil.copy(temp_care_map, output_file)
		return
		# output_file is a zip file
		care_map_path = "META/care_map.pb"
		if care_map_path in output_file.namelist():
		# Copy the temp file into the OPTIONS.input_tmp dir and update the
		# replace_updated_files_list used by add_img_to_target_files
		if not OPTIONS.replace_updated_files_list:
		OPTIONS.replace_updated_files_list = []
		shutil.copy(temp_care_map, os.path.join(OPTIONS.input_tmp, care_map_path))
		OPTIONS.replace_updated_files_list.append(care_map_path)
		else:
		ZipWrite(output_file, temp_care_map, arcname=care_map_path)


		def IsSparseImage(filepath):
		with open(filepath, 'rb') as fp:
		# Magic for android sparse image format

tools/releasetools/test_add_img_to_target_files.py

+17 −16

Original line number	Diff line number	Diff line
		@@ -22,9 +22,9 @@ import common
		import test_utils
		from add_img_to_target_files import (
		AddPackRadioImages,
		AddCareMapForAbOta, GetCareMap,
		CheckAbOtaImages)
		from rangelib import RangeSet
		from common import AddCareMapForAbOta, GetCareMap


		OPTIONS = common.OPTIONS
		@@ -255,7 +255,8 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase):
		AddCareMapForAbOta(care_map_file, ['system', 'vendor'], image_paths)

		expected = ['system', RangeSet("0-5 10-15").to_string_raw(), "unknown",
		"unknown", 'vendor', RangeSet("0-9").to_string_raw(), "unknown",
		"unknown", 'vendor', RangeSet(
		"0-9").to_string_raw(), "unknown",
		"unknown"]

		self._verifyCareMap(expected, care_map_file)