Merge "Use AVB footer to determine caremap" (fe21f99d) · Commits · e / os / android_build

tools/releasetools/Android.bp

+1 −0

Original line number	Diff line number	Diff line
		@@ -37,6 +37,7 @@ python_defaults {
		"releasetools_build_image",
		"releasetools_build_super_image",
		"releasetools_common",
		"libavbtool",
		],
		required: [
		"care_map_generator",

tools/releasetools/add_img_to_target_files.py

+39 −11

Original line number	Diff line number	Diff line
		@@ -46,6 +46,7 @@ Usage: add_img_to_target_files [flag] target_files

		from __future__ import print_function

		import avbtool
		import datetime
		import logging
		import os
		@@ -62,12 +63,12 @@ import build_super_image
		import common
		import verity_utils
		import ota_metadata_pb2

		from apex_utils import GetApexInfoFromTargetFiles
		from common import ZipDelete, PARTITIONS_WITH_CARE_MAP, ExternalError, RunAndCheckOutput, MakeTempFile, ZipWrite
		import rangelib
		import sparse_img

		from apex_utils import GetApexInfoFromTargetFiles
		from common import ZipDelete, PARTITIONS_WITH_CARE_MAP, ExternalError, RunAndCheckOutput, IsSparseImage, MakeTempFile, ZipWrite

		if sys.hexversion < 0x02070000:
		print("Python 2.7 or newer is required.", file=sys.stderr)
		sys.exit(1)
		@@ -87,6 +88,13 @@ FIXED_FILE_TIMESTAMP = int((
		datetime.datetime.utcfromtimestamp(0)).total_seconds())


		def ParseAvbFooter(img_path) -> avbtool.AvbFooter:
		with open(img_path, 'rb') as fp:
		fp.seek(-avbtool.AvbFooter.SIZE, os.SEEK_END)
		data = fp.read(avbtool.AvbFooter.SIZE)
		return avbtool.AvbFooter(data)


		def GetCareMap(which, imgname):
		"""Returns the care_map string for the given partition.

		@@ -100,15 +108,35 @@ def GetCareMap(which, imgname):
		"""
		assert which in PARTITIONS_WITH_CARE_MAP

		# which + "_image_size" contains the size that the actual filesystem image
		# resides in, which is all that needs to be verified. The additional blocks in
		# the image file contain verity metadata, by reading which would trigger
		# invalid reads.
		image_size = OPTIONS.info_dict.get(which + "_image_size")
		if not image_size:
		is_sparse_img = IsSparseImage(imgname)
		unsparsed_image_size = os.path.getsize(imgname)

		# A verified image contains original image + hash tree data + FEC data
		# + AVB footer, all concatenated together. The caremap specifies a range
		# of blocks that update_verifier should read on top of dm-verity device
		# to verify correctness of OTA updates. When reading off of dm-verity device,
		# the hashtree and FEC part of image isn't available. So caremap should
		# only contain the original image blocks.
		try:
		avbfooter = None
		if is_sparse_img:
		with tempfile.NamedTemporaryFile() as tmpfile:
		img = sparse_img.SparseImage(imgname)
		unsparsed_image_size = img.total_blocks * img.blocksize
		for data in img.ReadBlocks(img.total_blocks - 1, 1):
		tmpfile.write(data)
		tmpfile.flush()
		avbfooter = ParseAvbFooter(tmpfile.name)
		else:
		avbfooter = ParseAvbFooter(imgname)
		except LookupError as e:
		logger.warning(
		"Failed to parse avbfooter for partition %s image %s, %s", which, imgname, e)
		return None

		disable_sparse = OPTIONS.info_dict.get(which + "_disable_sparse")
		image_size = avbfooter.original_image_size
		assert image_size < unsparsed_image_size, f"AVB footer's original image size {image_size} is larger than or equal to image size on disk {unsparsed_image_size}, this can't happen because a verified image = original image + hash tree data + FEC data + avbfooter."
		assert image_size > 0

		image_blocks = int(image_size) // 4096 - 1
		# It's OK for image_blocks to be 0, because care map ranges are inclusive.
		@@ -118,7 +146,7 @@ def GetCareMap(which, imgname):

		# For sparse images, we will only check the blocks that are listed in the care
		# map, i.e. the ones with meaningful data.
		if "extfs_sparse_flag" in OPTIONS.info_dict and not disable_sparse:
		if is_sparse_img:
		simg = sparse_img.SparseImage(imgname)
		care_map_ranges = simg.care_map.intersect(
		rangelib.RangeSet("0-{}".format(image_blocks)))

tools/releasetools/common.py

+12 −3

Original line number	Diff line number	Diff line
		@@ -2865,7 +2865,7 @@ def ZipWriteStr(zip_file, zinfo_or_arcname, data, perms=None,
		zipfile.ZIP64_LIMIT = saved_zip64_limit


		def ZipDelete(zip_filename, entries):
		def ZipDelete(zip_filename, entries, force=False):
		"""Deletes entries from a ZIP file.

		Since deleting entries from a ZIP file is not supported, it shells out to
		@@ -2883,7 +2883,14 @@ def ZipDelete(zip_filename, entries):
		# If list is empty, nothing to do
		if not entries:
		return
		if force:
		cmd = ["zip", "-q", "-d", zip_filename] + entries
		else:
		cmd = ["zip", "-d", zip_filename] + entries
		if force:
		p = Run(cmd)
		p.wait()
		else:
		RunAndCheckOutput(cmd)


		@@ -3979,6 +3986,8 @@ def GetBootImageTimestamp(boot_img):


		def IsSparseImage(filepath):
		if not os.path.exists(filepath):
		return False
		with open(filepath, 'rb') as fp:
		# Magic for android sparse image format
		# https://source.android.com/devices/bootloader/images

tools/releasetools/sparse_img.py

+6 −1

Original line number	Diff line number	Diff line
		@@ -80,7 +80,7 @@ class SparseImage(object):
		self.offset_map = offset_map = []
		self.clobbered_blocks = rangelib.RangeSet(data=clobbered_blocks)

		for i in range(total_chunks):
		for _ in range(total_chunks):
		header_bin = f.read(12)
		header = struct.unpack("<2H2I", header_bin)
		chunk_type = header[0]
		@@ -166,6 +166,11 @@ class SparseImage(object):
		def ReadRangeSet(self, ranges):
		return [d for d in self._GetRangeData(ranges)]

		def ReadBlocks(self, start=0, num_blocks=None):
		if num_blocks is None:
		num_blocks = self.total_blocks
		return self._GetRangeData([(start, start + num_blocks)])

		def TotalSha1(self, include_clobbered_blocks=False):
		"""Return the SHA-1 hash of all data in the 'care' regions.

tools/releasetools/test_add_img_to_target_files.py

+14 −46

Original line number	Diff line number	Diff line
		@@ -16,6 +16,7 @@

		import os
		import os.path
		import tempfile
		import zipfile

		import common
		@@ -124,9 +125,6 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase):
		def _test_AddCareMapForAbOta():
		"""Helper function to set up the test for test_AddCareMapForAbOta()."""
		OPTIONS.info_dict = {
		'extfs_sparse_flag': '-s',
		'system_image_size': 65536,
		'vendor_image_size': 40960,
		'system_verity_block_device': '/dev/block/system',
		'vendor_verity_block_device': '/dev/block/vendor',
		'system.build.prop': common.PartitionBuildProps.FromDictionary(
		@@ -149,9 +147,9 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase):
		system_image = test_utils.construct_sparse_image([
		(0xCAC1, 6),
		(0xCAC3, 4),
		(0xCAC1, 8)])
		(0xCAC1, 6)], "system")
		vendor_image = test_utils.construct_sparse_image([
		(0xCAC2, 12)])
		(0xCAC2, 10)], "vendor")

		image_paths = {
		'system': system_image,
		@@ -210,9 +208,6 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase):
		"""Tests the case for device using AVB."""
		image_paths = self._test_AddCareMapForAbOta()
		OPTIONS.info_dict = {
		'extfs_sparse_flag': '-s',
		'system_image_size': 65536,
		'vendor_image_size': 40960,
		'avb_system_hashtree_enable': 'true',
		'avb_vendor_hashtree_enable': 'true',
		'system.build.prop': common.PartitionBuildProps.FromDictionary(
		@@ -244,9 +239,6 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase):
		"""Tests the case for partitions without fingerprint."""
		image_paths = self._test_AddCareMapForAbOta()
		OPTIONS.info_dict = {
		'extfs_sparse_flag': '-s',
		'system_image_size': 65536,
		'vendor_image_size': 40960,
		'system_verity_block_device': '/dev/block/system',
		'vendor_verity_block_device': '/dev/block/vendor',
		}
		@@ -266,9 +258,6 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase):
		"""Tests the case for partitions with thumbprint."""
		image_paths = self._test_AddCareMapForAbOta()
		OPTIONS.info_dict = {
		'extfs_sparse_flag': '-s',
		'system_image_size': 65536,
		'vendor_image_size': 40960,
		'system_verity_block_device': '/dev/block/system',
		'vendor_verity_block_device': '/dev/block/vendor',
		'system.build.prop': common.PartitionBuildProps.FromDictionary(
		@@ -298,9 +287,7 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase):
		@test_utils.SkipIfExternalToolsUnavailable()
		def test_AddCareMapForAbOta_skipPartition(self):
		image_paths = self._test_AddCareMapForAbOta()

		# Remove vendor_image_size to invalidate the care_map for vendor.img.
		del OPTIONS.info_dict['vendor_image_size']
		test_utils.erase_avb_footer(image_paths["vendor"])

		care_map_file = os.path.join(OPTIONS.input_tmp, 'META', 'care_map.pb')
		AddCareMapForAbOta(care_map_file, ['system', 'vendor'], image_paths)
		@@ -314,10 +301,8 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase):
		@test_utils.SkipIfExternalToolsUnavailable()
		def test_AddCareMapForAbOta_skipAllPartitions(self):
		image_paths = self._test_AddCareMapForAbOta()

		# Remove the image_size properties for all the partitions.
		del OPTIONS.info_dict['system_image_size']
		del OPTIONS.info_dict['vendor_image_size']
		test_utils.erase_avb_footer(image_paths["system"])
		test_utils.erase_avb_footer(image_paths["vendor"])

		care_map_file = os.path.join(OPTIONS.input_tmp, 'META', 'care_map.pb')
		AddCareMapForAbOta(care_map_file, ['system', 'vendor'], image_paths)
		@@ -396,35 +381,18 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase):
		sparse_image = test_utils.construct_sparse_image([
		(0xCAC1, 6),
		(0xCAC3, 4),
		(0xCAC1, 6)])
		OPTIONS.info_dict = {
		'extfs_sparse_flag': '-s',
		'system_image_size': 53248,
		}
		(0xCAC1, 6)], "system")
		name, care_map = GetCareMap('system', sparse_image)
		self.assertEqual('system', name)
		self.assertEqual(RangeSet("0-5 10-12").to_string_raw(), care_map)
		self.assertEqual(RangeSet("0-5 10-15").to_string_raw(), care_map)

		def test_GetCareMap_invalidPartition(self):
		self.assertRaises(AssertionError, GetCareMap, 'oem', None)

		def test_GetCareMap_invalidAdjustedPartitionSize(self):
		sparse_image = test_utils.construct_sparse_image([
		(0xCAC1, 6),
		(0xCAC3, 4),
		(0xCAC1, 6)])
		OPTIONS.info_dict = {
		'extfs_sparse_flag': '-s',
		'system_image_size': -45056,
		}
		self.assertRaises(AssertionError, GetCareMap, 'system', sparse_image)

		def test_GetCareMap_nonSparseImage(self):
		OPTIONS.info_dict = {
		'system_image_size': 53248,
		}
		# 'foo' is the image filename, which is expected to be not used by
		# GetCareMap().
		name, care_map = GetCareMap('system', 'foo')
		with tempfile.NamedTemporaryFile() as tmpfile:
		tmpfile.truncate(4096 * 13)
		test_utils.append_avb_footer(tmpfile.name, "system")
		name, care_map = GetCareMap('system', tmpfile.name)
		self.assertEqual('system', name)
		self.assertEqual(RangeSet("0-12").to_string_raw(), care_map)