Use AVB footer to determine caremap (1caead09) · Commits · e / os / android_build

tools/releasetools/Android.bp

+1 −0

Original line number	Original line	Diff line number	Diff line
	@@ -37,6 +37,7 @@ python_defaults {
	"releasetools_build_image",		"releasetools_build_image",
	"releasetools_build_super_image",		"releasetools_build_super_image",
	"releasetools_common",		"releasetools_common",
			"libavbtool",
	],		],
	required: [		required: [
	"care_map_generator",		"care_map_generator",

tools/releasetools/add_img_to_target_files.py

+39 −11

Original line number	Original line	Diff line number	Diff line
	@@ -46,6 +46,7 @@ Usage: add_img_to_target_files [flag] target_files

	from __future__ import print_function		from __future__ import print_function

			import avbtool
	import datetime		import datetime
	import logging		import logging
	import os		import os
	@@ -62,12 +63,12 @@ import build_super_image
	import common		import common
	import verity_utils		import verity_utils
	import ota_metadata_pb2		import ota_metadata_pb2

	from apex_utils import GetApexInfoFromTargetFiles
	from common import ZipDelete, PARTITIONS_WITH_CARE_MAP, ExternalError, RunAndCheckOutput, MakeTempFile, ZipWrite
	import rangelib		import rangelib
	import sparse_img		import sparse_img

			from apex_utils import GetApexInfoFromTargetFiles
			from common import ZipDelete, PARTITIONS_WITH_CARE_MAP, ExternalError, RunAndCheckOutput, IsSparseImage, MakeTempFile, ZipWrite

	if sys.hexversion < 0x02070000:		if sys.hexversion < 0x02070000:
	print("Python 2.7 or newer is required.", file=sys.stderr)		print("Python 2.7 or newer is required.", file=sys.stderr)
	sys.exit(1)		sys.exit(1)
	@@ -87,6 +88,13 @@ FIXED_FILE_TIMESTAMP = int((
	datetime.datetime.utcfromtimestamp(0)).total_seconds())		datetime.datetime.utcfromtimestamp(0)).total_seconds())


			def ParseAvbFooter(img_path) -> avbtool.AvbFooter:
			with open(img_path, 'rb') as fp:
			fp.seek(-avbtool.AvbFooter.SIZE, os.SEEK_END)
			data = fp.read(avbtool.AvbFooter.SIZE)
			return avbtool.AvbFooter(data)


	def GetCareMap(which, imgname):		def GetCareMap(which, imgname):
	"""Returns the care_map string for the given partition.		"""Returns the care_map string for the given partition.

	@@ -100,15 +108,35 @@ def GetCareMap(which, imgname):
	"""		"""
	assert which in PARTITIONS_WITH_CARE_MAP		assert which in PARTITIONS_WITH_CARE_MAP

	# which + "_image_size" contains the size that the actual filesystem image		is_sparse_img = IsSparseImage(imgname)
	# resides in, which is all that needs to be verified. The additional blocks in		unsparsed_image_size = os.path.getsize(imgname)
	# the image file contain verity metadata, by reading which would trigger
	# invalid reads.		# A verified image contains original image + hash tree data + FEC data
	image_size = OPTIONS.info_dict.get(which + "_image_size")		# + AVB footer, all concatenated together. The caremap specifies a range
	if not image_size:		# of blocks that update_verifier should read on top of dm-verity device
			# to verify correctness of OTA updates. When reading off of dm-verity device,
			# the hashtree and FEC part of image isn't available. So caremap should
			# only contain the original image blocks.
			try:
			avbfooter = None
			if is_sparse_img:
			with tempfile.NamedTemporaryFile() as tmpfile:
			img = sparse_img.SparseImage(imgname)
			unsparsed_image_size = img.total_blocks * img.blocksize
			for data in img.ReadBlocks(img.total_blocks - 1, 1):
			tmpfile.write(data)
			tmpfile.flush()
			avbfooter = ParseAvbFooter(tmpfile.name)
			else:
			avbfooter = ParseAvbFooter(imgname)
			except LookupError as e:
			logger.warning(
			"Failed to parse avbfooter for partition %s image %s, %s", which, imgname, e)
	return None		return None

	disable_sparse = OPTIONS.info_dict.get(which + "_disable_sparse")		image_size = avbfooter.original_image_size
			assert image_size < unsparsed_image_size, f"AVB footer's original image size {image_size} is larger than or equal to image size on disk {unsparsed_image_size}, this can't happen because a verified image = original image + hash tree data + FEC data + avbfooter."
			assert image_size > 0

	image_blocks = int(image_size) // 4096 - 1		image_blocks = int(image_size) // 4096 - 1
	# It's OK for image_blocks to be 0, because care map ranges are inclusive.		# It's OK for image_blocks to be 0, because care map ranges are inclusive.
	@@ -118,7 +146,7 @@ def GetCareMap(which, imgname):

	# For sparse images, we will only check the blocks that are listed in the care		# For sparse images, we will only check the blocks that are listed in the care
	# map, i.e. the ones with meaningful data.		# map, i.e. the ones with meaningful data.
	if "extfs_sparse_flag" in OPTIONS.info_dict and not disable_sparse:		if is_sparse_img:
	simg = sparse_img.SparseImage(imgname)		simg = sparse_img.SparseImage(imgname)
	care_map_ranges = simg.care_map.intersect(		care_map_ranges = simg.care_map.intersect(
	rangelib.RangeSet("0-{}".format(image_blocks)))		rangelib.RangeSet("0-{}".format(image_blocks)))

tools/releasetools/common.py

+12 −3

Original line number	Original line	Diff line number	Diff line
	@@ -2865,7 +2865,7 @@ def ZipWriteStr(zip_file, zinfo_or_arcname, data, perms=None,
	zipfile.ZIP64_LIMIT = saved_zip64_limit		zipfile.ZIP64_LIMIT = saved_zip64_limit


	def ZipDelete(zip_filename, entries):		def ZipDelete(zip_filename, entries, force=False):
	"""Deletes entries from a ZIP file.		"""Deletes entries from a ZIP file.

	Since deleting entries from a ZIP file is not supported, it shells out to		Since deleting entries from a ZIP file is not supported, it shells out to
	@@ -2883,7 +2883,14 @@ def ZipDelete(zip_filename, entries):
	# If list is empty, nothing to do		# If list is empty, nothing to do
	if not entries:		if not entries:
	return		return
			if force:
			cmd = ["zip", "-q", "-d", zip_filename] + entries
			else:
	cmd = ["zip", "-d", zip_filename] + entries		cmd = ["zip", "-d", zip_filename] + entries
			if force:
			p = Run(cmd)
			p.wait()
			else:
	RunAndCheckOutput(cmd)		RunAndCheckOutput(cmd)


	@@ -3979,6 +3986,8 @@ def GetBootImageTimestamp(boot_img):


	def IsSparseImage(filepath):		def IsSparseImage(filepath):
			if not os.path.exists(filepath):
			return False
	with open(filepath, 'rb') as fp:		with open(filepath, 'rb') as fp:
	# Magic for android sparse image format		# Magic for android sparse image format
	# https://source.android.com/devices/bootloader/images		# https://source.android.com/devices/bootloader/images

tools/releasetools/sparse_img.py

+6 −1

Original line number	Original line	Diff line number	Diff line
	@@ -80,7 +80,7 @@ class SparseImage(object):
	self.offset_map = offset_map = []		self.offset_map = offset_map = []
	self.clobbered_blocks = rangelib.RangeSet(data=clobbered_blocks)		self.clobbered_blocks = rangelib.RangeSet(data=clobbered_blocks)

	for i in range(total_chunks):		for _ in range(total_chunks):
	header_bin = f.read(12)		header_bin = f.read(12)
	header = struct.unpack("<2H2I", header_bin)		header = struct.unpack("<2H2I", header_bin)
	chunk_type = header[0]		chunk_type = header[0]
	@@ -166,6 +166,11 @@ class SparseImage(object):
	def ReadRangeSet(self, ranges):		def ReadRangeSet(self, ranges):
	return [d for d in self._GetRangeData(ranges)]		return [d for d in self._GetRangeData(ranges)]

			def ReadBlocks(self, start=0, num_blocks=None):
			if num_blocks is None:
			num_blocks = self.total_blocks
			return self._GetRangeData([(start, start + num_blocks)])

	def TotalSha1(self, include_clobbered_blocks=False):		def TotalSha1(self, include_clobbered_blocks=False):
	"""Return the SHA-1 hash of all data in the 'care' regions.		"""Return the SHA-1 hash of all data in the 'care' regions.

tools/releasetools/test_add_img_to_target_files.py

+14 −46

Original line number	Original line	Diff line number	Diff line
	@@ -16,6 +16,7 @@

	import os		import os
	import os.path		import os.path
			import tempfile
	import zipfile		import zipfile

	import common		import common
	@@ -124,9 +125,6 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase):
	def _test_AddCareMapForAbOta():		def _test_AddCareMapForAbOta():
	"""Helper function to set up the test for test_AddCareMapForAbOta()."""		"""Helper function to set up the test for test_AddCareMapForAbOta()."""
	OPTIONS.info_dict = {		OPTIONS.info_dict = {
	'extfs_sparse_flag': '-s',
	'system_image_size': 65536,
	'vendor_image_size': 40960,
	'system_verity_block_device': '/dev/block/system',		'system_verity_block_device': '/dev/block/system',
	'vendor_verity_block_device': '/dev/block/vendor',		'vendor_verity_block_device': '/dev/block/vendor',
	'system.build.prop': common.PartitionBuildProps.FromDictionary(		'system.build.prop': common.PartitionBuildProps.FromDictionary(
	@@ -149,9 +147,9 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase):
	system_image = test_utils.construct_sparse_image([		system_image = test_utils.construct_sparse_image([
	(0xCAC1, 6),		(0xCAC1, 6),
	(0xCAC3, 4),		(0xCAC3, 4),
	(0xCAC1, 8)])		(0xCAC1, 6)], "system")
	vendor_image = test_utils.construct_sparse_image([		vendor_image = test_utils.construct_sparse_image([
	(0xCAC2, 12)])		(0xCAC2, 10)], "vendor")

	image_paths = {		image_paths = {
	'system': system_image,		'system': system_image,
	@@ -210,9 +208,6 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase):
	"""Tests the case for device using AVB."""		"""Tests the case for device using AVB."""
	image_paths = self._test_AddCareMapForAbOta()		image_paths = self._test_AddCareMapForAbOta()
	OPTIONS.info_dict = {		OPTIONS.info_dict = {
	'extfs_sparse_flag': '-s',
	'system_image_size': 65536,
	'vendor_image_size': 40960,
	'avb_system_hashtree_enable': 'true',		'avb_system_hashtree_enable': 'true',
	'avb_vendor_hashtree_enable': 'true',		'avb_vendor_hashtree_enable': 'true',
	'system.build.prop': common.PartitionBuildProps.FromDictionary(		'system.build.prop': common.PartitionBuildProps.FromDictionary(
	@@ -244,9 +239,6 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase):
	"""Tests the case for partitions without fingerprint."""		"""Tests the case for partitions without fingerprint."""
	image_paths = self._test_AddCareMapForAbOta()		image_paths = self._test_AddCareMapForAbOta()
	OPTIONS.info_dict = {		OPTIONS.info_dict = {
	'extfs_sparse_flag': '-s',
	'system_image_size': 65536,
	'vendor_image_size': 40960,
	'system_verity_block_device': '/dev/block/system',		'system_verity_block_device': '/dev/block/system',
	'vendor_verity_block_device': '/dev/block/vendor',		'vendor_verity_block_device': '/dev/block/vendor',
	}		}
	@@ -266,9 +258,6 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase):
	"""Tests the case for partitions with thumbprint."""		"""Tests the case for partitions with thumbprint."""
	image_paths = self._test_AddCareMapForAbOta()		image_paths = self._test_AddCareMapForAbOta()
	OPTIONS.info_dict = {		OPTIONS.info_dict = {
	'extfs_sparse_flag': '-s',
	'system_image_size': 65536,
	'vendor_image_size': 40960,
	'system_verity_block_device': '/dev/block/system',		'system_verity_block_device': '/dev/block/system',
	'vendor_verity_block_device': '/dev/block/vendor',		'vendor_verity_block_device': '/dev/block/vendor',
	'system.build.prop': common.PartitionBuildProps.FromDictionary(		'system.build.prop': common.PartitionBuildProps.FromDictionary(
	@@ -298,9 +287,7 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase):
	@test_utils.SkipIfExternalToolsUnavailable()		@test_utils.SkipIfExternalToolsUnavailable()
	def test_AddCareMapForAbOta_skipPartition(self):		def test_AddCareMapForAbOta_skipPartition(self):
	image_paths = self._test_AddCareMapForAbOta()		image_paths = self._test_AddCareMapForAbOta()
			test_utils.erase_avb_footer(image_paths["vendor"])
	# Remove vendor_image_size to invalidate the care_map for vendor.img.
	del OPTIONS.info_dict['vendor_image_size']

	care_map_file = os.path.join(OPTIONS.input_tmp, 'META', 'care_map.pb')		care_map_file = os.path.join(OPTIONS.input_tmp, 'META', 'care_map.pb')
	AddCareMapForAbOta(care_map_file, ['system', 'vendor'], image_paths)		AddCareMapForAbOta(care_map_file, ['system', 'vendor'], image_paths)
	@@ -314,10 +301,8 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase):
	@test_utils.SkipIfExternalToolsUnavailable()		@test_utils.SkipIfExternalToolsUnavailable()
	def test_AddCareMapForAbOta_skipAllPartitions(self):		def test_AddCareMapForAbOta_skipAllPartitions(self):
	image_paths = self._test_AddCareMapForAbOta()		image_paths = self._test_AddCareMapForAbOta()
			test_utils.erase_avb_footer(image_paths["system"])
	# Remove the image_size properties for all the partitions.		test_utils.erase_avb_footer(image_paths["vendor"])
	del OPTIONS.info_dict['system_image_size']
	del OPTIONS.info_dict['vendor_image_size']

	care_map_file = os.path.join(OPTIONS.input_tmp, 'META', 'care_map.pb')		care_map_file = os.path.join(OPTIONS.input_tmp, 'META', 'care_map.pb')
	AddCareMapForAbOta(care_map_file, ['system', 'vendor'], image_paths)		AddCareMapForAbOta(care_map_file, ['system', 'vendor'], image_paths)
	@@ -396,35 +381,18 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase):
	sparse_image = test_utils.construct_sparse_image([		sparse_image = test_utils.construct_sparse_image([
	(0xCAC1, 6),		(0xCAC1, 6),
	(0xCAC3, 4),		(0xCAC3, 4),
	(0xCAC1, 6)])		(0xCAC1, 6)], "system")
	OPTIONS.info_dict = {
	'extfs_sparse_flag': '-s',
	'system_image_size': 53248,
	}
	name, care_map = GetCareMap('system', sparse_image)		name, care_map = GetCareMap('system', sparse_image)
	self.assertEqual('system', name)		self.assertEqual('system', name)
	self.assertEqual(RangeSet("0-5 10-12").to_string_raw(), care_map)		self.assertEqual(RangeSet("0-5 10-15").to_string_raw(), care_map)

	def test_GetCareMap_invalidPartition(self):		def test_GetCareMap_invalidPartition(self):
	self.assertRaises(AssertionError, GetCareMap, 'oem', None)		self.assertRaises(AssertionError, GetCareMap, 'oem', None)

	def test_GetCareMap_invalidAdjustedPartitionSize(self):
	sparse_image = test_utils.construct_sparse_image([
	(0xCAC1, 6),
	(0xCAC3, 4),
	(0xCAC1, 6)])
	OPTIONS.info_dict = {
	'extfs_sparse_flag': '-s',
	'system_image_size': -45056,
	}
	self.assertRaises(AssertionError, GetCareMap, 'system', sparse_image)

	def test_GetCareMap_nonSparseImage(self):		def test_GetCareMap_nonSparseImage(self):
	OPTIONS.info_dict = {		with tempfile.NamedTemporaryFile() as tmpfile:
	'system_image_size': 53248,		tmpfile.truncate(4096 * 13)
	}		test_utils.append_avb_footer(tmpfile.name, "system")
	# 'foo' is the image filename, which is expected to be not used by		name, care_map = GetCareMap('system', tmpfile.name)
	# GetCareMap().
	name, care_map = GetCareMap('system', 'foo')
	self.assertEqual('system', name)		self.assertEqual('system', name)
	self.assertEqual(RangeSet("0-12").to_string_raw(), care_map)		self.assertEqual(RangeSet("0-12").to_string_raw(), care_map)