Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 1caead09 authored by Kelvin Zhang's avatar Kelvin Zhang
Browse files

Use AVB footer to determine caremap 

Care maps need to store the original image size, which excludes bytes
used by hash tree or FEC code.

We used to propagate original image size using the global OPTIONS
dictionary. This is bad coding practice, and also fragile because we
have to make sure what's stored in dictionary and what's on disk are
consistent. Instead, let's read the content of images on disk, and parse
the AVB footer. The AVB footer contains the ground truth original image
size.

Test: build OTA, make sure the care maps have valid range
Bug: 246504616
Change-Id: I9250b478ab34dda60578a6b8c23ae6d7a9385788
parent c6ea3aa6

tools/releasetools/Android.bp

+1 −0
Original line number Diff line number Diff line
@@ -37,6 +37,7 @@ python_defaults { 
        "releasetools_build_image",

        "releasetools_build_super_image",

        "releasetools_common",

        "libavbtool",

    ],

    required: [

        "care_map_generator",

tools/releasetools/add_img_to_target_files.py

+39 −11
Original line number Diff line number Diff line
@@ -46,6 +46,7 @@ Usage: add_img_to_target_files [flag] target_files 


from __future__ import print_function



import avbtool

import datetime

import logging

import os
@@ -62,12 +63,12 @@ import build_super_image 
import common

import verity_utils

import ota_metadata_pb2



from apex_utils import GetApexInfoFromTargetFiles

from common import ZipDelete, PARTITIONS_WITH_CARE_MAP, ExternalError, RunAndCheckOutput, MakeTempFile, ZipWrite

import rangelib

import sparse_img



from apex_utils import GetApexInfoFromTargetFiles

from common import ZipDelete, PARTITIONS_WITH_CARE_MAP, ExternalError, RunAndCheckOutput, IsSparseImage, MakeTempFile, ZipWrite



if sys.hexversion < 0x02070000:

  print("Python 2.7 or newer is required.", file=sys.stderr)

  sys.exit(1)
@@ -87,6 +88,13 @@ FIXED_FILE_TIMESTAMP = int(( 
    datetime.datetime.utcfromtimestamp(0)).total_seconds())





def ParseAvbFooter(img_path) -> avbtool.AvbFooter:

  with open(img_path, 'rb') as fp:

    fp.seek(-avbtool.AvbFooter.SIZE, os.SEEK_END)

    data = fp.read(avbtool.AvbFooter.SIZE)

    return avbtool.AvbFooter(data)





def GetCareMap(which, imgname):

  """Returns the care_map string for the given partition.
@@ -100,15 +108,35 @@ def GetCareMap(which, imgname): 
  """

  assert which in PARTITIONS_WITH_CARE_MAP



  # which + "_image_size" contains the size that the actual filesystem image

  # resides in, which is all that needs to be verified. The additional blocks in

  # the image file contain verity metadata, by reading which would trigger

  # invalid reads.

  image_size = OPTIONS.info_dict.get(which + "_image_size")

  if not image_size:

  is_sparse_img = IsSparseImage(imgname)

  unsparsed_image_size = os.path.getsize(imgname)



  # A verified image contains original image + hash tree data + FEC data

  # + AVB footer, all concatenated together. The caremap specifies a range

  # of blocks that update_verifier should read on top of dm-verity device

  # to verify correctness of OTA updates. When reading off of dm-verity device,

  # the hashtree and FEC part of image isn't available. So caremap should

  # only contain the original image blocks.

  try:

    avbfooter = None

    if is_sparse_img:

      with tempfile.NamedTemporaryFile() as tmpfile:

        img = sparse_img.SparseImage(imgname)

        unsparsed_image_size = img.total_blocks * img.blocksize

        for data in img.ReadBlocks(img.total_blocks - 1, 1):

          tmpfile.write(data)

        tmpfile.flush()

        avbfooter = ParseAvbFooter(tmpfile.name)

    else:

      avbfooter = ParseAvbFooter(imgname)

  except LookupError as e:

    logger.warning(

        "Failed to parse avbfooter for partition %s image %s, %s", which, imgname, e)

    return None



  disable_sparse = OPTIONS.info_dict.get(which + "_disable_sparse")

  image_size = avbfooter.original_image_size

  assert image_size < unsparsed_image_size, f"AVB footer's original image size {image_size} is larger than or equal to image size on disk {unsparsed_image_size}, this can't happen because a verified image = original image + hash tree data + FEC data + avbfooter."

  assert image_size > 0



  image_blocks = int(image_size) // 4096 - 1

  # It's OK for image_blocks to be 0, because care map ranges are inclusive.
@@ -118,7 +146,7 @@ def GetCareMap(which, imgname): 


  # For sparse images, we will only check the blocks that are listed in the care

  # map, i.e. the ones with meaningful data.

  if "extfs_sparse_flag" in OPTIONS.info_dict and not disable_sparse:

  if is_sparse_img:

    simg = sparse_img.SparseImage(imgname)

    care_map_ranges = simg.care_map.intersect(

        rangelib.RangeSet("0-{}".format(image_blocks)))

tools/releasetools/common.py

+12 −3
Original line number Diff line number Diff line
@@ -2865,7 +2865,7 @@ def ZipWriteStr(zip_file, zinfo_or_arcname, data, perms=None, 
  zipfile.ZIP64_LIMIT = saved_zip64_limit





def ZipDelete(zip_filename, entries):

def ZipDelete(zip_filename, entries, force=False):

  """Deletes entries from a ZIP file.



  Since deleting entries from a ZIP file is not supported, it shells out to
@@ -2883,7 +2883,14 @@ def ZipDelete(zip_filename, entries): 
  # If list is empty, nothing to do

  if not entries:

    return

  if force:

    cmd = ["zip", "-q", "-d", zip_filename] + entries

  else:

    cmd = ["zip", "-d", zip_filename] + entries

  if force:

    p = Run(cmd)

    p.wait()

  else:

    RunAndCheckOutput(cmd)
@@ -3979,6 +3986,8 @@ def GetBootImageTimestamp(boot_img): 




def IsSparseImage(filepath):

  if not os.path.exists(filepath):

    return False

  with open(filepath, 'rb') as fp:

    # Magic for android sparse image format

    # https://source.android.com/devices/bootloader/images

tools/releasetools/sparse_img.py

+6 −1
Original line number Diff line number Diff line
@@ -80,7 +80,7 @@ class SparseImage(object): 
    self.offset_map = offset_map = []

    self.clobbered_blocks = rangelib.RangeSet(data=clobbered_blocks)



    for i in range(total_chunks):

    for _ in range(total_chunks):

      header_bin = f.read(12)

      header = struct.unpack("<2H2I", header_bin)

      chunk_type = header[0]
@@ -166,6 +166,11 @@ class SparseImage(object): 
  def ReadRangeSet(self, ranges):

    return [d for d in self._GetRangeData(ranges)]



  def ReadBlocks(self, start=0, num_blocks=None):

    if num_blocks is None:

      num_blocks = self.total_blocks

    return self._GetRangeData([(start, start + num_blocks)])



  def TotalSha1(self, include_clobbered_blocks=False):

    """Return the SHA-1 hash of all data in the 'care' regions.

tools/releasetools/test_add_img_to_target_files.py

+14 −46
Original line number Diff line number Diff line
@@ -16,6 +16,7 @@ 


import os

import os.path

import tempfile

import zipfile



import common
@@ -124,9 +125,6 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase): 
  def _test_AddCareMapForAbOta():

    """Helper function to set up the test for test_AddCareMapForAbOta()."""

    OPTIONS.info_dict = {

        'extfs_sparse_flag': '-s',

        'system_image_size': 65536,

        'vendor_image_size': 40960,

        'system_verity_block_device': '/dev/block/system',

        'vendor_verity_block_device': '/dev/block/vendor',

        'system.build.prop': common.PartitionBuildProps.FromDictionary(
@@ -149,9 +147,9 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase): 
    system_image = test_utils.construct_sparse_image([

        (0xCAC1, 6),

        (0xCAC3, 4),

        (0xCAC1, 8)])

        (0xCAC1, 6)], "system")

    vendor_image = test_utils.construct_sparse_image([

        (0xCAC2, 12)])

        (0xCAC2, 10)], "vendor")



    image_paths = {

        'system': system_image,
@@ -210,9 +208,6 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase): 
    """Tests the case for device using AVB."""

    image_paths = self._test_AddCareMapForAbOta()

    OPTIONS.info_dict = {

        'extfs_sparse_flag': '-s',

        'system_image_size': 65536,

        'vendor_image_size': 40960,

        'avb_system_hashtree_enable': 'true',

        'avb_vendor_hashtree_enable': 'true',

        'system.build.prop': common.PartitionBuildProps.FromDictionary(
@@ -244,9 +239,6 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase): 
    """Tests the case for partitions without fingerprint."""

    image_paths = self._test_AddCareMapForAbOta()

    OPTIONS.info_dict = {

        'extfs_sparse_flag': '-s',

        'system_image_size': 65536,

        'vendor_image_size': 40960,

        'system_verity_block_device': '/dev/block/system',

        'vendor_verity_block_device': '/dev/block/vendor',

    }
@@ -266,9 +258,6 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase): 
    """Tests the case for partitions with thumbprint."""

    image_paths = self._test_AddCareMapForAbOta()

    OPTIONS.info_dict = {

        'extfs_sparse_flag': '-s',

        'system_image_size': 65536,

        'vendor_image_size': 40960,

        'system_verity_block_device': '/dev/block/system',

        'vendor_verity_block_device': '/dev/block/vendor',

        'system.build.prop': common.PartitionBuildProps.FromDictionary(
@@ -298,9 +287,7 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase): 
  @test_utils.SkipIfExternalToolsUnavailable()

  def test_AddCareMapForAbOta_skipPartition(self):

    image_paths = self._test_AddCareMapForAbOta()



    # Remove vendor_image_size to invalidate the care_map for vendor.img.

    del OPTIONS.info_dict['vendor_image_size']

    test_utils.erase_avb_footer(image_paths["vendor"])



    care_map_file = os.path.join(OPTIONS.input_tmp, 'META', 'care_map.pb')

    AddCareMapForAbOta(care_map_file, ['system', 'vendor'], image_paths)
@@ -314,10 +301,8 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase): 
  @test_utils.SkipIfExternalToolsUnavailable()

  def test_AddCareMapForAbOta_skipAllPartitions(self):

    image_paths = self._test_AddCareMapForAbOta()



    # Remove the image_size properties for all the partitions.

    del OPTIONS.info_dict['system_image_size']

    del OPTIONS.info_dict['vendor_image_size']

    test_utils.erase_avb_footer(image_paths["system"])

    test_utils.erase_avb_footer(image_paths["vendor"])



    care_map_file = os.path.join(OPTIONS.input_tmp, 'META', 'care_map.pb')

    AddCareMapForAbOta(care_map_file, ['system', 'vendor'], image_paths)
@@ -396,35 +381,18 @@ class AddImagesToTargetFilesTest(test_utils.ReleaseToolsTestCase): 
    sparse_image = test_utils.construct_sparse_image([

        (0xCAC1, 6),

        (0xCAC3, 4),

        (0xCAC1, 6)])

    OPTIONS.info_dict = {

        'extfs_sparse_flag': '-s',

        'system_image_size': 53248,

    }

        (0xCAC1, 6)], "system")

    name, care_map = GetCareMap('system', sparse_image)

    self.assertEqual('system', name)

    self.assertEqual(RangeSet("0-5 10-12").to_string_raw(), care_map)

    self.assertEqual(RangeSet("0-5 10-15").to_string_raw(), care_map)



  def test_GetCareMap_invalidPartition(self):

    self.assertRaises(AssertionError, GetCareMap, 'oem', None)



  def test_GetCareMap_invalidAdjustedPartitionSize(self):

    sparse_image = test_utils.construct_sparse_image([

        (0xCAC1, 6),

        (0xCAC3, 4),

        (0xCAC1, 6)])

    OPTIONS.info_dict = {

        'extfs_sparse_flag': '-s',

        'system_image_size': -45056,

    }

    self.assertRaises(AssertionError, GetCareMap, 'system', sparse_image)



  def test_GetCareMap_nonSparseImage(self):

    OPTIONS.info_dict = {

        'system_image_size': 53248,

    }

    # 'foo' is the image filename, which is expected to be not used by

    # GetCareMap().

    name, care_map = GetCareMap('system', 'foo')

    with tempfile.NamedTemporaryFile() as tmpfile:

      tmpfile.truncate(4096 * 13)

      test_utils.append_avb_footer(tmpfile.name, "system")

      name, care_map = GetCareMap('system', tmpfile.name)

      self.assertEqual('system', name)

      self.assertEqual(RangeSet("0-12").to_string_raw(), care_map)