Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 93db9559 authored by Tao Bao's avatar Tao Bao Committed by Gerrit Code Review
Browse files

Merge "releasetools: Detect incomplete block ranges."

parents 80ac71ba c765cca3

tools/releasetools/common.py

+57 −0
Original line number Diff line number Diff line
@@ -25,6 +25,7 @@ import platform 
import re

import shlex

import shutil

import string

import subprocess

import sys

import tempfile
@@ -34,6 +35,7 @@ import zipfile 
from hashlib import sha1, sha256



import blockimgdiff

import sparse_img



class Options(object):

  def __init__(self):
@@ -124,6 +126,11 @@ def Run(args, verbose=None, **kwargs): 
  return subprocess.Popen(args, **kwargs)





def RoundUpTo4K(value):

  rounded_up = value + 4095

  return rounded_up - (rounded_up % 4096)





def CloseInheritedPipes():

  """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds

  before doing other work."""
@@ -618,6 +625,56 @@ def UnzipTemp(filename, pattern=None): 
  return tmp, zipfile.ZipFile(filename, "r")





def GetSparseImage(which, tmpdir, input_zip):

  """Returns a SparseImage object suitable for passing to BlockImageDiff.



  This function loads the specified sparse image from the given path, and

  performs additional processing for OTA purpose. For example, it always adds

  block 0 to clobbered blocks list. It also detects files that cannot be

  reconstructed from the block list, for whom we should avoid applying imgdiff.



  Args:

    which: The partition name, which must be "system" or "vendor".

    tmpdir: The directory that contains the prebuilt image and block map file.

    input_zip: The target-files ZIP archive.



  Returns:

    A SparseImage object, with file_map info loaded.

  """

  assert which in ("system", "vendor")



  path = os.path.join(tmpdir, "IMAGES", which + ".img")

  mappath = os.path.join(tmpdir, "IMAGES", which + ".map")



  # The image and map files must have been created prior to calling

  # ota_from_target_files.py (since LMP).

  assert os.path.exists(path) and os.path.exists(mappath)



  # In ext4 filesystems, block 0 might be changed even being mounted R/O. We add

  # it to clobbered_blocks so that it will be written to the target

  # unconditionally. Note that they are still part of care_map. (Bug: 20939131)

  clobbered_blocks = "0"



  image = sparse_img.SparseImage(path, mappath, clobbered_blocks)



  # block.map may contain less blocks, because mke2fs may skip allocating blocks

  # if they contain all zeros. We can't reconstruct such a file from its block

  # list. Tag such entries accordingly. (Bug: 65213616)

  for entry in image.file_map:

    # "/system/framework/am.jar" => "SYSTEM/framework/am.jar".

    arcname = string.replace(entry, which, which.upper(), 1)[1:]

    # Skip artificial names, such as "__ZERO", "__NONZERO-1".

    if arcname not in input_zip.namelist():

      continue



    info = input_zip.getinfo(arcname)

    ranges = image.file_map[entry]

    if RoundUpTo4K(info.file_size) > ranges.size() * 4096:

      ranges.extra['incomplete'] = True



  return image





def GetKeyPasswords(keylist):

  """Given a list of keys, prompt the user to enter passwords for

  those which require them.  Return a {key: password} dict.  password

tools/releasetools/ota_from_target_files.py

+6 −32
Original line number Diff line number Diff line
@@ -140,7 +140,6 @@ import zipfile 


import common

import edify_generator

import sparse_img



if sys.hexversion < 0x02070000:

  print("Python 2.7 or newer is required.", file=sys.stderr)
@@ -568,31 +567,6 @@ def WriteFingerprintAssertion(script, target_info, source_info): 
        source_info.GetBuildProp("ro.build.thumbprint"))





def GetImage(which, tmpdir):

  """Returns an image object suitable for passing to BlockImageDiff.



  'which' partition must be "system" or "vendor". A prebuilt image and file

  map must already exist in tmpdir.

  """



  assert which in ("system", "vendor")



  path = os.path.join(tmpdir, "IMAGES", which + ".img")

  mappath = os.path.join(tmpdir, "IMAGES", which + ".map")



  # The image and map files must have been created prior to calling

  # ota_from_target_files.py (since LMP).

  assert os.path.exists(path) and os.path.exists(mappath)



  # Bug: http://b/20939131

  # In ext4 filesystems, block 0 might be changed even being mounted

  # R/O. We add it to clobbered_blocks so that it will be written to the

  # target unconditionally. Note that they are still part of care_map.

  clobbered_blocks = "0"



  return sparse_img.SparseImage(path, mappath, clobbered_blocks)





def AddCompatibilityArchiveIfTrebleEnabled(target_zip, output_zip, target_info,

                                           source_info=None):

  """Adds compatibility info into the output zip if it's Treble-enabled target.
@@ -778,7 +752,7 @@ else if get_stage("%(bcb_dev)s") == "3/3" then 
  # has the effect of writing new data from the package to the entire

  # partition, but lets us reuse the updater code that writes incrementals to

  # do it.

  system_tgt = GetImage("system", OPTIONS.input_tmp)

  system_tgt = common.GetSparseImage("system", OPTIONS.input_tmp, input_zip)

  system_tgt.ResetFileMap()

  system_diff = common.BlockDifference("system", system_tgt, src=None)

  system_diff.WriteScript(script, output_zip)
@@ -789,7 +763,7 @@ else if get_stage("%(bcb_dev)s") == "3/3" then 
  if HasVendorPartition(input_zip):

    script.ShowProgress(0.1, 0)



    vendor_tgt = GetImage("vendor", OPTIONS.input_tmp)

    vendor_tgt = common.GetSparseImage("vendor", OPTIONS.input_tmp, input_zip)

    vendor_tgt.ResetFileMap()

    vendor_diff = common.BlockDifference("vendor", vendor_tgt)

    vendor_diff.WriteScript(script, output_zip)
@@ -962,8 +936,8 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip): 
  target_recovery = common.GetBootableImage(

      "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY")



  system_src = GetImage("system", OPTIONS.source_tmp)

  system_tgt = GetImage("system", OPTIONS.target_tmp)

  system_src = common.GetSparseImage("system", OPTIONS.source_tmp, source_zip)

  system_tgt = common.GetSparseImage("system", OPTIONS.target_tmp, target_zip)



  blockimgdiff_version = max(

      int(i) for i in target_info.get("blockimgdiff_versions", "1").split(","))
@@ -988,8 +962,8 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip): 
  if HasVendorPartition(target_zip):

    if not HasVendorPartition(source_zip):

      raise RuntimeError("can't generate incremental that adds /vendor")

    vendor_src = GetImage("vendor", OPTIONS.source_tmp)

    vendor_tgt = GetImage("vendor", OPTIONS.target_tmp)

    vendor_src = common.GetSparseImage("vendor", OPTIONS.source_tmp, source_zip)

    vendor_tgt = common.GetSparseImage("vendor", OPTIONS.target_tmp, target_zip)



    # Check first block of vendor partition for remount R/W only if

    # disk type is ext4

tools/releasetools/rangelib.py

+5 −0
Original line number Diff line number Diff line
@@ -25,6 +25,7 @@ class RangeSet(object): 


  def __init__(self, data=None):

    self.monotonic = False

    self._extra = {}

    if isinstance(data, str):

      self._parse_internal(data)

    elif data:
@@ -56,6 +57,10 @@ class RangeSet(object): 
  def __repr__(self):

    return '<RangeSet("' + self.to_string() + '")>'



  @property

  def extra(self):

    return self._extra



  @classmethod

  def parse(cls, text):

    """Parse a text string consisting of a space-separated list of

tools/releasetools/validate_target_files.py

+9 −32
Original line number Diff line number Diff line
@@ -29,35 +29,17 @@ import re 
import sys



import common

import sparse_img





def _GetImage(which, tmpdir):

  assert which in ('system', 'vendor')



  path = os.path.join(tmpdir, 'IMAGES', which + '.img')

  mappath = os.path.join(tmpdir, 'IMAGES', which + '.map')



  # Map file must exist (allowed to be empty).

  assert os.path.exists(path) and os.path.exists(mappath)



  clobbered_blocks = '0'

  return sparse_img.SparseImage(path, mappath, clobbered_blocks)





def _ReadFile(file_name, unpacked_name, round_up=False):

  """Constructs and returns a File object. Rounds up its size if needed."""



  def RoundUpTo4K(value):

    rounded_up = value + 4095

    return rounded_up - (rounded_up % 4096)



  assert os.path.exists(unpacked_name)

  with open(unpacked_name, 'r') as f:

    file_data = f.read()

  file_size = len(file_data)

  if round_up:

    file_size_rounded_up = RoundUpTo4K(file_size)

    file_size_rounded_up = common.RoundUpTo4K(file_size)

    file_data += '\0' * (file_size_rounded_up - file_size)

  return common.File(file_name, file_data)
@@ -79,33 +61,28 @@ def ValidateFileConsistency(input_zip, input_tmp): 


  def CheckAllFiles(which):

    logging.info('Checking %s image.', which)

    image = _GetImage(which, input_tmp)

    image = common.GetSparseImage(which, input_tmp, input_zip)

    prefix = '/' + which

    for entry in image.file_map:

      # Skip entries like '__NONZERO-0'.

      if not entry.startswith(prefix):

        continue



      # Read the blocks that the file resides. Note that it will contain the

      # bytes past the file length, which is expected to be padded with '\0's.

      ranges = image.file_map[entry]



      incomplete = ranges.extra.get('incomplete', False)

      if incomplete:

        logging.warning('Skipping %s that has incomplete block list', entry)

        continue



      blocks_sha1 = image.RangeSha1(ranges)



      # The filename under unpacked directory, such as SYSTEM/bin/sh.

      unpacked_name = os.path.join(

          input_tmp, which.upper(), entry[(len(prefix) + 1):])

      unpacked_file = _ReadFile(entry, unpacked_name, True)

      file_size = unpacked_file.size



      # block.map may contain less blocks, because mke2fs may skip allocating

      # blocks if they contain all zeros. We can't reconstruct such a file from

      # its block list. (Bug: 65213616)

      if file_size > ranges.size() * 4096:

        logging.warning(

            'Skipping %s that has less blocks: file size %d-byte,'

            ' ranges %s (%d-byte)', entry, file_size, ranges,

            ranges.size() * 4096)

        continue



      file_sha1 = unpacked_file.sha1

      assert blocks_sha1 == file_sha1, \

          'file: %s, range: %s, blocks_sha1: %s, file_sha1: %s' % (