Merge "releasetools: Support verifying files with non-monotonic ranges."

    with open(fn) as f:

      for line in f:

        fn, ranges = line.split(None, 1)

        ranges = rangelib.RangeSet.parse(ranges)

        fn, ranges_text = line.rstrip().split(None, 1)

        ranges = rangelib.RangeSet.parse(ranges_text)

        ranges.extra['text_str'] = ranges_text

        if allow_shared_blocks:

          # Find the shared blocks that have been claimed by others. If so, tag

            if not non_shared:

              continue

            # There shouldn't anything in the extra dict yet.

            assert not ranges.extra, "Non-empty RangeSet.extra"

            # Put the non-shared RangeSet as the value in the block map, which

            # has a copy of the original RangeSet.

            non_shared.extra['uses_shared_blocks'] = ranges

    system_root = os.path.join(input_tmp, "SYSTEM")

    os.mkdir(system_root)

    # Write the test file that contain multiple blocks of zeros, and these

    # zero blocks will be omitted by kernel. And the test files will occupy one

    # block range each in the final system image.

    # Write test files that contain multiple blocks of zeros, and these zero

    # blocks will be omitted by kernel. Each test file will occupy one block in

    # the final system image.

    with open(os.path.join(system_root, 'a'), 'w') as f:

      f.write("aaa")

      f.write('aaa')

      f.write('\0' * 4096 * 3)

    with open(os.path.join(system_root, 'b'), 'w') as f:

      f.write("bbb")

      f.write('bbb')

      f.write('\0' * 4096 * 3)

    raw_file_map = os.path.join(input_tmp, 'IMAGES', 'raw_system.map')

    # Parse the generated file map and update the block ranges for each file.

    file_map_list = {}

    image_ranges = RangeSet()

    with open(raw_file_map, 'r') as f:

    with open(raw_file_map) as f:

      for line in f.readlines():

        info = line.split()

        self.assertEqual(2, len(info))

    mock_shared_block = RangeSet("10-20").subtract(image_ranges).first(1)

    with open(os.path.join(input_tmp, 'IMAGES', 'system.map'), 'w') as f:

      for key in sorted(file_map_list.keys()):

        line = "{} {}\n".format(

        line = '{} {}\n'.format(

            key, file_map_list[key].union(mock_shared_block))

        f.write(line)

      for name in all_entries:

        input_zip.write(os.path.join(input_tmp, name), arcname=name)

    input_zip = zipfile.ZipFile(input_file, 'r')

    info_dict = {'extfs_sparse_flag': '-s'}

    # Expect the validation to pass and both files are skipped due to

    # 'incomplete' block range.

    with zipfile.ZipFile(input_file) as input_zip:

      info_dict = {'extfs_sparse_flag': '-s'}

      ValidateFileConsistency(input_zip, input_tmp, info_dict)

  @test_utils.SkipIfExternalToolsUnavailable()

  def test_ValidateFileConsistency_nonMonotonicRanges(self):

    input_tmp = common.MakeTempDir()

    os.mkdir(os.path.join(input_tmp, 'IMAGES'))

    system_image = os.path.join(input_tmp, 'IMAGES', 'system.img')

    system_root = os.path.join(input_tmp, "SYSTEM")

    os.mkdir(system_root)

    # Write the test file that contain three blocks of 'a', 'b', 'c'.

    with open(os.path.join(system_root, 'abc'), 'w') as f:

      f.write('a' * 4096 + 'b' * 4096 + 'c' * 4096)

    raw_file_map = os.path.join(input_tmp, 'IMAGES', 'raw_system.map')

    self._generate_system_image(system_image, system_root, raw_file_map)

    # Parse the generated file map and manipulate the block ranges of 'abc' to

    # be 'cba'.

    file_map_list = {}

    with open(raw_file_map) as f:

      for line in f.readlines():

        info = line.split()

        self.assertEqual(2, len(info))

        ranges = RangeSet(info[1])

        self.assertTrue(ranges.monotonic)

        blocks = reversed(list(ranges.next_item()))

        file_map_list[info[0]] = ' '.join([str(block) for block in blocks])

    # Update the contents of 'abc' to be 'cba'.

    with open(os.path.join(system_root, 'abc'), 'w') as f:

      f.write('c' * 4096 + 'b' * 4096 + 'a' * 4096)

    # Update the system.map.

    with open(os.path.join(input_tmp, 'IMAGES', 'system.map'), 'w') as f:

      for key in sorted(file_map_list.keys()):

        f.write('{} {}\n'.format(key, file_map_list[key]))

    # Get the target zip file.

    input_file = common.MakeTempFile()

    all_entries = ['SYSTEM/', 'SYSTEM/abc', 'IMAGES/',

                   'IMAGES/system.map', 'IMAGES/system.img']

    with zipfile.ZipFile(input_file, 'w') as input_zip:

      for name in all_entries:

        input_zip.write(os.path.join(input_tmp, name), arcname=name)

    with zipfile.ZipFile(input_file) as input_zip:

      info_dict = {'extfs_sparse_flag': '-s'}

      ValidateFileConsistency(input_zip, input_tmp, info_dict)

import os.path

import re

import zipfile

from hashlib import sha1

import common

import rangelib

def _ReadFile(file_name, unpacked_name, round_up=False):

  """Constructs and returns a File object. Rounds up its size if needed."""

  assert os.path.exists(unpacked_name)

  with open(unpacked_name, 'rb') as f:

    file_data = f.read()

  file_size = len(file_data)

  if round_up:

    file_size_rounded_up = common.RoundUpTo4K(file_size)

    file_data += '\0' * (file_size_rounded_up - file_size)

    file_data += b'\0' * (file_size_rounded_up - file_size)

  return common.File(file_name, file_data)

        logging.warning('Skipping %s that has incomplete block list', entry)

        continue

      # TODO(b/79951650): Handle files with non-monotonic ranges.

      # If the file has non-monotonic ranges, read each range in order.

      if not file_ranges.monotonic:

        logging.warning(

            'Skipping %s that has non-monotonic ranges: %s', entry, file_ranges)

        continue

        h = sha1()

        for file_range in file_ranges.extra['text_str'].split(' '):

          for data in image.ReadRangeSet(rangelib.RangeSet(file_range)):

            h.update(data)

        blocks_sha1 = h.hexdigest()

      else:

        blocks_sha1 = image.RangeSha1(file_ranges)

      # The filename under unpacked directory, such as SYSTEM/bin/sh.