Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 5df29604 authored by Yifan Hong's avatar Yifan Hong Committed by Gerrit Code Review
Browse files

Merge changes from topic "super_target_files" 

* changes:
  add_img_to_target_files uses build_super_image
  Build super_empty with build_super_image.py
  Build super.img from images in target_files
parents 5cf55948 055e6cf5

core/Makefile

+66 −84
Original line number Diff line number Diff line
@@ -2921,7 +2921,12 @@ vbmetaimage-nodeps: 
endif # BOARD_AVB_ENABLE



# -----------------------------------------------------------------

# super partition image

# Check image sizes <= size of super partition



ifeq (,$(TARGET_BUILD_APPS))

# Do not check for apps-only build



ifeq (true,$(PRODUCT_BUILD_SUPER_PARTITION))



# (1): list of items like "system", "vendor", "product", "product_services"

# return: map each item into a command ( wrapped in $$() ) that reads the size
@@ -2929,75 +2934,10 @@ define read-size-of-partitions 
$(foreach p,$(1),$(call read-image-prop-dictionary,$($(p)image_intermediates)/generated_$(p)_image_info.txt,$(p)_size))

endef



ifeq (true,$(PRODUCT_BUILD_SUPER_PARTITION))



# BOARD_SUPER_PARTITION_SIZE must be defined to build super image.

ifneq ($(BOARD_SUPER_PARTITION_SIZE),)



define super-slot-suffix

$(if $(filter true,$(AB_OTA_UPDATER)),$(if $(filter true,$(PRODUCT_RETROFIT_DYNAMIC_PARTITIONS)),,_a))

endef



# $(1): slot A suffix (_a or empty)

# $(2): include images or not (true or empty)

define build-superimage-target-args

  $(if $(2), --sparse) \

  --metadata-size 65536 \

  --metadata-slots $(if $(filter true,$(AB_OTA_UPDATER)),2,1) \

  --super-name $(BOARD_SUPER_PARTITION_METADATA_DEVICE) \

  $(if $(filter true,$(PRODUCT_RETROFIT_DYNAMIC_PARTITIONS)), $(if $(filter true,$(AB_OTA_UPDATER)), --auto-slot-suffixing)) \

  $(foreach device,$(BOARD_SUPER_PARTITION_BLOCK_DEVICES), \

    --device $(device):$(BOARD_SUPER_PARTITION_$(call to-upper,$(device))_DEVICE_SIZE)) \

  $(foreach group,$(BOARD_SUPER_PARTITION_GROUPS), \

    --group $(group)$(1):$(BOARD_$(call to-upper,$(group))_SIZE) \

    $(if $(1), --group $(group)_b:$(BOARD_$(call to-upper,$(group))_SIZE)) \

    $(foreach name,$(BOARD_$(call to-upper,$(group))_PARTITION_LIST), \

      --partition $(name)$(1):readonly:$(if $(2),$(call read-size-of-partitions,$(name)),0):$(group)$(1) \

      $(if $(2), --image $(name)$(1)=$(call images-for-partitions,$(name))) \

      $(if $(1), --partition $(name)_b:readonly:0:$(group)_b) \

  ))

endef



# $(1): output image path

# $(2): slot A suffix (_a or empty)

# $(3): include images or not (true or empty)

define build-superimage-target

  $(LPMAKE) \

    $(call build-superimage-target-args,$(2),$(3)) \

    --output $(1)

endef



# For A/B devices, super partition always contains sub-partitions in the _a slot, because this

# image should only be used for bootstrapping / initializing the device. When flashing the image,

# bootloader fastboot should always mark _a slot as bootable.



ifneq (true,$(PRODUCT_RETROFIT_DYNAMIC_PARTITIONS))

INSTALLED_SUPERIMAGE_TARGET := $(PRODUCT_OUT)/super.img

$(INSTALLED_SUPERIMAGE_TARGET): $(LPMAKE) $(call images-for-partitions,$(BOARD_SUPER_PARTITION_PARTITION_LIST))

	$(call pretty,"Target super fs image: $@")

	$(call build-superimage-target,$@,$(call super-slot-suffix),true)

endif



$(call dist-for-goals,dist_files,$(INSTALLED_SUPERIMAGE_TARGET))



INSTALLED_SUPERIMAGE_EMPTY_TARGET := $(PRODUCT_OUT)/super_empty.img

$(INSTALLED_SUPERIMAGE_EMPTY_TARGET): $(LPMAKE)

	$(call pretty,"Target empty super fs image: $@")

	$(call build-superimage-target,$@,$(call super-slot-suffix))



$(call dist-for-goals,dist_files,$(INSTALLED_SUPERIMAGE_EMPTY_TARGET))



endif # BOARD_SUPER_PARTITION_SIZE

endif # PRODUCT_BUILD_SUPER_PARTITION



# -----------------------------------------------------------------

# Check image sizes <= size of super partition



ifeq (,$(TARGET_BUILD_APPS))

# Do not check for apps-only build



ifeq (true,$(PRODUCT_BUILD_SUPER_PARTITION))



droid_targets: check-all-partition-sizes



.PHONY: check-all-partition-sizes check-all-partition-sizes-nodeps
@@ -3329,6 +3269,30 @@ define fs_config 
(cd $(1); find . -type d | sed 's,$$,/,'; find . \! -type d) | cut -c 3- | sort | sed 's,^,$(2),' | $(HOST_OUT_EXECUTABLES)/fs_config -C -D $(TARGET_OUT) -S $(SELINUX_FC) -R "$(2)"

endef



# $(1): file

define dump-dynamic-partitions-info

  $(if $(filter true,$(PRODUCT_USE_DYNAMIC_PARTITIONS)), \

    echo "use_dynamic_partitions=true" >> $(1))

  $(if $(filter true,$(PRODUCT_RETROFIT_DYNAMIC_PARTITIONS)), \

    echo "dynamic_partition_retrofit=true" >> $(1))

  echo "lpmake=$(notdir $(LPMAKE))" >> $(1)

  $(if $(filter true,$(PRODUCT_BUILD_SUPER_PARTITION)), $(if $(BOARD_SUPER_PARTITION_SIZE), \

    echo "build_super_partition=true" >> $(1)))

  echo "super_metadata_device=$(BOARD_SUPER_PARTITION_METADATA_DEVICE)" >> $(1)

  $(if $(BOARD_SUPER_PARTITION_BLOCK_DEVICES), \

    echo "super_block_devices=$(BOARD_SUPER_PARTITION_BLOCK_DEVICES)" >> $(1))

  $(foreach device,$(BOARD_SUPER_PARTITION_BLOCK_DEVICES), \

    echo "super_$(device)_device_size=$(BOARD_SUPER_PARTITION_$(call to-upper,$(device))_DEVICE_SIZE)" >> $(1);)

  $(if $(BOARD_SUPER_PARTITION_PARTITION_LIST), \

    echo "dynamic_partition_list=$(BOARD_SUPER_PARTITION_PARTITION_LIST)" >> $(1))

  $(if $(BOARD_SUPER_PARTITION_GROUPS),

    echo "super_partition_groups=$(BOARD_SUPER_PARTITION_GROUPS)" >> $(1))

  $(foreach group,$(BOARD_SUPER_PARTITION_GROUPS), \

    echo "super_$(group)_group_size=$(BOARD_$(call to-upper,$(group))_SIZE)" >> $(1); \

    $(if $(BOARD_$(call to-upper,$(group))_PARTITION_LIST), \

      echo "super_$(group)_partition_list=$(BOARD_$(call to-upper,$(group))_PARTITION_LIST)" >> $(1);))

endef



# Depending on the various images guarantees that the underlying

# directories are up-to-date.

$(BUILT_TARGET_FILES_PACKAGE): \
@@ -3644,6 +3608,7 @@ ifdef BOARD_AVB_DTBO_KEY_PATH 
endif # BOARD_AVB_DTBO_KEY_PATH

endif # BOARD_AVB_ENABLE

endif # BOARD_PREBUILT_DTBOIMAGE

	$(call dump-dynamic-partitions-info,$(zip_root)/META/misc_info.txt)

	@# The radio images in BOARD_PACK_RADIOIMAGES will be additionally copied from RADIO/ into

	@# IMAGES/, which then will be added into <product>-img.zip. Such images must be listed in

	@# INSTALLED_RADIOIMAGE_TARGET.
@@ -3689,24 +3654,6 @@ endif 
ifdef BUILT_VENDOR_MATRIX

	$(hide) cp $(BUILT_VENDOR_MATRIX) $(zip_root)/META/vendor_matrix.xml

endif

ifeq ($(PRODUCT_USE_DYNAMIC_PARTITIONS),true)

	$(hide) echo "use_dynamic_partitions=true" >> $(zip_root)/META/misc_info.txt

endif

ifeq ($(PRODUCT_RETROFIT_DYNAMIC_PARTITIONS),true)

	$(hide) echo "dynamic_partition_retrofit=true" >> $(zip_root)/META/misc_info.txt

endif

ifneq ($(BOARD_SUPER_PARTITION_SIZE),)

	$(hide) echo "lpmake=$(notdir $(LPMAKE))" >> $(zip_root)/META/misc_info.txt

	$(hide) echo -n "lpmake_args=" >> $(zip_root)/META/misc_info.txt

	$(hide) echo $(call build-superimage-target-args,$(call super-slot-suffix)) \

	    >> $(zip_root)/META/misc_info.txt

endif

ifneq ($(BOARD_SUPER_PARTITION_BLOCK_DEVICES),)

	$(hide) echo "super_block_devices=$(BOARD_SUPER_PARTITION_BLOCK_DEVICES)" >> $(zip_root)/META/misc_info.txt

endif

ifneq ($(BOARD_SUPER_PARTITION_PARTITION_LIST),)

	$(hide) echo "dynamic_partition_list=$(BOARD_SUPER_PARTITION_PARTITION_LIST)" >> $(zip_root)/META/misc_info.txt

endif

ifneq ($(BOARD_SUPER_PARTITION_GROUPS),)

	$(hide) echo "super_partition_groups=$(BOARD_SUPER_PARTITION_GROUPS)" > $(zip_root)/META/dynamic_partitions_info.txt

	$(foreach group,$(BOARD_SUPER_PARTITION_GROUPS), \
@@ -3974,6 +3921,41 @@ $(PROGUARD_DICT_ZIP) : 


endif # TARGET_BUILD_APPS



# -----------------------------------------------------------------

# super partition image



ifeq (true,$(PRODUCT_BUILD_SUPER_PARTITION))



# BOARD_SUPER_PARTITION_SIZE must be defined to build super image.

ifneq ($(BOARD_SUPER_PARTITION_SIZE),)



ifneq (true,$(PRODUCT_RETROFIT_DYNAMIC_PARTITIONS))

INSTALLED_SUPERIMAGE_TARGET := $(PRODUCT_OUT)/super.img

$(INSTALLED_SUPERIMAGE_TARGET): extracted_input_target_files := $(patsubst %.zip,%,$(BUILT_TARGET_FILES_PACKAGE))

$(INSTALLED_SUPERIMAGE_TARGET): $(LPMAKE) $(BUILT_TARGET_FILES_PACKAGE) $(BUILD_SUPER_IMAGE)

	$(call pretty,"Target super fs image: $@")

	$(BUILD_SUPER_IMAGE) -v $(extracted_input_target_files) $@

endif



$(call dist-for-goals,dist_files,$(INSTALLED_SUPERIMAGE_TARGET))



INSTALLED_SUPERIMAGE_EMPTY_TARGET := $(PRODUCT_OUT)/super_empty.img

$(INSTALLED_SUPERIMAGE_EMPTY_TARGET): intermediates := $(call intermediates-dir-for,PACKAGING,super_empty)

$(INSTALLED_SUPERIMAGE_EMPTY_TARGET): $(LPMAKE) $(BUILD_SUPER_IMAGE)

	$(call pretty,"Target empty super fs image: $@")

	mkdir -p $(intermediates)

	rm -rf $(intermediates)/misc_info.txt

	$(call dump-dynamic-partitions-info,$(intermediates)/misc_info.txt)

ifeq ($(AB_OTA_UPDATER),true)

	$(hide) echo "ab_update=true" >> $(intermediates)/misc_info.txt

endif

	$(BUILD_SUPER_IMAGE) -v $(intermediates)/misc_info.txt $@



$(call dist-for-goals,dist_files,$(INSTALLED_SUPERIMAGE_EMPTY_TARGET))



endif # BOARD_SUPER_PARTITION_SIZE != ""

endif # PRODUCT_BUILD_SUPER_PARTITION == "true"



# -----------------------------------------------------------------

# dalvik something

.PHONY: dalvikfiles

core/config.mk

+1 −0
Original line number Diff line number Diff line
@@ -698,6 +698,7 @@ DATA_BINDING_COMPILER := $(HOST_OUT_JAVA_LIBRARIES)/databinding-compiler.jar 
FAT16COPY := build/make/tools/fat16copy.py

CHECK_LINK_TYPE := build/make/tools/check_link_type.py

LPMAKE := $(HOST_OUT_EXECUTABLES)/lpmake$(HOST_EXECUTABLE_SUFFIX)

BUILD_SUPER_IMAGE := build/make/tools/releasetools/build_super_image.py



PROGUARD := external/proguard/bin/proguard.sh

JAVATAGS := build/make/tools/java-event-log-tags.py

tools/releasetools/add_img_to_target_files.py

+5 −54
Original line number Diff line number Diff line
@@ -55,6 +55,7 @@ import uuid 
import zipfile



import build_image

import build_super_image

import common

import rangelib

import sparse_img
@@ -648,64 +649,15 @@ def AddSuperEmpty(output_zip): 
  """Create a super_empty.img and store it in output_zip."""



  img = OutputFile(output_zip, OPTIONS.input_tmp, "IMAGES", "super_empty.img")

  cmd = [OPTIONS.info_dict['lpmake']]

  cmd += shlex.split(OPTIONS.info_dict['lpmake_args'].strip())

  cmd += ['--output', img.name]

  common.RunAndCheckOutput(cmd)



  build_super_image.BuildSuperImage(OPTIONS.info_dict, img.name)

  img.Write()





def AddSuperSplit(output_zip):

  """Create split super_*.img and store it in output_zip."""



  def GetPartitionSizeFromImage(img):

    try:

      simg = sparse_img.SparseImage(img)

      return simg.blocksize * simg.total_blocks

    except ValueError:

      return os.path.getsize(img)



  def TransformPartitionArg(arg):

    lst = arg.split(':')

    # Because --auto-slot-suffixing for A/B, there is no need to remove suffix.

    name = lst[0]

    if name + '_size' in OPTIONS.info_dict:

      size = str(OPTIONS.info_dict[name + '_size'])

      logger.info("Using %s_size = %s", name, size)

    else:

      size = str(GetPartitionSizeFromImage(

          os.path.join(OPTIONS.input_tmp, "IMAGES", '{}.img'.format(name))))

      logger.info("Using size of prebuilt %s = %s", name, size)

    lst[2] = size

    return ':'.join(lst)



  def GetLpmakeArgsWithSizes():

    lpmake_args = shlex.split(OPTIONS.info_dict['lpmake_args'].strip())



    for i, arg in enumerate(lpmake_args):

      if arg == '--partition':

        assert i + 1 < len(lpmake_args), \

          'lpmake_args has --partition without value'

        lpmake_args[i + 1] = TransformPartitionArg(lpmake_args[i + 1])



    return lpmake_args



  outdir = OutputFile(output_zip, OPTIONS.input_tmp, "OTA", "")

  cmd = [OPTIONS.info_dict['lpmake']]

  cmd += GetLpmakeArgsWithSizes()



  source = OPTIONS.info_dict.get('dynamic_partition_list', '').strip()

  if source:

    cmd.append('--sparse')

    for name in shlex.split(source):

      img = os.path.join(OPTIONS.input_tmp, "IMAGES", '{}.img'.format(name))

      # Because --auto-slot-suffixing for A/B, there is no need to add suffix.

      cmd += ['--image', '{}={}'.format(name, img)]



  cmd += ['--output', outdir.name]



  common.RunAndCheckOutput(cmd)

  outdir = os.path.join(OPTIONS.input_tmp, "OTA")

  build_super_image.BuildSuperImage(OPTIONS.input_tmp, outdir)



  for dev in OPTIONS.info_dict['super_block_devices'].strip().split():

    img = OutputFile(output_zip, OPTIONS.input_tmp, "OTA",
@@ -906,14 +858,13 @@ def AddImagesToTargetFiles(filename): 
    banner("vbmeta")

    AddVBMeta(output_zip, partitions, "vbmeta", vbmeta_partitions)



  if OPTIONS.info_dict.get("lpmake_args"):

  if OPTIONS.info_dict.get("build_super_partition"):

    banner("super_empty")

    AddSuperEmpty(output_zip)



    if OPTIONS.info_dict.get("dynamic_partition_retrofit") == "true":

      banner("super split images")

      AddSuperSplit(output_zip)

    # TODO(b/119322123): Add super.img to target_files for non-retrofit



  banner("radio")

  ab_partitions_txt = os.path.join(OPTIONS.input_tmp, "META",

tools/releasetools/build_super_image.py

0 → 100755
+202 −0
Original line number Diff line number Diff line 
#!/usr/bin/env python

#

# Copyright (C) 2018 The Android Open Source Project

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.



"""

Usage: build_super_image input_file output_dir_or_file



input_file: one of the following:

  - directory containing extracted target files. It will load info from

    META/misc_info.txt and build full super image / split images using source

    images from IMAGES/.

  - target files package. Same as above, but extracts the archive before

    building super image.

  - a dictionary file containing input arguments to build. Check

    `dump_dynamic_partitions_info' for details.

    In addition:

    - "ab_update" needs to be true for A/B devices.

    - If source images should be included in the output image (for super.img

      and super split images), a list of "*_image" should be paths of each

      source images.



output_dir_or_file:

    If a single super image is built (for super_empty.img, or super.img for

    launch devices), this argument is the output file.

    If a collection of split images are built (for retrofit devices), this

    argument is the output directory.

"""



from __future__ import print_function



import logging

import os.path

import shlex

import sys

import zipfile



import common

import sparse_img



if sys.hexversion < 0x02070000:

  print("Python 2.7 or newer is required.", file=sys.stderr)

  sys.exit(1)



logger = logging.getLogger(__name__)





UNZIP_PATTERN = ["IMAGES/*", "META/*"]





def GetPartitionSizeFromImage(img):

  try:

    simg = sparse_img.SparseImage(img)

    return simg.blocksize * simg.total_blocks

  except ValueError:

    return os.path.getsize(img)





def BuildSuperImageFromDict(info_dict, output):



  cmd = [info_dict["lpmake"],

         "--metadata-size", "65536",

         "--super-name", info_dict["super_metadata_device"]]



  ab_update = info_dict.get("ab_update") == "true"

  retrofit = info_dict.get("dynamic_partition_retrofit") == "true"

  block_devices = shlex.split(info_dict.get("super_block_devices", "").strip())

  groups = shlex.split(info_dict.get("super_partition_groups", "").strip())



  if ab_update:

    cmd += ["--metadata-slots", "2"]

  else:

    cmd += ["--metadata-slots", "1"]



  if ab_update and retrofit:

    cmd.append("--auto-slot-suffixing")



  for device in block_devices:

    size = info_dict["super_{}_device_size".format(device)]

    cmd += ["--device", "{}:{}".format(device, size)]



  append_suffix = ab_update and not retrofit

  has_image = False

  for group in groups:

    group_size = info_dict["super_{}_group_size".format(group)]

    if append_suffix:

      cmd += ["--group", "{}_a:{}".format(group, group_size),

              "--group", "{}_b:{}".format(group, group_size)]

    else:

      cmd += ["--group", "{}:{}".format(group, group_size)]



    partition_list = shlex.split(

        info_dict["super_{}_partition_list".format(group)].strip())



    for partition in partition_list:

      image = info_dict.get("{}_image".format(partition))

      image_size = 0

      if image:

        image_size = GetPartitionSizeFromImage(image)

        has_image = True

      if append_suffix:

        cmd += ["--partition",

                "{}_a:readonly:{}:{}_a".format(partition, image_size, group),

                "--partition",

                "{}_b:readonly:0:{}_b".format(partition, group)]

        if image:

          # For A/B devices, super partition always contains sub-partitions in

          # the _a slot, because this image should only be used for

          # bootstrapping / initializing the device. When flashing the image,

          # bootloader fastboot should always mark _a slot as bootable.

          cmd += ["--image", "{}_a={}".format(partition, image)]

      else:

        cmd += ["--partition",

                "{}:readonly:{}:{}".format(partition, image_size, group)]

        if image:

          cmd += ["--image", "{}={}".format(partition, image)]



  if has_image:

    cmd.append("--sparse")



  cmd += ["--output", output]



  common.RunAndCheckOutput(cmd)



  if retrofit and has_image:

    logger.info("Done writing images to directory %s", output)

  else:

    logger.info("Done writing image %s", output)





def BuildSuperImageFromExtractedTargetFiles(inp, out):

  info_dict = common.LoadInfoDict(inp)

  partition_list = shlex.split(

      info_dict.get("dynamic_partition_list", "").strip())

  for partition in partition_list:

    info_dict["{}_image".format(partition)] = os.path.join(

        inp, "IMAGES", "{}.img".format(partition))

  return BuildSuperImageFromDict(info_dict, out)





def BuildSuperImageFromTargetFiles(inp, out):

  input_tmp = common.UnzipTemp(inp, UNZIP_PATTERN)

  return BuildSuperImageFromExtractedTargetFiles(input_tmp, out)





def BuildSuperImage(inp, out):



  if isinstance(inp, dict):

    logger.info("Building super image from info dict...")

    return BuildSuperImageFromDict(inp, out)



  if isinstance(inp, str):

    if os.path.isdir(inp):

      logger.info("Building super image from extracted target files...")

      return BuildSuperImageFromExtractedTargetFiles(inp, out)



    if zipfile.is_zipfile(inp):

      logger.info("Building super image from target files...")

      return BuildSuperImageFromTargetFiles(inp, out)



    if os.path.isfile(inp):

      with open(inp) as f:

        lines = f.read()

      logger.info("Building super image from info dict...")

      return BuildSuperImageFromDict(common.LoadDictionaryFromLines(lines.split("\n")), out)



  raise ValueError("{} is not a dictionary or a valid path".format(inp))





def main(argv):



  args = common.ParseOptions(argv, __doc__)



  if len(args) != 2:

    common.Usage(__doc__)

    sys.exit(1)



  common.InitLogging()



  BuildSuperImage(args[0], args[1])





if __name__ == "__main__":

  try:

    common.CloseInheritedPipes()

    main(sys.argv[1:])

  except common.ExternalError:

    logger.exception("\n   ERROR:\n")

    sys.exit(1)

  finally:

    common.Cleanup()