Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 3d92e99a authored by Wei Li's avatar Wei Li Committed by Android (Google) Code Review
Browse files

Merge changes from topic "cherrypicker-L97600000961895565:N98400001386758173" into udc-dev 

* changes:
  Support SBOM generation of m build unbundled APEXs.
  Include static libraries information in Android SBOM.
parents e0d8de90 9b228201

core/main.mk

+64 −12
Original line number Diff line number Diff line
@@ -2157,13 +2157,21 @@ endif # TARGET_BUILD_APPS 
#       is_kernel_modules_blocklist: modules.blocklist created for _dlkm partitions, see macro build-image-kernel-modules-dir in Makefile.

#       is_fsverity_build_manifest_apk: BuildManifest<part>.apk files for system and system_ext partition, see ALL_FSVERITY_BUILD_MANIFEST_APK in Makefile.

#       is_linker_config: see SYSTEM_LINKER_CONFIG and vendor_linker_config_file in Makefile.



#   build_output_path: the path of the built file, used to calculate checksum

#   static_libraries/whole_static_libraries: list of module name of the static libraries the file links against, e.g. libclang_rt.builtins or libclang_rt.builtins_32

#       Info of all static libraries of all installed files are collected in variable _all_static_libs that is used to list all the static library files in sbom-metadata.csv.

#       See the second foreach loop in the rule of sbom-metadata.csv for the detailed info of static libraries collected in _all_static_libs.

#   is_static_lib: whether the file is a static library



metadata_list := $(OUT_DIR)/.module_paths/METADATA.list

metadata_files := $(subst $(newline),$(space),$(file <$(metadata_list)))

# (TODO: b/272358583 find another way of always rebuilding this target)

# Remove the sbom-metadata.csv whenever makefile is evaluated

$(shell rm $(PRODUCT_OUT)/sbom-metadata.csv >/dev/null 2>&1)

$(PRODUCT_OUT)/sbom-metadata.csv: $(installed_files)

$(PRODUCT_OUT)/sbom-metadata.csv: $(installed_files) $(metadata_list) $(metadata_files)

	rm -f $@

	@echo installed_file$(comma)module_path$(comma)soong_module_type$(comma)is_prebuilt_make_module$(comma)product_copy_files$(comma)kernel_module_copy_files$(comma)is_platform_generated,build_output_path >> $@

	echo installed_file,module_path,soong_module_type,is_prebuilt_make_module,product_copy_files,kernel_module_copy_files,is_platform_generated,build_output_path,static_libraries,whole_static_libraries,is_static_lib >> $@

	$(eval _all_static_libs :=)

	$(foreach f,$(installed_files),\

	  $(eval _module_name := $(ALL_INSTALLED_FILES.$f)) \

	  $(eval _path_on_device := $(patsubst $(PRODUCT_OUT)/%,%,$f)) \
@@ -2185,11 +2193,25 @@ $(PRODUCT_OUT)/sbom-metadata.csv: $(installed_files) 
	  $(eval _is_linker_config := $(if $(findstring $f,$(SYSTEM_LINKER_CONFIG) $(vendor_linker_config_file)),Y)) \

	  $(eval _is_partition_compat_symlink := $(if $(findstring $f,$(PARTITION_COMPAT_SYMLINKS)),Y)) \

	  $(eval _is_platform_generated := $(_is_build_prop)$(_is_notice_file)$(_is_dexpreopt_image_profile)$(_is_product_system_other_avbkey)$(_is_event_log_tags_file)$(_is_system_other_odex_marker)$(_is_kernel_modules_blocklist)$(_is_fsverity_build_manifest_apk)$(_is_linker_config)$(_is_partition_compat_symlink)) \

	  @echo /$(_path_on_device)$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated)$(comma)$(_build_output_path) >> $@ $(newline) \

	  $(eval _static_libs := $(ALL_INSTALLED_FILES.$f.STATIC_LIBRARIES)) \

	  $(eval _whole_static_libs := $(ALL_INSTALLED_FILES.$f.WHOLE_STATIC_LIBRARIES)) \

	  $(foreach l,$(_static_libs),$(eval _all_static_libs += $l:$(strip $(sort $(ALL_MODULES.$l.PATH))):$(strip $(sort $(ALL_MODULES.$l.SOONG_MODULE_TYPE))):$(ALL_STATIC_LIBRARIES.$l.BUILT_FILE))) \

	  $(foreach l,$(_whole_static_libs),$(eval _all_static_libs += $l:$(strip $(sort $(ALL_MODULES.$l.PATH))):$(strip $(sort $(ALL_MODULES.$l.SOONG_MODULE_TYPE))):$(ALL_STATIC_LIBRARIES.$l.BUILT_FILE))) \

	  echo /$(_path_on_device),$(_module_path),$(_soong_module_type),$(_is_prebuilt_make_module),$(_product_copy_files),$(_kernel_module_copy_files),$(_is_platform_generated),$(_build_output_path),$(_static_libs),$(_whole_static_libs), >> $@; \

	  $(if $(_post_installed_dexpreopt_zip), \

	  for i in $$(zipinfo -1 $(_post_installed_dexpreopt_zip)); do echo /$$i$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated)$(comma)$(PRODUCT_OUT)/$$i >> $@ ; done $(newline) \

	  for i in $$(zipinfo -1 $(_post_installed_dexpreopt_zip)); do echo /$$i$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated)$(comma)$(PRODUCT_OUT)/$$i$(comma)$(_static_libs)$(comma)$(_whole_static_libs)$(comma) >> $@ ; done ; \

	  ) \

	)

	$(foreach l,$(sort $(_all_static_libs)), \

	  $(eval _lib_stem := $(call word-colon,1,$l)) \

	  $(eval _module_path := $(call word-colon,2,$l)) \

	  $(eval _soong_module_type := $(call word-colon,3,$l)) \

	  $(eval _built_file := $(call word-colon,4,$l)) \

	  $(eval _static_libs := $(ALL_STATIC_LIBRARIES.$l.STATIC_LIBRARIES)) \

	  $(eval _whole_static_libs := $(ALL_STATIC_LIBRARIES.$l.WHOLE_STATIC_LIBRARIES)) \

	  $(eval _is_static_lib := Y) \

	  echo $(_lib_stem).a,$(_module_path),$(_soong_module_type),,,,,$(_built_file),$(_static_libs),$(_whole_static_libs),$(_is_static_lib) >> $@; \

	)



.PHONY: sbom

ifeq ($(TARGET_BUILD_APPS),)
@@ -2201,17 +2223,47 @@ $(PRODUCT_OUT)/sbom.spdx: $(PRODUCT_OUT)/sbom-metadata.csv $(GEN_SBOM) 


$(call dist-for-goals,droid,$(PRODUCT_OUT)/sbom.spdx.json:sbom/sbom.spdx.json)

else

apps_only_sbom_files := $(sort $(patsubst %,%.spdx.json,$(filter %.apk,$(apps_only_installed_files))))

$(apps_only_sbom_files): $(PRODUCT_OUT)/sbom-metadata.csv $(GEN_SBOM)

	rm -rf $@

	$(GEN_SBOM) --output_file $@ --metadata $(PRODUCT_OUT)/sbom-metadata.csv --build_version $(BUILD_FINGERPRINT_FROM_FILE) --product_mfr "$(PRODUCT_MANUFACTURER)" --unbundled_apk

# Create build rules for generating SBOMs of unbundled APKs and APEXs

# $1: sbom file

# $2: sbom fragment file

# $3: installed file

# $4: sbom-metadata.csv file

define generate-app-sbom

$(eval _path_on_device := $(patsubst $(PRODUCT_OUT)/%,%,$(3)))

$(eval _module_name := $(ALL_INSTALLED_FILES.$(3)))

$(eval _module_path := $(strip $(sort $(ALL_MODULES.$(_module_name).PATH))))

$(eval _soong_module_type := $(strip $(sort $(ALL_MODULES.$(_module_name).SOONG_MODULE_TYPE))))

$(eval _dep_modules := $(filter %.$(_module_name),$(ALL_MODULES)) $(filter %.$(_module_name)$(TARGET_2ND_ARCH_MODULE_SUFFIX),$(ALL_MODULES)))

$(eval _is_apex := $(filter %.apex,$(3)))



$(4): $(3) $(metadata_list) $(metadata_files)

	rm -rf $$@

	echo installed_file,module_path,soong_module_type,is_prebuilt_make_module,product_copy_files,kernel_module_copy_files,is_platform_generated,build_output_path,static_libraries,whole_static_libraries,is_static_lib >> $$@

	echo /$(_path_on_device),$(_module_path),$(_soong_module_type),,,,,$(3),,, >> $$@

	$(if $(filter %.apex,$(3)),\

	  $(foreach m,$(_dep_modules),\

	    echo $(patsubst $(PRODUCT_OUT)/apex/$(_module_name)/%,%,$(ALL_MODULES.$m.INSTALLED)),$(sort $(ALL_MODULES.$m.PATH)),$(sort $(ALL_MODULES.$m.SOONG_MODULE_TYPE)),,,,,$(strip $(ALL_MODULES.$m.BUILT)),,, >> $$@;))



$(2): $(1)

$(1): $(4) $(GEN_SBOM)

	rm -rf $$@

	$(GEN_SBOM) --output_file $$@ --metadata $(4) --build_version $$(BUILD_FINGERPRINT_FROM_FILE) --product_mfr "$(PRODUCT_MANUFACTURER)" --json $(if $(filter %.apk,$(3)),--unbundled_apk,--unbundled_apex)

endef



apps_only_sbom_files :=

apps_only_fragment_files :=

$(foreach f,$(filter %.apk %.apex,$(installed_files)), \

  $(eval _metadata_csv_file := $(patsubst %,%-sbom-metadata.csv,$f)) \

  $(eval _sbom_file := $(patsubst %,%.spdx.json,$f)) \

  $(eval _fragment_file := $(patsubst %,%-fragment.spdx,$f)) \

  $(eval apps_only_sbom_files += $(_sbom_file)) \

  $(eval apps_only_fragment_files += $(_fragment_file)) \

  $(eval $(call generate-app-sbom,$(_sbom_file),$(_fragment_file),$f,$(_metadata_csv_file))) \

)



sbom: $(apps_only_sbom_files)



$(foreach f,$(apps_only_sbom_files),$(eval $(patsubst %.spdx.json,%-fragment.spdx,$f): $f))

apps_only_fragment_files := $(patsubst %.spdx.json,%-fragment.spdx,$(apps_only_sbom_files))

$(foreach f,$(apps_only_fragment_files),$(eval apps_only_fragment_dist_files += :sbom/$(notdir $f)))



$(foreach f,$(apps_only_sbom_files),$(eval apps_only_sbom_dist_files += :sbom/$(notdir $f)))

$(call dist-for-goals,apps_only,$(join $(apps_only_sbom_files),$(apps_only_sbom_dist_files)) $(join $(apps_only_fragment_files),$(apps_only_fragment_dist_files)))

endif

core/sbom.mk

+11 −0
Original line number Diff line number Diff line
@@ -3,9 +3,20 @@ 
# unless a .mk file changes its installed file after including base_rules.mk.



ifdef my_register_name

  # ALL_INSTALLED_FILES.$(installed_file).STATIC_LIBRARIES: list of module name of static libraries, e.g. libc++demangle libclang_rt.builtins, for primary arch

  # ALL_INSTALLED_FILES.$(installed_file).WHOLE_STATIC_LIBRARIES: list of module name of static libraries, e.g. libc++demangle_32 libclang_rt.builtins_32, for 2nd arch.

  ifneq (, $(strip $(ALL_MODULES.$(my_register_name).INSTALLED)))

    $(foreach installed_file,$(ALL_MODULES.$(my_register_name).INSTALLED),\

      $(eval ALL_INSTALLED_FILES.$(installed_file) := $(my_register_name))\

      $(eval ALL_INSTALLED_FILES.$(installed_file).STATIC_LIBRARIES := $(foreach l,$(strip $(sort $(LOCAL_STATIC_LIBRARIES))),$l$(if $(LOCAL_2ND_ARCH_VAR_PREFIX),$($(my_prefix)2ND_ARCH_MODULE_SUFFIX))))\

      $(eval ALL_INSTALLED_FILES.$(installed_file).WHOLE_STATIC_LIBRARIES := $(foreach l,$(strip $(sort $(LOCAL_WHOLE_STATIC_LIBRARIES))),$l$(if $(LOCAL_2ND_ARCH_VAR_PREFIX),$($(my_prefix)2ND_ARCH_MODULE_SUFFIX))))\

    )

  endif

  ifeq (STATIC_LIBRARIES,$(LOCAL_MODULE_CLASS))

  ALL_STATIC_LIBRARIES.$(my_register_name).STATIC_LIBRARIES := $(foreach l,$(strip $(sort $(LOCAL_STATIC_LIBRARIES))),$l$($(my_prefix)2ND_ARCH_MODULE_SUFFIX))

  ALL_STATIC_LIBRARIES.$(my_register_name).WHOLE_STATIC_LIBRARIES := $(foreach l,$(strip $(sort $(LOCAL_WHOLE_STATIC_LIBRARIES))),$l$($(my_prefix)2ND_ARCH_MODULE_SUFFIX))

  ifdef LOCAL_SOONG_MODULE_TYPE

    ALL_STATIC_LIBRARIES.$(my_register_name).BUILT_FILE := $(LOCAL_PREBUILT_MODULE_FILE)

  endif

  endif

endif
 
 No newline at end of file

tools/sbom/generate-sbom.py

+29 −17
Original line number Diff line number Diff line
@@ -332,14 +332,6 @@ def get_sbom_fragments(installed_file_metadata, metadata_file_path): 
  return external_doc_ref, packages, relationships





def generate_package_verification_code(files):

  checksums = [file.checksum for file in files]

  checksums.sort()

  h = hashlib.sha1()

  h.update(''.join(checksums).encode(encoding='utf-8'))

  return h.hexdigest()





def save_report(report_file_path, report):

  with open(report_file_path, 'w', encoding='utf-8') as report_file:

    for type, issues in report.items():
@@ -487,16 +479,28 @@ def main(): 
      product_copy_files = installed_file_metadata['product_copy_files']

      kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files']

      build_output_path = installed_file_metadata['build_output_path']

      is_static_lib = installed_file_metadata['is_static_lib']



      if not installed_file_has_metadata(installed_file_metadata, report):

        continue

      if not (os.path.islink(build_output_path) or os.path.isfile(build_output_path)):

      if not is_static_lib and not (os.path.islink(build_output_path) or os.path.isfile(build_output_path)):

        # Ignore non-existing static library files for now since they are not shipped on devices.

        report[ISSUE_INSTALLED_FILE_NOT_EXIST].append(installed_file)

        continue



      file_id = new_file_id(installed_file)

      doc.files.append(

        sbom_data.File(id=file_id, name=installed_file, checksum=checksum(build_output_path)))

      # TODO(b/285453664): Soong should report the information of statically linked libraries to Make.

      # This happens when a different sanitized version of static libraries is used in linking.

      # As a workaround, use the following SHA1 checksum for static libraries created by Soong, if .a files could not be

      # located correctly because Soong doesn't report the information to Make.

      sha1 = 'SHA1: da39a3ee5e6b4b0d3255bfef95601890afd80709'  # SHA1 of empty string

      if os.path.islink(build_output_path) or os.path.isfile(build_output_path):

        sha1 = checksum(build_output_path)

      doc.files.append(sbom_data.File(id=file_id,

                                      name=installed_file,

                                      checksum=sha1))



      if not is_static_lib:

        if not args.unbundled_apex:

          product_package.file_ids.append(file_id)

        elif len(doc.files) > 1:
@@ -544,13 +548,21 @@ def main(): 
                                                    relationship=sbom_data.RelationshipType.GENERATED_FROM,

                                                    id2=sbom_data.SPDXID_PLATFORM))



  if not args.unbundled_apex:

    product_package.verification_code = generate_package_verification_code(doc.files)

      # Process static libraries and whole static libraries the installed file links to

      static_libs = installed_file_metadata['static_libraries']

      whole_static_libs = installed_file_metadata['whole_static_libraries']

      all_static_libs = (static_libs + ' ' + whole_static_libs).strip()

      if all_static_libs:

        for lib in all_static_libs.split(' '):

          doc.add_relationship(sbom_data.Relationship(id1=file_id,

                                                      relationship=sbom_data.RelationshipType.STATIC_LINK,

                                                      id2=new_file_id(lib + '.a')))



  if args.unbundled_apex:

    doc.describes = doc.files[0].id



  # Save SBOM records to output file

  doc.generate_packages_verification_code()

  doc.created = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')

  prefix = args.output_file

  if prefix.endswith('.spdx'):

tools/sbom/sbom_data.py

+16 −0
Original line number Diff line number Diff line
@@ -25,6 +25,7 @@ fields in each data class. 


from dataclasses import dataclass, field

from typing import List

import hashlib



SPDXID_DOC = 'SPDXRef-DOCUMENT'

SPDXID_PRODUCT = 'SPDXRef-PRODUCT'
@@ -81,6 +82,7 @@ class RelationshipType: 
  VARIANT_OF = 'VARIANT_OF'

  GENERATED_FROM = 'GENERATED_FROM'

  CONTAINS = 'CONTAINS'

  STATIC_LINK = 'STATIC_LINK'





@dataclass
@@ -122,3 +124,17 @@ class Document: 
    if not any(rel.id1 == r.id1 and rel.id2 == r.id2 and rel.relationship == r.relationship

               for r in self.relationships):

      self.relationships.append(rel)



  def generate_packages_verification_code(self):

    for package in self.packages:

      if not package.file_ids:

        continue



      checksums = []

      for file in self.files:

        if file.id in package.file_ids:

          checksums.append(file.checksum)

      checksums.sort()

      h = hashlib.sha1()

      h.update(''.join(checksums).encode(encoding='utf-8'))

      package.verification_code = h.hexdigest()

tools/sbom/sbom_writers.py

+29 −42
Original line number Diff line number Diff line
@@ -85,7 +85,7 @@ class TagValueWriter: 
    return headers



  @staticmethod

  def marshal_package(package):

  def marshal_package(sbom_doc, package, fragment):

    download_location = sbom_data.VALUE_NOASSERTION

    if package.download_location:

      download_location = package.download_location
@@ -107,50 +107,32 @@ class TagValueWriter: 
          f'{Tags.PACKAGE_EXTERNAL_REF}: {external_ref.category} {external_ref.type} {external_ref.locator}')



    tagvalues.append('')

    return tagvalues



  @staticmethod

  def marshal_described_element(sbom_doc, fragment):

    if not sbom_doc.describes:

      return None



    product_package = [p for p in sbom_doc.packages if p.id == sbom_doc.describes]

    if product_package:

      tagvalues = TagValueWriter.marshal_package(product_package[0])

      if not fragment:

    if package.id == sbom_doc.describes and not fragment:

      tagvalues.append(

          f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}')



      tagvalues.append('')

      return tagvalues



    file = [f for f in sbom_doc.files if f.id == sbom_doc.describes]

    if file:

      tagvalues = TagValueWriter.marshal_file(file[0])

      if not fragment:

        tagvalues.append(

            f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}')

    for file in sbom_doc.files:

      if file.id in package.file_ids:

        tagvalues += TagValueWriter.marshal_file(file)



    return tagvalues



    return None



  @staticmethod

  def marshal_packages(sbom_doc):

  def marshal_packages(sbom_doc, fragment):

    tagvalues = []

    marshaled_relationships = []

    i = 0

    packages = sbom_doc.packages

    while i < len(packages):

      if packages[i].id == sbom_doc.describes:

        i += 1

        continue



      if i + 1 < len(packages) \

          and packages[i].id.startswith('SPDXRef-SOURCE-') \

          and packages[i + 1].id.startswith('SPDXRef-UPSTREAM-'):

        tagvalues += TagValueWriter.marshal_package(packages[i])

        tagvalues += TagValueWriter.marshal_package(packages[i + 1])

      if (i + 1 < len(packages)

          and packages[i].id.startswith('SPDXRef-SOURCE-')

          and packages[i + 1].id.startswith('SPDXRef-UPSTREAM-')):

        # Output SOURCE, UPSTREAM packages and their VARIANT_OF relationship together, so they are close to each other

        # in SBOMs in tagvalue format.

        tagvalues += TagValueWriter.marshal_package(sbom_doc, packages[i], fragment)

        tagvalues += TagValueWriter.marshal_package(sbom_doc, packages[i + 1], fragment)

        rel = next((r for r in sbom_doc.relationships if

                    r.id1 == packages[i].id and

                    r.id2 == packages[i + 1].id and
@@ -162,7 +144,7 @@ class TagValueWriter: 


        i += 2

      else:

        tagvalues += TagValueWriter.marshal_package(packages[i])

        tagvalues += TagValueWriter.marshal_package(sbom_doc, packages[i], fragment)

        i += 1



    return tagvalues, marshaled_relationships
@@ -179,12 +161,20 @@ class TagValueWriter: 
    return tagvalues



  @staticmethod

  def marshal_files(sbom_doc):

  def marshal_files(sbom_doc, fragment):

    tagvalues = []

    files_in_packages = []

    for package in sbom_doc.packages:

      files_in_packages += package.file_ids

    for file in sbom_doc.files:

      if file.id == sbom_doc.describes:

      if file.id in files_in_packages:

        continue

      tagvalues += TagValueWriter.marshal_file(file)

      if file.id == sbom_doc.describes and not fragment:

        # Fragment is not a full SBOM document so the relationship DESCRIBES is not applicable.

        tagvalues.append(

            f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}')

        tagvalues.append('')

    return tagvalues



  @staticmethod
@@ -208,11 +198,8 @@ class TagValueWriter: 
    content = []

    if not fragment:

      content += TagValueWriter.marshal_doc_headers(sbom_doc)

    described_element = TagValueWriter.marshal_described_element(sbom_doc, fragment)

    if described_element:

      content += described_element

    content += TagValueWriter.marshal_files(sbom_doc)

    tagvalues, marshaled_relationships = TagValueWriter.marshal_packages(sbom_doc)

    content += TagValueWriter.marshal_files(sbom_doc, fragment)

    tagvalues, marshaled_relationships = TagValueWriter.marshal_packages(sbom_doc, fragment)

    content += tagvalues

    content += TagValueWriter.marshal_relationships(sbom_doc, marshaled_relationships)

    file.write('\n'.join(content))