Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c2017515 authored by Wei Li's avatar Wei Li Committed by Gerrit Code Review
Browse files

Merge "Include static libraries information in Android SBOM."

parents fb8c8651 d263695c

core/main.mk

+23 −3
Original line number Diff line number Diff line
@@ -2151,13 +2151,19 @@ endif # TARGET_BUILD_APPS 
#       is_kernel_modules_blocklist: modules.blocklist created for _dlkm partitions, see macro build-image-kernel-modules-dir in Makefile.

#       is_fsverity_build_manifest_apk: BuildManifest<part>.apk files for system and system_ext partition, see ALL_FSVERITY_BUILD_MANIFEST_APK in Makefile.

#       is_linker_config: see SYSTEM_LINKER_CONFIG and vendor_linker_config_file in Makefile.

#   build_output_path: the path of the built file, used to calculate checksum

#   static_libraries/whole_static_libraries: list of module name of the static libraries the file links against, e.g. libclang_rt.builtins or libclang_rt.builtins_32

#       Info of all static libraries of all installed files are collected in variable _all_static_libs that is used to list all the static library files in sbom-metadata.csv.

#       See the second foreach loop in the rule of sbom-metadata.csv for the detailed info of static libraries collected in _all_static_libs.

#   is_static_lib: whether the file is a static library



# (TODO: b/272358583 find another way of always rebuilding this target)

# Remove the sbom-metadata.csv whenever makefile is evaluated

$(shell rm $(PRODUCT_OUT)/sbom-metadata.csv >/dev/null 2>&1)

$(PRODUCT_OUT)/sbom-metadata.csv: $(installed_files)

	rm -f $@

	@echo installed_file$(comma)module_path$(comma)soong_module_type$(comma)is_prebuilt_make_module$(comma)product_copy_files$(comma)kernel_module_copy_files$(comma)is_platform_generated,build_output_path >> $@

	echo installed_file,module_path,soong_module_type,is_prebuilt_make_module,product_copy_files,kernel_module_copy_files,is_platform_generated,build_output_path,static_libraries,whole_static_libraries,is_static_lib >> $@

	$(eval _all_static_libs :=)

	$(foreach f,$(installed_files),\

	  $(eval _module_name := $(ALL_INSTALLED_FILES.$f)) \

	  $(eval _path_on_device := $(patsubst $(PRODUCT_OUT)/%,%,$f)) \
@@ -2179,11 +2185,25 @@ $(PRODUCT_OUT)/sbom-metadata.csv: $(installed_files) 
	  $(eval _is_linker_config := $(if $(findstring $f,$(SYSTEM_LINKER_CONFIG) $(vendor_linker_config_file)),Y)) \

	  $(eval _is_partition_compat_symlink := $(if $(findstring $f,$(PARTITION_COMPAT_SYMLINKS)),Y)) \

	  $(eval _is_platform_generated := $(_is_build_prop)$(_is_notice_file)$(_is_dexpreopt_image_profile)$(_is_product_system_other_avbkey)$(_is_event_log_tags_file)$(_is_system_other_odex_marker)$(_is_kernel_modules_blocklist)$(_is_fsverity_build_manifest_apk)$(_is_linker_config)$(_is_partition_compat_symlink)) \

	  @echo /$(_path_on_device)$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated)$(comma)$(_build_output_path) >> $@ $(newline) \

	  $(eval _static_libs := $(ALL_INSTALLED_FILES.$f.STATIC_LIBRARIES)) \

	  $(eval _whole_static_libs := $(ALL_INSTALLED_FILES.$f.WHOLE_STATIC_LIBRARIES)) \

	  $(foreach l,$(_static_libs),$(eval _all_static_libs += $l:$(strip $(sort $(ALL_MODULES.$l.PATH))):$(strip $(sort $(ALL_MODULES.$l.SOONG_MODULE_TYPE))):$(ALL_STATIC_LIBRARIES.$l.BUILT_FILE))) \

	  $(foreach l,$(_whole_static_libs),$(eval _all_static_libs += $l:$(strip $(sort $(ALL_MODULES.$l.PATH))):$(strip $(sort $(ALL_MODULES.$l.SOONG_MODULE_TYPE))):$(ALL_STATIC_LIBRARIES.$l.BUILT_FILE))) \

	  echo /$(_path_on_device),$(_module_path),$(_soong_module_type),$(_is_prebuilt_make_module),$(_product_copy_files),$(_kernel_module_copy_files),$(_is_platform_generated),$(_build_output_path),$(_static_libs),$(_whole_static_libs), >> $@; \

	  $(if $(_post_installed_dexpreopt_zip), \

	  for i in $$(zipinfo -1 $(_post_installed_dexpreopt_zip)); do echo /$$i$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated)$(comma)$(PRODUCT_OUT)/$$i >> $@ ; done $(newline) \

	  for i in $$(zipinfo -1 $(_post_installed_dexpreopt_zip)); do echo /$$i$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated)$(comma)$(PRODUCT_OUT)/$$i$(comma)$(_static_libs)$(comma)$(_whole_static_libs)$(comma) >> $@ ; done ; \

	  ) \

	)

	$(foreach l,$(sort $(_all_static_libs)), \

	  $(eval _lib_stem := $(call word-colon,1,$l)) \

	  $(eval _module_path := $(call word-colon,2,$l)) \

	  $(eval _soong_module_type := $(call word-colon,3,$l)) \

	  $(eval _built_file := $(call word-colon,4,$l)) \

	  $(eval _static_libs := $(ALL_STATIC_LIBRARIES.$l.STATIC_LIBRARIES)) \

	  $(eval _whole_static_libs := $(ALL_STATIC_LIBRARIES.$l.WHOLE_STATIC_LIBRARIES)) \

	  $(eval _is_static_lib := Y) \

	  echo $(_lib_stem).a,$(_module_path),$(_soong_module_type),,,,,$(_built_file),$(_static_libs),$(_whole_static_libs),$(_is_static_lib) >> $@; \

	)



.PHONY: sbom

ifeq ($(TARGET_BUILD_APPS),)

core/sbom.mk

+11 −0
Original line number Diff line number Diff line
@@ -3,9 +3,20 @@ 
# unless a .mk file changes its installed file after including base_rules.mk.



ifdef my_register_name

  # ALL_INSTALLED_FILES.$(installed_file).STATIC_LIBRARIES: list of module name of static libraries, e.g. libc++demangle libclang_rt.builtins, for primary arch

  # ALL_INSTALLED_FILES.$(installed_file).WHOLE_STATIC_LIBRARIES: list of module name of static libraries, e.g. libc++demangle_32 libclang_rt.builtins_32, for 2nd arch.

  ifneq (, $(strip $(ALL_MODULES.$(my_register_name).INSTALLED)))

    $(foreach installed_file,$(ALL_MODULES.$(my_register_name).INSTALLED),\

      $(eval ALL_INSTALLED_FILES.$(installed_file) := $(my_register_name))\

      $(eval ALL_INSTALLED_FILES.$(installed_file).STATIC_LIBRARIES := $(foreach l,$(strip $(sort $(LOCAL_STATIC_LIBRARIES))),$l$(if $(LOCAL_2ND_ARCH_VAR_PREFIX),$($(my_prefix)2ND_ARCH_MODULE_SUFFIX))))\

      $(eval ALL_INSTALLED_FILES.$(installed_file).WHOLE_STATIC_LIBRARIES := $(foreach l,$(strip $(sort $(LOCAL_WHOLE_STATIC_LIBRARIES))),$l$(if $(LOCAL_2ND_ARCH_VAR_PREFIX),$($(my_prefix)2ND_ARCH_MODULE_SUFFIX))))\

    )

  endif

  ifeq (STATIC_LIBRARIES,$(LOCAL_MODULE_CLASS))

  ALL_STATIC_LIBRARIES.$(my_register_name).STATIC_LIBRARIES := $(foreach l,$(strip $(sort $(LOCAL_STATIC_LIBRARIES))),$l$($(my_prefix)2ND_ARCH_MODULE_SUFFIX))

  ALL_STATIC_LIBRARIES.$(my_register_name).WHOLE_STATIC_LIBRARIES := $(foreach l,$(strip $(sort $(LOCAL_WHOLE_STATIC_LIBRARIES))),$l$($(my_prefix)2ND_ARCH_MODULE_SUFFIX))

  ifdef LOCAL_SOONG_MODULE_TYPE

    ALL_STATIC_LIBRARIES.$(my_register_name).BUILT_FILE := $(LOCAL_PREBUILT_MODULE_FILE)

  endif

  endif

endif
 
 No newline at end of file

tools/sbom/generate-sbom.py

+29 −17
Original line number Diff line number Diff line
@@ -332,14 +332,6 @@ def get_sbom_fragments(installed_file_metadata, metadata_file_path): 
  return external_doc_ref, packages, relationships





def generate_package_verification_code(files):

  checksums = [file.checksum for file in files]

  checksums.sort()

  h = hashlib.sha1()

  h.update(''.join(checksums).encode(encoding='utf-8'))

  return h.hexdigest()





def save_report(report_file_path, report):

  with open(report_file_path, 'w', encoding='utf-8') as report_file:

    for type, issues in report.items():
@@ -487,16 +479,28 @@ def main(): 
      product_copy_files = installed_file_metadata['product_copy_files']

      kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files']

      build_output_path = installed_file_metadata['build_output_path']

      is_static_lib = installed_file_metadata['is_static_lib']



      if not installed_file_has_metadata(installed_file_metadata, report):

        continue

      if not (os.path.islink(build_output_path) or os.path.isfile(build_output_path)):

      if not is_static_lib and not (os.path.islink(build_output_path) or os.path.isfile(build_output_path)):

        # Ignore non-existing static library files for now since they are not shipped on devices.

        report[ISSUE_INSTALLED_FILE_NOT_EXIST].append(installed_file)

        continue



      file_id = new_file_id(installed_file)

      doc.files.append(

        sbom_data.File(id=file_id, name=installed_file, checksum=checksum(build_output_path)))

      # TODO(b/285453664): Soong should report the information of statically linked libraries to Make.

      # This happens when a different sanitized version of static libraries is used in linking.

      # As a workaround, use the following SHA1 checksum for static libraries created by Soong, if .a files could not be

      # located correctly because Soong doesn't report the information to Make.

      sha1 = 'SHA1: da39a3ee5e6b4b0d3255bfef95601890afd80709'  # SHA1 of empty string

      if os.path.islink(build_output_path) or os.path.isfile(build_output_path):

        sha1 = checksum(build_output_path)

      doc.files.append(sbom_data.File(id=file_id,

                                      name=installed_file,

                                      checksum=sha1))



      if not is_static_lib:

        if not args.unbundled_apex:

          product_package.file_ids.append(file_id)

        elif len(doc.files) > 1:
@@ -544,13 +548,21 @@ def main(): 
                                                    relationship=sbom_data.RelationshipType.GENERATED_FROM,

                                                    id2=sbom_data.SPDXID_PLATFORM))



  if not args.unbundled_apex:

    product_package.verification_code = generate_package_verification_code(doc.files)

      # Process static libraries and whole static libraries the installed file links to

      static_libs = installed_file_metadata['static_libraries']

      whole_static_libs = installed_file_metadata['whole_static_libraries']

      all_static_libs = (static_libs + ' ' + whole_static_libs).strip()

      if all_static_libs:

        for lib in all_static_libs.split(' '):

          doc.add_relationship(sbom_data.Relationship(id1=file_id,

                                                      relationship=sbom_data.RelationshipType.STATIC_LINK,

                                                      id2=new_file_id(lib + '.a')))



  if args.unbundled_apex:

    doc.describes = doc.files[0].id



  # Save SBOM records to output file

  doc.generate_packages_verification_code()

  doc.created = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')

  prefix = args.output_file

  if prefix.endswith('.spdx'):

tools/sbom/sbom_data.py

+16 −0
Original line number Diff line number Diff line
@@ -25,6 +25,7 @@ fields in each data class. 


from dataclasses import dataclass, field

from typing import List

import hashlib



SPDXID_DOC = 'SPDXRef-DOCUMENT'

SPDXID_PRODUCT = 'SPDXRef-PRODUCT'
@@ -81,6 +82,7 @@ class RelationshipType: 
  VARIANT_OF = 'VARIANT_OF'

  GENERATED_FROM = 'GENERATED_FROM'

  CONTAINS = 'CONTAINS'

  STATIC_LINK = 'STATIC_LINK'





@dataclass
@@ -122,3 +124,17 @@ class Document: 
    if not any(rel.id1 == r.id1 and rel.id2 == r.id2 and rel.relationship == r.relationship

               for r in self.relationships):

      self.relationships.append(rel)



  def generate_packages_verification_code(self):

    for package in self.packages:

      if not package.file_ids:

        continue



      checksums = []

      for file in self.files:

        if file.id in package.file_ids:

          checksums.append(file.checksum)

      checksums.sort()

      h = hashlib.sha1()

      h.update(''.join(checksums).encode(encoding='utf-8'))

      package.verification_code = h.hexdigest()

tools/sbom/sbom_writers.py

+29 −42
Original line number Diff line number Diff line
@@ -85,7 +85,7 @@ class TagValueWriter: 
    return headers



  @staticmethod

  def marshal_package(package):

  def marshal_package(sbom_doc, package, fragment):

    download_location = sbom_data.VALUE_NOASSERTION

    if package.download_location:

      download_location = package.download_location
@@ -107,50 +107,32 @@ class TagValueWriter: 
          f'{Tags.PACKAGE_EXTERNAL_REF}: {external_ref.category} {external_ref.type} {external_ref.locator}')



    tagvalues.append('')

    return tagvalues



  @staticmethod

  def marshal_described_element(sbom_doc, fragment):

    if not sbom_doc.describes:

      return None



    product_package = [p for p in sbom_doc.packages if p.id == sbom_doc.describes]

    if product_package:

      tagvalues = TagValueWriter.marshal_package(product_package[0])

      if not fragment:

    if package.id == sbom_doc.describes and not fragment:

      tagvalues.append(

          f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}')



      tagvalues.append('')

      return tagvalues



    file = [f for f in sbom_doc.files if f.id == sbom_doc.describes]

    if file:

      tagvalues = TagValueWriter.marshal_file(file[0])

      if not fragment:

        tagvalues.append(

            f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}')

    for file in sbom_doc.files:

      if file.id in package.file_ids:

        tagvalues += TagValueWriter.marshal_file(file)



    return tagvalues



    return None



  @staticmethod

  def marshal_packages(sbom_doc):

  def marshal_packages(sbom_doc, fragment):

    tagvalues = []

    marshaled_relationships = []

    i = 0

    packages = sbom_doc.packages

    while i < len(packages):

      if packages[i].id == sbom_doc.describes:

        i += 1

        continue



      if i + 1 < len(packages) \

          and packages[i].id.startswith('SPDXRef-SOURCE-') \

          and packages[i + 1].id.startswith('SPDXRef-UPSTREAM-'):

        tagvalues += TagValueWriter.marshal_package(packages[i])

        tagvalues += TagValueWriter.marshal_package(packages[i + 1])

      if (i + 1 < len(packages)

          and packages[i].id.startswith('SPDXRef-SOURCE-')

          and packages[i + 1].id.startswith('SPDXRef-UPSTREAM-')):

        # Output SOURCE, UPSTREAM packages and their VARIANT_OF relationship together, so they are close to each other

        # in SBOMs in tagvalue format.

        tagvalues += TagValueWriter.marshal_package(sbom_doc, packages[i], fragment)

        tagvalues += TagValueWriter.marshal_package(sbom_doc, packages[i + 1], fragment)

        rel = next((r for r in sbom_doc.relationships if

                    r.id1 == packages[i].id and

                    r.id2 == packages[i + 1].id and
@@ -162,7 +144,7 @@ class TagValueWriter: 


        i += 2

      else:

        tagvalues += TagValueWriter.marshal_package(packages[i])

        tagvalues += TagValueWriter.marshal_package(sbom_doc, packages[i], fragment)

        i += 1



    return tagvalues, marshaled_relationships
@@ -179,12 +161,20 @@ class TagValueWriter: 
    return tagvalues



  @staticmethod

  def marshal_files(sbom_doc):

  def marshal_files(sbom_doc, fragment):

    tagvalues = []

    files_in_packages = []

    for package in sbom_doc.packages:

      files_in_packages += package.file_ids

    for file in sbom_doc.files:

      if file.id == sbom_doc.describes:

      if file.id in files_in_packages:

        continue

      tagvalues += TagValueWriter.marshal_file(file)

      if file.id == sbom_doc.describes and not fragment:

        # Fragment is not a full SBOM document so the relationship DESCRIBES is not applicable.

        tagvalues.append(

            f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}')

        tagvalues.append('')

    return tagvalues



  @staticmethod
@@ -208,11 +198,8 @@ class TagValueWriter: 
    content = []

    if not fragment:

      content += TagValueWriter.marshal_doc_headers(sbom_doc)

    described_element = TagValueWriter.marshal_described_element(sbom_doc, fragment)

    if described_element:

      content += described_element

    content += TagValueWriter.marshal_files(sbom_doc)

    tagvalues, marshaled_relationships = TagValueWriter.marshal_packages(sbom_doc)

    content += TagValueWriter.marshal_files(sbom_doc, fragment)

    tagvalues, marshaled_relationships = TagValueWriter.marshal_packages(sbom_doc, fragment)

    content += tagvalues

    content += TagValueWriter.marshal_relationships(sbom_doc, marshaled_relationships)

    file.write('\n'.join(content))