Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e6927235 authored by Wei Li's avatar Wei Li
Browse files

Generate SBOM of mainline modules using compliance metadata database 

Bug: 324467268
Test: presubmits
Test: build/soong/tests/sbom_test.sh
Change-Id: Ia6857b19d5ecf1445ead9514614bf49bb616e699
parent 50709b91

core/main.mk

+0 −47
Original line number Diff line number Diff line
@@ -1882,53 +1882,6 @@ $(shell rm -f $(PRODUCT_OUT)/always_dirty_file.txt) 
$(PRODUCT_OUT)/always_dirty_file.txt:

	touch $@



.PHONY: sbom

ifneq ($(TARGET_BUILD_APPS),)

# Create build rules for generating SBOMs of unbundled APKs and APEXs

# $1: sbom file

# $2: sbom fragment file

# $3: installed file

# $4: sbom-metadata.csv file

define generate-app-sbom

$(eval _path_on_device := $(patsubst $(PRODUCT_OUT)/%,%,$(3)))

$(eval _module_name := $(ALL_INSTALLED_FILES.$(3)))

$(eval _module_path := $(strip $(sort $(ALL_MODULES.$(_module_name).PATH))))

$(eval _soong_module_type := $(strip $(sort $(ALL_MODULES.$(_module_name).SOONG_MODULE_TYPE))))

$(eval _dep_modules := $(filter %.$(_module_name),$(ALL_MODULES)) $(filter %.$(_module_name)$(TARGET_2ND_ARCH_MODULE_SUFFIX),$(ALL_MODULES)))

$(eval _is_apex := $(filter %.apex,$(3)))



$(4):

	rm -rf $$@

	echo installed_file,module_path,soong_module_type,is_prebuilt_make_module,product_copy_files,kernel_module_copy_files,is_platform_generated,build_output_path,static_libraries,whole_static_libraries,is_static_lib >> $$@

	echo /$(_path_on_device),$(_module_path),$(_soong_module_type),,,,,$(3),,, >> $$@

	$(if $(filter %.apex,$(3)),\

	  $(foreach m,$(_dep_modules),\

	    echo $(patsubst $(PRODUCT_OUT)/apex/$(_module_name)/%,%,$(ALL_MODULES.$m.INSTALLED)),$(sort $(ALL_MODULES.$m.PATH)),$(sort $(ALL_MODULES.$m.SOONG_MODULE_TYPE)),,,,,$(strip $(ALL_MODULES.$m.INSTALLED)),,, >> $$@;))



$(2): $(1)

$(1): $(4) $(3) $(GEN_SBOM) $(installed_files) $(metadata_list) $(metadata_files)

	rm -rf $$@

	$(GEN_SBOM) --output_file $$@ --metadata $(4) --build_version $$(BUILD_FINGERPRINT_FROM_FILE) --product_mfr "$(PRODUCT_MANUFACTURER)" --json $(if $(filter %.apk,$(3)),--unbundled_apk,--unbundled_apex)

endef



apps_only_sbom_files :=

apps_only_fragment_files :=

$(foreach f,$(filter %.apk %.apex,$(installed_files)), \

  $(eval _metadata_csv_file := $(patsubst %,%-sbom-metadata.csv,$f)) \

  $(eval _sbom_file := $(patsubst %,%.spdx.json,$f)) \

  $(eval _fragment_file := $(patsubst %,%-fragment.spdx,$f)) \

  $(eval apps_only_sbom_files += $(_sbom_file)) \

  $(eval apps_only_fragment_files += $(_fragment_file)) \

  $(eval $(call generate-app-sbom,$(_sbom_file),$(_fragment_file),$f,$(_metadata_csv_file))) \

)



sbom: $(apps_only_sbom_files)



$(foreach f,$(apps_only_fragment_files),$(eval apps_only_fragment_dist_files += :sbom/$(notdir $f)))

$(foreach f,$(apps_only_sbom_files),$(eval apps_only_sbom_dist_files += :sbom/$(notdir $f)))

$(call dist-for-goals,apps_only,$(join $(apps_only_sbom_files),$(apps_only_sbom_dist_files)) $(join $(apps_only_fragment_files),$(apps_only_fragment_dist_files)))

endif



$(call dist-write-file,$(KATI_PACKAGE_MK_DIR)/dist.mk)



$(info [$(include_makefiles_total)/$(include_makefiles_total)] writing make module actions ...)

tools/sbom/compliance_metadata.py

+11 −0
Original line number Diff line number Diff line
@@ -132,6 +132,17 @@ class MetadataDb: 
      installed_files_metadata.append(metadata)

    return installed_files_metadata



  def get_installed_files_of_module(self, module_name):

    # Get install file list of a module

    cursor = self.conn.execute(f'select installed_file, build_output_path from "{module_name}"')

    rows = cursor.fetchall()

    cursor.close()

    installed_files_metadata = []

    for row in rows:

      metadata = dict(zip(row.keys(), row))

      installed_files_metadata.append(metadata)

    return installed_files_metadata



  def get_installed_file_in_dir(self, dir):

    dir = dir.removesuffix('/')

    cursor = self.conn.execute(

tools/sbom/gen_sbom.py

+85 −52
Original line number Diff line number Diff line
@@ -140,6 +140,7 @@ def get_args(): 
  parser.add_argument('--build_version', required=True, help='The build version.')

  parser.add_argument('--product_mfr', required=True, help='The product manufacturer.')

  parser.add_argument('--json', action='store_true', default=False, help='Generated SBOM file in SPDX JSON format')

  parser.add_argument('--unbundled_module', help='Module name, e.g. com.google.android.adbd, for which SBOM is generated')



  return parser.parse_args()
@@ -179,7 +180,11 @@ def is_soong_prebuilt_module(file_metadata): 


def is_source_package(file_metadata):

  module_path = file_metadata['module_path']

  return module_path.startswith('external/') and not is_prebuilt_package(file_metadata)

  is_source_package = False

  if args.unbundled_module and os.path.exists(module_path + '/METADATA'):

    # See b/272356272, change the logic of identifying source package for mainline modules for now.

    is_source_package = True

  return (module_path.startswith('external/') or is_source_package) and not is_prebuilt_package(file_metadata)





def is_prebuilt_package(file_metadata):
@@ -187,7 +192,7 @@ def is_prebuilt_package(file_metadata): 
  if module_path:

    return (module_path.startswith('prebuilts/') or

            is_soong_prebuilt_module(file_metadata) or

            file_metadata['is_prebuilt_make_module'])

            file_metadata.get('is_prebuilt_make_module'))



  kernel_module_copy_files = file_metadata['kernel_module_copy_files']

  if kernel_module_copy_files and not kernel_module_copy_files.startswith('ANDROID-GEN:'):
@@ -615,18 +620,23 @@ def main(): 
                                      supplier='Organization: ' + args.product_mfr,

                                      files_analyzed=True)

  doc_name = args.build_version

  if args.unbundled_module:

    doc_name = f'{args.build_version}/{args.unbundled_module}'

  doc = sbom_data.Document(name=doc_name,

                           namespace=f'https://www.google.com/sbom/spdx/android/{doc_name}',

                           creators=['Organization: ' + args.product_mfr],

                           describes=product_package_id)



  if not args.unbundled_module:

    doc.packages.append(product_package)

  doc.packages.append(sbom_data.Package(id=sbom_data.SPDXID_PLATFORM,



  platform_package = sbom_data.Package(id=sbom_data.SPDXID_PLATFORM,

                                       name=sbom_data.PACKAGE_NAME_PLATFORM,

                                       download_location=sbom_data.VALUE_NONE,

                                       version=args.build_version,

                                       supplier='Organization: ' + args.product_mfr,

                                        declared_license_ids=[sbom_data.SPDXID_LICENSE_APACHE]))

                                       declared_license_ids=[sbom_data.SPDXID_LICENSE_APACHE])

  doc.packages.append(platform_package)



  # Report on some issues and information

  report = {
@@ -639,12 +649,17 @@ def main(): 
      INFO_METADATA_FOUND_FOR_PACKAGE: [],

  }



  if args.unbundled_module:

    installed_files_metadata = db.get_installed_files_of_module(args.unbundled_module)

  else:

    # Get installed files and corresponding make modules' metadata if an installed file is from a make module.

    installed_files_metadata = db.get_installed_files()



  # Find which Soong module an installed file is from and merge metadata from Make and Soong

  for installed_file_metadata in installed_files_metadata:

    soong_module = db.get_soong_module_of_installed_file(installed_file_metadata['installed_file'])

    if not soong_module and args.unbundled_module:

      soong_module = db.get_soong_module_of_built_file(installed_file_metadata['build_output_path'])

    if soong_module:

      # Merge soong metadata to make metadata

      installed_file_metadata.update(soong_module)
@@ -655,15 +670,19 @@ def main(): 
      installed_file_metadata['whole_static_dep_files'] = ''



  # Scan the metadata and create the corresponding package and file records in SPDX

  for installed_file_metadata in installed_files_metadata:

  include_static_deps = True

  for index, installed_file_metadata in enumerate(installed_files_metadata):

    installed_file = installed_file_metadata['installed_file']

    module_path = installed_file_metadata['module_path']

    product_copy_files = installed_file_metadata['product_copy_files']

    kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files']

    product_copy_files = installed_file_metadata.get('product_copy_files')

    kernel_module_copy_files = installed_file_metadata.get('kernel_module_copy_files')

    is_platform_generated = installed_file_metadata.get('is_platform_generated')

    build_output_path = installed_file

    if args.unbundled_module:

      build_output_path = installed_file_metadata['build_output_path']

    installed_file = installed_file.removeprefix(args.product_out)



    if not installed_file_has_metadata(installed_file_metadata, report):

    if not args.unbundled_module and not installed_file_has_metadata(installed_file_metadata, report):

      continue

    if not (os.path.islink(build_output_path) or os.path.isfile(build_output_path)):

      report[ISSUE_INSTALLED_FILE_NOT_EXIST].append(installed_file)
@@ -674,16 +693,28 @@ def main(): 
    f = sbom_data.File(id=file_id, name=installed_file, checksum=sha1)

    doc.files.append(f)

    product_package.file_ids.append(file_id)

    if args.unbundled_module:

      if index == 0:

        # The generated SBOM describes the APEX file which is the first record

        doc.describes = file_id

        # Do not include static deps in SBOM of APKs

        if installed_file.endswith('.apk'):

          include_static_deps = False

      else:

        # All other files are contained by the APEX file, so add a CONTAINS relationship for them

        doc.add_relationship(sbom_data.Relationship(id1=product_package.file_ids[0],

                                                    relationship=sbom_data.RelationshipType.CONTAINS,

                                                    id2=file_id))



    if is_source_package(installed_file_metadata) or is_prebuilt_package(installed_file_metadata):

      add_package_of_file(file_id, installed_file_metadata, doc, report)



    elif module_path or installed_file_metadata['is_platform_generated']:

    elif module_path or is_platform_generated:

      # File from PLATFORM package

      doc.add_relationship(sbom_data.Relationship(id1=file_id,

                                                  relationship=sbom_data.RelationshipType.GENERATED_FROM,

                                                  id2=sbom_data.SPDXID_PLATFORM))

      if installed_file_metadata['is_platform_generated']:

      if is_platform_generated:

        f.concluded_license_ids = [sbom_data.SPDXID_LICENSE_APACHE]



    elif product_copy_files:
@@ -713,12 +744,14 @@ def main(): 
                                                  id2=sbom_data.SPDXID_PLATFORM))



    # Process static dependencies of the installed file

    if include_static_deps:

      add_static_deps_of_file(file_id, installed_file_metadata, doc)



    # Add licenses of the installed file

    add_licenses_of_file(file_id, installed_file_metadata, doc)



  # Add all static library files to SBOM

  if include_static_deps:

    for dep_file in get_all_transitive_static_dep_files_of_installed_files(installed_files_metadata, db, report):

      filepath = dep_file.removeprefix(args.soong_out + '/.intermediates/')

      file_id = new_file_id(filepath)