Merge "Revert "Generate SBOM of mainline modules using compliance metad..."" into main (b4d6ac97) · Commits · e / os / android_build

core/main.mk

+47 −0

Original line number	Diff line number	Diff line
		@@ -1882,6 +1882,53 @@ $(shell rm -f $(PRODUCT_OUT)/always_dirty_file.txt)
		$(PRODUCT_OUT)/always_dirty_file.txt:
		touch $@

		.PHONY: sbom
		ifneq ($(TARGET_BUILD_APPS),)
		# Create build rules for generating SBOMs of unbundled APKs and APEXs
		# $1: sbom file
		# $2: sbom fragment file
		# $3: installed file
		# $4: sbom-metadata.csv file
		define generate-app-sbom
		$(eval _path_on_device := $(patsubst $(PRODUCT_OUT)/%,%,$(3)))
		$(eval _module_name := $(ALL_INSTALLED_FILES.$(3)))
		$(eval _module_path := $(strip $(sort $(ALL_MODULES.$(_module_name).PATH))))
		$(eval _soong_module_type := $(strip $(sort $(ALL_MODULES.$(_module_name).SOONG_MODULE_TYPE))))
		$(eval _dep_modules := $(filter %.$(_module_name),$(ALL_MODULES)) $(filter %.$(_module_name)$(TARGET_2ND_ARCH_MODULE_SUFFIX),$(ALL_MODULES)))
		$(eval _is_apex := $(filter %.apex,$(3)))

		$(4):
		rm -rf $$@
		echo installed_file,module_path,soong_module_type,is_prebuilt_make_module,product_copy_files,kernel_module_copy_files,is_platform_generated,build_output_path,static_libraries,whole_static_libraries,is_static_lib >> $$@
		echo /$(_path_on_device),$(_module_path),$(_soong_module_type),,,,,$(3),,, >> $$@
		$(if $(filter %.apex,$(3)),\
		$(foreach m,$(_dep_modules),\
		echo $(patsubst $(PRODUCT_OUT)/apex/$(_module_name)/%,%,$(ALL_MODULES.$m.INSTALLED)),$(sort $(ALL_MODULES.$m.PATH)),$(sort $(ALL_MODULES.$m.SOONG_MODULE_TYPE)),,,,,$(strip $(ALL_MODULES.$m.INSTALLED)),,, >> $$@;))

		$(2): $(1)
		$(1): $(4) $(3) $(GEN_SBOM) $(installed_files) $(metadata_list) $(metadata_files)
		rm -rf $$@
		$(GEN_SBOM) --output_file $$@ --metadata $(4) --build_version $$(BUILD_FINGERPRINT_FROM_FILE) --product_mfr "$(PRODUCT_MANUFACTURER)" --json $(if $(filter %.apk,$(3)),--unbundled_apk,--unbundled_apex)
		endef

		apps_only_sbom_files :=
		apps_only_fragment_files :=
		$(foreach f,$(filter %.apk %.apex,$(installed_files)), \
		$(eval _metadata_csv_file := $(patsubst %,%-sbom-metadata.csv,$f)) \
		$(eval _sbom_file := $(patsubst %,%.spdx.json,$f)) \
		$(eval _fragment_file := $(patsubst %,%-fragment.spdx,$f)) \
		$(eval apps_only_sbom_files += $(_sbom_file)) \
		$(eval apps_only_fragment_files += $(_fragment_file)) \
		$(eval $(call generate-app-sbom,$(_sbom_file),$(_fragment_file),$f,$(_metadata_csv_file))) \
		)

		sbom: $(apps_only_sbom_files)

		$(foreach f,$(apps_only_fragment_files),$(eval apps_only_fragment_dist_files += :sbom/$(notdir $f)))
		$(foreach f,$(apps_only_sbom_files),$(eval apps_only_sbom_dist_files += :sbom/$(notdir $f)))
		$(call dist-for-goals,apps_only,$(join $(apps_only_sbom_files),$(apps_only_sbom_dist_files)) $(join $(apps_only_fragment_files),$(apps_only_fragment_dist_files)))
		endif

		$(call dist-write-file,$(KATI_PACKAGE_MK_DIR)/dist.mk)

		$(info [$(include_makefiles_total)/$(include_makefiles_total)] writing make module actions ...)

tools/sbom/compliance_metadata.py

+0 −11

Original line number	Diff line number	Diff line
		@@ -132,17 +132,6 @@ class MetadataDb:
		installed_files_metadata.append(metadata)
		return installed_files_metadata

		def get_installed_files_of_module(self, module_name):
		# Get install file list of a module
		cursor = self.conn.execute(f'select installed_file, build_output_path from "{module_name}"')
		rows = cursor.fetchall()
		cursor.close()
		installed_files_metadata = []
		for row in rows:
		metadata = dict(zip(row.keys(), row))
		installed_files_metadata.append(metadata)
		return installed_files_metadata

		def get_installed_file_in_dir(self, dir):
		dir = dir.removesuffix('/')
		cursor = self.conn.execute(

tools/sbom/gen_sbom.py

+52 −85

Original line number	Diff line number	Diff line
		@@ -140,7 +140,6 @@ def get_args():
		parser.add_argument('--build_version', required=True, help='The build version.')
		parser.add_argument('--product_mfr', required=True, help='The product manufacturer.')
		parser.add_argument('--json', action='store_true', default=False, help='Generated SBOM file in SPDX JSON format')
		parser.add_argument('--unbundled_module', help='Module name, e.g. com.google.android.adbd, for which SBOM is generated')

		return parser.parse_args()

		@@ -180,11 +179,7 @@ def is_soong_prebuilt_module(file_metadata):

		def is_source_package(file_metadata):
		module_path = file_metadata['module_path']
		is_source_package = False
		if args.unbundled_module and os.path.exists(module_path + '/METADATA'):
		# See b/272356272, change the logic of identifying source package for mainline modules for now.
		is_source_package = True
		return (module_path.startswith('external/') or is_source_package) and not is_prebuilt_package(file_metadata)
		return module_path.startswith('external/') and not is_prebuilt_package(file_metadata)


		def is_prebuilt_package(file_metadata):
		@@ -192,7 +187,7 @@ def is_prebuilt_package(file_metadata):
		if module_path:
		return (module_path.startswith('prebuilts/') or
		is_soong_prebuilt_module(file_metadata) or
		file_metadata.get('is_prebuilt_make_module'))
		file_metadata['is_prebuilt_make_module'])

		kernel_module_copy_files = file_metadata['kernel_module_copy_files']
		if kernel_module_copy_files and not kernel_module_copy_files.startswith('ANDROID-GEN:'):
		@@ -620,23 +615,18 @@ def main():
		supplier='Organization: ' + args.product_mfr,
		files_analyzed=True)
		doc_name = args.build_version
		if args.unbundled_module:
		doc_name = f'{args.build_version}/{args.unbundled_module}'
		doc = sbom_data.Document(name=doc_name,
		namespace=f'https://www.google.com/sbom/spdx/android/{doc_name}',
		creators=['Organization: ' + args.product_mfr],
		describes=product_package_id)

		if not args.unbundled_module:
		doc.packages.append(product_package)

		platform_package = sbom_data.Package(id=sbom_data.SPDXID_PLATFORM,
		doc.packages.append(sbom_data.Package(id=sbom_data.SPDXID_PLATFORM,
		name=sbom_data.PACKAGE_NAME_PLATFORM,
		download_location=sbom_data.VALUE_NONE,
		version=args.build_version,
		supplier='Organization: ' + args.product_mfr,
		declared_license_ids=[sbom_data.SPDXID_LICENSE_APACHE])
		doc.packages.append(platform_package)
		declared_license_ids=[sbom_data.SPDXID_LICENSE_APACHE]))

		# Report on some issues and information
		report = {
		@@ -649,17 +639,12 @@ def main():
		INFO_METADATA_FOUND_FOR_PACKAGE: [],
		}

		if args.unbundled_module:
		installed_files_metadata = db.get_installed_files_of_module(args.unbundled_module)
		else:
		# Get installed files and corresponding make modules' metadata if an installed file is from a make module.
		installed_files_metadata = db.get_installed_files()

		# Find which Soong module an installed file is from and merge metadata from Make and Soong
		for installed_file_metadata in installed_files_metadata:
		soong_module = db.get_soong_module_of_installed_file(installed_file_metadata['installed_file'])
		if not soong_module and args.unbundled_module:
		soong_module = db.get_soong_module_of_built_file(installed_file_metadata['build_output_path'])
		if soong_module:
		# Merge soong metadata to make metadata
		installed_file_metadata.update(soong_module)
		@@ -670,19 +655,15 @@ def main():
		installed_file_metadata['whole_static_dep_files'] = ''

		# Scan the metadata and create the corresponding package and file records in SPDX
		include_static_deps = True
		for index, installed_file_metadata in enumerate(installed_files_metadata):
		for installed_file_metadata in installed_files_metadata:
		installed_file = installed_file_metadata['installed_file']
		module_path = installed_file_metadata['module_path']
		product_copy_files = installed_file_metadata.get('product_copy_files')
		kernel_module_copy_files = installed_file_metadata.get('kernel_module_copy_files')
		is_platform_generated = installed_file_metadata.get('is_platform_generated')
		product_copy_files = installed_file_metadata['product_copy_files']
		kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files']
		build_output_path = installed_file
		if args.unbundled_module:
		build_output_path = installed_file_metadata['build_output_path']
		installed_file = installed_file.removeprefix(args.product_out)

		if not args.unbundled_module and not installed_file_has_metadata(installed_file_metadata, report):
		if not installed_file_has_metadata(installed_file_metadata, report):
		continue
		if not (os.path.islink(build_output_path) or os.path.isfile(build_output_path)):
		report[ISSUE_INSTALLED_FILE_NOT_EXIST].append(installed_file)
		@@ -693,28 +674,16 @@ def main():
		f = sbom_data.File(id=file_id, name=installed_file, checksum=sha1)
		doc.files.append(f)
		product_package.file_ids.append(file_id)
		if args.unbundled_module:
		if index == 0:
		# The generated SBOM describes the APEX file which is the first record
		doc.describes = file_id
		# Do not include static deps in SBOM of APKs
		if installed_file.endswith('.apk'):
		include_static_deps = False
		else:
		# All other files are contained by the APEX file, so add a CONTAINS relationship for them
		doc.add_relationship(sbom_data.Relationship(id1=product_package.file_ids[0],
		relationship=sbom_data.RelationshipType.CONTAINS,
		id2=file_id))

		if is_source_package(installed_file_metadata) or is_prebuilt_package(installed_file_metadata):
		add_package_of_file(file_id, installed_file_metadata, doc, report)

		elif module_path or is_platform_generated:
		elif module_path or installed_file_metadata['is_platform_generated']:
		# File from PLATFORM package
		doc.add_relationship(sbom_data.Relationship(id1=file_id,
		relationship=sbom_data.RelationshipType.GENERATED_FROM,
		id2=sbom_data.SPDXID_PLATFORM))
		if is_platform_generated:
		if installed_file_metadata['is_platform_generated']:
		f.concluded_license_ids = [sbom_data.SPDXID_LICENSE_APACHE]

		elif product_copy_files:
		@@ -744,14 +713,12 @@ def main():
		id2=sbom_data.SPDXID_PLATFORM))

		# Process static dependencies of the installed file
		if include_static_deps:
		add_static_deps_of_file(file_id, installed_file_metadata, doc)

		# Add licenses of the installed file
		add_licenses_of_file(file_id, installed_file_metadata, doc)

		# Add all static library files to SBOM
		if include_static_deps:
		for dep_file in get_all_transitive_static_dep_files_of_installed_files(installed_files_metadata, db, report):
		filepath = dep_file.removeprefix(args.soong_out + '/.intermediates/')
		file_id = new_file_id(filepath)