Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit f96858a8 authored by Wei Li's avatar Wei Li Committed by Android (Google) Code Review
Browse files

Merge changes from topic "cherrypicker-L80700000960770298:N69600001370607907" into udc-dev 

* changes:
  Some changes to support SBOM generation for b build unbundled APEXs.
  Export some make variables to soong.variables.
parents 98b4c2d9 233d5b97

core/main.mk

+6 −5
Original line number Diff line number Diff line
@@ -2163,10 +2163,11 @@ endif # TARGET_BUILD_APPS 
$(shell rm $(PRODUCT_OUT)/sbom-metadata.csv >/dev/null 2>&1)

$(PRODUCT_OUT)/sbom-metadata.csv: $(installed_files)

	rm -f $@

	@echo installed_file$(comma)module_path$(comma)soong_module_type$(comma)is_prebuilt_make_module$(comma)product_copy_files$(comma)kernel_module_copy_files$(comma)is_platform_generated >> $@

	@echo installed_file$(comma)module_path$(comma)soong_module_type$(comma)is_prebuilt_make_module$(comma)product_copy_files$(comma)kernel_module_copy_files$(comma)is_platform_generated,build_output_path >> $@

	$(foreach f,$(installed_files),\

	  $(eval _module_name := $(ALL_INSTALLED_FILES.$f)) \

	  $(eval _path_on_device := $(patsubst $(PRODUCT_OUT)/%,%,$f)) \

	  $(eval _build_output_path := $(PRODUCT_OUT)/$(_path_on_device)) \

	  $(eval _module_path := $(strip $(sort $(ALL_MODULES.$(_module_name).PATH)))) \

	  $(eval _soong_module_type := $(strip $(sort $(ALL_MODULES.$(_module_name).SOONG_MODULE_TYPE)))) \

	  $(eval _is_prebuilt_make_module := $(ALL_MODULES.$(_module_name).IS_PREBUILT_MAKE_MODULE)) \
@@ -2184,9 +2185,9 @@ $(PRODUCT_OUT)/sbom-metadata.csv: $(installed_files) 
	  $(eval _is_linker_config := $(if $(findstring $f,$(SYSTEM_LINKER_CONFIG) $(vendor_linker_config_file)),Y)) \

	  $(eval _is_partition_compat_symlink := $(if $(findstring $f,$(PARTITION_COMPAT_SYMLINKS)),Y)) \

	  $(eval _is_platform_generated := $(_is_build_prop)$(_is_notice_file)$(_is_dexpreopt_image_profile)$(_is_product_system_other_avbkey)$(_is_event_log_tags_file)$(_is_system_other_odex_marker)$(_is_kernel_modules_blocklist)$(_is_fsverity_build_manifest_apk)$(_is_linker_config)$(_is_partition_compat_symlink)) \

	  @echo /$(_path_on_device)$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated) >> $@ $(newline) \

	  @echo /$(_path_on_device)$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated)$(comma)$(_build_output_path) >> $@ $(newline) \

	  $(if $(_post_installed_dexpreopt_zip), \

	  for i in $$(zipinfo -1 $(_post_installed_dexpreopt_zip)); do echo /$$i$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated) >> $@ ; done $(newline) \

	  for i in $$(zipinfo -1 $(_post_installed_dexpreopt_zip)); do echo /$$i$(comma)$(_module_path)$(comma)$(_soong_module_type)$(comma)$(_is_prebuilt_make_module)$(comma)$(_product_copy_files)$(comma)$(_kernel_module_copy_files)$(comma)$(_is_platform_generated)$(comma)$(PRODUCT_OUT)/$$i >> $@ ; done $(newline) \

	  ) \

	)
@@ -2196,14 +2197,14 @@ sbom: $(PRODUCT_OUT)/sbom.spdx.json 
$(PRODUCT_OUT)/sbom.spdx.json: $(PRODUCT_OUT)/sbom.spdx

$(PRODUCT_OUT)/sbom.spdx: $(PRODUCT_OUT)/sbom-metadata.csv $(GEN_SBOM)

	rm -rf $@

	$(GEN_SBOM) --output_file $@ --metadata $(PRODUCT_OUT)/sbom-metadata.csv --product_out_dir=$(PRODUCT_OUT) --build_version $(BUILD_FINGERPRINT_FROM_FILE) --product_mfr="$(PRODUCT_MANUFACTURER)" --json

	$(GEN_SBOM) --output_file $@ --metadata $(PRODUCT_OUT)/sbom-metadata.csv --build_version $(BUILD_FINGERPRINT_FROM_FILE) --product_mfr "$(PRODUCT_MANUFACTURER)" --json



$(call dist-for-goals,droid,$(PRODUCT_OUT)/sbom.spdx.json:sbom/sbom.spdx.json)

else

apps_only_sbom_files := $(sort $(patsubst %,%.spdx.json,$(filter %.apk,$(apps_only_installed_files))))

$(apps_only_sbom_files): $(PRODUCT_OUT)/sbom-metadata.csv $(GEN_SBOM)

	rm -rf $@

	$(GEN_SBOM) --output_file $@ --metadata $(PRODUCT_OUT)/sbom-metadata.csv --product_out_dir=$(PRODUCT_OUT) --build_version $(BUILD_FINGERPRINT_FROM_FILE) --product_mfr="$(PRODUCT_MANUFACTURER)" --unbundled

	$(GEN_SBOM) --output_file $@ --metadata $(PRODUCT_OUT)/sbom-metadata.csv --build_version $(BUILD_FINGERPRINT_FROM_FILE) --product_mfr "$(PRODUCT_MANUFACTURER)" --unbundled_apk



sbom: $(apps_only_sbom_files)

core/soong_config.mk

+4 −0
Original line number Diff line number Diff line
@@ -323,6 +323,10 @@ $(call add_json_list, SourceRootDirs, $(PRODUCT_SOURCE_ROOT_DIRS)) 


$(call add_json_list, AfdoProfiles,                $(ALL_AFDO_PROFILES))



$(call add_json_str,  ProductManufacturer, $(PRODUCT_MANUFACTURER))

$(call add_json_str,  ProductBrand,        $(PRODUCT_BRAND))

$(call add_json_list, BuildVersionTags,    $(BUILD_VERSION_TAGS))



$(call json_end)



$(file >$(SOONG_VARIABLES).tmp,$(json_contents))

tools/sbom/generate-sbom.py

+43 −25
Original line number Diff line number Diff line
@@ -19,7 +19,6 @@ Generate the SBOM of the current target product in SPDX format. 
Usage example:

  generate-sbom.py --output_file out/target/product/vsoc_x86_64/sbom.spdx \

                   --metadata out/target/product/vsoc_x86_64/sbom-metadata.csv \

                   --product_out_dir=out/target/product/vsoc_x86_64 \

                   --build_version $(cat out/target/product/vsoc_x86_64/build_fingerprint.txt) \

                   --product_mfr=Google

"""
@@ -89,11 +88,11 @@ def get_args(): 
  parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Print more information.')

  parser.add_argument('--output_file', required=True, help='The generated SBOM file in SPDX format.')

  parser.add_argument('--metadata', required=True, help='The SBOM metadata file path.')

  parser.add_argument('--product_out_dir', required=True, help='The parent directory of all the installed files.')

  parser.add_argument('--build_version', required=True, help='The build version.')

  parser.add_argument('--product_mfr', required=True, help='The product manufacturer.')

  parser.add_argument('--json', action='store_true', default=False, help='Generated SBOM file in SPDX JSON format')

  parser.add_argument('--unbundled', action='store_true', default=False, help='Generate SBOM file for unbundled module')

  parser.add_argument('--unbundled_apk', action='store_true', default=False, help='Generate SBOM for unbundled APKs')

  parser.add_argument('--unbundled_apex', action='store_true', default=False, help='Generate SBOM for unbundled APEXs')



  return parser.parse_args()
@@ -127,7 +126,6 @@ def new_file_id(file_path): 




def checksum(file_path):

  file_path = args.product_out_dir + '/' + file_path

  h = hashlib.sha1()

  if os.path.islink(file_path):

    h.update(os.readlink(file_path).encode('utf-8'))
@@ -334,9 +332,8 @@ def generate_package_verification_code(files): 
  return h.hexdigest()





def save_report(report):

  prefix, _ = os.path.splitext(args.output_file)

  with open(prefix + '-gen-report.txt', 'w', encoding='utf-8') as report_file:

def save_report(report_file_path, report):

  with open(report_file_path, 'w', encoding='utf-8') as report_file:

    for type, issues in report.items():

      report_file.write(type + '\n')

      for issue in issues:
@@ -394,7 +391,7 @@ def report_metadata_file(metadata_file_path, installed_file_metadata, report): 
            installed_file_metadata['installed_file'], installed_file_metadata['module_path']))





def generate_sbom_for_unbundled():

def generate_sbom_for_unbundled_apk():

  with open(args.metadata, newline='') as sbom_metadata_file:

    reader = csv.DictReader(sbom_metadata_file)

    doc = sbom_data.Document(name=args.build_version,
@@ -402,7 +399,7 @@ def generate_sbom_for_unbundled(): 
                             creators=['Organization: ' + args.product_mfr])

    for installed_file_metadata in reader:

      installed_file = installed_file_metadata['installed_file']

      if args.output_file != args.product_out_dir + installed_file + '.spdx.json':

      if args.output_file != installed_file_metadata['build_output_path'] + '.spdx.json':

        continue



      module_path = installed_file_metadata['module_path']
@@ -412,7 +409,9 @@ def generate_sbom_for_unbundled(): 
                                  version=args.build_version,

                                  supplier='Organization: ' + args.product_mfr)

      file_id = new_file_id(installed_file)

      file = sbom_data.File(id=file_id, name=installed_file, checksum=checksum(installed_file))

      file = sbom_data.File(id=file_id,

                            name=installed_file,

                            checksum=checksum(installed_file_metadata['build_output_path']))

      relationship = sbom_data.Relationship(id1=file_id,

                                            relationship=sbom_data.RelationshipType.GENERATED_FROM,

                                            id2=package_id)
@@ -435,23 +434,24 @@ def main(): 
  args = get_args()

  log('Args:', vars(args))



  if args.unbundled:

    generate_sbom_for_unbundled()

  if args.unbundled_apk:

    generate_sbom_for_unbundled_apk()

    return



  global metadata_file_protos

  metadata_file_protos = {}



  doc = sbom_data.Document(name=args.build_version,

                           namespace=f'https://www.google.com/sbom/spdx/android/{args.build_version}',

                           creators=['Organization: ' + args.product_mfr])



  product_package = sbom_data.Package(id=sbom_data.SPDXID_PRODUCT,

                                      name=sbom_data.PACKAGE_NAME_PRODUCT,

                                      download_location=sbom_data.VALUE_NONE,

                                      version=args.build_version,

                                      supplier='Organization: ' + args.product_mfr,

                                      files_analyzed=True)



  doc = sbom_data.Document(name=args.build_version,

                           namespace=f'https://www.google.com/sbom/spdx/android/{args.build_version}',

                           creators=['Organization: ' + args.product_mfr])

  if not args.unbundled_apex:

    doc.packages.append(product_package)



  doc.packages.append(sbom_data.Package(id=sbom_data.SPDXID_PLATFORM,
@@ -478,18 +478,21 @@ def main(): 
      module_path = installed_file_metadata['module_path']

      product_copy_files = installed_file_metadata['product_copy_files']

      kernel_module_copy_files = installed_file_metadata['kernel_module_copy_files']

      build_output_path = installed_file_metadata['build_output_path']



      if not installed_file_has_metadata(installed_file_metadata, report):

        continue

      file_path = args.product_out_dir + '/' + installed_file

      if not (os.path.islink(file_path) or os.path.isfile(file_path)):

      if not (os.path.islink(build_output_path) or os.path.isfile(build_output_path)):

        report[ISSUE_INSTALLED_FILE_NOT_EXIST].append(installed_file)

        continue



      file_id = new_file_id(installed_file)

      doc.files.append(

        sbom_data.File(id=file_id, name=installed_file, checksum=checksum(installed_file)))

        sbom_data.File(id=file_id, name=installed_file, checksum=checksum(build_output_path)))

      if not args.unbundled_apex:

        product_package.file_ids.append(file_id)

      elif len(doc.files) > 1:

          doc.add_relationship(sbom_data.Relationship(doc.files[0].id, sbom_data.RelationshipType.CONTAINS, file_id))



      if is_source_package(installed_file_metadata) or is_prebuilt_package(installed_file_metadata):

        metadata_file_path = get_metadata_file_path(installed_file_metadata)
@@ -533,16 +536,31 @@ def main(): 
                                                    relationship=sbom_data.RelationshipType.GENERATED_FROM,

                                                    id2=sbom_data.SPDXID_PLATFORM))



  if not args.unbundled_apex:

    product_package.verification_code = generate_package_verification_code(doc.files)



  if args.unbundled_apex:

    doc.describes = doc.files[0].id



  # Save SBOM records to output file

  doc.created = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')

  with open(args.output_file, 'w', encoding="utf-8") as file:

    sbom_writers.TagValueWriter.write(doc, file)

  prefix = args.output_file

  if prefix.endswith('.spdx'):

    prefix = prefix.removesuffix('.spdx')

  elif prefix.endswith('.spdx.json'):

    prefix = prefix.removesuffix('.spdx.json')



  output_file = prefix + '.spdx'

  if args.unbundled_apex:

    output_file = prefix + '-fragment.spdx'

  with open(output_file, 'w', encoding="utf-8") as file:

    sbom_writers.TagValueWriter.write(doc, file, fragment=args.unbundled_apex)

  if args.json:

    with open(args.output_file+'.json', 'w', encoding="utf-8") as file:

    with open(prefix + '.spdx.json', 'w', encoding="utf-8") as file:

      sbom_writers.JSONWriter.write(doc, file)



  save_report(prefix + '-gen-report.txt', report)





if __name__ == '__main__':

  main()

tools/sbom/sbom_data.py

+1 −0
Original line number Diff line number Diff line
@@ -80,6 +80,7 @@ class RelationshipType: 
  DESCRIBES = 'DESCRIBES'

  VARIANT_OF = 'VARIANT_OF'

  GENERATED_FROM = 'GENERATED_FROM'

  CONTAINS = 'CONTAINS'





@dataclass

tools/sbom/sbom_writers.py

+13 −9
Original line number Diff line number Diff line
@@ -110,13 +110,14 @@ class TagValueWriter: 
    return tagvalues



  @staticmethod

  def marshal_described_element(sbom_doc):

  def marshal_described_element(sbom_doc, fragment):

    if not sbom_doc.describes:

      return None



    product_package = [p for p in sbom_doc.packages if p.id == sbom_doc.describes]

    if product_package:

      tagvalues = TagValueWriter.marshal_package(product_package[0])

      if not fragment:

        tagvalues.append(

            f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}')
@@ -125,9 +126,10 @@ class TagValueWriter: 


    file = [f for f in sbom_doc.files if f.id == sbom_doc.describes]

    if file:

      tagvalues = [

        f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}'

      ]

      tagvalues = TagValueWriter.marshal_file(file[0])

      if not fragment:

        tagvalues.append(

            f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}')



      return tagvalues
@@ -180,6 +182,8 @@ class TagValueWriter: 
  def marshal_files(sbom_doc):

    tagvalues = []

    for file in sbom_doc.files:

      if file.id == sbom_doc.describes:

        continue

      tagvalues += TagValueWriter.marshal_file(file)

    return tagvalues
@@ -204,7 +208,7 @@ class TagValueWriter: 
    content = []

    if not fragment:

      content += TagValueWriter.marshal_doc_headers(sbom_doc)

      described_element = TagValueWriter.marshal_described_element(sbom_doc)

    described_element = TagValueWriter.marshal_described_element(sbom_doc, fragment)

    if described_element:

      content += described_element

    content += TagValueWriter.marshal_files(sbom_doc)