Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 55decf2e authored by Treehugger Robot's avatar Treehugger Robot Committed by Automerger Merge Worker
Browse files

Merge "Create separate python libraries for the following logic and refactor...

Merge "Create separate python libraries for the following logic and refactor SBOM generation script accordingly." am: ebf41e9a am: 28fc5a97 am: 074bfc12

Original change: https://android-review.googlesource.com/c/platform/build/+/2525827



Change-Id: I0391804c0924a64ae58b71e99815cecd6c078309
Signed-off-by: default avatarAutomerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
parents 3bc1d4b9 074bfc12
Loading
Loading
Loading
Loading
+0 −16
Original line number Diff line number Diff line
@@ -70,22 +70,6 @@ python_binary_host {
  srcs: ["generate_gts_shared_report.py"],
}

python_binary_host {
    name: "generate-sbom",
    srcs: [
        "generate-sbom.py",
    ],
    version: {
        py3: {
            embedded_launcher: true,
        },
    },
    libs: [
        "metadata_file_proto_py",
        "libprotobuf-python",
    ],
}

python_binary_host {
    name: "list_files",
    main: "list_files.py",

tools/sbom/Android.bp

0 → 100644
+53 −0
Original line number Diff line number Diff line
// Copyright (C) 2023 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

python_binary_host {
    name: "generate-sbom",
    srcs: [
        "generate-sbom.py",
    ],
    version: {
        py3: {
            embedded_launcher: true,
        },
    },
    libs: [
        "metadata_file_proto_py",
        "libprotobuf-python",
        "sbom_lib",
    ],
}

python_library_host {
    name: "sbom_lib",
    srcs: [
        "sbom_data.py",
        "sbom_writers.py",
    ],
}

python_test_host {
    name: "sbom_writers_test",
    main: "sbom_writers_test.py",
    srcs: [
        "sbom_writers_test.py",
    ],
    data: [
        "testdata/*",
    ],
    libs: [
        "sbom_lib",
    ],
    test_suites: ["general-tests"],
}
+115 −269

File changed and moved.

Preview size limit exceeded, changes collapsed.

+120 −0
Original line number Diff line number Diff line
#!/usr/bin/env python3
#
# Copyright (C) 2023 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Define data classes that model SBOMs defined by SPDX. The data classes could be
written out to different formats (tagvalue, JSON, etc) of SPDX with corresponding
writer utilities.

Rrefer to SPDX 2.3 spec: https://spdx.github.io/spdx-spec/v2.3/ and go/android-spdx for details of
fields in each data class.
"""

from dataclasses import dataclass, field
from typing import List

SPDXID_DOC = 'SPDXRef-DOCUMENT'
SPDXID_PRODUCT = 'SPDXRef-PRODUCT'
SPDXID_PLATFORM = 'SPDXRef-PLATFORM'

PACKAGE_NAME_PRODUCT = 'PRODUCT'
PACKAGE_NAME_PLATFORM = 'PLATFORM'


class PackageExternalRefCategory:
  SECURITY = 'SECURITY'
  PACKAGE_MANAGER = 'PACKAGE-MANAGER'
  PERSISTENT_ID = 'PERSISTENT-ID'
  OTHER = 'OTHER'


class PackageExternalRefType:
  cpe22Type = 'cpe22Type'
  cpe23Type = 'cpe23Type'


@dataclass
class PackageExternalRef:
  category: PackageExternalRefCategory
  type: PackageExternalRefType
  locator: str


@dataclass
class Package:
  name: str
  id: str
  version: str = None
  supplier: str = None
  download_location: str = None
  files_analyzed: bool = False
  verification_code: str = None
  file_ids: List[str] = field(default_factory=list)
  external_refs: List[PackageExternalRef] = field(default_factory=list)


@dataclass
class File:
  id: str
  name: str
  checksum: str


class RelationshipType:
  DESCRIBES = 'DESCRIBES'
  VARIANT_OF = 'VARIANT_OF'
  GENERATED_FROM = 'GENERATED_FROM'


@dataclass
class Relationship:
  id1: str
  relationship: RelationshipType
  id2: str


@dataclass
class DocumentExternalReference:
  id: str
  uri: str
  checksum: str


@dataclass
class Document:
  name: str
  namespace: str
  id: str = SPDXID_DOC
  describes: str = SPDXID_PRODUCT
  creators: List[str] = field(default_factory=list)
  created: str = None
  external_refs: List[DocumentExternalReference] = field(default_factory=list)
  packages: List[Package] = field(default_factory=list)
  files: List[File] = field(default_factory=list)
  relationships: List[Relationship] = field(default_factory=list)

  def add_external_ref(self, external_ref):
    if not any(external_ref.uri == ref.uri for ref in self.external_refs):
      self.external_refs.append(external_ref)

  def add_package(self, package):
    if not any(package.id == p.id for p in self.packages):
      self.packages.append(package)

  def add_relationship(self, rel):
    if not any(rel.id1 == r.id1 and rel.id2 == r.id2 and rel.relationship == r.relationship
               for r in self.relationships):
      self.relationships.append(rel)
+365 −0
Original line number Diff line number Diff line
#!/usr/bin/env python3
#
# Copyright (C) 2023 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Serialize objects defined in package sbom_data to SPDX format: tagvalue, JSON.
"""

import json
import sbom_data

SPDX_VER = 'SPDX-2.3'
DATA_LIC = 'CC0-1.0'


class Tags:
  # Common
  SPDXID = 'SPDXID'
  SPDX_VERSION = 'SPDXVersion'
  DATA_LICENSE = 'DataLicense'
  DOCUMENT_NAME = 'DocumentName'
  DOCUMENT_NAMESPACE = 'DocumentNamespace'
  CREATED = 'Created'
  CREATOR = 'Creator'
  EXTERNAL_DOCUMENT_REF = 'ExternalDocumentRef'

  # Package
  PACKAGE_NAME = 'PackageName'
  PACKAGE_DOWNLOAD_LOCATION = 'PackageDownloadLocation'
  PACKAGE_VERSION = 'PackageVersion'
  PACKAGE_SUPPLIER = 'PackageSupplier'
  FILES_ANALYZED = 'FilesAnalyzed'
  PACKAGE_VERIFICATION_CODE = 'PackageVerificationCode'
  PACKAGE_EXTERNAL_REF = 'ExternalRef'
  # Package license
  PACKAGE_LICENSE_CONCLUDED = 'PackageLicenseConcluded'
  PACKAGE_LICENSE_INFO_FROM_FILES = 'PackageLicenseInfoFromFiles'
  PACKAGE_LICENSE_DECLARED = 'PackageLicenseDeclared'
  PACKAGE_LICENSE_COMMENTS = 'PackageLicenseComments'

  # File
  FILE_NAME = 'FileName'
  FILE_CHECKSUM = 'FileChecksum'
  # File license
  FILE_LICENSE_CONCLUDED = 'LicenseConcluded'
  FILE_LICENSE_INFO_IN_FILE = 'LicenseInfoInFile'
  FILE_LICENSE_COMMENTS = 'LicenseComments'
  FILE_COPYRIGHT_TEXT = 'FileCopyrightText'
  FILE_NOTICE = 'FileNotice'
  FILE_ATTRIBUTION_TEXT = 'FileAttributionText'

  # Relationship
  RELATIONSHIP = 'Relationship'


class TagValueWriter:
  @staticmethod
  def marshal_doc_headers(sbom_doc):
    headers = [
      f'{Tags.SPDX_VERSION}: {SPDX_VER}',
      f'{Tags.DATA_LICENSE}: {DATA_LIC}',
      f'{Tags.SPDXID}: {sbom_doc.id}',
      f'{Tags.DOCUMENT_NAME}: {sbom_doc.name}',
      f'{Tags.DOCUMENT_NAMESPACE}: {sbom_doc.namespace}',
    ]
    for creator in sbom_doc.creators:
      headers.append(f'{Tags.CREATOR}: {creator}')
    headers.append(f'{Tags.CREATED}: {sbom_doc.created}')
    for doc_ref in sbom_doc.external_refs:
      headers.append(
        f'{Tags.EXTERNAL_DOCUMENT_REF}: {doc_ref.id} {doc_ref.uri} {doc_ref.checksum}')
    headers.append('')
    return headers

  @staticmethod
  def marshal_package(package):
    download_location = 'NONE'
    if package.download_location:
      download_location = package.download_location
    tagvalues = [
      f'{Tags.PACKAGE_NAME}: {package.name}',
      f'{Tags.SPDXID}: {package.id}',
      f'{Tags.PACKAGE_DOWNLOAD_LOCATION}: {download_location}',
      f'{Tags.FILES_ANALYZED}: {str(package.files_analyzed).lower()}',
    ]
    if package.version:
      tagvalues.append(f'{Tags.PACKAGE_VERSION}: {package.version}')
    if package.supplier:
      tagvalues.append(f'{Tags.PACKAGE_SUPPLIER}: {package.supplier}')
    if package.verification_code:
      tagvalues.append(f'{Tags.PACKAGE_VERIFICATION_CODE}: {package.verification_code}')
    if package.external_refs:
      for external_ref in package.external_refs:
        tagvalues.append(
          f'{Tags.PACKAGE_EXTERNAL_REF}: {external_ref.category} {external_ref.type} {external_ref.locator}')

    tagvalues.append('')
    return tagvalues

  @staticmethod
  def marshal_described_element(sbom_doc):
    if not sbom_doc.describes:
      return None

    product_package = [p for p in sbom_doc.packages if p.id == sbom_doc.describes]
    if product_package:
      tagvalues = TagValueWriter.marshal_package(product_package[0])
      tagvalues.append(
        f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}')

      tagvalues.append('')
      return tagvalues

    file = [f for f in sbom_doc.files if f.id == sbom_doc.describes]
    if file:
      tagvalues = [
        f'{Tags.RELATIONSHIP}: {sbom_doc.id} {sbom_data.RelationshipType.DESCRIBES} {sbom_doc.describes}'
      ]

      return tagvalues

    return None

  @staticmethod
  def marshal_packages(sbom_doc):
    tagvalues = []
    marshaled_relationships = []
    i = 0
    packages = sbom_doc.packages
    while i < len(packages):
      if packages[i].id == sbom_doc.describes:
        i += 1
        continue

      if i + 1 < len(packages) \
          and packages[i].id.startswith('SPDXRef-SOURCE-') \
          and packages[i + 1].id.startswith('SPDXRef-UPSTREAM-'):
        tagvalues += TagValueWriter.marshal_package(packages[i])
        tagvalues += TagValueWriter.marshal_package(packages[i + 1])
        rel = next((r for r in sbom_doc.relationships if
                    r.id1 == packages[i].id and
                    r.id2 == packages[i + 1].id and
                    r.relationship == sbom_data.RelationshipType.VARIANT_OF), None)
        if rel:
          marshaled_relationships.append(rel)
          tagvalues.append(TagValueWriter.marshal_relationship(rel))
          tagvalues.append('')

        i += 2
      else:
        tagvalues += TagValueWriter.marshal_package(packages[i])
        i += 1

    return tagvalues, marshaled_relationships

  @staticmethod
  def marshal_file(file):
    tagvalues = [
      f'{Tags.FILE_NAME}: {file.name}',
      f'{Tags.SPDXID}: {file.id}',
      f'{Tags.FILE_CHECKSUM}: {file.checksum}',
      '',
    ]

    return tagvalues

  @staticmethod
  def marshal_files(sbom_doc):
    tagvalues = []
    for file in sbom_doc.files:
      tagvalues += TagValueWriter.marshal_file(file)
    return tagvalues

  @staticmethod
  def marshal_relationship(rel):
    return f'{Tags.RELATIONSHIP}: {rel.id1} {rel.relationship} {rel.id2}'

  @staticmethod
  def marshal_relationships(sbom_doc, marshaled_rels):
    tagvalues = []
    sorted_rels = sorted(sbom_doc.relationships, key=lambda r: r.id2 + r.id1)
    for rel in sorted_rels:
      if any(r.id1 == rel.id1 and r.id2 == rel.id2 and r.relationship == rel.relationship
             for r in marshaled_rels):
        continue
      tagvalues.append(TagValueWriter.marshal_relationship(rel))
    tagvalues.append('')
    return tagvalues

  @staticmethod
  def write(sbom_doc, file, fragment=False):
    content = []
    if not fragment:
      content += TagValueWriter.marshal_doc_headers(sbom_doc)
      described_element = TagValueWriter.marshal_described_element(sbom_doc)
      if described_element:
        content += described_element
    content += TagValueWriter.marshal_files(sbom_doc)
    tagvalues, marshaled_relationships = TagValueWriter.marshal_packages(sbom_doc)
    content += tagvalues
    content += TagValueWriter.marshal_relationships(sbom_doc, marshaled_relationships)
    file.write('\n'.join(content))


class PropNames:
  # Common
  SPDXID = 'SPDXID'
  SPDX_VERSION = 'spdxVersion'
  DATA_LICENSE = 'dataLicense'
  NAME = 'name'
  DOCUMENT_NAMESPACE = 'documentNamespace'
  CREATION_INFO = 'creationInfo'
  CREATORS = 'creators'
  CREATED = 'created'
  EXTERNAL_DOCUMENT_REF = 'externalDocumentRefs'
  DOCUMENT_DESCRIBES = 'documentDescribes'
  EXTERNAL_DOCUMENT_ID = 'externalDocumentId'
  EXTERNAL_DOCUMENT_URI = 'spdxDocument'
  EXTERNAL_DOCUMENT_CHECKSUM = 'checksum'
  ALGORITHM = 'algorithm'
  CHECKSUM_VALUE = 'checksumValue'

  # Package
  PACKAGES = 'packages'
  PACKAGE_DOWNLOAD_LOCATION = 'downloadLocation'
  PACKAGE_VERSION = 'versionInfo'
  PACKAGE_SUPPLIER = 'supplier'
  FILES_ANALYZED = 'filesAnalyzed'
  PACKAGE_VERIFICATION_CODE = 'packageVerificationCode'
  PACKAGE_VERIFICATION_CODE_VALUE = 'packageVerificationCodeValue'
  PACKAGE_EXTERNAL_REFS = 'externalRefs'
  PACKAGE_EXTERNAL_REF_CATEGORY = 'referenceCategory'
  PACKAGE_EXTERNAL_REF_TYPE = 'referenceType'
  PACKAGE_EXTERNAL_REF_LOCATOR = 'referenceLocator'
  PACKAGE_HAS_FILES = 'hasFiles'

  # File
  FILES = 'files'
  FILE_NAME = 'fileName'
  FILE_CHECKSUMS = 'checksums'

  # Relationship
  RELATIONSHIPS = 'relationships'
  REL_ELEMENT_ID = 'spdxElementId'
  REL_RELATED_ELEMENT_ID = 'relatedSpdxElement'
  REL_TYPE = 'relationshipType'


class JSONWriter:
  @staticmethod
  def marshal_doc_headers(sbom_doc):
    headers = {
      PropNames.SPDX_VERSION: SPDX_VER,
      PropNames.DATA_LICENSE: DATA_LIC,
      PropNames.SPDXID: sbom_doc.id,
      PropNames.NAME: sbom_doc.name,
      PropNames.DOCUMENT_NAMESPACE: sbom_doc.namespace,
      PropNames.CREATION_INFO: {}
    }
    creators = [creator for creator in sbom_doc.creators]
    headers[PropNames.CREATION_INFO][PropNames.CREATORS] = creators
    headers[PropNames.CREATION_INFO][PropNames.CREATED] = sbom_doc.created
    external_refs = []
    for doc_ref in sbom_doc.external_refs:
      checksum = doc_ref.checksum.split(': ')
      external_refs.append({
        PropNames.EXTERNAL_DOCUMENT_ID: f'{doc_ref.id}',
        PropNames.EXTERNAL_DOCUMENT_URI: doc_ref.uri,
        PropNames.EXTERNAL_DOCUMENT_CHECKSUM: {
          PropNames.ALGORITHM: checksum[0],
          PropNames.CHECKSUM_VALUE: checksum[1]
        }
      })
    if external_refs:
      headers[PropNames.EXTERNAL_DOCUMENT_REF] = external_refs
    headers[PropNames.DOCUMENT_DESCRIBES] = [sbom_doc.describes]

    return headers

  @staticmethod
  def marshal_packages(sbom_doc):
    packages = []
    for p in sbom_doc.packages:
      package = {
        PropNames.NAME: p.name,
        PropNames.SPDXID: p.id,
        PropNames.PACKAGE_DOWNLOAD_LOCATION: p.download_location if p.download_location else 'NONE',
        PropNames.FILES_ANALYZED: p.files_analyzed
      }
      if p.version:
        package[PropNames.PACKAGE_VERSION] = p.version
      if p.supplier:
        package[PropNames.PACKAGE_SUPPLIER] = p.supplier
      if p.verification_code:
        package[PropNames.PACKAGE_VERIFICATION_CODE] = {
          PropNames.PACKAGE_VERIFICATION_CODE_VALUE: p.verification_code
        }
      if p.external_refs:
        package[PropNames.PACKAGE_EXTERNAL_REFS] = []
        for ref in p.external_refs:
          ext_ref = {
            PropNames.PACKAGE_EXTERNAL_REF_CATEGORY: ref.category,
            PropNames.PACKAGE_EXTERNAL_REF_TYPE: ref.type,
            PropNames.PACKAGE_EXTERNAL_REF_LOCATOR: ref.locator,
          }
          package[PropNames.PACKAGE_EXTERNAL_REFS].append(ext_ref)
      if p.file_ids:
        package[PropNames.PACKAGE_HAS_FILES] = []
        for file_id in p.file_ids:
          package[PropNames.PACKAGE_HAS_FILES].append(file_id)

      packages.append(package)

    return {PropNames.PACKAGES: packages}

  @staticmethod
  def marshal_files(sbom_doc):
    files = []
    for f in sbom_doc.files:
      file = {
        PropNames.FILE_NAME: f.name,
        PropNames.SPDXID: f.id
      }
      checksum = f.checksum.split(': ')
      file[PropNames.FILE_CHECKSUMS] = [{
        PropNames.ALGORITHM: checksum[0],
        PropNames.CHECKSUM_VALUE: checksum[1],
      }]
      files.append(file)
    return {PropNames.FILES: files}

  @staticmethod
  def marshal_relationships(sbom_doc):
    relationships = []
    sorted_rels = sorted(sbom_doc.relationships, key=lambda r: r.relationship + r.id2 + r.id1)
    for r in sorted_rels:
      rel = {
        PropNames.REL_ELEMENT_ID: r.id1,
        PropNames.REL_RELATED_ELEMENT_ID: r.id2,
        PropNames.REL_TYPE: r.relationship,
      }
      relationships.append(rel)

    return {PropNames.RELATIONSHIPS: relationships}

  @staticmethod
  def write(sbom_doc, file):
    doc = {}
    doc.update(JSONWriter.marshal_doc_headers(sbom_doc))
    doc.update(JSONWriter.marshal_packages(sbom_doc))
    doc.update(JSONWriter.marshal_files(sbom_doc))
    doc.update(JSONWriter.marshal_relationships(sbom_doc))
    file.write(json.dumps(doc, indent=4))
Loading