Merge "Support license information in SBOM writers library." into main (8715d314) · Commits · e / os / android_build

tools/sbom/sbom_data.py

+24 −4

Original line number	Diff line number	Diff line
		@@ -30,6 +30,7 @@ import hashlib
		SPDXID_DOC = 'SPDXRef-DOCUMENT'
		SPDXID_PRODUCT = 'SPDXRef-PRODUCT'
		SPDXID_PLATFORM = 'SPDXRef-PLATFORM'
		SPDXID_LICENSE_APACHE = 'LicenseRef-Android-Apache-2.0'

		PACKAGE_NAME_PRODUCT = 'PRODUCT'
		PACKAGE_NAME_PLATFORM = 'PLATFORM'
		@@ -50,7 +51,7 @@ class PackageExternalRefType:
		cpe23Type = 'cpe23Type'


		@dataclass
		@dataclass(frozen=True)
		class PackageExternalRef:
		category: PackageExternalRefCategory
		type: PackageExternalRefType
		@@ -68,6 +69,7 @@ class Package:
		verification_code: str = None
		file_ids: List[str] = field(default_factory=list)
		external_refs: List[PackageExternalRef] = field(default_factory=list)
		declared_license_ids: List[str] = field(default_factory=list)


		@dataclass
		@@ -75,6 +77,7 @@ class File:
		id: str
		name: str
		checksum: str
		concluded_license_ids: List[str] = field(default_factory=list)


		class RelationshipType:
		@@ -85,20 +88,27 @@ class RelationshipType:
		STATIC_LINK = 'STATIC_LINK'


		@dataclass
		@dataclass(frozen=True)
		class Relationship:
		id1: str
		relationship: RelationshipType
		id2: str


		@dataclass
		@dataclass(frozen=True)
		class DocumentExternalReference:
		id: str
		uri: str
		checksum: str


		@dataclass(frozen=True)
		class License:
		id: str
		text: str
		name: str


		@dataclass
		class Document:
		name: str
		@@ -111,20 +121,30 @@ class Document:
		packages: List[Package] = field(default_factory=list)
		files: List[File] = field(default_factory=list)
		relationships: List[Relationship] = field(default_factory=list)
		licenses: List[License] = field(default_factory=list)

		def add_external_ref(self, external_ref):
		if not any(external_ref.uri == ref.uri for ref in self.external_refs):
		self.external_refs.append(external_ref)

		def add_package(self, package):
		if not any(package.id == p.id for p in self.packages):
		p = next((p for p in self.packages if package.id == p.id), None)
		if not p:
		self.packages.append(package)
		else:
		for license_id in package.declared_license_ids:
		if license_id not in p.declared_license_ids:
		p.declared_license_ids.append(license_id)

		def add_relationship(self, rel):
		if not any(rel.id1 == r.id1 and rel.id2 == r.id2 and rel.relationship == r.relationship
		for r in self.relationships):
		self.relationships.append(rel)

		def add_license(self, license):
		if not any(license.id == l.id for l in self.licenses):
		self.licenses.append(license)

		def generate_packages_verification_code(self):
		for package in self.packages:
		if not package.file_ids:

tools/sbom/sbom_data_test.py

+45 −0

Original line number	Diff line number	Diff line
		@@ -23,6 +23,7 @@ SUPPLIER_GOOGLE = 'Organization: Google'
		SUPPLIER_UPSTREAM = 'Organization: upstream'

		SPDXID_PREBUILT_PACKAGE1 = 'SPDXRef-PREBUILT-package1'
		SPDXID_PREBUILT_PACKAGE2 = 'SPDXRef-PREBUILT-package2'
		SPDXID_SOURCE_PACKAGE1 = 'SPDXRef-SOURCE-package1'
		SPDXID_UPSTREAM_PACKAGE1 = 'SPDXRef-UPSTREAM-package1'

		@@ -31,6 +32,9 @@ SPDXID_FILE2 = 'SPDXRef-file2'
		SPDXID_FILE3 = 'SPDXRef-file3'
		SPDXID_FILE4 = 'SPDXRef-file4'

		SPDXID_LICENSE1 = "SPDXRef-License-1"
		SPDXID_LICENSE2 = "SPDXRef-License-2"


		class SBOMDataTest(unittest.TestCase):

		@@ -134,6 +138,47 @@ class SBOMDataTest(unittest.TestCase):
		self.sbom_doc.generate_packages_verification_code()
		self.assertEqual(expected_package_verification_code, self.sbom_doc.packages[0].verification_code)

		def test_add_package_(self):
		self.sbom_doc.add_package(sbom_data.Package(id=SPDXID_PREBUILT_PACKAGE2,
		name='Prebuilt package2',
		download_location=sbom_data.VALUE_NONE,
		supplier=SUPPLIER_GOOGLE,
		version=BUILD_FINGER_PRINT,
		))
		p = next((p for p in self.sbom_doc.packages if p.id == SPDXID_PREBUILT_PACKAGE2), None)
		self.assertNotEqual(p, None)
		self.assertEqual(p.declared_license_ids, [])

		# Add same package with license 1
		self.sbom_doc.add_package(sbom_data.Package(id=SPDXID_PREBUILT_PACKAGE2,
		name='Prebuilt package2',
		download_location=sbom_data.VALUE_NONE,
		supplier=SUPPLIER_GOOGLE,
		version=BUILD_FINGER_PRINT,
		declared_license_ids=[SPDXID_LICENSE1]
		))
		self.assertEqual(p.declared_license_ids, [SPDXID_LICENSE1])

		# Add same package with license 2
		self.sbom_doc.add_package(sbom_data.Package(id=SPDXID_PREBUILT_PACKAGE2,
		name='Prebuilt package2',
		download_location=sbom_data.VALUE_NONE,
		supplier=SUPPLIER_GOOGLE,
		version=BUILD_FINGER_PRINT,
		declared_license_ids=[SPDXID_LICENSE2]
		))
		self.assertEqual(p.declared_license_ids, [SPDXID_LICENSE1, SPDXID_LICENSE2])

		# Add same package with license 2 again
		self.sbom_doc.add_package(sbom_data.Package(id=SPDXID_PREBUILT_PACKAGE2,
		name='Prebuilt package2',
		download_location=sbom_data.VALUE_NONE,
		supplier=SUPPLIER_GOOGLE,
		version=BUILD_FINGER_PRINT,
		declared_license_ids=[SPDXID_LICENSE2]
		))
		self.assertEqual(p.declared_license_ids, [SPDXID_LICENSE1, SPDXID_LICENSE2])


		if __name__ == '__main__':
		unittest.main(verbosity=2)

tools/sbom/sbom_writers.py

+59 −1

Original line number	Diff line number	Diff line
		@@ -64,6 +64,11 @@ class Tags:
		# Relationship
		RELATIONSHIP = 'Relationship'

		# License
		LICENSE_ID = 'LicenseID'
		LICENSE_NAME = 'LicenseName'
		LICENSE_EXTRACTED_TEXT = 'ExtractedText'


		class TagValueWriter:
		@staticmethod
		@@ -99,6 +104,12 @@ class TagValueWriter:
		tagvalues.append(f'{Tags.PACKAGE_VERSION}: {package.version}')
		if package.supplier:
		tagvalues.append(f'{Tags.PACKAGE_SUPPLIER}: {package.supplier}')

		license = sbom_data.VALUE_NOASSERTION
		if package.declared_license_ids:
		license = ' OR '.join(package.declared_license_ids)
		tagvalues.append(f'{Tags.PACKAGE_LICENSE_DECLARED}: {license}')

		if package.verification_code:
		tagvalues.append(f'{Tags.PACKAGE_VERIFICATION_CODE}: {package.verification_code}')
		if package.external_refs:
		@@ -155,8 +166,12 @@ class TagValueWriter:
		f'{Tags.FILE_NAME}: {file.name}',
		f'{Tags.SPDXID}: {file.id}',
		f'{Tags.FILE_CHECKSUM}: {file.checksum}',
		'',
		]
		license = sbom_data.VALUE_NOASSERTION
		if file.concluded_license_ids:
		license = ' OR '.join(file.concluded_license_ids)
		tagvalues.append(f'{Tags.FILE_LICENSE_CONCLUDED}: {license}')
		tagvalues.append('')

		return tagvalues

		@@ -193,6 +208,22 @@ class TagValueWriter:
		tagvalues.append('')
		return tagvalues

		@staticmethod
		def marshal_license(license):
		tagvalues = []
		tagvalues.append(f'{Tags.LICENSE_ID}: {license.id}')
		tagvalues.append(f'{Tags.LICENSE_NAME}: {license.name}')
		tagvalues.append(f'{Tags.LICENSE_EXTRACTED_TEXT}: <text>{license.text}</text>')
		return tagvalues

		@staticmethod
		def marshal_licenses(sbom_doc):
		tagvalues = []
		for license in sbom_doc.licenses:
		tagvalues += TagValueWriter.marshal_license(license)
		tagvalues.append('')
		return tagvalues

		@staticmethod
		def write(sbom_doc, file, fragment=False):
		content = []
		@@ -202,6 +233,7 @@ class TagValueWriter:
		tagvalues, marshaled_relationships = TagValueWriter.marshal_packages(sbom_doc, fragment)
		content += tagvalues
		content += TagValueWriter.marshal_relationships(sbom_doc, marshaled_relationships)
		content += TagValueWriter.marshal_licenses(sbom_doc)
		file.write('\n'.join(content))


		@@ -236,11 +268,13 @@ class PropNames:
		PACKAGE_EXTERNAL_REF_TYPE = 'referenceType'
		PACKAGE_EXTERNAL_REF_LOCATOR = 'referenceLocator'
		PACKAGE_HAS_FILES = 'hasFiles'
		PACKAGE_LICENSE_DECLARED = 'licenseDeclared'

		# File
		FILES = 'files'
		FILE_NAME = 'fileName'
		FILE_CHECKSUMS = 'checksums'
		FILE_LICENSE_CONCLUDED = 'licenseConcluded'

		# Relationship
		RELATIONSHIPS = 'relationships'
		@@ -248,6 +282,12 @@ class PropNames:
		REL_RELATED_ELEMENT_ID = 'relatedSpdxElement'
		REL_TYPE = 'relationshipType'

		# License
		LICENSES = 'hasExtractedLicensingInfos'
		LICENSE_ID = 'licenseId'
		LICENSE_NAME = 'name'
		LICENSE_EXTRACTED_TEXT = 'extractedText'


		class JSONWriter:
		@staticmethod
		@@ -294,6 +334,9 @@ class JSONWriter:
		package[PropNames.PACKAGE_VERSION] = p.version
		if p.supplier:
		package[PropNames.PACKAGE_SUPPLIER] = p.supplier
		package[PropNames.PACKAGE_LICENSE_DECLARED] = sbom_data.VALUE_NOASSERTION
		if p.declared_license_ids:
		package[PropNames.PACKAGE_LICENSE_DECLARED] = ' OR '.join(p.declared_license_ids)
		if p.verification_code:
		package[PropNames.PACKAGE_VERIFICATION_CODE] = {
		PropNames.PACKAGE_VERIFICATION_CODE_VALUE: p.verification_code
		@@ -329,6 +372,9 @@ class JSONWriter:
		PropNames.ALGORITHM: checksum[0],
		PropNames.CHECKSUM_VALUE: checksum[1],
		}]
		file[PropNames.FILE_LICENSE_CONCLUDED] = sbom_data.VALUE_NOASSERTION
		if f.concluded_license_ids:
		file[PropNames.FILE_LICENSE_CONCLUDED] = ' OR '.join(f.concluded_license_ids)
		files.append(file)
		return {PropNames.FILES: files}

		@@ -346,6 +392,17 @@ class JSONWriter:

		return {PropNames.RELATIONSHIPS: relationships}

		@staticmethod
		def marshal_licenses(sbom_doc):
		licenses = []
		for l in sbom_doc.licenses:
		licenses.append({
		PropNames.LICENSE_ID: l.id,
		PropNames.LICENSE_NAME: l.name,
		PropNames.LICENSE_EXTRACTED_TEXT: f'<text>{l.text}</text>'
		})
		return {PropNames.LICENSES: licenses}

		@staticmethod
		def write(sbom_doc, file):
		doc = {}
		@@ -353,4 +410,5 @@ class JSONWriter:
		doc.update(JSONWriter.marshal_packages(sbom_doc))
		doc.update(JSONWriter.marshal_files(sbom_doc))
		doc.update(JSONWriter.marshal_relationships(sbom_doc))
		doc.update(JSONWriter.marshal_licenses(sbom_doc))
		file.write(json.dumps(doc, indent=4))

tools/sbom/sbom_writers_test.py

+20 −3

Original line number	Diff line number	Diff line
		@@ -33,6 +33,14 @@ SPDXID_FILE2 = 'SPDXRef-file2'
		SPDXID_FILE3 = 'SPDXRef-file3'
		SPDXID_FILE4 = 'SPDXRef-file4'

		SPDXID_LICENSE_1 = 'LicenseRef-Android-License-1'
		SPDXID_LICENSE_2 = 'LicenseRef-Android-License-2'
		SPDXID_LICENSE_3 = 'LicenseRef-Android-License-3'

		LICENSE_APACHE_TEXT = "LICENSE_APACHE"
		LICENSE1_TEXT = 'LICENSE 1'
		LICENSE2_TEXT = 'LICENSE 2'
		LICENSE3_TEXT = 'LICENSE 3'

		class SBOMWritersTest(unittest.TestCase):

		@@ -63,6 +71,7 @@ class SBOMWritersTest(unittest.TestCase):
		download_location=sbom_data.VALUE_NONE,
		supplier=SUPPLIER_GOOGLE,
		version=BUILD_FINGER_PRINT,
		declared_license_ids=[sbom_data.SPDXID_LICENSE_APACHE]
		))

		self.sbom_doc.add_package(
		@@ -71,6 +80,7 @@ class SBOMWritersTest(unittest.TestCase):
		download_location=sbom_data.VALUE_NONE,
		supplier=SUPPLIER_GOOGLE,
		version=BUILD_FINGER_PRINT,
		declared_license_ids=[SPDXID_LICENSE_1],
		))

		self.sbom_doc.add_package(
		@@ -79,6 +89,7 @@ class SBOMWritersTest(unittest.TestCase):
		download_location=sbom_data.VALUE_NONE,
		supplier=SUPPLIER_GOOGLE,
		version=BUILD_FINGER_PRINT,
		declared_license_ids=[SPDXID_LICENSE_2, SPDXID_LICENSE_3],
		external_refs=[sbom_data.PackageExternalRef(
		category=sbom_data.PackageExternalRefCategory.SECURITY,
		type=sbom_data.PackageExternalRefType.cpe22Type,
		@@ -90,6 +101,7 @@ class SBOMWritersTest(unittest.TestCase):
		name='Upstream package1',
		supplier=SUPPLIER_UPSTREAM,
		version='1.1',
		declared_license_ids=[SPDXID_LICENSE_2, SPDXID_LICENSE_3],
		))

		self.sbom_doc.add_relationship(sbom_data.Relationship(id1=SPDXID_SOURCE_PACKAGE1,
		@@ -97,11 +109,11 @@ class SBOMWritersTest(unittest.TestCase):
		id2=SPDXID_UPSTREAM_PACKAGE1))

		self.sbom_doc.files.append(
		sbom_data.File(id=SPDXID_FILE1, name='/bin/file1', checksum='SHA1: 11111'))
		sbom_data.File(id=SPDXID_FILE1, name='/bin/file1', checksum='SHA1: 11111', concluded_license_ids=[sbom_data.SPDXID_LICENSE_APACHE]))
		self.sbom_doc.files.append(
		sbom_data.File(id=SPDXID_FILE2, name='/bin/file2', checksum='SHA1: 22222'))
		sbom_data.File(id=SPDXID_FILE2, name='/bin/file2', checksum='SHA1: 22222', concluded_license_ids=[SPDXID_LICENSE_1]))
		self.sbom_doc.files.append(
		sbom_data.File(id=SPDXID_FILE3, name='/bin/file3', checksum='SHA1: 33333'))
		sbom_data.File(id=SPDXID_FILE3, name='/bin/file3', checksum='SHA1: 33333', concluded_license_ids=[SPDXID_LICENSE_2, SPDXID_LICENSE_3]))
		self.sbom_doc.files.append(
		sbom_data.File(id=SPDXID_FILE4, name='file4.a', checksum='SHA1: 44444'))

		@@ -120,6 +132,11 @@ class SBOMWritersTest(unittest.TestCase):
		id2=SPDXID_FILE4
		))

		self.sbom_doc.add_license(sbom_data.License(sbom_data.SPDXID_LICENSE_APACHE, LICENSE_APACHE_TEXT, "License-Apache"))
		self.sbom_doc.add_license(sbom_data.License(SPDXID_LICENSE_1, LICENSE1_TEXT, "License-1"))
		self.sbom_doc.add_license(sbom_data.License(SPDXID_LICENSE_2, LICENSE2_TEXT, "License-2"))
		self.sbom_doc.add_license(sbom_data.License(SPDXID_LICENSE_3, LICENSE3_TEXT, "License-3"))

		# SBOM fragment of a APK
		self.unbundled_sbom_doc = sbom_data.Document(name='test doc',
		namespace='http://www.google.com/sbom/spdx/android',

tools/sbom/testdata/expected_json_sbom.spdx.json

+38 −7

Original line number	Diff line number	Diff line
		@@ -31,6 +31,7 @@
		"filesAnalyzed": true,
		"versionInfo": "build_finger_print",
		"supplier": "Organization: Google",
		"licenseDeclared": "NOASSERTION",
		"packageVerificationCode": {
		"packageVerificationCodeValue": "123456"
		},
		@@ -46,7 +47,8 @@
		"downloadLocation": "NONE",
		"filesAnalyzed": false,
		"versionInfo": "build_finger_print",
		"supplier": "Organization: Google"
		"supplier": "Organization: Google",
		"licenseDeclared": "LicenseRef-Android-Apache-2.0"
		},
		{
		"name": "Prebuilt package1",
		@@ -54,7 +56,8 @@
		"downloadLocation": "NONE",
		"filesAnalyzed": false,
		"versionInfo": "build_finger_print",
		"supplier": "Organization: Google"
		"supplier": "Organization: Google",
		"licenseDeclared": "LicenseRef-Android-License-1"
		},
		{
		"name": "Source package1",
		@@ -63,6 +66,7 @@
		"filesAnalyzed": false,
		"versionInfo": "build_finger_print",
		"supplier": "Organization: Google",
		"licenseDeclared": "LicenseRef-Android-License-2 OR LicenseRef-Android-License-3",
		"externalRefs": [
		{
		"referenceCategory": "SECURITY",
		@@ -77,7 +81,8 @@
		"downloadLocation": "NOASSERTION",
		"filesAnalyzed": false,
		"versionInfo": "1.1",
		"supplier": "Organization: upstream"
		"supplier": "Organization: upstream",
		"licenseDeclared": "LicenseRef-Android-License-2 OR LicenseRef-Android-License-3"
		}
		],
		"files": [
		@@ -89,7 +94,8 @@
		"algorithm": "SHA1",
		"checksumValue": "11111"
		}
		]
		],
		"licenseConcluded": "LicenseRef-Android-Apache-2.0"
		},
		{
		"fileName": "/bin/file2",
		@@ -99,7 +105,8 @@
		"algorithm": "SHA1",
		"checksumValue": "22222"
		}
		]
		],
		"licenseConcluded": "LicenseRef-Android-License-1"
		},
		{
		"fileName": "/bin/file3",
		@@ -109,7 +116,8 @@
		"algorithm": "SHA1",
		"checksumValue": "33333"
		}
		]
		],
		"licenseConcluded": "LicenseRef-Android-License-2 OR LicenseRef-Android-License-3"
		},
		{
		"fileName": "file4.a",
		@@ -119,7 +127,8 @@
		"algorithm": "SHA1",
		"checksumValue": "44444"
		}
		]
		],
		"licenseConcluded": "NOASSERTION"
		}
		],
		"relationships": [
		@@ -148,5 +157,27 @@
		"relatedSpdxElement": "SPDXRef-UPSTREAM-package1",
		"relationshipType": "VARIANT_OF"
		}
		],
		"hasExtractedLicensingInfos": [
		{
		"licenseId": "LicenseRef-Android-Apache-2.0",
		"name": "License-Apache",
		"extractedText": "<text>LICENSE_APACHE</text>"
		},
		{
		"licenseId": "LicenseRef-Android-License-1",
		"name": "License-1",
		"extractedText": "<text>LICENSE 1</text>"
		},
		{
		"licenseId": "LicenseRef-Android-License-2",
		"name": "License-2",
		"extractedText": "<text>LICENSE 2</text>"
		},
		{
		"licenseId": "LicenseRef-Android-License-3",
		"name": "License-3",
		"extractedText": "<text>LICENSE 3</text>"
		}
		]
		}
		No newline at end of file