Merge "Revert "Revert "Updated SBOM generator module to generate JSON spdx utility bill of""" (960a4700) · Commits · e / os / android_build

tools/compliance/Android.bp

+4 −0

Original line number	Diff line number	Diff line
		@@ -138,6 +138,10 @@ blueprint_go_binary {
		"compliance-module",
		"blueprint-deptools",
		"soong-response",
		"spdx-tools-spdxv2_2",
		"spdx-tools-builder2v2",
		"spdx-tools-spdxcommon",
		"spdx-tools-spdx-json",
		],
		testSrcs: ["cmd/sbom/sbom_test.go"],
		}

tools/compliance/cmd/sbom/sbom.go

+99 −45

Original line number	Diff line number	Diff line
		@@ -31,6 +31,11 @@ import (
		"android/soong/tools/compliance/projectmetadata"

		"github.com/google/blueprint/deptools"

		"github.com/spdx/tools-golang/builder/builder2v2"
		"github.com/spdx/tools-golang/json"
		"github.com/spdx/tools-golang/spdx/common"
		spdx "github.com/spdx/tools-golang/spdx/v2_2"
		)

		var (
		@@ -38,6 +43,8 @@ var (
		failNoLicenses = fmt.Errorf("No licenses found")
		)

		const NOASSERTION = "NOASSERTION"

		type context struct {
		stdout io.Writer
		stderr io.Writer
		@@ -154,7 +161,8 @@ Options:

		ctx := &context{ofile, os.Stderr, compliance.FS, product, stripPrefix, actualTime}

		deps, err := sbomGenerator(ctx, flags.Args()...)
		spdxDoc, deps, err := sbomGenerator(ctx, flags.Args()...)

		if err != nil {
		if err == failNoneRequested {
		flags.Usage()
		@@ -163,6 +171,11 @@ Options:
		os.Exit(1)
		}

		if err := spdx_json.Save2_2(spdxDoc, ofile); err != nil {
		fmt.Fprintf(os.Stderr, "failed to write document to %v: %v", *outputFile, err)
		os.Exit(1)
		}

		if *outputFile != "-" {
		err := os.WriteFile(*outputFile, obuf.Bytes(), 0666)
		if err != nil {
		@@ -218,17 +231,17 @@ func getDocumentName(ctx context, tn compliance.TargetNode, pm *projectmetadat
		// or NOASSERTION if not available, none determined or ambiguous
		func getDownloadUrl(_ context, pm projectmetadata.ProjectMetadata) string {
		if pm == nil {
		return "NOASSERTION"
		return NOASSERTION
		}

		urlsByTypeName := pm.UrlsByTypeName()
		if urlsByTypeName == nil {
		return "NOASSERTION"
		return NOASSERTION
		}

		url := urlsByTypeName.DownloadUrl()
		if url == "" {
		return "NOASSERTION"
		return NOASSERTION
		}
		return url
		}
		@@ -289,10 +302,10 @@ func inputFiles(lg compliance.LicenseGraph, pmix projectmetadata.Index, licens

		// sbomGenerator uses the SPDX standard, see the SPDX specification (https://spdx.github.io/spdx-spec/)
		// sbomGenerator is also following the internal google SBOM styleguide (http://goto.google.com/spdx-style-guide)
		func sbomGenerator(ctx *context, files ...string) ([]string, error) {
		func sbomGenerator(ctx context, files ...string) (spdx.Document, []string, error) {
		// Must be at least one root file.
		if len(files) < 1 {
		return nil, failNoneRequested
		return nil, nil, failNoneRequested
		}

		pmix := projectmetadata.NewIndex(ctx.rootFS)
		@@ -300,9 +313,21 @@ func sbomGenerator(ctx *context, files ...string) ([]string, error) {
		lg, err := compliance.ReadLicenseGraph(ctx.rootFS, ctx.stderr, files)

		if err != nil {
		return nil, fmt.Errorf("Unable to read license text file(s) for %q: %v\n", files, err)
		return nil, nil, fmt.Errorf("Unable to read license text file(s) for %q: %v\n", files, err)
		}

		// creating the packages section
		pkgs := []*spdx.Package{}

		// creating the relationship section
		relationships := []*spdx.Relationship{}

		// creating the license section
		otherLicenses := []*spdx.OtherLicense{}

		// main package name
		var mainPkgName string

		// implementing the licenses references for the packages
		licenses := make(map[string]string)
		concludedLicenses := func(licenseTexts []string) string {
		@@ -325,7 +350,6 @@ func sbomGenerator(ctx *context, files ...string) ([]string, error) {
		}

		isMainPackage := true
		var mainPackage string
		visitedNodes := make(map[*compliance.TargetNode]struct{})

		// performing a Breadth-first top down walk of licensegraph and building package information
		@@ -341,45 +365,50 @@ func sbomGenerator(ctx *context, files ...string) ([]string, error) {
		}

		if isMainPackage {
		mainPackage = getDocumentName(ctx, tn, pm)
		fmt.Fprintf(ctx.stdout, "SPDXVersion: SPDX-2.2\n")
		fmt.Fprintf(ctx.stdout, "DataLicense: CC0-1.0\n")
		fmt.Fprintf(ctx.stdout, "DocumentName: %s\n", mainPackage)
		fmt.Fprintf(ctx.stdout, "SPDXID: SPDXRef-DOCUMENT\n")
		fmt.Fprintf(ctx.stdout, "DocumentNamespace: Android\n")
		fmt.Fprintf(ctx.stdout, "Creator: Organization: Google LLC\n")
		fmt.Fprintf(ctx.stdout, "Created: %s\n", ctx.creationTime().Format("2006-01-02T15:04:05Z"))
		mainPkgName = replaceSlashes(getPackageName(ctx, tn))
		isMainPackage = false
		}

		relationships := make([]string, 0, 1)
		defer func() {
		if r := recover(); r != nil {
		panic(r)
		}
		for _, relationship := range relationships {
		fmt.Fprintln(ctx.stdout, relationship)
		}
		}()
		if len(path) == 0 {
		relationships = append(relationships,
		fmt.Sprintf("Relationship: SPDXRef-DOCUMENT DESCRIBES SPDXRef-Package-%s",
		getPackageName(ctx, tn)))
		// Add the describe relationship for the main package
		rln := &spdx.Relationship{
		RefA: common.MakeDocElementID("" /* this document */, "DOCUMENT"),
		RefB: common.MakeDocElementID("", mainPkgName),
		Relationship: "DESCRIBES",
		}
		relationships = append(relationships, rln)

		} else {
		// Check parent and identify annotation
		parent := path[len(path)-1]
		targetEdge := parent.Edge()
		if targetEdge.IsRuntimeDependency() {
		// Adding the dynamic link annotation RUNTIME_DEPENDENCY_OF relationship
		relationships = append(relationships, fmt.Sprintf("Relationship: SPDXRef-Package-%s RUNTIME_DEPENDENCY_OF SPDXRef-Package-%s", getPackageName(ctx, tn), getPackageName(ctx, targetEdge.Target())))
		rln := &spdx.Relationship{
		RefA: common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, tn))),
		RefB: common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, targetEdge.Target()))),
		Relationship: "RUNTIME_DEPENDENCY_OF",
		}
		relationships = append(relationships, rln)

		} else if targetEdge.IsDerivation() {
		// Adding the derivation annotation as a CONTAINS relationship
		relationships = append(relationships, fmt.Sprintf("Relationship: SPDXRef-Package-%s CONTAINS SPDXRef-Package-%s", getPackageName(ctx, targetEdge.Target()), getPackageName(ctx, tn)))
		rln := &spdx.Relationship{
		RefA: common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, targetEdge.Target()))),
		RefB: common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, tn))),
		Relationship: "CONTAINS",
		}
		relationships = append(relationships, rln)

		} else if targetEdge.IsBuildTool() {
		// Adding the toolchain annotation as a BUILD_TOOL_OF relationship
		relationships = append(relationships, fmt.Sprintf("Relationship: SPDXRef-Package-%s BUILD_TOOL_OF SPDXRef-Package-%s", getPackageName(ctx, tn), getPackageName(ctx, targetEdge.Target())))
		rln := &spdx.Relationship{
		RefA: common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, tn))),
		RefB: common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, targetEdge.Target()))),
		Relationship: "BUILD_TOOL_OF",
		}
		relationships = append(relationships, rln)

		} else {
		panic(fmt.Errorf("Unknown dependency type: %v", targetEdge.Annotations()))
		}
		@@ -390,18 +419,27 @@ func sbomGenerator(ctx *context, files ...string) ([]string, error) {
		}
		visitedNodes[tn] = struct{}{}
		pkgName := getPackageName(ctx, tn)
		fmt.Fprintf(ctx.stdout, "##### Package: %s\n", strings.Replace(pkgName, "-", "/", -2))
		fmt.Fprintf(ctx.stdout, "PackageName: %s\n", pkgName)

		// Making an spdx package and adding it to pkgs
		pkg := &spdx.Package{
		PackageName: replaceSlashes(pkgName),
		PackageDownloadLocation: getDownloadUrl(ctx, pm),
		PackageSPDXIdentifier: common.ElementID(replaceSlashes(pkgName)),
		PackageLicenseConcluded: concludedLicenses(tn.LicenseTexts()),
		}

		if pm != nil && pm.Version() != "" {
		fmt.Fprintf(ctx.stdout, "PackageVersion: %s\n", pm.Version())
		pkg.PackageVersion = pm.Version()
		} else {
		pkg.PackageVersion = NOASSERTION
		}
		fmt.Fprintf(ctx.stdout, "SPDXID: SPDXRef-Package-%s\n", pkgName)
		fmt.Fprintf(ctx.stdout, "PackageDownloadLocation: %s\n", getDownloadUrl(ctx, pm))
		fmt.Fprintf(ctx.stdout, "PackageLicenseConcluded: %s\n", concludedLicenses(tn.LicenseTexts()))

		pkgs = append(pkgs, pkg)

		return true
		})

		fmt.Fprintf(ctx.stdout, "##### Non-standard license:\n")
		// Adding Non-standard licenses

		licenseTexts := make([]string, 0, len(licenses))

		@@ -412,23 +450,39 @@ func sbomGenerator(ctx *context, files ...string) ([]string, error) {
		sort.Strings(licenseTexts)

		for _, licenseText := range licenseTexts {
		fmt.Fprintf(ctx.stdout, "LicenseID: %s\n", licenses[licenseText])
		// open the file
		f, err := ctx.rootFS.Open(filepath.Clean(licenseText))
		if err != nil {
		return nil, fmt.Errorf("error opening license text file %q: %w", licenseText, err)
		return nil, nil, fmt.Errorf("error opening license text file %q: %w", licenseText, err)
		}

		// read the file
		text, err := io.ReadAll(f)
		if err != nil {
		return nil, fmt.Errorf("error reading license text file %q: %w", licenseText, err)
		return nil, nil, fmt.Errorf("error reading license text file %q: %w", licenseText, err)
		}
		// adding the extracted license text
		fmt.Fprintf(ctx.stdout, "ExtractedText: <text>%v</text>\n", string(text))
		// Making an spdx License and adding it to otherLicenses
		otherLicenses = append(otherLicenses, &spdx.OtherLicense{
		LicenseName: strings.Replace(licenses[licenseText], "LicenseRef-", "", -1),
		LicenseIdentifier: string(licenses[licenseText]),
		ExtractedText: string(text),
		})
		}

		deps := inputFiles(lg, pmix, licenseTexts)
		sort.Strings(deps)
		return deps, nil

		// Making the SPDX doc
		ci, err := builder2v2.BuildCreationInfoSection2_2("Organization", "Google LLC", nil)
		if err != nil {
		return nil, nil, fmt.Errorf("Unable to build creation info section for SPDX doc: %v\n", err)
		}

		return &spdx.Document{
		SPDXIdentifier: "DOCUMENT",
		CreationInfo: ci,
		Packages: pkgs,
		Relationships: relationships,
		OtherLicenses: otherLicenses,
		}, deps, nil
		}

tools/compliance/cmd/sbom/sbom_test.go

+1866 −1292

File changed.

Preview size limit exceeded, changes collapsed.