Merge "Added SBOM generator module to implement the spdx utility bill of material" (1bb27903) · Commits · e / os / android_build

tools/compliance/Android.bp

+11 −0

Original line number	Diff line number	Diff line
		@@ -131,6 +131,17 @@ blueprint_go_binary {
		testSrcs: ["cmd/xmlnotice/xmlnotice_test.go"],
		}

		blueprint_go_binary {
		name: "compliance_sbom",
		srcs: ["cmd/sbom/sbom.go"],
		deps: [
		"compliance-module",
		"blueprint-deptools",
		"soong-response",
		],
		testSrcs: ["cmd/sbom/sbom_test.go"],
		}

		bootstrap_go_package {
		name: "compliance-module",
		srcs: [

tools/compliance/cmd/sbom/sbom.go

0 → 100644

+399 −0

Original line number	Diff line number	Diff line
		// Copyright 2022 Google LLC
		//
		// Licensed under the Apache License, Version 2.0 (the "License");
		// you may not use this file except in compliance with the License.
		// You may obtain a copy of the License at
		//
		// http://www.apache.org/licenses/LICENSE-2.0
		//
		// Unless required by applicable law or agreed to in writing, software
		// distributed under the License is distributed on an "AS IS" BASIS,
		// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		// See the License for the specific language governing permissions and
		// limitations under the License.

		package main

		import (
		"bytes"
		"flag"
		"fmt"
		"io"
		"io/fs"
		"os"
		"path/filepath"
		"sort"
		"strings"
		"time"

		"android/soong/response"
		"android/soong/tools/compliance"
		"android/soong/tools/compliance/projectmetadata"

		"github.com/google/blueprint/deptools"
		)

		var (
		failNoneRequested = fmt.Errorf("\nNo license metadata files requested")
		failNoLicenses = fmt.Errorf("No licenses found")
		)

		type context struct {
		stdout io.Writer
		stderr io.Writer
		rootFS fs.FS
		product string
		stripPrefix []string
		creationTime creationTimeGetter
		}

		func (ctx context) strip(installPath string) string {
		for _, prefix := range ctx.stripPrefix {
		if strings.HasPrefix(installPath, prefix) {
		p := strings.TrimPrefix(installPath, prefix)
		if 0 == len(p) {
		p = ctx.product
		}
		if 0 == len(p) {
		continue
		}
		return p
		}
		}
		return installPath
		}

		// newMultiString creates a flag that allows multiple values in an array.
		func newMultiString(flags flag.FlagSet, name, usage string) multiString {
		var f multiString
		flags.Var(&f, name, usage)
		return &f
		}

		// multiString implements the flag `Value` interface for multiple strings.
		type multiString []string

		func (ms multiString) String() string { return strings.Join(ms, ", ") }
		func (ms multiString) Set(s string) error { ms = append(*ms, s); return nil }

		func main() {
		var expandedArgs []string
		for _, arg := range os.Args[1:] {
		if strings.HasPrefix(arg, "@") {
		f, err := os.Open(strings.TrimPrefix(arg, "@"))
		if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
		}

		respArgs, err := response.ReadRspFile(f)
		f.Close()
		if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
		}
		expandedArgs = append(expandedArgs, respArgs...)
		} else {
		expandedArgs = append(expandedArgs, arg)
		}
		}

		flags := flag.NewFlagSet("flags", flag.ExitOnError)

		flags.Usage = func() {
		fmt.Fprintf(os.Stderr, `Usage: %s {options} file.meta_lic {file.meta_lic...}

		Outputs an SBOM.spdx.

		Options:
		`, filepath.Base(os.Args[0]))
		flags.PrintDefaults()
		}

		outputFile := flags.String("o", "-", "Where to write the SBOM spdx file. (default stdout)")
		depsFile := flags.String("d", "", "Where to write the deps file")
		product := flags.String("product", "", "The name of the product for which the notice is generated.")
		stripPrefix := newMultiString(flags, "strip_prefix", "Prefix to remove from paths. i.e. path to root (multiple allowed)")

		flags.Parse(expandedArgs)

		// Must specify at least one root target.
		if flags.NArg() == 0 {
		flags.Usage()
		os.Exit(2)
		}

		if len(*outputFile) == 0 {
		flags.Usage()
		fmt.Fprintf(os.Stderr, "must specify file for -o; use - for stdout\n")
		os.Exit(2)
		} else {
		dir, err := filepath.Abs(filepath.Dir(*outputFile))
		if err != nil {
		fmt.Fprintf(os.Stderr, "cannot determine path to %q: %s\n", *outputFile, err)
		os.Exit(1)
		}
		fi, err := os.Stat(dir)
		if err != nil {
		fmt.Fprintf(os.Stderr, "cannot read directory %q of %q: %s\n", dir, *outputFile, err)
		os.Exit(1)
		}
		if !fi.IsDir() {
		fmt.Fprintf(os.Stderr, "parent %q of %q is not a directory\n", dir, *outputFile)
		os.Exit(1)
		}
		}

		var ofile io.Writer
		ofile = os.Stdout
		var obuf *bytes.Buffer
		if *outputFile != "-" {
		obuf = &bytes.Buffer{}
		ofile = obuf
		}

		ctx := &context{ofile, os.Stderr, compliance.FS, product, stripPrefix, actualTime}

		deps, err := sbomGenerator(ctx, flags.Args()...)
		if err != nil {
		if err == failNoneRequested {
		flags.Usage()
		}
		fmt.Fprintf(os.Stderr, "%s\n", err.Error())
		os.Exit(1)
		}

		if *outputFile != "-" {
		err := os.WriteFile(*outputFile, obuf.Bytes(), 0666)
		if err != nil {
		fmt.Fprintf(os.Stderr, "could not write output to %q: %s\n", *outputFile, err)
		os.Exit(1)
		}
		}

		if *depsFile != "" {
		err := deptools.WriteDepFile(depsFile, outputFile, deps)
		if err != nil {
		fmt.Fprintf(os.Stderr, "could not write deps to %q: %s\n", *depsFile, err)
		os.Exit(1)
		}
		}
		os.Exit(0)
		}

		type creationTimeGetter func() time.Time

		// actualTime returns current time in UTC
		func actualTime() time.Time {
		return time.Now().UTC()
		}

		// replaceSlashes replaces "/" by "-" for the library path to be used for packages & files SPDXID
		func replaceSlashes(x string) string {
		return strings.ReplaceAll(x, "/", "-")
		}

		// getPackageName returns a package name of a target Node
		func getPackageName(_ context, tn compliance.TargetNode) string {
		return replaceSlashes(tn.Name())
		}

		// getDocumentName returns a package name of a target Node
		func getDocumentName(ctx context, tn compliance.TargetNode, pm *projectmetadata.ProjectMetadata) string {
		if len(ctx.product) > 0 {
		return replaceSlashes(ctx.product)
		}
		if len(tn.ModuleName()) > 0 {
		if pm != nil {
		return replaceSlashes(pm.Name() + ":" + tn.ModuleName())
		}
		return replaceSlashes(tn.ModuleName())
		}

		// TO DO: Replace tn.Name() with pm.Name() + parts of the target name
		return replaceSlashes(tn.Name())
		}

		// getDownloadUrl returns the download URL if available (GIT, SVN, etc..),
		// or NOASSERTION if not available, none determined or ambiguous
		func getDownloadUrl(_ context, pm projectmetadata.ProjectMetadata) string {
		if pm == nil {
		return "NOASSERTION"
		}

		urlsByTypeName := pm.UrlsByTypeName()
		if urlsByTypeName == nil {
		return "NOASSERTION"
		}

		url := urlsByTypeName.DownloadUrl()
		if url == "" {
		return "NOASSERTION"
		}
		return url
		}

		// getProjectMetadata returns the project metadata for the target node
		func getProjectMetadata(_ context, pmix projectmetadata.Index,
		tn compliance.TargetNode) (projectmetadata.ProjectMetadata, error) {
		pms, err := pmix.MetadataForProjects(tn.Projects()...)
		if err != nil {
		return nil, fmt.Errorf("Unable to read projects for %q: %w\n", tn, err)
		}
		if len(pms) == 0 {
		return nil, nil
		}

		// TO DO: skip first element if it doesn't have one of the three info needed
		return pms[0], nil
		}

		// sbomGenerator implements the spdx bom utility

		// SBOM is part of the new government regulation issued to improve national cyber security
		// and enhance software supply chain and transparency, see https://www.cisa.gov/sbom

		// sbomGenerator uses the SPDX standard, see the SPDX specification (https://spdx.github.io/spdx-spec/)
		// sbomGenerator is also following the internal google SBOM styleguide (http://goto.google.com/spdx-style-guide)
		func sbomGenerator(ctx *context, files ...string) ([]string, error) {
		// Must be at least one root file.
		if len(files) < 1 {
		return nil, failNoneRequested
		}

		pmix := projectmetadata.NewIndex(ctx.rootFS)

		lg, err := compliance.ReadLicenseGraph(ctx.rootFS, ctx.stderr, files)

		if err != nil {
		return nil, fmt.Errorf("Unable to read license text file(s) for %q: %v\n", files, err)
		}

		// implementing the licenses references for the packages
		licenses := make(map[string]string)
		concludedLicenses := func(licenseTexts []string) string {
		licenseRefs := make([]string, 0, len(licenseTexts))
		for _, licenseText := range licenseTexts {
		license := strings.SplitN(licenseText, ":", 2)[0]
		if _, ok := licenses[license]; !ok {
		licenseRef := "LicenseRef-" + replaceSlashes(license)
		licenses[license] = licenseRef
		}

		licenseRefs = append(licenseRefs, licenses[license])
		}
		if len(licenseRefs) > 1 {
		return "(" + strings.Join(licenseRefs, " AND ") + ")"
		} else if len(licenseRefs) == 1 {
		return licenseRefs[0]
		}
		return "NONE"
		}

		isMainPackage := true
		var mainPackage string
		visitedNodes := make(map[*compliance.TargetNode]struct{})

		// performing a Breadth-first top down walk of licensegraph and building package information
		compliance.WalkTopDownBreadthFirst(nil, lg,
		func(lg compliance.LicenseGraph, tn compliance.TargetNode, path compliance.TargetEdgePath) bool {
		if err != nil {
		return false
		}
		var pm *projectmetadata.ProjectMetadata
		pm, err = getProjectMetadata(ctx, pmix, tn)
		if err != nil {
		return false
		}

		if isMainPackage {
		mainPackage = getDocumentName(ctx, tn, pm)
		fmt.Fprintf(ctx.stdout, "SPDXVersion: SPDX-2.2\n")
		fmt.Fprintf(ctx.stdout, "DataLicense: CC-1.0\n")
		fmt.Fprintf(ctx.stdout, "DocumentName: %s\n", mainPackage)
		fmt.Fprintf(ctx.stdout, "SPDXID: SPDXRef-DOCUMENT-%s\n", mainPackage)
		fmt.Fprintf(ctx.stdout, "DocumentNamespace: Android\n")
		fmt.Fprintf(ctx.stdout, "Creator: Organization: Google LLC\n")
		fmt.Fprintf(ctx.stdout, "Created: %s\n", ctx.creationTime().Format("2006-01-02T15:04:05Z"))
		isMainPackage = false
		}

		relationships := make([]string, 0, 1)
		defer func() {
		if r := recover(); r != nil {
		panic(r)
		}
		for _, relationship := range relationships {
		fmt.Fprintln(ctx.stdout, relationship)
		}
		}()
		if len(path) == 0 {
		relationships = append(relationships,
		fmt.Sprintf("Relationship: SPDXRef-DOCUMENT-%s DESCRIBES SPDXRef-Package-%s",
		mainPackage, getPackageName(ctx, tn)))
		} else {
		// Check parent and identify annotation
		parent := path[len(path)-1]
		targetEdge := parent.Edge()
		if targetEdge.IsRuntimeDependency() {
		// Adding the dynamic link annotation RUNTIME_DEPENDENCY_OF relationship
		relationships = append(relationships, fmt.Sprintf("Relationship: SPDXRef-Package-%s RUNTIME_DEPENDENCY_OF SPDXRef-Package-%s", getPackageName(ctx, tn), getPackageName(ctx, targetEdge.Target())))

		} else if targetEdge.IsDerivation() {
		// Adding the derivation annotation as a CONTAINS relationship
		relationships = append(relationships, fmt.Sprintf("Relationship: SPDXRef-Package-%s CONTAINS SPDXRef-Package-%s", getPackageName(ctx, targetEdge.Target()), getPackageName(ctx, tn)))

		} else if targetEdge.IsBuildTool() {
		// Adding the toolchain annotation as a BUILD_TOOL_OF relationship
		relationships = append(relationships, fmt.Sprintf("Relationship: SPDXRef-Package-%s BUILD_TOOL_OF SPDXRef-Package-%s", getPackageName(ctx, tn), getPackageName(ctx, targetEdge.Target())))
		} else {
		panic(fmt.Errorf("Unknown dependency type: %v", targetEdge.Annotations()))
		}
		}

		if _, alreadyVisited := visitedNodes[tn]; alreadyVisited {
		return false
		}
		visitedNodes[tn] = struct{}{}
		pkgName := getPackageName(ctx, tn)
		fmt.Fprintf(ctx.stdout, "##### Package: %s\n", strings.Replace(pkgName, "-", "/", -2))
		fmt.Fprintf(ctx.stdout, "PackageName: %s\n", pkgName)
		if pm != nil && pm.Version() != "" {
		fmt.Fprintf(ctx.stdout, "PackageVersion: %s\n", pm.Version())
		}
		fmt.Fprintf(ctx.stdout, "SPDXID: SPDXRef-Package-%s\n", pkgName)
		fmt.Fprintf(ctx.stdout, "PackageDownloadLocation: %s\n", getDownloadUrl(ctx, pm))
		fmt.Fprintf(ctx.stdout, "PackageLicenseConcluded: %s\n", concludedLicenses(tn.LicenseTexts()))
		return true
		})

		fmt.Fprintf(ctx.stdout, "##### Non-standard license:\n")

		licenseTexts := make([]string, 0, len(licenses))

		for licenseText := range licenses {
		licenseTexts = append(licenseTexts, licenseText)
		}

		sort.Strings(licenseTexts)

		for _, licenseText := range licenseTexts {
		fmt.Fprintf(ctx.stdout, "LicenseID: %s\n", licenses[licenseText])
		// open the file
		f, err := ctx.rootFS.Open(filepath.Clean(licenseText))
		if err != nil {
		return nil, fmt.Errorf("error opening license text file %q: %w", licenseText, err)
		}

		// read the file
		text, err := io.ReadAll(f)
		if err != nil {
		return nil, fmt.Errorf("error reading license text file %q: %w", licenseText, err)
		}
		// adding the extracted license text
		fmt.Fprintf(ctx.stdout, "ExtractedText: <text>%v</text>\n", string(text))
		}

		deps := licenseTexts
		return deps, nil
		}

tools/compliance/cmd/sbom/sbom_test.go

0 → 100644

+1629 −0

File added.

Preview size limit exceeded, changes collapsed.

tools/compliance/graph.go

+28 −0

Original line number	Diff line number	Diff line
		@@ -137,6 +137,24 @@ func (e *TargetEdge) Annotations() TargetEdgeAnnotations {
		return e.annotations
		}

		// IsRuntimeDependency returns true for edges representing shared libraries
		// linked dynamically at runtime.
		func (e *TargetEdge) IsRuntimeDependency() bool {
		return edgeIsDynamicLink(e)
		}

		// IsDerivation returns true for edges where the target is a derivative
		// work of dependency.
		func (e *TargetEdge) IsDerivation() bool {
		return edgeIsDerivation(e)
		}

		// IsBuildTool returns true for edges where the target is built
		// by dependency.
		func (e *TargetEdge) IsBuildTool() bool {
		return !edgeIsDerivation(e) && !edgeIsDynamicLink(e)
		}

		// String returns a human-readable string representation of the edge.
		func (e *TargetEdge) String() string {
		return fmt.Sprintf("%s -[%s]> %s", e.target.name, strings.Join(e.annotations.AsList(), ", "), e.dependency.name)
		@@ -186,6 +204,11 @@ func (s TargetEdgePathSegment) Dependency() *TargetNode {
		return s.edge.dependency
		}

		// Edge describes the target edge.
		func (s TargetEdgePathSegment) Edge() *TargetEdge {
		return s.edge
		}

		// Annotations describes the type of edge by the set of annotations attached to
		// it.
		//
		@@ -298,6 +321,11 @@ func (tn *TargetNode) PackageName() string {
		return tn.proto.GetPackageName()
		}

		// ModuleName returns the module name of the target.
		func (tn *TargetNode) ModuleName() string {
		return tn.proto.GetModuleName()
		}

		// Projects returns the projects defining the target node. (unordered)
		//
		// In an ideal world, only 1 project defines a target, but the interaction