Merge "Remove unused compliance_sbom tool" into main am: 3fb0fc6a7e am: 02cbbe8a93 (e8b6c8e5) · Commits · e / os / android_build

tools/compliance/Android.bp

+0 −16

Original line number	Diff line number	Diff line
		@@ -131,22 +131,6 @@ blueprint_go_binary {
		testSrcs: ["cmd/xmlnotice/xmlnotice_test.go"],
		}

		blueprint_go_binary {
		name: "compliance_sbom",
		srcs: ["cmd/sbom/sbom.go"],
		deps: [
		"compliance-module",
		"blueprint-deptools",
		"soong-response",
		"spdx-tools-spdxv2_2",
		"spdx-tools-builder2v2",
		"spdx-tools-spdxcommon",
		"spdx-tools-spdx-json",
		"spdx-tools-spdxlib",
		],
		testSrcs: ["cmd/sbom/sbom_test.go"],
		}

		bootstrap_go_package {
		name: "compliance-module",
		srcs: [

tools/compliance/cmd/sbom/sbom.go

deleted100644 → 0

+0 −547

Original line number	Diff line number	Diff line
		// Copyright 2022 Google LLC
		//
		// Licensed under the Apache License, Version 2.0 (the "License");
		// you may not use this file except in compliance with the License.
		// You may obtain a copy of the License at
		//
		// http://www.apache.org/licenses/LICENSE-2.0
		//
		// Unless required by applicable law or agreed to in writing, software
		// distributed under the License is distributed on an "AS IS" BASIS,
		// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		// See the License for the specific language governing permissions and
		// limitations under the License.

		package main

		import (
		"bytes"
		"crypto/sha1"
		"encoding/hex"
		"flag"
		"fmt"
		"io"
		"io/fs"
		"os"
		"path/filepath"
		"sort"
		"strings"
		"time"

		"android/soong/response"
		"android/soong/tools/compliance"
		"android/soong/tools/compliance/projectmetadata"

		"github.com/google/blueprint/deptools"

		"github.com/spdx/tools-golang/builder/builder2v2"
		spdx_json "github.com/spdx/tools-golang/json"
		"github.com/spdx/tools-golang/spdx/common"
		spdx "github.com/spdx/tools-golang/spdx/v2_2"
		"github.com/spdx/tools-golang/spdxlib"
		)

		var (
		failNoneRequested = fmt.Errorf("\nNo license metadata files requested")
		failNoLicenses = fmt.Errorf("No licenses found")
		)

		const NOASSERTION = "NOASSERTION"

		type context struct {
		stdout io.Writer
		stderr io.Writer
		rootFS fs.FS
		product string
		stripPrefix []string
		creationTime creationTimeGetter
		buildid string
		}

		func (ctx context) strip(installPath string) string {
		for _, prefix := range ctx.stripPrefix {
		if strings.HasPrefix(installPath, prefix) {
		p := strings.TrimPrefix(installPath, prefix)
		if 0 == len(p) {
		p = ctx.product
		}
		if 0 == len(p) {
		continue
		}
		return p
		}
		}
		return installPath
		}

		// newMultiString creates a flag that allows multiple values in an array.
		func newMultiString(flags flag.FlagSet, name, usage string) multiString {
		var f multiString
		flags.Var(&f, name, usage)
		return &f
		}

		// multiString implements the flag `Value` interface for multiple strings.
		type multiString []string

		func (ms multiString) String() string { return strings.Join(ms, ", ") }
		func (ms multiString) Set(s string) error { ms = append(*ms, s); return nil }

		func main() {
		var expandedArgs []string
		for _, arg := range os.Args[1:] {
		if strings.HasPrefix(arg, "@") {
		f, err := os.Open(strings.TrimPrefix(arg, "@"))
		if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
		}

		respArgs, err := response.ReadRspFile(f)
		f.Close()
		if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
		}
		expandedArgs = append(expandedArgs, respArgs...)
		} else {
		expandedArgs = append(expandedArgs, arg)
		}
		}

		flags := flag.NewFlagSet("flags", flag.ExitOnError)

		flags.Usage = func() {
		fmt.Fprintf(os.Stderr, `Usage: %s {options} file.meta_lic {file.meta_lic...}

		Outputs an SBOM.spdx.

		Options:
		`, filepath.Base(os.Args[0]))
		flags.PrintDefaults()
		}

		outputFile := flags.String("o", "-", "Where to write the SBOM spdx file. (default stdout)")
		depsFile := flags.String("d", "", "Where to write the deps file")
		product := flags.String("product", "", "The name of the product for which the notice is generated.")
		stripPrefix := newMultiString(flags, "strip_prefix", "Prefix to remove from paths. i.e. path to root (multiple allowed)")
		buildid := flags.String("build_id", "", "Uniquely identifies the build. (default timestamp)")

		flags.Parse(expandedArgs)

		// Must specify at least one root target.
		if flags.NArg() == 0 {
		flags.Usage()
		os.Exit(2)
		}

		if len(*outputFile) == 0 {
		flags.Usage()
		fmt.Fprintf(os.Stderr, "must specify file for -o; use - for stdout\n")
		os.Exit(2)
		} else {
		dir, err := filepath.Abs(filepath.Dir(*outputFile))
		if err != nil {
		fmt.Fprintf(os.Stderr, "cannot determine path to %q: %s\n", *outputFile, err)
		os.Exit(1)
		}
		fi, err := os.Stat(dir)
		if err != nil {
		fmt.Fprintf(os.Stderr, "cannot read directory %q of %q: %s\n", dir, *outputFile, err)
		os.Exit(1)
		}
		if !fi.IsDir() {
		fmt.Fprintf(os.Stderr, "parent %q of %q is not a directory\n", dir, *outputFile)
		os.Exit(1)
		}
		}

		var ofile io.Writer
		ofile = os.Stdout
		var obuf *bytes.Buffer
		if *outputFile != "-" {
		obuf = &bytes.Buffer{}
		ofile = obuf
		}

		ctx := &context{ofile, os.Stderr, compliance.FS, product, stripPrefix, actualTime, *buildid}

		spdxDoc, deps, err := sbomGenerator(ctx, flags.Args()...)

		if err != nil {
		if err == failNoneRequested {
		flags.Usage()
		}
		fmt.Fprintf(os.Stderr, "%s\n", err.Error())
		os.Exit(1)
		}

		// writing the spdx Doc created
		if err := spdx_json.Save2_2(spdxDoc, ofile); err != nil {
		fmt.Fprintf(os.Stderr, "failed to write document to %v: %v", *outputFile, err)
		os.Exit(1)
		}

		if *outputFile != "-" {
		err := os.WriteFile(*outputFile, obuf.Bytes(), 0666)
		if err != nil {
		fmt.Fprintf(os.Stderr, "could not write output to %q: %s\n", *outputFile, err)
		os.Exit(1)
		}
		}

		if *depsFile != "" {
		err := deptools.WriteDepFile(depsFile, outputFile, deps)
		if err != nil {
		fmt.Fprintf(os.Stderr, "could not write deps to %q: %s\n", *depsFile, err)
		os.Exit(1)
		}
		}
		os.Exit(0)
		}

		type creationTimeGetter func() string

		// actualTime returns current time in UTC
		func actualTime() string {
		t := time.Now().UTC()
		return t.UTC().Format("2006-01-02T15:04:05Z")
		}

		// replaceSlashes replaces "/" by "-" for the library path to be used for packages & files SPDXID
		func replaceSlashes(x string) string {
		return strings.ReplaceAll(x, "/", "-")
		}

		// stripDocName removes the outdir prefix and meta_lic suffix from a target Name
		func stripDocName(name string) string {
		// remove outdir prefix
		if strings.HasPrefix(name, "out/") {
		name = name[4:]
		}

		// remove suffix
		if strings.HasSuffix(name, ".meta_lic") {
		name = name[:len(name)-9]
		} else if strings.HasSuffix(name, "/meta_lic") {
		name = name[:len(name)-9] + "/"
		}

		return name
		}

		// getPackageName returns a package name of a target Node
		func getPackageName(_ context, tn compliance.TargetNode) string {
		return replaceSlashes(tn.Name())
		}

		// getDocumentName returns a package name of a target Node
		func getDocumentName(ctx context, tn compliance.TargetNode, pm *projectmetadata.ProjectMetadata) string {
		if len(ctx.product) > 0 {
		return replaceSlashes(ctx.product)
		}
		if len(tn.ModuleName()) > 0 {
		if pm != nil {
		return replaceSlashes(pm.Name() + ":" + tn.ModuleName())
		}
		return replaceSlashes(tn.ModuleName())
		}

		return stripDocName(replaceSlashes(tn.Name()))
		}

		// getDownloadUrl returns the download URL if available (GIT, SVN, etc..),
		// or NOASSERTION if not available, none determined or ambiguous
		func getDownloadUrl(_ context, pm projectmetadata.ProjectMetadata) string {
		if pm == nil {
		return NOASSERTION
		}

		urlsByTypeName := pm.UrlsByTypeName()
		if urlsByTypeName == nil {
		return NOASSERTION
		}

		url := urlsByTypeName.DownloadUrl()
		if url == "" {
		return NOASSERTION
		}
		return url
		}

		// getProjectMetadata returns the optimal project metadata for the target node
		func getProjectMetadata(_ context, pmix projectmetadata.Index,
		tn compliance.TargetNode) (projectmetadata.ProjectMetadata, error) {
		pms, err := pmix.MetadataForProjects(tn.Projects()...)
		if err != nil {
		return nil, fmt.Errorf("Unable to read projects for %q: %w\n", tn.Name(), err)
		}
		if len(pms) == 0 {
		return nil, nil
		}

		// Getting the project metadata that contains most of the info needed for sbomGenerator
		score := -1
		index := -1
		for i := 0; i < len(pms); i++ {
		tempScore := 0
		if pms[i].Name() != "" {
		tempScore += 1
		}
		if pms[i].Version() != "" {
		tempScore += 1
		}
		if pms[i].UrlsByTypeName().DownloadUrl() != "" {
		tempScore += 1
		}

		if tempScore == score {
		if pms[i].Project() < pms[index].Project() {
		index = i
		}
		} else if tempScore > score {
		score = tempScore
		index = i
		}
		}
		return pms[index], nil
		}

		// inputFiles returns the complete list of files read
		func inputFiles(lg compliance.LicenseGraph, pmix projectmetadata.Index, licenseTexts []string) []string {
		projectMeta := pmix.AllMetadataFiles()
		targets := lg.TargetNames()
		files := make([]string, 0, len(licenseTexts)+len(targets)+len(projectMeta))
		files = append(files, licenseTexts...)
		files = append(files, targets...)
		files = append(files, projectMeta...)
		return files
		}

		// generateSPDXNamespace generates a unique SPDX Document Namespace using a SHA1 checksum
		func generateSPDXNamespace(buildid string, created string, files ...string) string {

		seed := strings.Join(files, "")

		if buildid == "" {
		seed += created
		} else {
		seed += buildid
		}

		// Compute a SHA1 checksum of the seed.
		hash := sha1.Sum([]byte(seed))
		uuid := hex.EncodeToString(hash[:])

		namespace := fmt.Sprintf("SPDXRef-DOCUMENT-%s", uuid)

		return namespace
		}

		// sbomGenerator implements the spdx bom utility

		// SBOM is part of the new government regulation issued to improve national cyber security
		// and enhance software supply chain and transparency, see https://www.cisa.gov/sbom

		// sbomGenerator uses the SPDX standard, see the SPDX specification (https://spdx.github.io/spdx-spec/)
		// sbomGenerator is also following the internal google SBOM styleguide (http://goto.google.com/spdx-style-guide)
		func sbomGenerator(ctx context, files ...string) (spdx.Document, []string, error) {
		// Must be at least one root file.
		if len(files) < 1 {
		return nil, nil, failNoneRequested
		}

		pmix := projectmetadata.NewIndex(ctx.rootFS)

		lg, err := compliance.ReadLicenseGraph(ctx.rootFS, ctx.stderr, files)

		if err != nil {
		return nil, nil, fmt.Errorf("Unable to read license text file(s) for %q: %v\n", files, err)
		}

		// creating the packages section
		pkgs := []*spdx.Package{}

		// creating the relationship section
		relationships := []*spdx.Relationship{}

		// creating the license section
		otherLicenses := []*spdx.OtherLicense{}

		// spdx document name
		var docName string

		// main package name
		var mainPkgName string

		// implementing the licenses references for the packages
		licenses := make(map[string]string)
		concludedLicenses := func(licenseTexts []string) string {
		licenseRefs := make([]string, 0, len(licenseTexts))
		for _, licenseText := range licenseTexts {
		license := strings.SplitN(licenseText, ":", 2)[0]
		if _, ok := licenses[license]; !ok {
		licenseRef := "LicenseRef-" + replaceSlashes(license)
		licenses[license] = licenseRef
		}

		licenseRefs = append(licenseRefs, licenses[license])
		}
		if len(licenseRefs) > 1 {
		return "(" + strings.Join(licenseRefs, " AND ") + ")"
		} else if len(licenseRefs) == 1 {
		return licenseRefs[0]
		}
		return "NONE"
		}

		isMainPackage := true
		visitedNodes := make(map[*compliance.TargetNode]struct{})

		// performing a Breadth-first top down walk of licensegraph and building package information
		compliance.WalkTopDownBreadthFirst(nil, lg,
		func(lg compliance.LicenseGraph, tn compliance.TargetNode, path compliance.TargetEdgePath) bool {
		if err != nil {
		return false
		}
		var pm *projectmetadata.ProjectMetadata
		pm, err = getProjectMetadata(ctx, pmix, tn)
		if err != nil {
		return false
		}

		if isMainPackage {
		docName = getDocumentName(ctx, tn, pm)
		mainPkgName = replaceSlashes(getPackageName(ctx, tn))
		isMainPackage = false
		}

		if len(path) == 0 {
		// Add the describe relationship for the main package
		rln := &spdx.Relationship{
		RefA: common.MakeDocElementID("" /* this document */, "DOCUMENT"),
		RefB: common.MakeDocElementID("", mainPkgName),
		Relationship: "DESCRIBES",
		}
		relationships = append(relationships, rln)

		} else {
		// Check parent and identify annotation
		parent := path[len(path)-1]
		targetEdge := parent.Edge()
		if targetEdge.IsRuntimeDependency() {
		// Adding the dynamic link annotation RUNTIME_DEPENDENCY_OF relationship
		rln := &spdx.Relationship{
		RefA: common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, tn))),
		RefB: common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, targetEdge.Target()))),
		Relationship: "RUNTIME_DEPENDENCY_OF",
		}
		relationships = append(relationships, rln)

		} else if targetEdge.IsDerivation() {
		// Adding the derivation annotation as a CONTAINS relationship
		rln := &spdx.Relationship{
		RefA: common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, targetEdge.Target()))),
		RefB: common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, tn))),
		Relationship: "CONTAINS",
		}
		relationships = append(relationships, rln)

		} else if targetEdge.IsBuildTool() {
		// Adding the toolchain annotation as a BUILD_TOOL_OF relationship
		rln := &spdx.Relationship{
		RefA: common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, tn))),
		RefB: common.MakeDocElementID("", replaceSlashes(getPackageName(ctx, targetEdge.Target()))),
		Relationship: "BUILD_TOOL_OF",
		}
		relationships = append(relationships, rln)

		} else {
		panic(fmt.Errorf("Unknown dependency type: %v", targetEdge.Annotations()))
		}
		}

		if _, alreadyVisited := visitedNodes[tn]; alreadyVisited {
		return false
		}
		visitedNodes[tn] = struct{}{}
		pkgName := getPackageName(ctx, tn)

		// Making an spdx package and adding it to pkgs
		pkg := &spdx.Package{
		PackageName: replaceSlashes(pkgName),
		PackageDownloadLocation: getDownloadUrl(ctx, pm),
		PackageSPDXIdentifier: common.ElementID(replaceSlashes(pkgName)),
		PackageLicenseConcluded: concludedLicenses(tn.LicenseTexts()),
		}

		if pm != nil && pm.Version() != "" {
		pkg.PackageVersion = pm.Version()
		} else {
		pkg.PackageVersion = NOASSERTION
		}

		pkgs = append(pkgs, pkg)

		return true
		})

		// Adding Non-standard licenses

		licenseTexts := make([]string, 0, len(licenses))

		for licenseText := range licenses {
		licenseTexts = append(licenseTexts, licenseText)
		}

		sort.Strings(licenseTexts)

		for _, licenseText := range licenseTexts {
		// open the file
		f, err := ctx.rootFS.Open(filepath.Clean(licenseText))
		if err != nil {
		return nil, nil, fmt.Errorf("error opening license text file %q: %w", licenseText, err)
		}

		// read the file
		text, err := io.ReadAll(f)
		if err != nil {
		return nil, nil, fmt.Errorf("error reading license text file %q: %w", licenseText, err)
		}
		// Making an spdx License and adding it to otherLicenses
		otherLicenses = append(otherLicenses, &spdx.OtherLicense{
		LicenseName: strings.Replace(licenses[licenseText], "LicenseRef-", "", -1),
		LicenseIdentifier: string(licenses[licenseText]),
		ExtractedText: string(text),
		})
		}

		deps := inputFiles(lg, pmix, licenseTexts)
		sort.Strings(deps)

		// Making the SPDX doc
		ci, err := builder2v2.BuildCreationInfoSection2_2("Organization", "Google LLC", nil)
		if err != nil {
		return nil, nil, fmt.Errorf("Unable to build creation info section for SPDX doc: %v\n", err)
		}

		ci.Created = ctx.creationTime()

		doc := &spdx.Document{
		SPDXVersion: "SPDX-2.2",
		DataLicense: "CC0-1.0",
		SPDXIdentifier: "DOCUMENT",
		DocumentName: docName,
		DocumentNamespace: generateSPDXNamespace(ctx.buildid, ci.Created, files...),
		CreationInfo: ci,
		Packages: pkgs,
		Relationships: relationships,
		OtherLicenses: otherLicenses,
		}

		if err := spdxlib.ValidateDocument2_2(doc); err != nil {
		return nil, nil, fmt.Errorf("Unable to validate the SPDX doc: %v\n", err)
		}

		return doc, deps, nil
		}