Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 9f135552 authored by Wei Li's avatar Wei Li Committed by Gerrit Code Review
Browse files

Merge "Generate SBOM of products in Soong." into main

parents a2f2e80c b85a178b

android/Android.bp

+1 −0
Original line number Diff line number Diff line
@@ -93,6 +93,7 @@ bootstrap_go_package { 
        "register.go",

        "rule_builder.go",

        "sandbox.go",

        "sbom.go",

        "sdk.go",

        "sdk_version.go",

        "shared_properties.go",

android/sbom.go

0 → 100644
+100 −0
Original line number Diff line number Diff line 
// Copyright 2024 Google Inc. All rights reserved.

//

// Licensed under the Apache License, Version 2.0 (the "License");

// you may not use this file except in compliance with the License.

// You may obtain a copy of the License at

//

//     http://www.apache.org/licenses/LICENSE-2.0

//

// Unless required by applicable law or agreed to in writing, software

// distributed under the License is distributed on an "AS IS" BASIS,

// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

// See the License for the specific language governing permissions and

// limitations under the License.



package android



import (

	"io"

	"path/filepath"

	"strings"



	"github.com/google/blueprint"

)



var (

	// Command line tool to generate SBOM in Soong

	genSbom = pctx.HostBinToolVariable("genSbom", "gen_sbom")



	// Command to generate SBOM in Soong.

	genSbomRule = pctx.AndroidStaticRule("genSbomRule", blueprint.RuleParams{

		Command:     "rm -rf $out && ${genSbom} --output_file ${out} --metadata ${in} --product_out ${productOut} --soong_out ${soongOut} --build_version \"$$(cat ${buildFingerprintFile})\" --product_mfr \"${productManufacturer}\" --json",

		CommandDeps: []string{"${genSbom}"},

	}, "productOut", "soongOut", "buildFingerprintFile", "productManufacturer")

)



func init() {

	RegisterSbomSingleton(InitRegistrationContext)

}



func RegisterSbomSingleton(ctx RegistrationContext) {

	ctx.RegisterParallelSingletonType("sbom_singleton", sbomSingletonFactory)

}



// sbomSingleton is used to generate build actions of generating SBOM of products.

type sbomSingleton struct{}



func sbomSingletonFactory() Singleton {

	return &sbomSingleton{}

}



// Generates SBOM of products

func (this *sbomSingleton) GenerateBuildActions(ctx SingletonContext) {

	if !ctx.Config().HasDeviceProduct() {

		return

	}

	// Get all METADATA files and add them as implicit input

	metadataFileListFile := PathForArbitraryOutput(ctx, ".module_paths", "METADATA.list")

	f, err := ctx.Config().fs.Open(metadataFileListFile.String())

	if err != nil {

		panic(err)

	}

	b, err := io.ReadAll(f)

	if err != nil {

		panic(err)

	}

	allMetadataFiles := strings.Split(string(b), "\n")

	implicits := []Path{metadataFileListFile}

	for _, path := range allMetadataFiles {

		implicits = append(implicits, PathForSource(ctx, path))

	}

	prodVars := ctx.Config().productVariables

	buildFingerprintFile := PathForArbitraryOutput(ctx, "target", "product", String(prodVars.DeviceName), "build_fingerprint.txt")

	implicits = append(implicits, buildFingerprintFile)



	// Add installed_files.stamp as implicit input, which depends on all installed files of the product.

	installedFilesStamp := PathForOutput(ctx, "compliance-metadata", ctx.Config().DeviceProduct(), "installed_files.stamp")

	implicits = append(implicits, installedFilesStamp)



	metadataDb := PathForOutput(ctx, "compliance-metadata", ctx.Config().DeviceProduct(), "compliance-metadata.db")

	sbomFile := PathForOutput(ctx, "sbom", ctx.Config().DeviceProduct(), "sbom.spdx.json")

	ctx.Build(pctx, BuildParams{

		Rule:      genSbomRule,

		Input:     metadataDb,

		Implicits: implicits,

		Output:    sbomFile,

		Args: map[string]string{

			"productOut":           filepath.Join(ctx.Config().OutDir(), "target", "product", String(prodVars.DeviceName)),

			"soongOut":             ctx.Config().soongOutDir,

			"buildFingerprintFile": buildFingerprintFile.String(),

			"productManufacturer":  ctx.Config().ProductVariables().ProductManufacturer,

		},

	})



	// Phony rule "soong-sbom". "m soong-sbom" to generate product SBOM in Soong.

	ctx.Build(pctx, BuildParams{

		Rule:   blueprint.Phony,

		Inputs: []Path{sbomFile},

		Output: PathForPhony(ctx, "soong-sbom"),

	})

}

tests/sbom_test.sh

+61 −3
Original line number Diff line number Diff line
@@ -70,13 +70,14 @@ function test_sbom_aosp_cf_x86_64_phone { 
  # m droid, build sbom later in case additional dependencies might be built and included in partition images.

  run_soong "${out_dir}" "droid dump.erofs lz4"



  soong_sbom_out=$out_dir/soong/sbom/$target_product

  product_out=$out_dir/target/product/vsoc_x86_64

  sbom_test=$product_out/sbom_test

  mkdir -p $sbom_test

  cp $product_out/*.img $sbom_test



  # m sbom

  run_soong "${out_dir}" sbom

  # m sbom soong-sbom

  run_soong "${out_dir}" "sbom soong-sbom"



  # Generate installed file list from .img files in PRODUCT_OUT

  dump_erofs=$out_dir/host/linux-x86/bin/dump.erofs
@@ -118,6 +119,7 @@ function test_sbom_aosp_cf_x86_64_phone { 
    partition_name=$(basename $f | cut -d. -f1)

    file_list_file="${sbom_test}/sbom-${partition_name}-files.txt"

    files_in_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-spdx.txt"

    files_in_soong_spdx_file="${sbom_test}/soong-sbom-${partition_name}-files-in-spdx.txt"

    rm "$file_list_file" > /dev/null 2>&1 || true

    all_dirs="/"

    while [ ! -z "$all_dirs" ]; do
@@ -145,6 +147,7 @@ function test_sbom_aosp_cf_x86_64_phone { 
    done

    sort -n -o "$file_list_file" "$file_list_file"



    # Diff the file list from image and file list in SBOM created by Make

    grep "FileName: /${partition_name}/" $product_out/sbom.spdx | sed 's/^FileName: //' > "$files_in_spdx_file"

    if [ "$partition_name" = "system" ]; then

      # system partition is mounted to /, so include FileName starts with /root/ too.
@@ -154,6 +157,17 @@ function test_sbom_aosp_cf_x86_64_phone { 


    echo ============ Diffing files in $f and SBOM

    diff_files "$file_list_file" "$files_in_spdx_file" "$partition_name" ""



    # Diff the file list from image and file list in SBOM created by Soong

    grep "FileName: /${partition_name}/" $soong_sbom_out/sbom.spdx | sed 's/^FileName: //' > "$files_in_soong_spdx_file"

        if [ "$partition_name" = "system" ]; then

          # system partition is mounted to /, so include FileName starts with /root/ too.

          grep "FileName: /root/" $soong_sbom_out/sbom.spdx | sed 's/^FileName: \/root//' >> "$files_in_soong_spdx_file"

        fi

        sort -n -o "$files_in_soong_spdx_file" "$files_in_soong_spdx_file"



        echo ============ Diffing files in $f and SBOM created by Soong

        diff_files "$file_list_file" "$files_in_soong_spdx_file" "$partition_name" ""

  done



  RAMDISK_IMAGES="$product_out/ramdisk.img"
@@ -161,6 +175,7 @@ function test_sbom_aosp_cf_x86_64_phone { 
    partition_name=$(basename $f | cut -d. -f1)

    file_list_file="${sbom_test}/sbom-${partition_name}-files.txt"

    files_in_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-spdx.txt"

    files_in_soong_spdx_file="${sbom_test}/sbom-${partition_name}-files-in-soong-spdx.txt"

    # lz4 decompress $f to stdout

    # cpio list all entries like ls -l

    # grep filter normal files and symlinks
@@ -170,11 +185,19 @@ function test_sbom_aosp_cf_x86_64_phone { 


    grep "FileName: /${partition_name}/" $product_out/sbom.spdx | sed 's/^FileName: //' | sort -n > "$files_in_spdx_file"



    grep "FileName: /${partition_name}/" $soong_sbom_out/sbom.spdx | sed 's/^FileName: //' | sort -n > "$files_in_soong_spdx_file"



    echo ============ Diffing files in $f and SBOM

    diff_files "$file_list_file" "$files_in_spdx_file" "$partition_name" ""



    echo ============ Diffing files in $f and SBOM created by Soong

    diff_files "$file_list_file" "$files_in_soong_spdx_file" "$partition_name" ""

  done



  verify_package_verification_code "$product_out/sbom.spdx"

  verify_package_verification_code "$soong_sbom_out/sbom.spdx"



  verify_packages_licenses "$soong_sbom_out/sbom.spdx"



  # Teardown

  cleanup "${out_dir}"
@@ -213,6 +236,41 @@ function verify_package_verification_code { 
  fi

}



function verify_packages_licenses {

  local sbom_file="$1"; shift



  num_of_packages=$(grep 'PackageName:' $sbom_file | wc -l)

  num_of_declared_licenses=$(grep 'PackageLicenseDeclared:' $sbom_file | wc -l)

  if [ "$num_of_packages" = "$num_of_declared_licenses" ]

  then

    echo "Number of packages with declared license is correct."

  else

    echo "Number of packages with declared license is WRONG."

    exit 1

  fi



  # PRODUCT and 7 prebuilt packages have "PackageLicenseDeclared: NOASSERTION"

  # All other packages have declared licenses

  num_of_packages_with_noassertion_license=$(grep 'PackageLicenseDeclared: NOASSERTION' $sbom_file | wc -l)

  if [ $num_of_packages_with_noassertion_license = 15 ]

  then

    echo "Number of packages with NOASSERTION license is correct."

  else

    echo "Number of packages with NOASSERTION license is WRONG."

    exit 1

  fi



  num_of_files=$(grep 'FileName:' $sbom_file | wc -l)

  num_of_concluded_licenses=$(grep 'LicenseConcluded:' $sbom_file | wc -l)

  if [ "$num_of_files" = "$num_of_concluded_licenses" ]

  then

    echo "Number of files with concluded license is correct."

  else

    echo "Number of files with concluded license is WRONG."

    exit 1

  fi

}



function test_sbom_unbundled_apex {

  # Setup

  out_dir="$(setup)"
@@ -274,7 +332,7 @@ function test_sbom_unbundled_apk { 


target_product=aosp_cf_x86_64_phone

target_release=trunk_staging

target_build_variant=userdebug

target_build_variant=eng

for i in "$@"; do

  case $i in

    TARGET_PRODUCT=*)

ui/build/test_build.go

+10 −3
Original line number Diff line number Diff line
@@ -15,14 +15,16 @@ 
package build



import (

	"android/soong/ui/metrics"

	"android/soong/ui/status"

	"bufio"

	"fmt"

	"path/filepath"

	"regexp"

	"runtime"

	"sort"

	"strings"



	"android/soong/ui/metrics"

	"android/soong/ui/status"

)



// Checks for files in the out directory that have a rule that depends on them but no rule to
@@ -84,6 +86,10 @@ func testForDanglingRules(ctx Context, config Config) { 
	// before running soong and ninja.

	releaseConfigDir := filepath.Join(outDir, "soong", "release-config")



	// out/target/product/<xxxxx>/build_fingerprint.txt is a source file created in sysprop.mk

	// ^out/target/product/[^/]+/build_fingerprint.txt$

	buildFingerPrintFilePattern := regexp.MustCompile("^" + filepath.Join(outDir, "target", "product") + "/[^/]+/build_fingerprint.txt$")



	danglingRules := make(map[string]bool)



	scanner := bufio.NewScanner(stdout)
@@ -100,7 +106,8 @@ func testForDanglingRules(ctx Context, config Config) { 
			line == dexpreoptConfigFilePath ||

			line == buildDatetimeFilePath ||

			line == bpglob ||

			strings.HasPrefix(line, releaseConfigDir) {

			strings.HasPrefix(line, releaseConfigDir) ||

			buildFingerPrintFilePattern.MatchString(line) {

			// Leaf node is in one of Soong's bootstrap directories, which do not have

			// full build rules in the primary build.ninja file.

			continue