Merge "Support long commands in RuleBuilder" into main (0674ba09) · Commits · e / os / android_build_soong

android/packaging.go

+4 −12

Original line number	Diff line number	Diff line
		@@ -18,7 +18,6 @@ import (
		"fmt"
		"path/filepath"
		"sort"
		"strings"

		"github.com/google/blueprint"
		"github.com/google/blueprint/gobtools"
		@@ -594,10 +593,6 @@ func (p PackagingBase) CopySpecsToDirs(ctx ModuleContext, builder RuleBuilder,
		}

		seenDir := make(map[string]bool)
		preparerPath := PathForModuleOut(ctx, "preparer.sh")
		cmd := builder.Command().Tool(preparerPath)
		var sb strings.Builder
		sb.WriteString("set -e\n")

		dirs := make([]WritablePath, 0, len(dirsToSpecs))
		for dir, _ := range dirsToSpecs {
		@@ -616,22 +611,19 @@ func (p PackagingBase) CopySpecsToDirs(ctx ModuleContext, builder RuleBuilder,
		entries = append(entries, ps.relPathInPackage)
		if _, ok := seenDir[destDir]; !ok {
		seenDir[destDir] = true
		sb.WriteString(fmt.Sprintf("mkdir -p %s\n", destDir))
		builder.Command().Textf("mkdir -p %s", destDir)
		}
		if ps.symlinkTarget == "" {
		cmd.Implicit(ps.srcPath)
		sb.WriteString(fmt.Sprintf("cp %s %s\n", ps.srcPath, destPath))
		builder.Command().Text("cp").Input(ps.srcPath).Text(destPath)
		} else {
		sb.WriteString(fmt.Sprintf("ln -sf %s %s\n", ps.symlinkTarget, destPath))
		builder.Command().Textf("ln -sf %s %s", ps.symlinkTarget, destPath)
		}
		if ps.executable {
		sb.WriteString(fmt.Sprintf("chmod a+x %s\n", destPath))
		builder.Command().Textf("chmod a+x %s", destPath)
		}
		}
		}

		WriteExecutableFileRuleVerbatim(ctx, preparerPath, sb.String())

		return entries
		}

android/rule_builder.go

+12 −0

Original line number	Diff line number	Diff line
		@@ -853,6 +853,18 @@ func (r *RuleBuilder) build(name string, desc string) {
		pool = localPool
		}

		// If the command length is getting close to linux's maximum, dump it to a file, which allows
		// for longer commands.
		if len(commandString) > 100000 {
		hasher := sha256.New()
		hasher.Write([]byte(output.String()))
		script := PathForOutput(r.ctx, "rule_builder_scripts", fmt.Sprintf("%x.sh", hasher.Sum(nil)))
		commandString = "set -eu\n\n" + commandString + "\n"
		WriteExecutableFileRuleVerbatim(r.ctx, script, commandString)
		inputs = append(inputs, script)
		commandString = script.String()
		}

		commandString = proptools.NinjaEscape(commandString)

		args_vars := make([]string, len(r.args))

filesystem/aconfig_files.go

+17 −43

Original line number	Diff line number	Diff line
		@@ -16,7 +16,6 @@ package filesystem

		import (
		"android/soong/android"
		"strings"

		"github.com/google/blueprint/proptools"
		)
		@@ -26,57 +25,34 @@ func (f filesystem) buildAconfigFlagsFiles(ctx android.ModuleContext, builder
		return
		}

		aconfigFlagsBuilderPath := android.PathForModuleOut(ctx, "aconfig_flags_builder.sh")
		aconfigToolPath := ctx.Config().HostToolPath(ctx, "aconfig")
		cmd := builder.Command().Tool(aconfigFlagsBuilderPath).Implicit(aconfigToolPath)

		var caches []string
		var caches []android.Path
		for _, ps := range specs {
		cmd.Implicits(ps.GetAconfigPaths())
		caches = append(caches, ps.GetAconfigPaths().Strings()...)
		caches = append(caches, ps.GetAconfigPaths()...)
		}
		caches = android.SortedUniqueStrings(caches)
		caches = android.SortedUniquePaths(caches)

		var sbCaches strings.Builder
		installAconfigFlagsPath := dir.Join(ctx, "etc", "aconfig_flags.pb")
		cmd := builder.Command().
		BuiltTool("aconfig").
		Text(" dump-cache --dedup --format protobuf --out").
		Output(installAconfigFlagsPath).
		Textf("--filter container:%s", f.PartitionType())
		for _, cache := range caches {
		sbCaches.WriteString(" --cache ")
		sbCaches.WriteString(cache)
		sbCaches.WriteString(" \\\n")
		cmd.FlagWithInput("--cache ", cache)
		}
		sbCaches.WriteRune('\n')

		var sb strings.Builder
		sb.WriteString("set -e\n")

		installAconfigFlagsPath := dir.Join(ctx, "etc", "aconfig_flags.pb")
		sb.WriteString(aconfigToolPath.String())
		sb.WriteString(" dump-cache --dedup --format protobuf --out ")
		sb.WriteString(installAconfigFlagsPath.String())
		sb.WriteString(" --filter container:")
		sb.WriteString(f.PartitionType())
		sb.WriteString(" \\\n")
		sb.WriteString(sbCaches.String())
		cmd.ImplicitOutput(installAconfigFlagsPath)
		f.appendToEntry(ctx, installAconfigFlagsPath)

		installAconfigStorageDir := dir.Join(ctx, "etc", "aconfig")
		sb.WriteString("mkdir -p ")
		sb.WriteString(installAconfigStorageDir.String())
		sb.WriteRune('\n')
		builder.Command().Text("mkdir -p").Text(installAconfigStorageDir.String())

		generatePartitionAconfigStorageFile := func(fileType, fileName string) {
		outputPath := installAconfigStorageDir.Join(ctx, fileName)
		sb.WriteString(aconfigToolPath.String())
		sb.WriteString(" create-storage --container ")
		sb.WriteString(f.PartitionType())
		sb.WriteString(" --file ")
		sb.WriteString(fileType)
		sb.WriteString(" --out ")
		sb.WriteString(outputPath.String())
		sb.WriteString(" --cache ")
		sb.WriteString(installAconfigFlagsPath.String())
		sb.WriteRune('\n')
		cmd.ImplicitOutput(outputPath)
		builder.Command().
		BuiltTool("aconfig").
		FlagWithArg("create-storage --container ", f.PartitionType()).
		FlagWithArg("--file ", fileType).
		FlagWithOutput("--out ", outputPath).
		FlagWithArg("--cache ", installAconfigFlagsPath.String())
		f.appendToEntry(ctx, outputPath)
		}

		@@ -86,6 +62,4 @@ func (f filesystem) buildAconfigFlagsFiles(ctx android.ModuleContext, builder
		generatePartitionAconfigStorageFile("flag_val", "flag.val")
		generatePartitionAconfigStorageFile("flag_info", "flag.info")
		}

		android.WriteExecutableFileRuleVerbatim(ctx, aconfigFlagsBuilderPath, sb.String())
		}

filesystem/filesystem_test.go

+7 −2

Original line number	Diff line number	Diff line
		@@ -16,6 +16,7 @@ package filesystem

		import (
		"os"
		"strings"
		"testing"

		"android/soong/android"
		@@ -182,10 +183,14 @@ func TestFileSystemFillsLinkerConfigWithStubLibs(t *testing.T) {
		module := result.ModuleForTests("myfilesystem", "android_common")
		output := module.Output("out/soong/.intermediates/myfilesystem/android_common/root/system/etc/linker.config.pb")

		fullCommand := output.RuleParams.Command
		startIndex := strings.Index(fullCommand, "conv_linker_config")
		linkerConfigCommand := fullCommand[startIndex:]

		android.AssertStringDoesContain(t, "linker.config.pb should have libfoo",
		output.RuleParams.Command, "libfoo.so")
		linkerConfigCommand, "libfoo.so")
		android.AssertStringDoesNotContain(t, "linker.config.pb should not have libbar",
		output.RuleParams.Command, "libbar.so")
		linkerConfigCommand, "libbar.so")
		}

		func registerComponent(ctx android.RegistrationContext) {

filesystem/fsverity_metadata.go

+41 −76

Original line number	Diff line number	Diff line
		@@ -66,62 +66,44 @@ func (f *filesystem) buildFsverityMetadataFiles(ctx android.ModuleContext, build
		return
		}

		fsverityBuilderPath := android.PathForModuleOut(ctx, "fsverity_builder.sh")
		metadataGeneratorPath := ctx.Config().HostToolPath(ctx, "fsverity_metadata_generator")
		fsverityPath := ctx.Config().HostToolPath(ctx, "fsverity")

		cmd := builder.Command().Tool(fsverityBuilderPath)

		// STEP 1: generate .fsv_meta
		var sb strings.Builder
		sb.WriteString("set -e\n")
		cmd.Implicit(metadataGeneratorPath).Implicit(fsverityPath)
		for _, spec := range matchedSpecs {
		// srcPath is copied by CopySpecsToDir()
		srcPath := rebasedDir.Join(ctx, spec.RelPathInPackage())
		destPath := rebasedDir.Join(ctx, spec.RelPathInPackage()+".fsv_meta")
		sb.WriteString(metadataGeneratorPath.String())
		sb.WriteString(" --fsverity-path ")
		sb.WriteString(fsverityPath.String())
		sb.WriteString(" --signature none --hash-alg sha256 --output ")
		sb.WriteString(destPath.String())
		sb.WriteRune(' ')
		sb.WriteString(srcPath.String())
		sb.WriteRune('\n')
		builder.Command().
		BuiltTool("fsverity_metadata_generator").
		FlagWithInput("--fsverity-path ", fsverityPath).
		FlagWithArg("--signature ", "none").
		FlagWithArg("--hash-alg ", "sha256").
		FlagWithArg("--output ", destPath.String()).
		Text(srcPath.String())
		f.appendToEntry(ctx, destPath)
		}

		// STEP 2: generate signed BuildManifest.apk
		// STEP 2-1: generate build_manifest.pb
		manifestGeneratorListPath := android.PathForModuleOut(ctx, "fsverity_manifest.list")
		f.writeManifestGeneratorListFile(ctx, manifestGeneratorListPath, matchedSpecs, rebasedDir)
		assetsPath := android.PathForModuleOut(ctx, "fsverity_manifest/assets")
		manifestPbPath := assetsPath.Join(ctx, "build_manifest.pb")
		manifestGeneratorPath := ctx.Config().HostToolPath(ctx, "fsverity_manifest_generator")
		cmd.Implicit(manifestGeneratorPath)
		sb.WriteString("rm -rf ")
		sb.WriteString(assetsPath.String())
		sb.WriteString(" && mkdir -p ")
		sb.WriteString(assetsPath.String())
		sb.WriteRune('\n')
		sb.WriteString(manifestGeneratorPath.String())
		sb.WriteString(" --fsverity-path ")
		sb.WriteString(fsverityPath.String())
		sb.WriteString(" --base-dir ")
		sb.WriteString(rootDir.String())
		sb.WriteString(" --output ")
		sb.WriteString(manifestPbPath.String())
		sb.WriteRune(' ')
		f.appendToEntry(ctx, manifestPbPath)
		builder.Command().Text("rm -rf " + assetsPath.String())
		builder.Command().Text("mkdir -p " + assetsPath.String())
		builder.Command().
		BuiltTool("fsverity_manifest_generator").
		FlagWithInput("--fsverity-path ", fsverityPath).
		FlagWithArg("--base-dir ", rootDir.String()).
		FlagWithArg("--output ", manifestPbPath.String()).
		FlagWithInput("@", manifestGeneratorListPath)

		manifestGeneratorListPath := android.PathForModuleOut(ctx, "fsverity_manifest.list")
		f.writeManifestGeneratorListFile(ctx, manifestGeneratorListPath, matchedSpecs, rebasedDir)
		sb.WriteRune('@')
		sb.WriteString(manifestGeneratorListPath.String())
		sb.WriteRune('\n')
		cmd.Implicit(manifestGeneratorListPath)
		f.appendToEntry(ctx, manifestPbPath)
		f.appendToEntry(ctx, manifestGeneratorListPath)

		// STEP 2-2: generate BuildManifest.apk (unsigned)
		aapt2Path := ctx.Config().HostToolPath(ctx, "aapt2")
		apkNameSuffix := ""
		if f.PartitionType() == "system_ext" {
		//https://source.corp.google.com/h/googleplex-android/platform/build/+/e392d2b486c2d4187b20a72b1c67cc737ecbcca5:core/Makefile;l=3410;drc=ea8f34bc1d6e63656b4ec32f2391e9d54b3ebb6b;bpv=1;bpt=0
		@@ -131,55 +113,38 @@ func (f *filesystem) buildFsverityMetadataFiles(ctx android.ModuleContext, build
		idsigPath := rebasedDir.Join(ctx, "etc", "security", "fsverity", fmt.Sprintf("BuildManifest%s.apk.idsig", apkNameSuffix))
		manifestTemplatePath := android.PathForSource(ctx, "system/security/fsverity/AndroidManifest.xml")
		libs := android.PathsForModuleSrc(ctx, f.properties.Fsverity.Libs)
		cmd.Implicit(aapt2Path)
		cmd.Implicit(manifestTemplatePath)
		cmd.Implicits(libs)
		cmd.ImplicitOutput(apkPath)

		sb.WriteString(aapt2Path.String())
		sb.WriteString(" link -o ")
		sb.WriteString(apkPath.String())
		sb.WriteString(" -A ")
		sb.WriteString(assetsPath.String())
		for _, lib := range libs {
		sb.WriteString(" -I ")
		sb.WriteString(lib.String())
		}

		minSdkVersion := ctx.Config().PlatformSdkCodename()
		if minSdkVersion == "REL" {
		minSdkVersion = ctx.Config().PlatformSdkVersion().String()
		}
		sb.WriteString(" --min-sdk-version ")
		sb.WriteString(minSdkVersion)
		sb.WriteString(" --version-code ")
		sb.WriteString(ctx.Config().PlatformSdkVersion().String())
		sb.WriteString(" --version-name ")
		sb.WriteString(ctx.Config().AppsDefaultVersionName())
		sb.WriteString(" --manifest ")
		sb.WriteString(manifestTemplatePath.String())
		sb.WriteString(" --rename-manifest-package com.android.security.fsverity_metadata.")
		sb.WriteString(f.partitionName())
		sb.WriteRune('\n')

		unsignedApkCommand := builder.Command().
		BuiltTool("aapt2").
		Text("link").
		FlagWithOutput("-o ", apkPath).
		FlagWithArg("-A ", assetsPath.String())
		for _, lib := range libs {
		unsignedApkCommand.FlagWithInput("-I ", lib)
		}
		unsignedApkCommand.
		FlagWithArg("--min-sdk-version ", minSdkVersion).
		FlagWithArg("--version-code ", ctx.Config().PlatformSdkVersion().String()).
		FlagWithArg("--version-name ", ctx.Config().AppsDefaultVersionName()).
		FlagWithInput("--manifest ", manifestTemplatePath).
		Text(" --rename-manifest-package com.android.security.fsverity_metadata." + f.partitionName())

		f.appendToEntry(ctx, apkPath)

		// STEP 2-3: sign BuildManifest.apk
		apksignerPath := ctx.Config().HostToolPath(ctx, "apksigner")
		pemPath, keyPath := ctx.Config().DefaultAppCertificate(ctx)
		cmd.Implicit(apksignerPath)
		cmd.Implicit(pemPath)
		cmd.Implicit(keyPath)
		cmd.ImplicitOutput(idsigPath)
		sb.WriteString(apksignerPath.String())
		sb.WriteString(" sign --in ")
		sb.WriteString(apkPath.String())
		sb.WriteString(" --cert ")
		sb.WriteString(pemPath.String())
		sb.WriteString(" --key ")
		sb.WriteString(keyPath.String())
		sb.WriteRune('\n')
		builder.Command().
		BuiltTool("apksigner").
		Text("sign").
		FlagWithArg("--in ", apkPath.String()).
		FlagWithInput("--cert ", pemPath).
		FlagWithInput("--key ", keyPath).
		ImplicitOutput(idsigPath)

		f.appendToEntry(ctx, idsigPath)

		android.WriteExecutableFileRuleVerbatim(ctx, fsverityBuilderPath, sb.String())
		}