Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit b664afc7 authored by Treehugger Robot's avatar Treehugger Robot Committed by Automerger Merge Worker
Browse files

Merge "Support long commands in RuleBuilder" into main am: 0674ba09 

Original change: https://android-review.googlesource.com/c/platform/build/soong/+/3387734



Change-Id: Ief34b4db7e68b2624f8c7b8518332d35286a2a04
Signed-off-by: default avatarAutomerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
parents c0f8cfda 0674ba09

android/packaging.go

+4 −12
Original line number Diff line number Diff line
@@ -18,7 +18,6 @@ import ( 
	"fmt"

	"path/filepath"

	"sort"

	"strings"



	"github.com/google/blueprint"

	"github.com/google/blueprint/gobtools"
@@ -594,10 +593,6 @@ func (p *PackagingBase) CopySpecsToDirs(ctx ModuleContext, builder *RuleBuilder, 
	}



	seenDir := make(map[string]bool)

	preparerPath := PathForModuleOut(ctx, "preparer.sh")

	cmd := builder.Command().Tool(preparerPath)

	var sb strings.Builder

	sb.WriteString("set -e\n")



	dirs := make([]WritablePath, 0, len(dirsToSpecs))

	for dir, _ := range dirsToSpecs {
@@ -616,22 +611,19 @@ func (p *PackagingBase) CopySpecsToDirs(ctx ModuleContext, builder *RuleBuilder, 
			entries = append(entries, ps.relPathInPackage)

			if _, ok := seenDir[destDir]; !ok {

				seenDir[destDir] = true

				sb.WriteString(fmt.Sprintf("mkdir -p %s\n", destDir))

				builder.Command().Textf("mkdir -p %s", destDir)

			}

			if ps.symlinkTarget == "" {

				cmd.Implicit(ps.srcPath)

				sb.WriteString(fmt.Sprintf("cp %s %s\n", ps.srcPath, destPath))

				builder.Command().Text("cp").Input(ps.srcPath).Text(destPath)

			} else {

				sb.WriteString(fmt.Sprintf("ln -sf %s %s\n", ps.symlinkTarget, destPath))

				builder.Command().Textf("ln -sf %s %s", ps.symlinkTarget, destPath)

			}

			if ps.executable {

				sb.WriteString(fmt.Sprintf("chmod a+x %s\n", destPath))

				builder.Command().Textf("chmod a+x %s", destPath)

			}

		}

	}



	WriteExecutableFileRuleVerbatim(ctx, preparerPath, sb.String())



	return entries

}

android/rule_builder.go

+12 −0
Original line number Diff line number Diff line
@@ -853,6 +853,18 @@ func (r *RuleBuilder) build(name string, desc string) { 
		pool = localPool

	}



	// If the command length is getting close to linux's maximum, dump it to a file, which allows

	// for longer commands.

	if len(commandString) > 100000 {

		hasher := sha256.New()

		hasher.Write([]byte(output.String()))

		script := PathForOutput(r.ctx, "rule_builder_scripts", fmt.Sprintf("%x.sh", hasher.Sum(nil)))

		commandString = "set -eu\n\n" + commandString + "\n"

		WriteExecutableFileRuleVerbatim(r.ctx, script, commandString)

		inputs = append(inputs, script)

		commandString = script.String()

	}



	commandString = proptools.NinjaEscape(commandString)



	args_vars := make([]string, len(r.args))

filesystem/aconfig_files.go

+17 −43
Original line number Diff line number Diff line
@@ -16,7 +16,6 @@ package filesystem 


import (

	"android/soong/android"

	"strings"



	"github.com/google/blueprint/proptools"

)
@@ -26,57 +25,34 @@ func (f *filesystem) buildAconfigFlagsFiles(ctx android.ModuleContext, builder * 
		return

	}



	aconfigFlagsBuilderPath := android.PathForModuleOut(ctx, "aconfig_flags_builder.sh")

	aconfigToolPath := ctx.Config().HostToolPath(ctx, "aconfig")

	cmd := builder.Command().Tool(aconfigFlagsBuilderPath).Implicit(aconfigToolPath)



	var caches []string

	var caches []android.Path

	for _, ps := range specs {

		cmd.Implicits(ps.GetAconfigPaths())

		caches = append(caches, ps.GetAconfigPaths().Strings()...)

		caches = append(caches, ps.GetAconfigPaths()...)

	}

	caches = android.SortedUniqueStrings(caches)

	caches = android.SortedUniquePaths(caches)



	var sbCaches strings.Builder

	installAconfigFlagsPath := dir.Join(ctx, "etc", "aconfig_flags.pb")

	cmd := builder.Command().

		BuiltTool("aconfig").

		Text(" dump-cache --dedup --format protobuf --out").

		Output(installAconfigFlagsPath).

		Textf("--filter container:%s", f.PartitionType())

	for _, cache := range caches {

		sbCaches.WriteString("  --cache ")

		sbCaches.WriteString(cache)

		sbCaches.WriteString(" \\\n")

		cmd.FlagWithInput("--cache ", cache)

	}

	sbCaches.WriteRune('\n')



	var sb strings.Builder

	sb.WriteString("set -e\n")



	installAconfigFlagsPath := dir.Join(ctx, "etc", "aconfig_flags.pb")

	sb.WriteString(aconfigToolPath.String())

	sb.WriteString(" dump-cache --dedup --format protobuf --out ")

	sb.WriteString(installAconfigFlagsPath.String())

	sb.WriteString(" --filter container:")

	sb.WriteString(f.PartitionType())

	sb.WriteString(" \\\n")

	sb.WriteString(sbCaches.String())

	cmd.ImplicitOutput(installAconfigFlagsPath)

	f.appendToEntry(ctx, installAconfigFlagsPath)



	installAconfigStorageDir := dir.Join(ctx, "etc", "aconfig")

	sb.WriteString("mkdir -p ")

	sb.WriteString(installAconfigStorageDir.String())

	sb.WriteRune('\n')

	builder.Command().Text("mkdir -p").Text(installAconfigStorageDir.String())



	generatePartitionAconfigStorageFile := func(fileType, fileName string) {

		outputPath := installAconfigStorageDir.Join(ctx, fileName)

		sb.WriteString(aconfigToolPath.String())

		sb.WriteString(" create-storage --container ")

		sb.WriteString(f.PartitionType())

		sb.WriteString(" --file ")

		sb.WriteString(fileType)

		sb.WriteString(" --out ")

		sb.WriteString(outputPath.String())

		sb.WriteString(" --cache ")

		sb.WriteString(installAconfigFlagsPath.String())

		sb.WriteRune('\n')

		cmd.ImplicitOutput(outputPath)

		builder.Command().

			BuiltTool("aconfig").

			FlagWithArg("create-storage --container ", f.PartitionType()).

			FlagWithArg("--file ", fileType).

			FlagWithOutput("--out ", outputPath).

			FlagWithArg("--cache ", installAconfigFlagsPath.String())

		f.appendToEntry(ctx, outputPath)

	}
@@ -86,6 +62,4 @@ func (f *filesystem) buildAconfigFlagsFiles(ctx android.ModuleContext, builder * 
		generatePartitionAconfigStorageFile("flag_val", "flag.val")

		generatePartitionAconfigStorageFile("flag_info", "flag.info")

	}



	android.WriteExecutableFileRuleVerbatim(ctx, aconfigFlagsBuilderPath, sb.String())

}

filesystem/filesystem_test.go

+7 −2
Original line number Diff line number Diff line
@@ -16,6 +16,7 @@ package filesystem 


import (

	"os"

	"strings"

	"testing"



	"android/soong/android"
@@ -182,10 +183,14 @@ func TestFileSystemFillsLinkerConfigWithStubLibs(t *testing.T) { 
	module := result.ModuleForTests("myfilesystem", "android_common")

	output := module.Output("out/soong/.intermediates/myfilesystem/android_common/root/system/etc/linker.config.pb")



	fullCommand := output.RuleParams.Command

	startIndex := strings.Index(fullCommand, "conv_linker_config")

	linkerConfigCommand := fullCommand[startIndex:]



	android.AssertStringDoesContain(t, "linker.config.pb should have libfoo",

		output.RuleParams.Command, "libfoo.so")

		linkerConfigCommand, "libfoo.so")

	android.AssertStringDoesNotContain(t, "linker.config.pb should not have libbar",

		output.RuleParams.Command, "libbar.so")

		linkerConfigCommand, "libbar.so")

}



func registerComponent(ctx android.RegistrationContext) {

filesystem/fsverity_metadata.go

+41 −76
Original line number Diff line number Diff line
@@ -66,62 +66,44 @@ func (f *filesystem) buildFsverityMetadataFiles(ctx android.ModuleContext, build 
		return

	}



	fsverityBuilderPath := android.PathForModuleOut(ctx, "fsverity_builder.sh")

	metadataGeneratorPath := ctx.Config().HostToolPath(ctx, "fsverity_metadata_generator")

	fsverityPath := ctx.Config().HostToolPath(ctx, "fsverity")



	cmd := builder.Command().Tool(fsverityBuilderPath)



	// STEP 1: generate .fsv_meta

	var sb strings.Builder

	sb.WriteString("set -e\n")

	cmd.Implicit(metadataGeneratorPath).Implicit(fsverityPath)

	for _, spec := range matchedSpecs {

		// srcPath is copied by CopySpecsToDir()

		srcPath := rebasedDir.Join(ctx, spec.RelPathInPackage())

		destPath := rebasedDir.Join(ctx, spec.RelPathInPackage()+".fsv_meta")

		sb.WriteString(metadataGeneratorPath.String())

		sb.WriteString(" --fsverity-path ")

		sb.WriteString(fsverityPath.String())

		sb.WriteString(" --signature none --hash-alg sha256 --output ")

		sb.WriteString(destPath.String())

		sb.WriteRune(' ')

		sb.WriteString(srcPath.String())

		sb.WriteRune('\n')

		builder.Command().

			BuiltTool("fsverity_metadata_generator").

			FlagWithInput("--fsverity-path ", fsverityPath).

			FlagWithArg("--signature ", "none").

			FlagWithArg("--hash-alg ", "sha256").

			FlagWithArg("--output ", destPath.String()).

			Text(srcPath.String())

		f.appendToEntry(ctx, destPath)

	}



	// STEP 2: generate signed BuildManifest.apk

	// STEP 2-1: generate build_manifest.pb

	manifestGeneratorListPath := android.PathForModuleOut(ctx, "fsverity_manifest.list")

	f.writeManifestGeneratorListFile(ctx, manifestGeneratorListPath, matchedSpecs, rebasedDir)

	assetsPath := android.PathForModuleOut(ctx, "fsverity_manifest/assets")

	manifestPbPath := assetsPath.Join(ctx, "build_manifest.pb")

	manifestGeneratorPath := ctx.Config().HostToolPath(ctx, "fsverity_manifest_generator")

	cmd.Implicit(manifestGeneratorPath)

	sb.WriteString("rm -rf ")

	sb.WriteString(assetsPath.String())

	sb.WriteString(" && mkdir -p ")

	sb.WriteString(assetsPath.String())

	sb.WriteRune('\n')

	sb.WriteString(manifestGeneratorPath.String())

	sb.WriteString(" --fsverity-path ")

	sb.WriteString(fsverityPath.String())

	sb.WriteString(" --base-dir ")

	sb.WriteString(rootDir.String())

	sb.WriteString(" --output ")

	sb.WriteString(manifestPbPath.String())

	sb.WriteRune(' ')

	f.appendToEntry(ctx, manifestPbPath)

	builder.Command().Text("rm -rf " + assetsPath.String())

	builder.Command().Text("mkdir -p " + assetsPath.String())

	builder.Command().

		BuiltTool("fsverity_manifest_generator").

		FlagWithInput("--fsverity-path ", fsverityPath).

		FlagWithArg("--base-dir ", rootDir.String()).

		FlagWithArg("--output ", manifestPbPath.String()).

		FlagWithInput("@", manifestGeneratorListPath)



	manifestGeneratorListPath := android.PathForModuleOut(ctx, "fsverity_manifest.list")

	f.writeManifestGeneratorListFile(ctx, manifestGeneratorListPath, matchedSpecs, rebasedDir)

	sb.WriteRune('@')

	sb.WriteString(manifestGeneratorListPath.String())

	sb.WriteRune('\n')

	cmd.Implicit(manifestGeneratorListPath)

	f.appendToEntry(ctx, manifestPbPath)

	f.appendToEntry(ctx, manifestGeneratorListPath)



	// STEP 2-2: generate BuildManifest.apk (unsigned)

	aapt2Path := ctx.Config().HostToolPath(ctx, "aapt2")

	apkNameSuffix := ""

	if f.PartitionType() == "system_ext" {

		//https://source.corp.google.com/h/googleplex-android/platform/build/+/e392d2b486c2d4187b20a72b1c67cc737ecbcca5:core/Makefile;l=3410;drc=ea8f34bc1d6e63656b4ec32f2391e9d54b3ebb6b;bpv=1;bpt=0
@@ -131,55 +113,38 @@ func (f *filesystem) buildFsverityMetadataFiles(ctx android.ModuleContext, build 
	idsigPath := rebasedDir.Join(ctx, "etc", "security", "fsverity", fmt.Sprintf("BuildManifest%s.apk.idsig", apkNameSuffix))

	manifestTemplatePath := android.PathForSource(ctx, "system/security/fsverity/AndroidManifest.xml")

	libs := android.PathsForModuleSrc(ctx, f.properties.Fsverity.Libs)

	cmd.Implicit(aapt2Path)

	cmd.Implicit(manifestTemplatePath)

	cmd.Implicits(libs)

	cmd.ImplicitOutput(apkPath)



	sb.WriteString(aapt2Path.String())

	sb.WriteString(" link -o ")

	sb.WriteString(apkPath.String())

	sb.WriteString(" -A ")

	sb.WriteString(assetsPath.String())

	for _, lib := range libs {

		sb.WriteString(" -I ")

		sb.WriteString(lib.String())

	}



	minSdkVersion := ctx.Config().PlatformSdkCodename()

	if minSdkVersion == "REL" {

		minSdkVersion = ctx.Config().PlatformSdkVersion().String()

	}

	sb.WriteString(" --min-sdk-version ")

	sb.WriteString(minSdkVersion)

	sb.WriteString(" --version-code ")

	sb.WriteString(ctx.Config().PlatformSdkVersion().String())

	sb.WriteString(" --version-name ")

	sb.WriteString(ctx.Config().AppsDefaultVersionName())

	sb.WriteString(" --manifest ")

	sb.WriteString(manifestTemplatePath.String())

	sb.WriteString(" --rename-manifest-package com.android.security.fsverity_metadata.")

	sb.WriteString(f.partitionName())

	sb.WriteRune('\n')



	unsignedApkCommand := builder.Command().

		BuiltTool("aapt2").

		Text("link").

		FlagWithOutput("-o ", apkPath).

		FlagWithArg("-A ", assetsPath.String())

	for _, lib := range libs {

		unsignedApkCommand.FlagWithInput("-I ", lib)

	}

	unsignedApkCommand.

		FlagWithArg("--min-sdk-version ", minSdkVersion).

		FlagWithArg("--version-code ", ctx.Config().PlatformSdkVersion().String()).

		FlagWithArg("--version-name ", ctx.Config().AppsDefaultVersionName()).

		FlagWithInput("--manifest ", manifestTemplatePath).

		Text(" --rename-manifest-package com.android.security.fsverity_metadata." + f.partitionName())



	f.appendToEntry(ctx, apkPath)



	// STEP 2-3: sign BuildManifest.apk

	apksignerPath := ctx.Config().HostToolPath(ctx, "apksigner")

	pemPath, keyPath := ctx.Config().DefaultAppCertificate(ctx)

	cmd.Implicit(apksignerPath)

	cmd.Implicit(pemPath)

	cmd.Implicit(keyPath)

	cmd.ImplicitOutput(idsigPath)

	sb.WriteString(apksignerPath.String())

	sb.WriteString(" sign --in ")

	sb.WriteString(apkPath.String())

	sb.WriteString(" --cert ")

	sb.WriteString(pemPath.String())

	sb.WriteString(" --key ")

	sb.WriteString(keyPath.String())

	sb.WriteRune('\n')

	builder.Command().

		BuiltTool("apksigner").

		Text("sign").

		FlagWithArg("--in ", apkPath.String()).

		FlagWithInput("--cert ", pemPath).

		FlagWithInput("--key ", keyPath).

		ImplicitOutput(idsigPath)



	f.appendToEntry(ctx, idsigPath)



	android.WriteExecutableFileRuleVerbatim(ctx, fsverityBuilderPath, sb.String())

}