Merge changes Iab4e09d9,Icf2f24dd,I15be5ef1,Ic0db9619 (d2db7953) · Commits · e / os / android_build_soong

android/Android.bp

+1 −0

Original line number	Diff line number	Diff line
		@@ -13,6 +13,7 @@ bootstrap_go_package {
		"soong-android-soongconfig",
		"soong-bazel",
		"soong-cquery",
		"soong-remoteexec",
		"soong-shared",
		"soong-ui-metrics_proto",
		],

android/config.go

+5 −0

Original line number	Diff line number	Diff line
		@@ -35,6 +35,7 @@ import (
		"github.com/google/blueprint/proptools"

		"android/soong/android/soongconfig"
		"android/soong/remoteexec"
		)

		// Bool re-exports proptools.Bool for the android package.
		@@ -1774,3 +1775,7 @@ func (c *config) NonUpdatableBootJars() ConfiguredJarList {
		func (c *config) UpdatableBootJars() ConfiguredJarList {
		return c.productVariables.UpdatableBootJars
		}

		func (c *config) RBEWrapper() string {
		return c.GetenvWithDefault("RBE_WRAPPER", remoteexec.DefaultWrapperPath)
		}

android/defs.go

+4 −0

Original line number	Diff line number	Diff line
		@@ -124,6 +124,10 @@ var (

		func init() {
		pctx.Import("github.com/google/blueprint/bootstrap")

		pctx.VariableFunc("RBEWrapper", func(ctx PackageVarContext) string {
		return ctx.Config().RBEWrapper()
		})
		}

		var (

android/package_ctx.go

+39 −0

Original line number	Diff line number	Diff line
		@@ -19,6 +19,8 @@ import (
		"strings"

		"github.com/google/blueprint"

		"android/soong/remoteexec"
		)

		// PackageContext is a wrapper for blueprint.PackageContext that adds
		@@ -260,3 +262,40 @@ func (p PackageContext) AndroidRemoteStaticRule(name string, supports RemoteRule
		return params, nil
		}, argNames...)
		}

		// RemoteStaticRules returns a pair of rules based on the given RuleParams, where the first rule is a
		// locally executable rule and the second rule is a remotely executable rule. commonArgs are args
		// used for both the local and remotely executable rules. reArgs are used only for remote
		// execution.
		func (p PackageContext) RemoteStaticRules(name string, ruleParams blueprint.RuleParams, reParams *remoteexec.REParams, commonArgs []string, reArgs []string) (blueprint.Rule, blueprint.Rule) {
		ruleParamsRE := ruleParams
		ruleParams.Command = strings.ReplaceAll(ruleParams.Command, "$reTemplate", "")
		ruleParamsRE.Command = strings.ReplaceAll(ruleParamsRE.Command, "$reTemplate", reParams.Template())

		return p.AndroidStaticRule(name, ruleParams, commonArgs...),
		p.AndroidRemoteStaticRule(name+"RE", RemoteRuleSupports{RBE: true}, ruleParamsRE, append(commonArgs, reArgs...)...)
		}

		// MultiCommandStaticRules returns a pair of rules based on the given RuleParams, where the first
		// rule is a locally executable rule and the second rule is a remotely executable rule. This
		// function supports multiple remote execution wrappers placed in the template when commands are
		// chained together with &&. commonArgs are args used for both the local and remotely executable
		// rules. reArgs are args used only for remote execution.
		func (p PackageContext) MultiCommandRemoteStaticRules(name string, ruleParams blueprint.RuleParams, reParams map[string]*remoteexec.REParams, commonArgs []string, reArgs []string) (blueprint.Rule, blueprint.Rule) {
		ruleParamsRE := ruleParams
		for k, v := range reParams {
		ruleParams.Command = strings.ReplaceAll(ruleParams.Command, k, "")
		ruleParamsRE.Command = strings.ReplaceAll(ruleParamsRE.Command, k, v.Template())
		}

		return p.AndroidStaticRule(name, ruleParams, commonArgs...),
		p.AndroidRemoteStaticRule(name+"RE", RemoteRuleSupports{RBE: true}, ruleParamsRE, append(commonArgs, reArgs...)...)
		}

		// StaticVariableWithEnvOverride creates a static variable that evaluates to the value of the given
		// environment variable if set, otherwise the given default.
		func (p PackageContext) StaticVariableWithEnvOverride(name, envVar, defaultVal string) blueprint.Variable {
		return p.VariableFunc(name, func(ctx PackageVarContext) string {
		return ctx.Config().GetenvWithDefault(envVar, defaultVal)
		})
		}

android/rule_builder.go

+166 −14

Original line number	Diff line number	Diff line
		@@ -27,6 +27,7 @@ import (
		"github.com/google/blueprint/proptools"

		"android/soong/cmd/sbox/sbox_proto"
		"android/soong/remoteexec"
		"android/soong/shared"
		)

		@@ -48,8 +49,10 @@ type RuleBuilder struct {
		sbox bool
		highmem bool
		remoteable RemoteRuleSupports
		rbeParams *remoteexec.REParams
		outDir WritablePath
		sboxTools bool
		sboxInputs bool
		sboxManifestPath WritablePath
		missingDeps []string
		}
		@@ -119,6 +122,18 @@ func (r RuleBuilder) Remoteable(supports RemoteRuleSupports) RuleBuilder {
		return r
		}

		// Rewrapper marks the rule as running inside rewrapper using the given params in order to support
		// running on RBE. During RuleBuilder.Build the params will be combined with the inputs, outputs
		// and tools known to RuleBuilder to prepend an appropriate rewrapper command line to the rule's
		// command line.
		func (r RuleBuilder) Rewrapper(params remoteexec.REParams) *RuleBuilder {
		if !r.sboxInputs {
		panic(fmt.Errorf("RuleBuilder.Rewrapper must be called after RuleBuilder.SandboxInputs"))
		}
		r.rbeParams = params
		return r
		}

		// Sbox marks the rule as needing to be wrapped by sbox. The outputDir should point to the output
		// directory that sbox will wipe. It should not be written to by any other rule. manifestPath should
		// point to a location where sbox's manifest will be written and must be outside outputDir. sbox
		@@ -155,6 +170,25 @@ func (r RuleBuilder) SandboxTools() RuleBuilder {
		return r
		}

		// SandboxInputs enables input sandboxing for the rule by copying any referenced inputs into the
		// sandbox. It also implies SandboxTools().
		//
		// Sandboxing inputs requires RuleBuilder to be aware of all references to input paths. Paths
		// that are passed to RuleBuilder outside of the methods that expect inputs, for example
		// FlagWithArg, must use RuleBuilderCommand.PathForInput to translate the path to one that matches
		// the sandbox layout.
		func (r RuleBuilder) SandboxInputs() RuleBuilder {
		if !r.sbox {
		panic("SandboxInputs() must be called after Sbox()")
		}
		if len(r.commands) > 0 {
		panic("SandboxInputs() may not be called after Command()")
		}
		r.sboxTools = true
		r.sboxInputs = true
		return r
		}

		// Install associates an output of the rule with an install location, which can be retrieved later using
		// RuleBuilder.Installs.
		func (r *RuleBuilder) Install(from Path, to string) {
		@@ -425,6 +459,26 @@ func (r RuleBuilder) depFileMergerCmd(depFiles WritablePaths) RuleBuilderComma
		Inputs(depFiles.Paths())
		}

		// composeRspFileContent returns a string that will serve as the contents of the rsp file to pass
		// the listed input files to the command running in the sandbox.
		func (r *RuleBuilder) composeRspFileContent(rspFileInputs Paths) string {
		if r.sboxInputs {
		if len(rspFileInputs) > 0 {
		// When SandboxInputs is used the paths need to be rewritten to be relative to the sandbox
		// directory so that they are valid after sbox chdirs into the sandbox directory.
		return proptools.NinjaEscape(strings.Join(r.sboxPathsForInputsRel(rspFileInputs), " "))
		} else {
		// If the list of inputs is empty fall back to "$in" so that the rspfilecontent Ninja
		// variable is set to something non-empty, otherwise ninja will complain. The inputs
		// will be empty (all the non-rspfile inputs are implicits), so $in will evaluate to
		// an empty string.
		return "$in"
		}
		} else {
		return "$in"
		}
		}

		// Build adds the built command line to the build graph, with dependencies on Inputs and Tools, and output files for
		// Outputs.
		func (r *RuleBuilder) Build(name string, desc string) {
		@@ -511,6 +565,27 @@ func (r *RuleBuilder) Build(name string, desc string) {
		}
		}

		// If sandboxing inputs is enabled, add copy rules to the manifest to copy each input
		// into the sbox directory.
		if r.sboxInputs {
		for _, input := range inputs {
		command.CopyBefore = append(command.CopyBefore, &sbox_proto.Copy{
		From: proto.String(input.String()),
		To: proto.String(r.sboxPathForInputRel(input)),
		})
		}

		// If using an rsp file copy it into the sbox directory.
		if rspFilePath != nil {
		command.CopyBefore = append(command.CopyBefore, &sbox_proto.Copy{
		From: proto.String(rspFilePath.String()),
		To: proto.String(r.sboxPathForInputRel(rspFilePath)),
		})
		}

		command.Chdir = proto.Bool(true)
		}

		// Add copy rules to the manifest to copy each output file from the sbox directory.
		// to the output directory after running the commands.
		sboxOutputs := make([]string, len(outputs))
		@@ -564,6 +639,25 @@ func (r *RuleBuilder) Build(name string, desc string) {
		commandString = sboxCmd.buf.String()
		tools = append(tools, sboxCmd.tools...)
		inputs = append(inputs, sboxCmd.inputs...)

		if r.rbeParams != nil {
		var remoteInputs []string
		remoteInputs = append(remoteInputs, inputs.Strings()...)
		remoteInputs = append(remoteInputs, tools.Strings()...)
		remoteInputs = append(remoteInputs, rspFileInputs.Strings()...)
		if rspFilePath != nil {
		remoteInputs = append(remoteInputs, rspFilePath.String())
		}
		inputsListFile := r.sboxManifestPath.ReplaceExtension(r.ctx, "rbe_inputs.list")
		inputsListContents := rspFileForInputs(remoteInputs)
		WriteFileRule(r.ctx, inputsListFile, inputsListContents)
		inputs = append(inputs, inputsListFile)

		r.rbeParams.OutputFiles = outputs.Strings()
		r.rbeParams.RSPFile = inputsListFile.String()
		rewrapperCommand := r.rbeParams.NoVarTemplate(r.ctx.Config().RBEWrapper())
		commandString = rewrapperCommand + " bash -c '" + strings.ReplaceAll(commandString, `'`, `'\''`) + "'"
		}
		} else {
		// If not using sbox the rule will run the command directly, put the hash of the
		// list of input files in a comment at the end of the command line to ensure ninja
		@@ -580,7 +674,7 @@ func (r *RuleBuilder) Build(name string, desc string) {
		var rspFile, rspFileContent string
		if rspFilePath != nil {
		rspFile = rspFilePath.String()
		rspFileContent = "$in"
		rspFileContent = r.composeRspFileContent(rspFileInputs)
		}

		var pool blueprint.Pool
		@@ -636,27 +730,43 @@ type RuleBuilderCommand struct {
		}

		func (c *RuleBuilderCommand) addInput(path Path) string {
		if c.rule.sbox {
		if rel, isRel, _ := maybeRelErr(c.rule.outDir.String(), path.String()); isRel {
		return filepath.Join(sboxOutDir, rel)
		c.inputs = append(c.inputs, path)
		return c.PathForInput(path)
		}

		func (c *RuleBuilderCommand) addImplicit(path Path) {
		c.implicits = append(c.implicits, path)
		}
		c.inputs = append(c.inputs, path)
		return path.String()

		func (c *RuleBuilderCommand) addOrderOnly(path Path) {
		c.orderOnlys = append(c.orderOnlys, path)
		}

		func (c *RuleBuilderCommand) addImplicit(path Path) string {
		// PathForInput takes an input path and returns the appropriate path to use on the command line. If
		// sbox was enabled via a call to RuleBuilder.Sbox() and the path was an output path it returns a
		// path with the placeholder prefix used for outputs in sbox. If sbox is not enabled it returns the
		// original path.
		func (c *RuleBuilderCommand) PathForInput(path Path) string {
		if c.rule.sbox {
		if rel, isRel, _ := maybeRelErr(c.rule.outDir.String(), path.String()); isRel {
		return filepath.Join(sboxOutDir, rel)
		rel, inSandbox := c.rule._sboxPathForInputRel(path)
		if inSandbox {
		rel = filepath.Join(sboxSandboxBaseDir, rel)
		}
		return rel
		}
		c.implicits = append(c.implicits, path)
		return path.String()
		}

		func (c *RuleBuilderCommand) addOrderOnly(path Path) {
		c.orderOnlys = append(c.orderOnlys, path)
		// PathsForInputs takes a list of input paths and returns the appropriate paths to use on the
		// command line. If sbox was enabled via a call to RuleBuilder.Sbox() a path was an output path, it
		// returns the path with the placeholder prefix used for outputs in sbox. If sbox is not enabled it
		// returns the original paths.
		func (c *RuleBuilderCommand) PathsForInputs(paths Paths) []string {
		ret := make([]string, len(paths))
		for i, path := range paths {
		ret[i] = c.PathForInput(path)
		}
		return ret
		}

		// PathForOutput takes an output path and returns the appropriate path to use on the command
		@@ -690,6 +800,37 @@ func sboxPathForToolRel(ctx BuilderContext, path Path) string {
		return filepath.Join(sboxToolsSubDir, "src", path.String())
		}

		func (r *RuleBuilder) _sboxPathForInputRel(path Path) (rel string, inSandbox bool) {
		// Errors will be handled in RuleBuilder.Build where we have a context to report them
		rel, isRelSboxOut, _ := maybeRelErr(r.outDir.String(), path.String())
		if isRelSboxOut {
		return filepath.Join(sboxOutSubDir, rel), true
		}
		if r.sboxInputs {
		// When sandboxing inputs all inputs have to be copied into the sandbox. Input files that
		// are outputs of other rules could be an arbitrary absolute path if OUT_DIR is set, so they
		// will be copied to relative paths under __SBOX_OUT_DIR__/out.
		rel, isRelOut, _ := maybeRelErr(PathForOutput(r.ctx).String(), path.String())
		if isRelOut {
		return filepath.Join(sboxOutSubDir, rel), true
		}
		}
		return path.String(), false
		}

		func (r *RuleBuilder) sboxPathForInputRel(path Path) string {
		rel, _ := r._sboxPathForInputRel(path)
		return rel
		}

		func (r *RuleBuilder) sboxPathsForInputsRel(paths Paths) []string {
		ret := make([]string, len(paths))
		for i, path := range paths {
		ret[i] = r.sboxPathForInputRel(path)
		}
		return ret
		}

		// SboxPathForPackagedTool takes a PackageSpec for a tool and returns the corresponding path for the
		// tool after copying it into the sandbox. This can be used on the RuleBuilder command line to
		// reference the tool.
		@@ -1053,7 +1194,7 @@ func (c *RuleBuilderCommand) FlagWithRspFileInputList(flag string, rspFile Writa
		}
		}

		c.FlagWithArg(flag, rspFile.String())
		c.FlagWithArg(flag, c.PathForInput(rspFile))
		return c
		}

		@@ -1122,3 +1263,14 @@ func (builderContextForTests) Rule(PackageContext, string, blueprint.RuleParams,
		return nil
		}
		func (builderContextForTests) Build(PackageContext, BuildParams) {}

		func rspFileForInputs(paths []string) string {
		s := strings.Builder{}
		for i, path := range paths {
		if i != 0 {
		s.WriteByte(' ')
		}
		s.WriteString(proptools.ShellEscape(path))
		}
		return s.String()
		}