Merge changes from topic "buildnumberfromfile" am: 223e6a65b6 (1ad74432) · Commits · e / os / android_build_soong

android/expand.go

+31 −10

Original line number	Diff line number	Diff line
		@@ -18,12 +18,30 @@ import (
		"fmt"
		"strings"
		"unicode"

		"github.com/google/blueprint/proptools"
		)

		// ExpandNinjaEscaped substitutes $() variables in a string
		// $(var) is passed to mapping(var), which should return the expanded value, a bool for whether the result should
		// be left unescaped when using in a ninja value (generally false, true if the expanded value is a ninja variable like
		// '${in}'), and an error.
		// $$ is converted to $, which is escaped back to $$.
		func ExpandNinjaEscaped(s string, mapping func(string) (string, bool, error)) (string, error) {
		return expand(s, true, mapping)
		}

		// Expand substitutes $() variables in a string
		// $(var) is passed to Expander(var)
		// $$ is converted to $
		// $(var) is passed to mapping(var), which should return the expanded value and an error.
		// $$ is converted to $.
		func Expand(s string, mapping func(string) (string, error)) (string, error) {
		return expand(s, false, func(s string) (string, bool, error) {
		s, err := mapping(s)
		return s, false, err
		})
		}

		func expand(s string, ninjaEscape bool, mapping func(string) (string, bool, error)) (string, error) {
		// based on os.Expand
		buf := make([]byte, 0, 2*len(s))
		i := 0
		@@ -33,10 +51,13 @@ func Expand(s string, mapping func(string) (string, error)) (string, error) {
		return "", fmt.Errorf("expected character after '$'")
		}
		buf = append(buf, s[i:j]...)
		value, w, err := getMapping(s[j+1:], mapping)
		value, ninjaVariable, w, err := getMapping(s[j+1:], mapping)
		if err != nil {
		return "", err
		}
		if !ninjaVariable && ninjaEscape {
		value = proptools.NinjaEscape(value)
		}
		buf = append(buf, value...)
		j += w
		i = j + 1
		@@ -45,26 +66,26 @@ func Expand(s string, mapping func(string) (string, error)) (string, error) {
		return string(buf) + s[i:], nil
		}

		func getMapping(s string, mapping func(string) (string, error)) (string, int, error) {
		func getMapping(s string, mapping func(string) (string, bool, error)) (string, bool, int, error) {
		switch s[0] {
		case '(':
		// Scan to closing brace
		for i := 1; i < len(s); i++ {
		if s[i] == ')' {
		ret, err := mapping(strings.TrimSpace(s[1:i]))
		return ret, i + 1, err
		ret, ninjaVariable, err := mapping(strings.TrimSpace(s[1:i]))
		return ret, ninjaVariable, i + 1, err
		}
		}
		return "", len(s), fmt.Errorf("missing )")
		return "", false, len(s), fmt.Errorf("missing )")
		case '$':
		return "$$", 1, nil
		return "$", false, 1, nil
		default:
		i := strings.IndexFunc(s, unicode.IsSpace)
		if i == 0 {
		return "", 0, fmt.Errorf("unexpected character '%c' after '$'", s[0])
		return "", false, 0, fmt.Errorf("unexpected character '%c' after '$'", s[0])
		} else if i == -1 {
		i = len(s)
		}
		return "", 0, fmt.Errorf("expected '(' after '$', did you mean $(%s)?", s[:i])
		return "", false, 0, fmt.Errorf("expected '(' after '$', did you mean $(%s)?", s[:i])
		}
		}

android/expand_test.go

+82 −43

Original line number	Diff line number	Diff line
		@@ -24,84 +24,107 @@ var vars = map[string]string{
		"var2": "",
		"var3": "def",
		"💩": "😃",
		"escape": "${in}",
		}

		func expander(s string) (string, error) {
		func expander(s string) (string, bool, error) {
		if val, ok := vars[s]; ok {
		return val, nil
		return val, s == "escape", nil
		} else {
		return "", fmt.Errorf("unknown variable %q", s)
		return "", false, fmt.Errorf("unknown variable %q", s)
		}
		}

		var expandTestCases = []struct {
		in string
		out string
		out_escaped string
		err bool
		}{
		{
		in: "$(var1)",
		out: "abc",
		out_escaped: "abc",
		},
		{
		in: "$( var1 )",
		out: "abc",
		out_escaped: "abc",
		},
		{
		in: "def$(var1)",
		out: "defabc",
		out_escaped: "defabc",
		},
		{
		in: "$(var1)def",
		out: "abcdef",
		out_escaped: "abcdef",
		},
		{
		in: "def$(var1)def",
		out: "defabcdef",
		out_escaped: "defabcdef",
		},
		{
		in: "$(var2)",
		out: "",
		out_escaped: "",
		},
		{
		in: "def$(var2)",
		out: "def",
		out_escaped: "def",
		},
		{
		in: "$(var2)def",
		out: "def",
		out_escaped: "def",
		},
		{
		in: "def$(var2)def",
		out: "defdef",
		out_escaped: "defdef",
		},
		{
		in: "$(var1)$(var3)",
		out: "abcdef",
		out_escaped: "abcdef",
		},
		{
		in: "$(var1)g$(var3)",
		out: "abcgdef",
		out_escaped: "abcgdef",
		},
		{
		in: "$$",
		out: "$$",
		out: "$",
		out_escaped: "$$",
		},
		{
		in: "$$(var1)",
		out: "$$(var1)",
		out: "$(var1)",
		out_escaped: "$$(var1)",
		},
		{
		in: "$$$(var1)",
		out: "$$abc",
		out: "$abc",
		out_escaped: "$$abc",
		},
		{
		in: "$(var1)$$",
		out: "abc$$",
		out: "abc$",
		out_escaped: "abc$$",
		},
		{
		in: "$(💩)",
		out: "😃",
		out_escaped: "😃",
		},
		{
		in: "$$a$(escape)$$b",
		out: "$a${in}$b",
		out_escaped: "$$a${in}$$b",
		},

		// Errors
		@@ -141,7 +164,10 @@ var expandTestCases = []struct {

		func TestExpand(t *testing.T) {
		for _, test := range expandTestCases {
		got, err := Expand(test.in, expander)
		got, err := Expand(test.in, func(s string) (string, error) {
		s, _, err := expander(s)
		return s, err
		})
		if err != nil && !test.err {
		t.Errorf("%q: unexpected error %s", test.in, err.Error())
		} else if err == nil && test.err {
		@@ -151,3 +177,16 @@ func TestExpand(t *testing.T) {
		}
		}
		}

		func TestExpandNinjaEscaped(t *testing.T) {
		for _, test := range expandTestCases {
		got, err := ExpandNinjaEscaped(test.in, expander)
		if err != nil && !test.err {
		t.Errorf("%q: unexpected error %s", test.in, err.Error())
		} else if err == nil && test.err {
		t.Errorf("%q: expected error, got %q", test.in, got)
		} else if !test.err && got != test.out_escaped {
		t.Errorf("%q: expected %q, got %q", test.in, test.out, got)
		}
		}
		}

android/proto.go

+2 −2

Original line number	Diff line number	Diff line
		@@ -135,7 +135,7 @@ func ProtoRule(ctx ModuleContext, rule *RuleBuilder, protoFile Path, flags Proto
		}

		rule.Command().
		Tool(ctx.Config().HostToolPath(ctx, "aprotoc")).
		BuiltTool(ctx, "aprotoc").
		FlagWithArg(flags.OutTypeFlag+"=", strings.Join(flags.OutParams, ",")+":"+outDir.String()).
		FlagWithDepFile("--dependency_out=", depFile).
		FlagWithArg("-I ", protoBase).
		@@ -145,5 +145,5 @@ func ProtoRule(ctx ModuleContext, rule *RuleBuilder, protoFile Path, flags Proto
		ImplicitOutputs(outputs)

		rule.Command().
		Tool(ctx.Config().HostToolPath(ctx, "dep_fixer")).Flag(depFile.String())
		BuiltTool(ctx, "dep_fixer").Flag(depFile.String())
		}

android/rule_builder.go

+118 −16

Original line number	Diff line number	Diff line
		@@ -263,11 +263,36 @@ func (r *RuleBuilder) Tools() Paths {
		return toolsList
		}

		// Commands returns a slice containing a the built command line for each call to RuleBuilder.Command.
		// RspFileInputs returns the list of paths that were passed to the RuleBuilderCommand.FlagWithRspFileInputList method.
		func (r *RuleBuilder) RspFileInputs() Paths {
		var rspFileInputs Paths
		for _, c := range r.commands {
		if c.rspFileInputs != nil {
		if rspFileInputs != nil {
		panic("Multiple commands in a rule may not have rsp file inputs")
		}
		rspFileInputs = c.rspFileInputs
		}
		}

		return rspFileInputs
		}

		// Commands returns a slice containing the built command line for each call to RuleBuilder.Command.
		func (r *RuleBuilder) Commands() []string {
		var commands []string
		for _, c := range r.commands {
		commands = append(commands, c.buf.String())
		commands = append(commands, c.String())
		}
		return commands
		}

		// NinjaEscapedCommands returns a slice containin the built command line after ninja escaping for each call to
		// RuleBuilder.Command.
		func (r *RuleBuilder) NinjaEscapedCommands() []string {
		var commands []string
		for _, c := range r.commands {
		commands = append(commands, c.NinjaEscapedString())
		}
		return commands
		}
		@@ -284,7 +309,7 @@ var _ BuilderContext = SingletonContext(nil)

		func (r RuleBuilder) depFileMergerCmd(ctx PathContext, depFiles WritablePaths) RuleBuilderCommand {
		return r.Command().
		Tool(ctx.Config().HostToolPath(ctx, "dep_fixer")).
		BuiltTool(ctx, "dep_fixer").
		Inputs(depFiles.Paths())
		}

		@@ -324,7 +349,7 @@ func (r *RuleBuilder) Build(pctx PackageContext, ctx BuilderContext, name string
		}

		tools := r.Tools()
		commands := r.Commands()
		commands := r.NinjaEscapedCommands()
		outputs := r.Outputs()

		if len(commands) == 0 {
		@@ -334,7 +359,7 @@ func (r *RuleBuilder) Build(pctx PackageContext, ctx BuilderContext, name string
		panic("No outputs specified from any Commands")
		}

		commandString := strings.Join(proptools.NinjaEscapeList(commands), " && ")
		commandString := strings.Join(commands, " && ")

		if r.sbox {
		sboxOutputs := make([]string, len(outputs))
		@@ -352,7 +377,7 @@ func (r *RuleBuilder) Build(pctx PackageContext, ctx BuilderContext, name string
		}

		sboxCmd := &RuleBuilderCommand{}
		sboxCmd.Tool(ctx.Config().HostToolPath(ctx, "sbox")).
		sboxCmd.BuiltTool(ctx, "sbox").
		Flag("-c").Text(commandString).
		Flag("--sandbox-path").Text(shared.TempDirForOutDir(PathForOutput(ctx).String())).
		Flag("--output-root").Text(r.sboxOutDir.String()).
		@@ -363,17 +388,27 @@ func (r *RuleBuilder) Build(pctx PackageContext, ctx BuilderContext, name string
		}

		// Ninja doesn't like multiple outputs when depfiles are enabled, move all but the first output to
		// ImplicitOutputs. RuleBuilder never uses "$out", so the distinction between Outputs and ImplicitOutputs
		// doesn't matter.
		// ImplicitOutputs. RuleBuilder only uses "$out" for the rsp file location, so the distinction between Outputs and
		// ImplicitOutputs doesn't matter.
		output := outputs[0]
		implicitOutputs := outputs[1:]

		var rspFile, rspFileContent string
		rspFileInputs := r.RspFileInputs()
		if rspFileInputs != nil {
		rspFile = "$out.rsp"
		rspFileContent = "$in"
		}

		ctx.Build(pctx, BuildParams{
		Rule: ctx.Rule(pctx, name, blueprint.RuleParams{
		Command: commandString,
		CommandDeps: tools.Strings(),
		Restat: r.restat,
		Rspfile: rspFile,
		RspfileContent: rspFileContent,
		}),
		Inputs: rspFileInputs,
		Implicits: r.Inputs(),
		Output: output,
		ImplicitOutputs: implicitOutputs,
		@@ -393,6 +428,10 @@ type RuleBuilderCommand struct {
		outputs WritablePaths
		depFiles WritablePaths
		tools Paths
		rspFileInputs Paths

		// spans [start,end) of the command that should not be ninja escaped
		unescapedSpans [][2]int

		sbox bool
		sboxOutDir WritablePath
		@@ -478,6 +517,24 @@ func (c RuleBuilderCommand) Tool(path Path) RuleBuilderCommand {
		return c.Text(path.String())
		}

		// BuiltTool adds the specified tool path that was built using a host Soong module to the command line. The path will
		// be also added to the dependencies returned by RuleBuilder.Tools.
		//
		// It is equivalent to:
		// cmd.Tool(ctx.Config().HostToolPath(ctx, tool))
		func (c RuleBuilderCommand) BuiltTool(ctx PathContext, tool string) RuleBuilderCommand {
		return c.Tool(ctx.Config().HostToolPath(ctx, tool))
		}

		// PrebuiltBuildTool adds the specified tool path from prebuils/build-tools. The path will be also added to the
		// dependencies returned by RuleBuilder.Tools.
		//
		// It is equivalent to:
		// cmd.Tool(ctx.Config().PrebuiltBuildTool(ctx, tool))
		func (c RuleBuilderCommand) PrebuiltBuildTool(ctx PathContext, tool string) RuleBuilderCommand {
		return c.Tool(ctx.Config().PrebuiltBuildTool(ctx, tool))
		}

		// Input adds the specified input path to the command line. The path will also be added to the dependencies returned by
		// RuleBuilder.Inputs.
		func (c RuleBuilderCommand) Input(path Path) RuleBuilderCommand {
		@@ -606,11 +663,56 @@ func (c RuleBuilderCommand) FlagWithDepFile(flag string, path WritablePath) Ru
		return c.Text(flag + c.outputStr(path))
		}

		// FlagWithRspFileInputList adds the specified flag and path to an rspfile to the command line, with no separator
		// between them. The paths will be written to the rspfile.
		func (c RuleBuilderCommand) FlagWithRspFileInputList(flag string, paths Paths) RuleBuilderCommand {
		if c.rspFileInputs != nil {
		panic("FlagWithRspFileInputList cannot be called if rsp file inputs have already been provided")
		}

		// Use an empty slice if paths is nil, the non-nil slice is used as an indicator that the rsp file must be
		// generated.
		if paths == nil {
		paths = Paths{}
		}

		c.rspFileInputs = paths

		rspFile := "$out.rsp"
		c.FlagWithArg(flag, rspFile)
		c.unescapedSpans = append(c.unescapedSpans, [2]int{c.buf.Len() - len(rspFile), c.buf.Len()})
		return c
		}

		// String returns the command line.
		func (c *RuleBuilderCommand) String() string {
		return c.buf.String()
		}

		// String returns the command line.
		func (c *RuleBuilderCommand) NinjaEscapedString() string {
		return ninjaEscapeExceptForSpans(c.String(), c.unescapedSpans)
		}

		func ninjaEscapeExceptForSpans(s string, spans [][2]int) string {
		if len(spans) == 0 {
		return proptools.NinjaEscape(s)
		}

		sb := strings.Builder{}
		sb.Grow(len(s) * 11 / 10)

		i := 0
		for _, span := range spans {
		sb.WriteString(proptools.NinjaEscape(s[i:span[0]]))
		sb.WriteString(s[span[0]:span[1]])
		i = span[1]
		}
		sb.WriteString(proptools.NinjaEscape(s[i:]))

		return sb.String()
		}

		func ninjaNameEscape(s string) string {
		b := []byte(s)
		escaped := false

android/rule_builder_test.go

+103 −0

Original line number	Diff line number	Diff line
		@@ -38,6 +38,7 @@ func pathContext() PathContext {
		"ls": nil,
		"turbine": nil,
		"java": nil,
		"javac": nil,
		})
		}

		@@ -235,6 +236,34 @@ func ExampleRuleBuilderCommand_FlagWithList() {
		// ls --sort=time,size
		}

		func ExampleRuleBuilderCommand_FlagWithRspFileInputList() {
		ctx := pathContext()
		fmt.Println(NewRuleBuilder().Command().
		Tool(PathForSource(ctx, "javac")).
		FlagWithRspFileInputList("@", PathsForTesting("a.java", "b.java")).
		NinjaEscapedString())
		// Output:
		// javac @$out.rsp
		}

		func ExampleRuleBuilderCommand_String() {
		fmt.Println(NewRuleBuilder().Command().
		Text("FOO=foo").
		Text("echo $FOO").
		String())
		// Output:
		// FOO=foo echo $FOO
		}

		func ExampleRuleBuilderCommand_NinjaEscapedString() {
		fmt.Println(NewRuleBuilder().Command().
		Text("FOO=foo").
		Text("echo $FOO").
		NinjaEscapedString())
		// Output:
		// FOO=foo echo $$FOO
		}

		func TestRuleBuilder(t *testing.T) {
		fs := map[string][]byte{
		"dep_fixer": nil,
		@@ -503,3 +532,77 @@ func TestRuleBuilder_Build(t *testing.T) {
		"cp bar "+outFile, outFile, outFile+".d", true, nil)
		})
		}

		func Test_ninjaEscapeExceptForSpans(t *testing.T) {
		type args struct {
		s string
		spans [][2]int
		}
		tests := []struct {
		name string
		args args
		want string
		}{
		{
		name: "empty",
		args: args{
		s: "",
		},
		want: "",
		},
		{
		name: "unescape none",
		args: args{
		s: "$abc",
		},
		want: "$$abc",
		},
		{
		name: "unescape all",
		args: args{
		s: "$abc",
		spans: [][2]int{{0, 4}},
		},
		want: "$abc",
		},
		{
		name: "unescape first",
		args: args{
		s: "$abc$",
		spans: [][2]int{{0, 1}},
		},
		want: "$abc$$",
		},
		{
		name: "unescape last",
		args: args{
		s: "$abc$",
		spans: [][2]int{{4, 5}},
		},
		want: "$$abc$",
		},
		{
		name: "unescape middle",
		args: args{
		s: "$a$b$c$",
		spans: [][2]int{{2, 5}},
		},
		want: "$$a$b$c$$",
		},
		{
		name: "unescape multiple",
		args: args{
		s: "$a$b$c$",
		spans: [][2]int{{2, 3}, {4, 5}},
		},
		want: "$$a$b$c$$",
		},
		}
		for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
		if got := ninjaEscapeExceptForSpans(tt.args.s, tt.args.spans); got != tt.want {
		t.Errorf("ninjaEscapeExceptForSpans() = %v, want %v", got, tt.want)
		}
		})
		}
		}