Merge "Initial implementation of the bazel sandwich" into main am: 61a27f80cd (80a77780) · Commits · e / os / android_build_soong

android/allowlists/allowlists.go

+10 −0

Original line number	Diff line number	Diff line
		@@ -1643,4 +1643,14 @@ var (
		"art_": DEFAULT_PRIORITIZED_WEIGHT,
		"ndk_library": DEFAULT_PRIORITIZED_WEIGHT,
		}

		BazelSandwichTargets = []struct {
		Label string
		Host bool
		}{
		{
		Label: "//build/bazel/examples/partitions:system_image",
		Host: false,
		},
		}
		)

android/bazel_handler.go

+84 −5

Original line number	Diff line number	Diff line
		@@ -186,6 +186,8 @@ type BazelContext interface {

		// Returns the depsets defined in Bazel's aquery response.
		AqueryDepsets() []bazel.AqueryDepset

		QueueBazelSandwichCqueryRequests(config Config) error
		}

		type bazelRunner interface {
		@@ -264,6 +266,10 @@ func (m MockBazelContext) QueueBazelRequest(label string, requestType cqueryRequ
		m.BazelRequests[key] = true
		}

		func (m MockBazelContext) QueueBazelSandwichCqueryRequests(config Config) error {
		panic("unimplemented")
		}

		func (m MockBazelContext) GetOutputFiles(label string, _ configKey) ([]string, error) {
		result, ok := m.LabelToOutputFiles[label]
		if !ok {
		@@ -424,6 +430,10 @@ func (n noopBazelContext) QueueBazelRequest(_ string, _ cqueryRequest, _ configK
		panic("unimplemented")
		}

		func (n noopBazelContext) QueueBazelSandwichCqueryRequests(config Config) error {
		panic("unimplemented")
		}

		func (n noopBazelContext) GetOutputFiles(_ string, _ configKey) ([]string, error) {
		panic("unimplemented")
		}
		@@ -1042,6 +1052,45 @@ var (
		allBazelCommands = []bazelCommand{aqueryCmd, cqueryCmd, buildCmd}
		)

		func GetBazelSandwichCqueryRequests(config Config) ([]cqueryKey, error) {
		result := make([]cqueryKey, 0, len(allowlists.BazelSandwichTargets))
		// Note that bazel "targets" are different from soong "targets", the bazel targets are
		// synonymous with soong modules, and soong targets are a configuration a module is built in.
		for _, target := range allowlists.BazelSandwichTargets {
		var soongTarget Target
		if target.Host {
		soongTarget = config.BuildOSTarget
		} else {
		soongTarget = config.AndroidCommonTarget
		}

		result = append(result, cqueryKey{
		label: target.Label,
		requestType: cquery.GetOutputFiles,
		configKey: configKey{
		arch: soongTarget.Arch.String(),
		osType: soongTarget.Os,
		},
		})
		}
		return result, nil
		}

		// QueueBazelSandwichCqueryRequests queues cquery requests for all the bazel labels in
		// bazel_sandwich_targets. These will later be given phony targets so that they can be built on the
		// command line.
		func (context *mixedBuildBazelContext) QueueBazelSandwichCqueryRequests(config Config) error {
		requests, err := GetBazelSandwichCqueryRequests(config)
		if err != nil {
		return err
		}
		for _, request := range requests {
		context.QueueBazelRequest(request.label, request.requestType, request.configKey)
		}

		return nil
		}

		// Issues commands to Bazel to receive results for all cquery requests
		// queued in the BazelContext.
		func (context *mixedBuildBazelContext) InvokeBazel(config Config, ctx invokeBazelContext) error {
		@@ -1255,6 +1304,11 @@ func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) {

		executionRoot := path.Join(ctx.Config().BazelContext.OutputBase(), "execroot", "__main__")
		bazelOutDir := path.Join(executionRoot, "bazel-out")
		rel, err := filepath.Rel(ctx.Config().OutDir(), executionRoot)
		if err != nil {
		ctx.Errorf("%s", err.Error())
		}
		dotdotsToOutRoot := strings.Repeat("../", strings.Count(rel, "/")+1)
		for index, buildStatement := range ctx.Config().BazelContext.BuildStatementsToRegister() {
		// nil build statements are a valid case where we do not create an action because it is
		// unnecessary or handled by other processing
		@@ -1286,7 +1340,8 @@ func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) {
		})
		}
		}
		createCommand(rule.Command(), buildStatement, executionRoot, bazelOutDir, ctx, depsetHashToDepset)
		createCommand(rule.Command(), buildStatement, executionRoot, bazelOutDir, ctx, depsetHashToDepset, dotdotsToOutRoot)

		desc := fmt.Sprintf("%s: %s", buildStatement.Mnemonic, buildStatement.OutputPaths)
		rule.Build(fmt.Sprintf("bazel %d", index), desc)
		continue
		@@ -1331,6 +1386,24 @@ func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) {
		panic(fmt.Sprintf("unhandled build statement: %v", buildStatement))
		}
		}

		// Create phony targets for all the bazel sandwich output files
		requests, err := GetBazelSandwichCqueryRequests(ctx.Config())
		if err != nil {
		ctx.Errorf(err.Error())
		}
		for _, request := range requests {
		files, err := ctx.Config().BazelContext.GetOutputFiles(request.label, request.configKey)
		if err != nil {
		ctx.Errorf(err.Error())
		}
		filesAsPaths := make([]Path, 0, len(files))
		for _, file := range files {
		filesAsPaths = append(filesAsPaths, PathForBazelOut(ctx, file))
		}
		ctx.Phony("bazel_sandwich", filesAsPaths...)
		}
		ctx.Phony("checkbuild", PathForPhony(ctx, "bazel_sandwich"))
		}

		// Returns a out dir path for a sandboxed mixed build action
		@@ -1344,7 +1417,7 @@ func intermediatePathForSboxMixedBuildAction(ctx PathContext, statement *bazel.B
		}

		// Register bazel-owned build statements (obtained from the aquery invocation).
		func createCommand(cmd RuleBuilderCommand, buildStatement bazel.BuildStatement, executionRoot string, bazelOutDir string, ctx BuilderContext, depsetHashToDepset map[string]bazel.AqueryDepset) {
		func createCommand(cmd RuleBuilderCommand, buildStatement bazel.BuildStatement, executionRoot string, bazelOutDir string, ctx BuilderContext, depsetHashToDepset map[string]bazel.AqueryDepset, dotdotsToOutRoot string) {
		// executionRoot is the action cwd.
		if buildStatement.ShouldRunInSbox {
		// mkdir -p ensures that the directory exists when run via sbox
		@@ -1367,14 +1440,17 @@ func createCommand(cmd RuleBuilderCommand, buildStatement bazel.BuildStatement
		cmd.Flag(pair.Key + "=" + pair.Value)
		}

		command := buildStatement.Command
		command = strings.ReplaceAll(command, "{DOTDOTS_TO_OUTPUT_ROOT}", dotdotsToOutRoot)

		// The actual Bazel action.
		if len(buildStatement.Command) > 16*1024 {
		if len(command) > 16*1024 {
		commandFile := PathForBazelOut(ctx, buildStatement.OutputPaths[0]+".sh")
		WriteFileRule(ctx, commandFile, buildStatement.Command)
		WriteFileRule(ctx, commandFile, command)

		cmd.Text("bash").Text(buildStatement.OutputPaths[0] + ".sh").Implicit(commandFile)
		} else {
		cmd.Text(buildStatement.Command)
		cmd.Text(command)
		}

		for _, outputPath := range buildStatement.OutputPaths {
		@@ -1403,6 +1479,9 @@ func createCommand(cmd RuleBuilderCommand, buildStatement bazel.BuildStatement
		cmd.Implicit(PathForPhony(ctx, otherDepsetName))
		}
		}
		for _, implicitPath := range buildStatement.ImplicitDeps {
		cmd.Implicit(PathForArbitraryOutput(ctx, implicitPath))
		}

		if depfile := buildStatement.Depfile; depfile != nil {
		// The paths in depfile are relative to `executionRoot`.

android/bazel_handler_test.go

+2 −2

Original line number	Diff line number	Diff line
		@@ -181,7 +181,7 @@ func TestInvokeBazelPopulatesBuildStatements(t *testing.T) {

		cmd := RuleBuilderCommand{}
		ctx := builderContextForTests{PathContextForTesting(TestConfig("out", nil, "", nil))}
		createCommand(&cmd, got[0], "test/exec_root", "test/bazel_out", ctx, map[string]bazel.AqueryDepset{})
		createCommand(&cmd, got[0], "test/exec_root", "test/bazel_out", ctx, map[string]bazel.AqueryDepset{}, "")
		if actual, expected := cmd.buf.String(), testCase.command; expected != actual {
		t.Errorf("expected: [%s], actual: [%s]", expected, actual)
		}
		@@ -224,7 +224,7 @@ func TestMixedBuildSandboxedAction(t *testing.T) {

		cmd := RuleBuilderCommand{}
		ctx := builderContextForTests{PathContextForTesting(TestConfig("out", nil, "", nil))}
		createCommand(&cmd, statement, "test/exec_root", "test/bazel_out", ctx, map[string]bazel.AqueryDepset{})
		createCommand(&cmd, statement, "test/exec_root", "test/bazel_out", ctx, map[string]bazel.AqueryDepset{}, "")
		// Assert that the output is generated in an intermediate directory
		// fe05bcdcdc4928012781a5f1a2a77cbb5398e106 is the sha1 checksum of "one"
		if actual, expected := cmd.outputs[0].String(), "out/soong/mixed_build_sbox_intermediates/fe05bcdcdc4928012781a5f1a2a77cbb5398e106/test/exec_root/one"; expected != actual {

android/paths.go

+15 −5

Original line number	Diff line number	Diff line
		@@ -1029,16 +1029,16 @@ func (p basePath) withRel(rel string) basePath {
		return p
		}

		func (p basePath) RelativeToTop() Path {
		ensureTestOnly()
		return p
		}

		// SourcePath is a Path representing a file path rooted from SrcDir
		type SourcePath struct {
		basePath
		}

		func (p SourcePath) RelativeToTop() Path {
		ensureTestOnly()
		return p
		}

		var _ Path = SourcePath{}

		func (p SourcePath) withRel(rel string) SourcePath {
		@@ -1126,6 +1126,16 @@ func PathForSource(ctx PathContext, pathComponents ...string) SourcePath {
		return path
		}

		// PathForArbitraryOutput creates a path for the given components. Unlike PathForOutput,
		// the path is relative to the root of the output folder, not the out/soong folder.
		func PathForArbitraryOutput(ctx PathContext, pathComponents ...string) Path {
		p, err := validatePath(pathComponents...)
		if err != nil {
		reportPathError(ctx, err)
		}
		return basePath{path: filepath.Join(ctx.Config().OutDir(), p)}
		}

		// MaybeExistentPathForSource joins the provided path components and validates that the result
		// neither escapes the source dir nor is in the out dir.
		// It does not validate whether the path exists.

bazel/aquery.go

+74 −2

Original line number	Diff line number	Diff line
		@@ -17,15 +17,15 @@ package bazel
		import (
		"crypto/sha256"
		"encoding/base64"
		"encoding/json"
		"fmt"
		"path/filepath"
		analysis_v2_proto "prebuilts/bazel/common/proto/analysis_v2"
		"reflect"
		"sort"
		"strings"
		"sync"

		analysis_v2_proto "prebuilts/bazel/common/proto/analysis_v2"

		"github.com/google/blueprint/metrics"
		"github.com/google/blueprint/proptools"
		"google.golang.org/protobuf/proto"
		@@ -119,6 +119,10 @@ type BuildStatement struct {
		// If ShouldRunInSbox is true, Soong will use sbox to created an isolated environment
		// and run the mixed build action there
		ShouldRunInSbox bool
		// A list of files to add as implicit deps to the outputs of this BuildStatement.
		// Unlike most properties in BuildStatement, these paths must be relative to the root of
		// the whole out/ folder, instead of relative to ctx.Config().BazelContext.OutputBase()
		ImplicitDeps []string
		}

		// A helper type for aquery processing which facilitates retrieval of path IDs from their
		@@ -581,6 +585,72 @@ func (a aqueryArtifactHandler) symlinkTreeActionBuildStatement(actionEntry ana
		}, nil
		}

		type bazelSandwichJson struct {
		Target string `json:"target"`
		DependOnTarget *bool `json:"depend_on_target,omitempty"`
		ImplicitDeps []string `json:"implicit_deps"`
		}

		func (a aqueryArtifactHandler) unresolvedSymlinkActionBuildStatement(actionEntry analysis_v2_proto.Action) (*BuildStatement, error) {
		outputPaths, depfile, err := a.getOutputPaths(actionEntry)
		if err != nil {
		return nil, err
		}
		if len(actionEntry.InputDepSetIds) != 0 \|\| len(outputPaths) != 1 {
		return nil, fmt.Errorf("expected 0 inputs and 1 output to symlink action, got: input %q, output %q", actionEntry.InputDepSetIds, outputPaths)
		}
		target := actionEntry.UnresolvedSymlinkTarget
		if target == "" {
		return nil, fmt.Errorf("expected an unresolved_symlink_target, but didn't get one")
		}
		if filepath.Clean(target) != target {
		return nil, fmt.Errorf("expected %q, got %q", filepath.Clean(target), target)
		}
		if strings.HasPrefix(target, "/") {
		return nil, fmt.Errorf("no absolute symlinks allowed: %s", target)
		}

		out := outputPaths[0]
		outDir := filepath.Dir(out)
		var implicitDeps []string
		if strings.HasPrefix(target, "bazel_sandwich:") {
		j := bazelSandwichJson{}
		err := json.Unmarshal([]byte(target[len("bazel_sandwich:"):]), &j)
		if err != nil {
		return nil, err
		}
		if proptools.BoolDefault(j.DependOnTarget, true) {
		implicitDeps = append(implicitDeps, j.Target)
		}
		implicitDeps = append(implicitDeps, j.ImplicitDeps...)
		dotDotsToReachCwd := ""
		if outDir != "." {
		dotDotsToReachCwd = strings.Repeat("../", strings.Count(outDir, "/")+1)
		}
		target = proptools.ShellEscapeIncludingSpaces(j.Target)
		target = "{DOTDOTS_TO_OUTPUT_ROOT}" + dotDotsToReachCwd + target
		} else {
		target = proptools.ShellEscapeIncludingSpaces(target)
		}

		outDir = proptools.ShellEscapeIncludingSpaces(outDir)
		out = proptools.ShellEscapeIncludingSpaces(out)
		// Use absolute paths, because some soong actions don't play well with relative paths (for example, `cp -d`).
		command := fmt.Sprintf("mkdir -p %[1]s && rm -f %[2]s && ln -sf %[3]s %[2]s", outDir, out, target)
		symlinkPaths := outputPaths[:]

		buildStatement := &BuildStatement{
		Command: command,
		Depfile: depfile,
		OutputPaths: outputPaths,
		Env: actionEntry.EnvironmentVariables,
		Mnemonic: actionEntry.Mnemonic,
		SymlinkPaths: symlinkPaths,
		ImplicitDeps: implicitDeps,
		}
		return buildStatement, nil
		}

		func (a aqueryArtifactHandler) symlinkActionBuildStatement(actionEntry analysis_v2_proto.Action) (*BuildStatement, error) {
		outputPaths, depfile, err := a.getOutputPaths(actionEntry)
		if err != nil {
		@@ -690,6 +760,8 @@ func (a aqueryArtifactHandler) actionToBuildStatement(actionEntry analysis_v2_
		return a.fileWriteActionBuildStatement(actionEntry)
		case "SymlinkTree":
		return a.symlinkTreeActionBuildStatement(actionEntry)
		case "UnresolvedSymlink":
		return a.unresolvedSymlinkActionBuildStatement(actionEntry)
		}

		if len(actionEntry.Arguments) < 1 {