Merge "Refactor mixed builds to only take one pass" (489128b8) · Commits · e / os / android_build_soong

android/bazel.go

+23 −2

Original line number	Diff line number	Diff line
		@@ -115,6 +115,27 @@ type Bazelable interface {
		SetBaseModuleType(baseModuleType string)
		}

		// MixedBuildBuildable is an interface that module types should implement in order
		// to be "handled by Bazel" in a mixed build.
		type MixedBuildBuildable interface {
		// IsMixedBuildSupported returns true if and only if this module should be
		// "handled by Bazel" in a mixed build.
		// This "escape hatch" allows modules with corner-case scenarios to opt out
		// of being built with Bazel.
		IsMixedBuildSupported(ctx BaseModuleContext) bool

		// QueueBazelCall invokes request-queueing functions on the BazelContext
		// so that these requests are handled when Bazel's cquery is invoked.
		QueueBazelCall(ctx BaseModuleContext)

		// ProcessBazelQueryResponse uses Bazel information (obtained from the BazelContext)
		// to set module fields and providers to propagate this module's metadata upstream.
		// This effectively "bridges the gap" between Bazel and Soong in a mixed build.
		// Soong modules depending on this module should be oblivious to the fact that
		// this module was handled by Bazel.
		ProcessBazelQueryResponse(ctx ModuleContext)
		}

		// BazelModule is a lightweight wrapper interface around Module for Bazel-convertible modules.
		type BazelModule interface {
		Module
		@@ -342,7 +363,7 @@ func shouldKeepExistingBuildFileForDir(allowlist bp2BuildConversionAllowlist, di
		// converted or handcrafted Bazel target. As a side effect, calling this
		// method will also log whether this module is mixed build enabled for
		// metrics reporting.
		func MixedBuildsEnabled(ctx ModuleContext) bool {
		func MixedBuildsEnabled(ctx BaseModuleContext) bool {
		mixedBuildEnabled := mixedBuildPossible(ctx)
		ctx.Config().LogMixedBuild(ctx, mixedBuildEnabled)
		return mixedBuildEnabled
		@@ -350,7 +371,7 @@ func MixedBuildsEnabled(ctx ModuleContext) bool {

		// mixedBuildPossible returns true if a module is ready to be replaced by a
		// converted or handcrafted Bazel target.
		func mixedBuildPossible(ctx ModuleContext) bool {
		func mixedBuildPossible(ctx BaseModuleContext) bool {
		if ctx.Os() == Windows {
		// Windows toolchains are not currently supported.
		return false

android/bazel_handler.go

+82 −70

Original line number	Diff line number	Diff line
		@@ -33,6 +33,26 @@ import (
		"android/soong/bazel"
		)

		func init() {
		RegisterMixedBuildsMutator(InitRegistrationContext)
		}

		func RegisterMixedBuildsMutator(ctx RegistrationContext) {
		ctx.PostDepsMutators(func(ctx RegisterMutatorsContext) {
		ctx.BottomUp("mixed_builds_prep", mixedBuildsPrepareMutator).Parallel()
		})
		}

		func mixedBuildsPrepareMutator(ctx BottomUpMutatorContext) {
		if m := ctx.Module(); m.Enabled() {
		if mixedBuildMod, ok := m.(MixedBuildBuildable); ok {
		if mixedBuildMod.IsMixedBuildSupported(ctx) && MixedBuildsEnabled(ctx) {
		mixedBuildMod.QueueBazelCall(ctx)
		}
		}
		}
		}

		type cqueryRequest interface {
		// Name returns a string name for this request type. Such request type names must be unique,
		// and must only consist of alphanumeric characters.
		@@ -62,33 +82,32 @@ type cqueryKey struct {
		configKey configKey
		}

		// bazelHandler is the interface for a helper object related to deferring to Bazel for
		// processing a module (during Bazel mixed builds). Individual module types should define
		// their own bazel handler if they support deferring to Bazel.
		type BazelHandler interface {
		// Issue query to Bazel to retrieve information about Bazel's view of the current module.
		// If Bazel returns this information, set module properties on the current module to reflect
		// the returned information.
		// Returns true if information was available from Bazel, false if bazel invocation still needs to occur.
		GenerateBazelBuildActions(ctx ModuleContext, label string) bool
		}

		// BazelContext is a context object useful for interacting with Bazel during
		// the course of a build. Use of Bazel to evaluate part of the build graph
		// is referred to as a "mixed build". (Some modules are managed by Soong,
		// some are managed by Bazel). To facilitate interop between these build
		// subgraphs, Soong may make requests to Bazel and evaluate their responses
		// so that Soong modules may accurately depend on Bazel targets.
		type BazelContext interface {
		// The methods below involve queuing cquery requests to be later invoked
		// by bazel. If any of these methods return (_, false), then the request
		// has been queued to be run later.
		// Add a cquery request to the bazel request queue. All queued requests
		// will be sent to Bazel on a subsequent invocation of InvokeBazel.
		QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey)

		// ** Cquery Results Retrieval Functions
		// The below functions pertain to retrieving cquery results from a prior
		// InvokeBazel function call and parsing the results.

		// Returns result files built by building the given bazel target label.
		GetOutputFiles(label string, cfgKey configKey) ([]string, bool)
		GetOutputFiles(label string, cfgKey configKey) ([]string, error)

		// TODO(cparsons): Other cquery-related methods should be added here.
		// Returns the results of GetOutputFiles and GetCcObjectFiles in a single query (in that order).
		GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, bool, error)
		GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error)

		// Returns the executable binary resultant from building together the python sources
		GetPythonBinary(label string, cfgKey configKey) (string, bool)
		// TODO(b/232976601): Remove.
		GetPythonBinary(label string, cfgKey configKey) (string, error)

		// ** End cquery methods
		// ** end Cquery Results Retrieval Functions

		// Issues commands to Bazel to receive results for all cquery requests
		// queued in the BazelContext.
		@@ -153,19 +172,23 @@ type MockBazelContext struct {
		LabelToPythonBinary map[string]string
		}

		func (m MockBazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, bool) {
		result, ok := m.LabelToOutputFiles[label]
		return result, ok
		func (m MockBazelContext) QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey) {
		panic("unimplemented")
		}

		func (m MockBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, bool, error) {
		result, ok := m.LabelToCcInfo[label]
		return result, ok, nil
		func (m MockBazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, error) {
		result, _ := m.LabelToOutputFiles[label]
		return result, nil
		}

		func (m MockBazelContext) GetPythonBinary(label string, cfgKey configKey) (string, bool) {
		result, ok := m.LabelToPythonBinary[label]
		return result, ok
		func (m MockBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) {
		result, _ := m.LabelToCcInfo[label]
		return result, nil
		}

		func (m MockBazelContext) GetPythonBinary(label string, cfgKey configKey) (string, error) {
		result, _ := m.LabelToPythonBinary[label]
		return result, nil
		}

		func (m MockBazelContext) InvokeBazel() error {
		@@ -188,46 +211,53 @@ func (m MockBazelContext) AqueryDepsets() []bazel.AqueryDepset {

		var _ BazelContext = MockBazelContext{}

		func (bazelCtx *bazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, bool) {
		rawString, ok := bazelCtx.cquery(label, cquery.GetOutputFiles, cfgKey)
		var ret []string
		if ok {
		func (bazelCtx *bazelContext) QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey) {
		key := cqueryKey{label, requestType, cfgKey}
		bazelCtx.requestMutex.Lock()
		defer bazelCtx.requestMutex.Unlock()
		bazelCtx.requests[key] = true
		}

		func (bazelCtx *bazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, error) {
		key := cqueryKey{label, cquery.GetOutputFiles, cfgKey}
		if rawString, ok := bazelCtx.results[key]; ok {
		bazelOutput := strings.TrimSpace(rawString)
		ret = cquery.GetOutputFiles.ParseResult(bazelOutput)
		return cquery.GetOutputFiles.ParseResult(bazelOutput), nil
		}
		return ret, ok
		return nil, fmt.Errorf("no bazel response found for %v", key)
		}

		func (bazelCtx *bazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, bool, error) {
		result, ok := bazelCtx.cquery(label, cquery.GetCcInfo, cfgKey)
		if !ok {
		return cquery.CcInfo{}, ok, nil
		func (bazelCtx *bazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) {
		key := cqueryKey{label, cquery.GetCcInfo, cfgKey}
		if rawString, ok := bazelCtx.results[key]; ok {
		bazelOutput := strings.TrimSpace(rawString)
		return cquery.GetCcInfo.ParseResult(bazelOutput)
		}

		bazelOutput := strings.TrimSpace(result)
		ret, err := cquery.GetCcInfo.ParseResult(bazelOutput)
		return ret, ok, err
		return cquery.CcInfo{}, fmt.Errorf("no bazel response found for %v", key)
		}

		func (bazelCtx *bazelContext) GetPythonBinary(label string, cfgKey configKey) (string, bool) {
		rawString, ok := bazelCtx.cquery(label, cquery.GetPythonBinary, cfgKey)
		var ret string
		if ok {
		func (bazelCtx *bazelContext) GetPythonBinary(label string, cfgKey configKey) (string, error) {
		key := cqueryKey{label, cquery.GetPythonBinary, cfgKey}
		if rawString, ok := bazelCtx.results[key]; ok {
		bazelOutput := strings.TrimSpace(rawString)
		ret = cquery.GetPythonBinary.ParseResult(bazelOutput)
		return cquery.GetPythonBinary.ParseResult(bazelOutput), nil
		}
		return "", fmt.Errorf("no bazel response found for %v", key)
		}
		return ret, ok

		func (n noopBazelContext) QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey) {
		panic("unimplemented")
		}

		func (n noopBazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, bool) {
		func (n noopBazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, error) {
		panic("unimplemented")
		}

		func (n noopBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, bool, error) {
		func (n noopBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) {
		panic("unimplemented")
		}

		func (n noopBazelContext) GetPythonBinary(label string, cfgKey configKey) (string, bool) {
		func (n noopBazelContext) GetPythonBinary(label string, cfgKey configKey) (string, error) {
		panic("unimplemented")
		}

		@@ -314,24 +344,6 @@ func (context *bazelContext) BazelEnabled() bool {
		return true
		}

		// Adds a cquery request to the Bazel request queue, to be later invoked, or
		// returns the result of the given request if the request was already made.
		// If the given request was already made (and the results are available), then
		// returns (result, true). If the request is queued but no results are available,
		// then returns ("", false).
		func (context *bazelContext) cquery(label string, requestType cqueryRequest,
		cfgKey configKey) (string, bool) {
		key := cqueryKey{label, requestType, cfgKey}
		if result, ok := context.results[key]; ok {
		return result, true
		} else {
		context.requestMutex.Lock()
		defer context.requestMutex.Unlock()
		context.requests[key] = true
		return "", false
		}
		}

		func pwdPrefix() string {
		// Darwin doesn't have /proc
		if runtime.GOOS != "darwin" {
		@@ -916,7 +928,7 @@ func getConfigString(key cqueryKey) string {
		return arch + "\|" + os
		}

		func GetConfigKey(ctx ModuleContext) configKey {
		func GetConfigKey(ctx BaseModuleContext) configKey {
		return configKey{
		// use string because Arch is not a valid key in go
		arch: ctx.Arch().String(),

android/bazel_handler_test.go

+6 −7

Original line number	Diff line number	Diff line
		@@ -5,6 +5,8 @@ import (
		"path/filepath"
		"reflect"
		"testing"

		"android/soong/bazel/cquery"
		)

		func TestRequestResultsAfterInvokeBazel(t *testing.T) {
		@@ -13,17 +15,14 @@ func TestRequestResultsAfterInvokeBazel(t *testing.T) {
		bazelContext, _ := testBazelContext(t, map[bazelCommand]string{
		bazelCommand{command: "cquery", expression: "deps(@soong_injection//mixed_builds:buildroot, 2)"}: `//foo:bar\|arm64_armv8-a\|android>>out/foo/bar.txt`,
		})
		g, ok := bazelContext.GetOutputFiles(label, cfg)
		if ok {
		t.Errorf("Did not expect cquery results prior to running InvokeBazel(), but got %s", g)
		}
		bazelContext.QueueBazelRequest(label, cquery.GetOutputFiles, cfg)
		err := bazelContext.InvokeBazel()
		if err != nil {
		t.Fatalf("Did not expect error invoking Bazel, but got %s", err)
		}
		g, ok = bazelContext.GetOutputFiles(label, cfg)
		if !ok {
		t.Errorf("Expected cquery results after running InvokeBazel(), but got none")
		g, err := bazelContext.GetOutputFiles(label, cfg)
		if err != nil {
		t.Errorf("Expected cquery results after running InvokeBazel(), but got err %v", err)
		} else if w := []string{"out/foo/bar.txt"}; !reflect.DeepEqual(w, g) {
		t.Errorf("Expected output %s, got %s", w, g)
		}

android/config.go

+1 −1

Original line number	Diff line number	Diff line
		@@ -2047,7 +2047,7 @@ func (c *config) UseHostMusl() bool {
		return Bool(c.productVariables.HostMusl)
		}

		func (c *config) LogMixedBuild(ctx ModuleContext, useBazel bool) {
		func (c *config) LogMixedBuild(ctx BaseModuleContext, useBazel bool) {
		moduleName := ctx.Module().Name()
		c.mixedBuildsLock.Lock()
		defer c.mixedBuildsLock.Unlock()

android/filegroup.go

+36 −30

Original line number	Diff line number	Diff line
		@@ -18,6 +18,7 @@ import (
		"strings"

		"android/soong/bazel"
		"android/soong/bazel/cquery"

		"github.com/google/blueprint"
		)
		@@ -101,6 +102,7 @@ type fileGroup struct {
		srcs Paths
		}

		var _ MixedBuildBuildable = (*fileGroup)(nil)
		var _ SourceFileProducer = (*fileGroup)(nil)

		// filegroup contains a list of files that are referenced by other modules
		@@ -114,50 +116,54 @@ func FileGroupFactory() Module {
		return module
		}

		func (fg *fileGroup) maybeGenerateBazelBuildActions(ctx ModuleContext) {
		if !MixedBuildsEnabled(ctx) {
		return
		func (fg *fileGroup) GenerateAndroidBuildActions(ctx ModuleContext) {
		fg.srcs = PathsForModuleSrcExcludes(ctx, fg.properties.Srcs, fg.properties.Exclude_srcs)
		if fg.properties.Path != nil {
		fg.srcs = PathsWithModuleSrcSubDir(ctx, fg.srcs, String(fg.properties.Path))
		}
		}

		archVariant := ctx.Arch().String()
		osVariant := ctx.Os()
		if len(fg.Srcs()) == 1 && fg.Srcs()[0].Base() == fg.Name() {
		// This will be a regular file target, not filegroup, in Bazel.
		// See FilegroupBp2Build for more information.
		archVariant = Common.String()
		osVariant = CommonOS
		func (fg *fileGroup) Srcs() Paths {
		return append(Paths{}, fg.srcs...)
		}

		bazelCtx := ctx.Config().BazelContext
		filePaths, ok := bazelCtx.GetOutputFiles(fg.GetBazelLabel(ctx, fg), configKey{archVariant, osVariant})
		if !ok {
		return
		func (fg *fileGroup) MakeVars(ctx MakeVarsModuleContext) {
		if makeVar := String(fg.properties.Export_to_make_var); makeVar != "" {
		ctx.StrictRaw(makeVar, strings.Join(fg.srcs.Strings(), " "))
		}
		}

		bazelOuts := make(Paths, 0, len(filePaths))
		for _, p := range filePaths {
		src := PathForBazelOut(ctx, p)
		bazelOuts = append(bazelOuts, src)
		func (fg *fileGroup) QueueBazelCall(ctx BaseModuleContext) {
		bazelCtx := ctx.Config().BazelContext

		bazelCtx.QueueBazelRequest(
		fg.GetBazelLabel(ctx, fg),
		cquery.GetOutputFiles,
		configKey{Common.String(), CommonOS})
		}

		fg.srcs = bazelOuts
		func (fg *fileGroup) IsMixedBuildSupported(ctx BaseModuleContext) bool {
		return true
		}

		func (fg *fileGroup) GenerateAndroidBuildActions(ctx ModuleContext) {
		func (fg *fileGroup) ProcessBazelQueryResponse(ctx ModuleContext) {
		fg.srcs = PathsForModuleSrcExcludes(ctx, fg.properties.Srcs, fg.properties.Exclude_srcs)
		if fg.properties.Path != nil {
		fg.srcs = PathsWithModuleSrcSubDir(ctx, fg.srcs, String(fg.properties.Path))
		}

		fg.maybeGenerateBazelBuildActions(ctx)
		bazelCtx := ctx.Config().BazelContext
		filePaths, err := bazelCtx.GetOutputFiles(fg.GetBazelLabel(ctx, fg), configKey{Common.String(), CommonOS})
		if err != nil {
		ctx.ModuleErrorf(err.Error())
		return
		}

		func (fg *fileGroup) Srcs() Paths {
		return append(Paths{}, fg.srcs...)
		bazelOuts := make(Paths, 0, len(filePaths))
		for _, p := range filePaths {
		src := PathForBazelOut(ctx, p)
		bazelOuts = append(bazelOuts, src)
		}

		func (fg *fileGroup) MakeVars(ctx MakeVarsModuleContext) {
		if makeVar := String(fg.properties.Export_to_make_var); makeVar != "" {
		ctx.StrictRaw(makeVar, strings.Join(fg.srcs.Strings(), " "))
		}
		fg.srcs = bazelOuts
		}