Merge "Support cc_object modules in mixed builds" (bc39e17a) · Commits · e / os / android_build_soong

android/bazel_handler.go

+206 −29

Original line number	Diff line number	Diff line
		@@ -36,12 +36,14 @@ type CqueryRequestType int

		const (
		getAllFiles CqueryRequestType = iota
		getCcObjectFiles
		)

		// Map key to describe bazel cquery requests.
		type cqueryKey struct {
		label string
		requestType CqueryRequestType
		archType ArchType
		}

		type BazelContext interface {
		@@ -50,7 +52,11 @@ type BazelContext interface {
		// has been queued to be run later.

		// Returns result files built by building the given bazel target label.
		GetAllFiles(label string) ([]string, bool)
		GetAllFiles(label string, archType ArchType) ([]string, bool)

		// Returns object files produced by compiling the given cc-related target.
		// Retrieves these files from Bazel's CcInfo provider.
		GetCcObjectFiles(label string, archType ArchType) ([]string, bool)

		// TODO(cparsons): Other cquery-related methods should be added here.
		// ** End cquery methods
		@@ -100,7 +106,12 @@ type MockBazelContext struct {
		AllFiles map[string][]string
		}

		func (m MockBazelContext) GetAllFiles(label string) ([]string, bool) {
		func (m MockBazelContext) GetAllFiles(label string, archType ArchType) ([]string, bool) {
		result, ok := m.AllFiles[label]
		return result, ok
		}

		func (m MockBazelContext) GetCcObjectFiles(label string, archType ArchType) ([]string, bool) {
		result, ok := m.AllFiles[label]
		return result, ok
		}
		@@ -123,8 +134,18 @@ func (m MockBazelContext) BuildStatementsToRegister() []bazel.BuildStatement {

		var _ BazelContext = MockBazelContext{}

		func (bazelCtx *bazelContext) GetAllFiles(label string) ([]string, bool) {
		result, ok := bazelCtx.cquery(label, getAllFiles)
		func (bazelCtx *bazelContext) GetAllFiles(label string, archType ArchType) ([]string, bool) {
		result, ok := bazelCtx.cquery(label, getAllFiles, archType)
		if ok {
		bazelOutput := strings.TrimSpace(result)
		return strings.Split(bazelOutput, ", "), true
		} else {
		return nil, false
		}
		}

		func (bazelCtx *bazelContext) GetCcObjectFiles(label string, archType ArchType) ([]string, bool) {
		result, ok := bazelCtx.cquery(label, getCcObjectFiles, archType)
		if ok {
		bazelOutput := strings.TrimSpace(result)
		return strings.Split(bazelOutput, ", "), true
		@@ -133,7 +154,11 @@ func (bazelCtx *bazelContext) GetAllFiles(label string) ([]string, bool) {
		}
		}

		func (n noopBazelContext) GetAllFiles(label string) ([]string, bool) {
		func (n noopBazelContext) GetAllFiles(label string, archType ArchType) ([]string, bool) {
		panic("unimplemented")
		}

		func (n noopBazelContext) GetCcObjectFiles(label string, archType ArchType) ([]string, bool) {
		panic("unimplemented")
		}

		@@ -207,8 +232,9 @@ func (context *bazelContext) BazelEnabled() bool {
		// If the given request was already made (and the results are available), then
		// returns (result, true). If the request is queued but no results are available,
		// then returns ("", false).
		func (context *bazelContext) cquery(label string, requestType CqueryRequestType) (string, bool) {
		key := cqueryKey{label, requestType}
		func (context *bazelContext) cquery(label string, requestType CqueryRequestType,
		archType ArchType) (string, bool) {
		key := cqueryKey{label, requestType, archType}
		if result, ok := context.results[key]; ok {
		return result, true
		} else {
		@@ -241,17 +267,21 @@ func (context *bazelContext) issueBazelCommand(runName bazel.RunName, command st
		fmt.Sprintf("--platforms=%s", canonicalizeLabel("//build/bazel/platforms:generic_x86_64")))
		cmdFlags = append(cmdFlags,
		fmt.Sprintf("--extra_toolchains=%s", canonicalizeLabel("//prebuilts/clang/host/linux-x86:all")))
		// Explicitly disable downloading rules (such as canonical C++ and Java rules) from the network.
		cmdFlags = append(cmdFlags, "--experimental_repository_disable_download")
		cmdFlags = append(cmdFlags, extraFlags...)

		bazelCmd := exec.Command(context.bazelPath, cmdFlags...)
		bazelCmd.Dir = context.workspaceDir
		bazelCmd.Env = append(os.Environ(), "HOME="+context.homeDir, pwdPrefix())

		bazelCmd.Env = append(os.Environ(), "HOME="+context.homeDir, pwdPrefix(),
		// Disables local host detection of gcc; toolchain information is defined
		// explicitly in BUILD files.
		"BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1")
		stderr := &bytes.Buffer{}
		bazelCmd.Stderr = stderr

		if output, err := bazelCmd.Output(); err != nil {
		return "", fmt.Errorf("bazel command failed. command: [%s], error [%s]", bazelCmd, stderr)
		return "", fmt.Errorf("bazel command failed. command: [%s], env: [%s], error [%s]", bazelCmd, bazelCmd.Env, stderr)
		} else {
		return string(output), nil
		}
		@@ -273,20 +303,81 @@ local_repository(
		}

		func (context *bazelContext) mainBzlFileContents() []byte {
		// TODO(cparsons): Define configuration transitions programmatically based
		// on available archs.
		contents := `
		#####################################################
		# This file is generated by soong_build. Do not edit.
		#####################################################

		def _x86_64_transition_impl(settings, attr):
		return {
		"//command_line_option:platforms": "@sourceroot//build/bazel/platforms:generic_x86_64",
		}

		def _x86_transition_impl(settings, attr):
		return {
		"//command_line_option:platforms": "@sourceroot//build/bazel/platforms:generic_x86",
		}

		def _arm64_transition_impl(settings, attr):
		return {
		"//command_line_option:platforms": "@sourceroot//build/bazel/platforms:generic_arm64",
		}

		def _arm_transition_impl(settings, attr):
		return {
		"//command_line_option:platforms": "@sourceroot//build/bazel/platforms:generic_arm",
		}

		x86_64_transition = transition(
		implementation = _x86_64_transition_impl,
		inputs = [],
		outputs = [
		"//command_line_option:platforms",
		],
		)

		x86_transition = transition(
		implementation = _x86_transition_impl,
		inputs = [],
		outputs = [
		"//command_line_option:platforms",
		],
		)

		arm64_transition = transition(
		implementation = _arm64_transition_impl,
		inputs = [],
		outputs = [
		"//command_line_option:platforms",
		],
		)

		arm_transition = transition(
		implementation = _arm_transition_impl,
		inputs = [],
		outputs = [
		"//command_line_option:platforms",
		],
		)

		def _mixed_build_root_impl(ctx):
		return [DefaultInfo(files = depset(ctx.files.deps))]
		all_files = ctx.files.deps_x86_64 + ctx.files.deps_x86 + ctx.files.deps_arm64 + ctx.files.deps_arm
		return [DefaultInfo(files = depset(all_files))]

		# Rule representing the root of the build, to depend on all Bazel targets that
		# are required for the build. Building this target will build the entire Bazel
		# build tree.
		mixed_build_root = rule(
		implementation = _mixed_build_root_impl,
		attrs = {"deps" : attr.label_list()},
		attrs = {
		"deps_x86_64" : attr.label_list(cfg = x86_64_transition),
		"deps_x86" : attr.label_list(cfg = x86_transition),
		"deps_arm64" : attr.label_list(cfg = arm64_transition),
		"deps_arm" : attr.label_list(cfg = arm_transition),
		"_allowlist_function_transition": attr.label(default = "@bazel_tools//tools/allowlists/function_transition_allowlist"),
		},
		)

		def _phony_root_impl(ctx):
		@@ -317,25 +408,48 @@ func canonicalizeLabel(label string) string {
		}

		func (context *bazelContext) mainBuildFileContents() []byte {
		// TODO(cparsons): Map label to attribute programmatically; don't use hard-coded
		// architecture mapping.
		formatString := `
		# This file is generated by soong_build. Do not edit.
		load(":main.bzl", "mixed_build_root", "phony_root")

		mixed_build_root(name = "buildroot",
		deps = [%s],
		deps_x86_64 = [%s],
		deps_x86 = [%s],
		deps_arm64 = [%s],
		deps_arm = [%s],
		)

		phony_root(name = "phonyroot",
		deps = [":buildroot"],
		)
		`
		var buildRootDeps []string = nil
		var deps_x86_64 []string = nil
		var deps_x86 []string = nil
		var deps_arm64 []string = nil
		var deps_arm []string = nil
		for val, _ := range context.requests {
		buildRootDeps = append(buildRootDeps, fmt.Sprintf("\"%s\"", canonicalizeLabel(val.label)))
		labelString := fmt.Sprintf("\"%s\"", canonicalizeLabel(val.label))
		switch getArchString(val) {
		case "x86_64":
		deps_x86_64 = append(deps_x86_64, labelString)
		case "x86":
		deps_x86 = append(deps_x86, labelString)
		case "arm64":
		deps_arm64 = append(deps_arm64, labelString)
		case "arm":
		deps_arm = append(deps_arm, labelString)
		default:
		panic(fmt.Sprintf("unhandled architecture %s for %s", getArchString(val), val))
		}
		}
		buildRootDepsString := strings.Join(buildRootDeps, ",\n ")

		return []byte(fmt.Sprintf(formatString, buildRootDepsString))
		return []byte(fmt.Sprintf(formatString,
		strings.Join(deps_x86_64, ",\n "),
		strings.Join(deps_x86, ",\n "),
		strings.Join(deps_arm64, ",\n "),
		strings.Join(deps_arm, ",\n ")))
		}

		func (context *bazelContext) cqueryStarlarkFileContents() []byte {
		@@ -345,23 +459,64 @@ getAllFilesLabels = {
		%s
		}

		getCcObjectFilesLabels = {
		%s
		}

		def get_cc_object_files(target):
		result = []
		linker_inputs = providers(target)["CcInfo"].linking_context.linker_inputs.to_list()

		for linker_input in linker_inputs:
		for library in linker_input.libraries:
		for object in library.objects:
		result += [object.path]
		return result

		def get_arch(target):
		buildoptions = build_options(target)
		platforms = build_options(target)["//command_line_option:platforms"]
		if len(platforms) != 1:
		# An individual configured target should have only one platform architecture.
		# Note that it's fine for there to be multiple architectures for the same label,
		# but each is its own configured target.
		fail("expected exactly 1 platform for " + str(target.label) + " but got " + str(platforms))
		platform_name = build_options(target)["//command_line_option:platforms"][0].name
		if platform_name == "host":
		return "HOST"
		elif not platform_name.startswith("generic_"):
		fail("expected platform name of the form 'generic_<arch>', but was " + str(platforms))
		return "UNKNOWN"
		return platform_name[len("generic_"):]

		def format(target):
		if str(target.label) in getAllFilesLabels:
		return str(target.label) + ">>" + ', '.join([f.path for f in target.files.to_list()])
		id_string = str(target.label) + "\|" + get_arch(target)
		if id_string in getAllFilesLabels:
		return id_string + ">>" + ', '.join([f.path for f in target.files.to_list()])
		elif id_string in getCcObjectFilesLabels:
		return id_string + ">>" + ', '.join(get_cc_object_files(target))
		else:
		# This target was not requested via cquery, and thus must be a dependency
		# of a requested target.
		return ""
		return id_string + ">>NONE"
		`
		var buildRootDeps []string = nil
		// TODO(cparsons): Sort by request type instead of assuming all requests
		// are of GetAllFiles type.
		var getAllFilesDeps []string = nil
		var getCcObjectFilesDeps []string = nil

		for val, _ := range context.requests {
		buildRootDeps = append(buildRootDeps, fmt.Sprintf("\"%s\" : True", canonicalizeLabel(val.label)))
		labelWithArch := getCqueryId(val)
		mapEntryString := fmt.Sprintf("%q : True", labelWithArch)
		switch val.requestType {
		case getAllFiles:
		getAllFilesDeps = append(getAllFilesDeps, mapEntryString)
		case getCcObjectFiles:
		getCcObjectFilesDeps = append(getCcObjectFilesDeps, mapEntryString)
		}
		}
		buildRootDepsString := strings.Join(buildRootDeps, ",\n ")
		getAllFilesDepsString := strings.Join(getAllFilesDeps, ",\n ")
		getCcObjectFilesDepsString := strings.Join(getCcObjectFilesDeps, ",\n ")

		return []byte(fmt.Sprintf(formatString, buildRootDepsString))
		return []byte(fmt.Sprintf(formatString, getAllFilesDepsString, getCcObjectFilesDepsString))
		}

		// Returns a workspace-relative path containing build-related metadata required
		@@ -414,9 +569,15 @@ func (context *bazelContext) InvokeBazel() error {
		}
		buildrootLabel := "//:buildroot"
		cqueryOutput, err = context.issueBazelCommand(bazel.CqueryBuildRootRunName, "cquery",
		[]string{fmt.Sprintf("deps(%s)", buildrootLabel)},
		[]string{fmt.Sprintf("kind(rule, deps(%s))", buildrootLabel)},
		"--output=starlark",
		"--starlark:file="+cqueryFileRelpath)
		err = ioutil.WriteFile(
		absolutePath(filepath.Join(context.intermediatesDir(), "cquery.out")),
		[]byte(cqueryOutput), 0666)
		if err != nil {
		return err
		}

		if err != nil {
		return err
		@@ -431,10 +592,10 @@ func (context *bazelContext) InvokeBazel() error {
		}

		for val, _ := range context.requests {
		if cqueryResult, ok := cqueryResults[canonicalizeLabel(val.label)]; ok {
		if cqueryResult, ok := cqueryResults[getCqueryId(val)]; ok {
		context.results[val] = string(cqueryResult)
		} else {
		return fmt.Errorf("missing result for bazel target %s", val.label)
		return fmt.Errorf("missing result for bazel target %s. query output: [%s]", getCqueryId(val), cqueryOutput)
		}
		}

		@@ -510,6 +671,9 @@ func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) {

		// Register bazel-owned build statements (obtained from the aquery invocation).
		for index, buildStatement := range ctx.Config().BazelContext.BuildStatementsToRegister() {
		if len(buildStatement.Command) < 1 {
		panic(fmt.Sprintf("unhandled build statement: %s", buildStatement))
		}
		rule := NewRuleBuilder(pctx, ctx)
		cmd := rule.Command()
		cmd.Text(fmt.Sprintf("cd %s/execroot/__main__ && %s",
		@@ -531,3 +695,16 @@ func (c *bazelSingleton) GenerateBuildActions(ctx SingletonContext) {
		rule.Build(fmt.Sprintf("bazel %d", index), buildStatement.Mnemonic)
		}
		}

		func getCqueryId(key cqueryKey) string {
		return canonicalizeLabel(key.label) + "\|" + getArchString(key)
		}

		func getArchString(key cqueryKey) string {
		arch := key.archType.Name
		if len(arch) > 0 {
		return arch
		} else {
		return "x86_64"
		}
		}

bazel/aquery.go

+46 −0

Original line number	Diff line number	Diff line
		@@ -115,7 +115,23 @@ func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, error) {
		// may be an expensive operation.
		depsetIdToArtifactIdsCache := map[int][]int{}

		// Do a pass through all actions to identify which artifacts are middleman artifacts.
		// These will be omitted from the inputs of other actions.
		// TODO(b/180945500): Handle middleman actions; without proper handling, depending on generated
		// headers may cause build failures.
		middlemanArtifactIds := map[int]bool{}
		for _, actionEntry := range aqueryResult.Actions {
		if actionEntry.Mnemonic == "Middleman" {
		for _, outputId := range actionEntry.OutputIds {
		middlemanArtifactIds[outputId] = true
		}
		}
		}

		for _, actionEntry := range aqueryResult.Actions {
		if shouldSkipAction(actionEntry) {
		continue
		}
		outputPaths := []string{}
		for _, outputId := range actionEntry.OutputIds {
		outputPath, exists := artifactIdToPath[outputId]
		@@ -132,6 +148,10 @@ func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, error) {
		return nil, err
		}
		for _, inputId := range inputArtifacts {
		if _, isMiddlemanArtifact := middlemanArtifactIds[inputId]; isMiddlemanArtifact {
		// Omit middleman artifacts.
		continue
		}
		inputPath, exists := artifactIdToPath[inputId]
		if !exists {
		return nil, fmt.Errorf("undefined input artifactId %d", inputId)
		@@ -145,12 +165,38 @@ func AqueryBuildStatements(aqueryJsonProto []byte) ([]BuildStatement, error) {
		InputPaths: inputPaths,
		Env: actionEntry.EnvironmentVariables,
		Mnemonic: actionEntry.Mnemonic}
		if len(actionEntry.Arguments) < 1 {
		return nil, fmt.Errorf("received action with no command: [%s]", buildStatement)
		continue
		}
		buildStatements = append(buildStatements, buildStatement)
		}

		return buildStatements, nil
		}

		func shouldSkipAction(a action) bool {
		// TODO(b/180945121): Handle symlink actions.
		if a.Mnemonic == "Symlink" \|\| a.Mnemonic == "SourceSymlinkManifest" \|\| a.Mnemonic == "SymlinkTree" {
		return true
		}
		// TODO(b/180945500): Handle middleman actions; without proper handling, depending on generated
		// headers may cause build failures.
		if a.Mnemonic == "Middleman" {
		return true
		}
		// Skip "Fail" actions, which are placeholder actions designed to always fail.
		if a.Mnemonic == "Fail" {
		return true
		}
		// TODO(b/180946980): Handle FileWrite. The aquery proto currently contains no information
		// about the contents that are written.
		if a.Mnemonic == "FileWrite" {
		return true
		}
		return false
		}

		func artifactIdsFromDepsetId(depsetIdToDepset map[int]depSetOfFiles,
		depsetIdToArtifactIdsCache map[int][]int, depsetId int) ([]int, error) {
		if result, exists := depsetIdToArtifactIdsCache[depsetId]; exists {

cc/cc.go

+53 −21

Original line number	Diff line number	Diff line
		@@ -577,6 +577,17 @@ type installer interface {
		makeUninstallable(mod *Module)
		}

		// bazelHandler is the interface for a helper object related to deferring to Bazel for
		// processing a module (during Bazel mixed builds). Individual module types should define
		// their own bazel handler if they support deferring to Bazel.
		type bazelHandler interface {
		// Issue query to Bazel to retrieve information about Bazel's view of the current module.
		// If Bazel returns this information, set module properties on the current module to reflect
		// the returned information.
		// Returns true if information was available from Bazel, false if bazel invocation still needs to occur.
		generateBazelBuildActions(ctx android.ModuleContext, label string) bool
		}

		type xref interface {
		XrefCcFiles() android.Paths
		}
		@@ -782,6 +793,7 @@ type Module struct {
		compiler compiler
		linker linker
		installer installer
		bazelHandler bazelHandler

		features []feature
		stl *stl
		@@ -1559,24 +1571,7 @@ func (c *Module) getNameSuffixWithVndkVersion(ctx android.ModuleContext) string
		return nameSuffix
		}

		func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) {
		// Handle the case of a test module split by `test_per_src` mutator.
		//
		// The `test_per_src` mutator adds an extra variation named "", depending on all the other
		// `test_per_src` variations of the test module. Set `outputFile` to an empty path for this
		// module and return early, as this module does not produce an output file per se.
		if c.IsTestPerSrcAllTestsVariation() {
		c.outputFile = android.OptionalPath{}
		return
		}

		apexInfo := actx.Provider(android.ApexInfoProvider).(android.ApexInfo)
		if !apexInfo.IsForPlatform() {
		c.hideApexVariantFromMake = true
		}

		c.makeLinkType = GetMakeLinkType(actx, c)

		func (c *Module) setSubnameProperty(actx android.ModuleContext) {
		c.Properties.SubName = ""

		if c.Target().NativeBridge == android.NativeBridgeEnabled {
		@@ -1606,6 +1601,43 @@ func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) {
		c.Properties.SubName += "." + c.SdkVersion()
		}
		}
		}

		// Returns true if Bazel was successfully used for the analysis of this module.
		func (c *Module) maybeGenerateBazelActions(actx android.ModuleContext) bool {
		bazelModuleLabel := c.GetBazelLabel()
		bazelActionsUsed := false
		if c.bazelHandler != nil && actx.Config().BazelContext.BazelEnabled() && len(bazelModuleLabel) > 0 {
		bazelActionsUsed = c.bazelHandler.generateBazelBuildActions(actx, bazelModuleLabel)
		}
		return bazelActionsUsed
		}

		func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) {
		// TODO(cparsons): Any logic in this method occurring prior to querying Bazel should be
		// requested from Bazel instead.

		// Handle the case of a test module split by `test_per_src` mutator.
		//
		// The `test_per_src` mutator adds an extra variation named "", depending on all the other
		// `test_per_src` variations of the test module. Set `outputFile` to an empty path for this
		// module and return early, as this module does not produce an output file per se.
		if c.IsTestPerSrcAllTestsVariation() {
		c.outputFile = android.OptionalPath{}
		return
		}

		c.setSubnameProperty(actx)
		apexInfo := actx.Provider(android.ApexInfoProvider).(android.ApexInfo)
		if !apexInfo.IsForPlatform() {
		c.hideApexVariantFromMake = true
		}

		if c.maybeGenerateBazelActions(actx) {
		return
		}

		c.makeLinkType = GetMakeLinkType(actx, c)

		ctx := &moduleContext{
		ModuleContext: actx,

cc/object.go

+21 −0

Original line number	Diff line number	Diff line
		@@ -46,6 +46,26 @@ type objectLinker struct {
		Properties ObjectLinkerProperties
		}

		type objectBazelHandler struct {
		bazelHandler

		module *Module
		}

		func (handler *objectBazelHandler) generateBazelBuildActions(ctx android.ModuleContext, label string) bool {
		bazelCtx := ctx.Config().BazelContext
		objPaths, ok := bazelCtx.GetCcObjectFiles(label, ctx.Arch().ArchType)
		if ok {
		if len(objPaths) != 1 {
		ctx.ModuleErrorf("expected exactly one object file for '%s', but got %s", label, objPaths)
		return false
		}

		handler.module.outputFile = android.OptionalPathForPath(android.PathForBazelOut(ctx, objPaths[0]))
		}
		return ok
		}

		type ObjectLinkerProperties struct {
		// list of modules that should only provide headers for this module.
		Header_libs []string `android:"arch_variant,variant_prepend"`
		@@ -80,6 +100,7 @@ func ObjectFactory() android.Module {
		baseLinker: NewBaseLinker(module.sanitize),
		}
		module.compiler = NewBaseCompiler()
		module.bazelHandler = &objectBazelHandler{module: module}

		// Clang's address-significance tables are incompatible with ld -r.
		module.compiler.appendCflags([]string{"-fno-addrsig"})

genrule/genrule.go

+1 −1

Original line number	Diff line number	Diff line
		@@ -206,7 +206,7 @@ func toolDepsMutator(ctx android.BottomUpMutatorContext) {
		// Returns true if information was available from Bazel, false if bazel invocation still needs to occur.
		func (c *Module) generateBazelBuildActions(ctx android.ModuleContext, label string) bool {
		bazelCtx := ctx.Config().BazelContext
		filePaths, ok := bazelCtx.GetAllFiles(label)
		filePaths, ok := bazelCtx.GetAllFiles(label, ctx.Arch().ArchType)
		if ok {
		var bazelOutputFiles android.Paths
		for _, bazelOutputFile := range filePaths {