Implement mixed builds for apex modules. (6a2b7c40) · Commits · e / os / android_build_soong

android/bazel_handler.go

+49 −11

Original line number	Diff line number	Diff line
		@@ -91,6 +91,10 @@ type configKey struct {
		osType OsType
		}

		func (c configKey) String() string {
		return fmt.Sprintf("%s::%s", c.arch, c.osType)
		}

		// Map key to describe bazel cquery requests.
		type cqueryKey struct {
		label string
		@@ -98,6 +102,11 @@ type cqueryKey struct {
		configKey configKey
		}

		func (c cqueryKey) String() string {
		return fmt.Sprintf("cquery(%s,%s,%s)", c.label, c.requestType.Name(), c.configKey)

		}

		// BazelContext is a context object useful for interacting with Bazel during
		// the course of a build. Use of Bazel to evaluate part of the build graph
		// is referred to as a "mixed build". (Some modules are managed by Soong,
		@@ -123,6 +132,9 @@ type BazelContext interface {
		// TODO(b/232976601): Remove.
		GetPythonBinary(label string, cfgKey configKey) (string, error)

		// Returns the results of the GetApexInfo query (including output files)
		GetApexInfo(label string, cfgkey configKey) (cquery.ApexCqueryInfo, error)

		// ** end Cquery Results Retrieval Functions

		// Issues commands to Bazel to receive results for all cquery requests
		@@ -186,6 +198,7 @@ type MockBazelContext struct {
		LabelToOutputFiles map[string][]string
		LabelToCcInfo map[string]cquery.CcInfo
		LabelToPythonBinary map[string]string
		LabelToApexInfo map[string]cquery.ApexCqueryInfo
		}

		func (m MockBazelContext) QueueBazelRequest(_ string, _ cqueryRequest, _ configKey) {
		@@ -207,6 +220,10 @@ func (m MockBazelContext) GetPythonBinary(label string, _ configKey) (string, er
		return result, nil
		}

		func (n MockBazelContext) GetApexInfo(_ string, _ configKey) (cquery.ApexCqueryInfo, error) {
		panic("unimplemented")
		}

		func (m MockBazelContext) InvokeBazel(_ Config) error {
		panic("unimplemented")
		}
		@@ -261,6 +278,14 @@ func (bazelCtx *bazelContext) GetPythonBinary(label string, cfgKey configKey) (s
		return "", fmt.Errorf("no bazel response found for %v", key)
		}

		func (bazelCtx *bazelContext) GetApexInfo(label string, cfgKey configKey) (cquery.ApexCqueryInfo, error) {
		key := cqueryKey{label, cquery.GetApexInfo, cfgKey}
		if rawString, ok := bazelCtx.results[key]; ok {
		return cquery.GetApexInfo.ParseResult(strings.TrimSpace(rawString)), nil
		}
		return cquery.ApexCqueryInfo{}, fmt.Errorf("no bazel response found for %v", key)
		}

		func (n noopBazelContext) QueueBazelRequest(_ string, _ cqueryRequest, _ configKey) {
		panic("unimplemented")
		}
		@@ -277,6 +302,10 @@ func (n noopBazelContext) GetPythonBinary(_ string, _ configKey) (string, error)
		panic("unimplemented")
		}

		func (n noopBazelContext) GetApexInfo(_ string, _ configKey) (cquery.ApexCqueryInfo, error) {
		panic("unimplemented")
		}

		func (n noopBazelContext) InvokeBazel(_ Config) error {
		panic("unimplemented")
		}
		@@ -401,9 +430,7 @@ func (r builtinBazelRunner) issueBazelCommand(paths bazelPaths, runName bazel.
		cmdFlags := []string{
		"--output_base=" + absolutePath(paths.outputBase),
		command.command,
		}
		cmdFlags = append(cmdFlags, command.expression)
		cmdFlags = append(cmdFlags,
		command.expression,
		// TODO(asmundak): is it needed in every build?
		"--profile=" + shared.BazelMetricsFilename(paths, runName),

		@@ -426,12 +453,12 @@ func (r builtinBazelRunner) issueBazelCommand(paths bazelPaths, runName bazel.

		// Suppress noise
		"--ui_event_filters=-INFO",
		"--noshow_progress")
		"--noshow_progress"}
		cmdFlags = append(cmdFlags, extraFlags...)

		bazelCmd := exec.Command(paths.bazelPath, cmdFlags...)
		bazelCmd.Dir = absolutePath(paths.syntheticWorkspaceDir())
		bazelCmd.Env = append(os.Environ(),
		extraEnv := []string{
		"HOME=" + paths.homeDir,
		pwdPrefix(),
		"BUILD_DIR=" + absolutePath(paths.soongOutDir),
		@@ -440,7 +467,9 @@ func (r builtinBazelRunner) issueBazelCommand(paths bazelPaths, runName bazel.
		"OUT_DIR=" + absolutePath(paths.outDir()),
		// Disables local host detection of gcc; toolchain information is defined
		// explicitly in BUILD files.
		"BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1")
		"BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1",
		}
		bazelCmd.Env = append(os.Environ(), extraEnv...)
		stderr := &bytes.Buffer{}
		bazelCmd.Stderr = stderr

		@@ -651,6 +680,15 @@ def get_arch(target):
		fail("expected platform name of the form 'android_<arch>' or 'linux_<arch>', but was " + str(platforms))
		return "UNKNOWN"

		def json_for_file(key, file):
		return '"' + key + '":"' + file.path + '"'

		def json_for_files(key, files):
		return '"' + key + '":[' + ",".join(['"' + f.path + '"' for f in files]) + ']'

		def json_for_labels(key, ll):
		return '"' + key + '":[' + ",".join(['"' + str(x) + '"' for x in ll]) + ']'

		def format(target):
		id_string = str(target.label) + "\|" + get_arch(target)

		@@ -728,7 +766,7 @@ func (context *bazelContext) InvokeBazel(config Config) error {
		cqueryOutput, cqueryErr, err := context.issueBazelCommand(context.paths, bazel.CqueryBuildRootRunName, cqueryCmd,
		"--output=starlark", "--starlark:file="+absolutePath(cqueryFileRelpath))
		if err != nil {
		err = ioutil.WriteFile(filepath.Join(soongInjectionPath, "cquery.out"), []byte(cqueryOutput), 0666)
		_ = ioutil.WriteFile(filepath.Join(soongInjectionPath, "cquery.out"), []byte(cqueryOutput), 0666)
		}
		if err != nil {
		return err

apex/apex.go

+180 −76

Original line number	Diff line number	Diff line
		@@ -17,6 +17,7 @@
		package apex

		import (
		"android/soong/bazel/cquery"
		"fmt"
		"path/filepath"
		"regexp"
		@@ -1803,6 +1804,181 @@ func (f fsType) string() string {
		}
		}

		var _ android.MixedBuildBuildable = (*apexBundle)(nil)

		func (a *apexBundle) IsMixedBuildSupported(ctx android.BaseModuleContext) bool {
		return ctx.ModuleType() == "apex" && a.properties.ApexType == imageApex
		}

		func (a *apexBundle) QueueBazelCall(ctx android.BaseModuleContext) {
		bazelCtx := ctx.Config().BazelContext
		bazelCtx.QueueBazelRequest(a.GetBazelLabel(ctx, a), cquery.GetApexInfo, android.GetConfigKey(ctx))
		}

		func (a *apexBundle) ProcessBazelQueryResponse(ctx android.ModuleContext) {
		if !a.commonBuildActions(ctx) {
		return
		}

		a.setApexTypeAndSuffix(ctx)
		a.setPayloadFsType(ctx)
		a.setSystemLibLink(ctx)

		if a.properties.ApexType != zipApex {
		a.compatSymlinks = makeCompatSymlinks(a.BaseModuleName(), ctx, a.primaryApexType)
		}

		bazelCtx := ctx.Config().BazelContext
		outputs, err := bazelCtx.GetApexInfo(a.GetBazelLabel(ctx, a), android.GetConfigKey(ctx))
		if err != nil {
		ctx.ModuleErrorf(err.Error())
		return
		}
		a.installDir = android.PathForModuleInstall(ctx, "apex")
		a.outputApexFile = android.PathForBazelOut(ctx, outputs.SignedOutput)
		a.outputFile = a.outputApexFile
		a.setCompression(ctx)

		a.publicKeyFile = android.PathForBazelOut(ctx, outputs.BundleKeyPair[0])
		a.privateKeyFile = android.PathForBazelOut(ctx, outputs.BundleKeyPair[1])
		a.containerCertificateFile = android.PathForBazelOut(ctx, outputs.ContainerKeyPair[0])
		a.containerPrivateKeyFile = android.PathForBazelOut(ctx, outputs.ContainerKeyPair[1])
		apexType := a.properties.ApexType
		switch apexType {
		case imageApex:
		// TODO(asmundak): Bazel does not create these files yet.
		// b/190817312
		a.htmlGzNotice = android.PathForBazelOut(ctx, "NOTICE.html.gz")
		// b/239081457
		a.bundleModuleFile = android.PathForBazelOut(ctx, a.Name()+apexType.suffix()+"-base.zip")
		// b/239081455
		a.nativeApisUsedByModuleFile = android.ModuleOutPath(android.PathForBazelOut(ctx, a.Name()+"_using.txt"))
		// b/239081456
		a.nativeApisBackedByModuleFile = android.ModuleOutPath(android.PathForBazelOut(ctx, a.Name()+"_backing.txt"))
		// b/239084755
		a.javaApisUsedByModuleFile = android.ModuleOutPath(android.PathForBazelOut(ctx, a.Name()+"_using.xml"))
		a.installedFile = ctx.InstallFile(a.installDir, a.Name()+a.installSuffix(), a.outputFile,
		a.compatSymlinks.Paths()...)
		default:
		panic(fmt.Errorf("unexpected apex_type for the ProcessBazelQuery: %v", a.properties.ApexType))
		}

		/*
		TODO(asmundak): compared to building an APEX with Soong, building it with Bazel does not
		return filesInfo and requiredDeps fields (in the Soong build the latter is updated).
		Fix this, as these fields are subsequently used in apex/androidmk.go and in apex/builder/go
		To find out what Soong build puts there, run:
		vctx := visitorContext{handleSpecialLibs: !android.Bool(a.properties.Ignore_system_library_special_case)}
		ctx.WalkDepsBlueprint(func(child, parent blueprint.Module) bool {
		return a.depVisitor(&vctx, ctx, child, parent)
		})
		vctx.normalizeFileInfo()
		*/

		}

		func (a *apexBundle) setCompression(ctx android.ModuleContext) {
		a.isCompressed = (a.properties.ApexType == imageApex) &&
		((ctx.Config().CompressedApex() &&
		proptools.BoolDefault(a.overridableProperties.Compressible, false) &&
		!a.testApex && !ctx.Config().UnbundledBuildApps()) \|\|
		a.testOnlyShouldForceCompression())
		}

		func (a apexBundle) installSuffix() string {
		if a.isCompressed {
		return imageCapexSuffix
		}
		return imageApexSuffix
		}

		func (a *apexBundle) setSystemLibLink(ctx android.ModuleContext) {
		// Optimization. If we are building bundled APEX, for the files that are gathered due to the
		// transitive dependencies, don't place them inside the APEX, but place a symlink pointing
		// the same library in the system partition, thus effectively sharing the same libraries
		// across the APEX boundary. For unbundled APEX, all the gathered files are actually placed
		// in the APEX.
		a.linkToSystemLib = !ctx.Config().UnbundledBuild() && a.installable()

		// APEXes targeting other than system/system_ext partitions use vendor/product variants.
		// So we can't link them to /system/lib libs which are core variants.
		if a.SocSpecific() \|\| a.DeviceSpecific() \|\| (a.ProductSpecific() && ctx.Config().EnforceProductPartitionInterface()) {
		a.linkToSystemLib = false
		}

		forced := ctx.Config().ForceApexSymlinkOptimization()
		updatable := a.Updatable() \|\| a.FutureUpdatable()

		// We don't need the optimization for updatable APEXes, as it might give false signal
		// to the system health when the APEXes are still bundled (b/149805758).
		if !forced && updatable && a.properties.ApexType == imageApex {
		a.linkToSystemLib = false
		}

		// We also don't want the optimization for host APEXes, because it doesn't make sense.
		if ctx.Host() {
		a.linkToSystemLib = false
		}
		}

		func (a *apexBundle) setPayloadFsType(ctx android.ModuleContext) {
		switch proptools.StringDefault(a.properties.Payload_fs_type, ext4FsType) {
		case ext4FsType:
		a.payloadFsType = ext4
		case f2fsFsType:
		a.payloadFsType = f2fs
		case erofsFsType:
		a.payloadFsType = erofs
		default:
		ctx.PropertyErrorf("payload_fs_type", "%q is not a valid filesystem for apex [ext4, f2fs, erofs]", *a.properties.Payload_fs_type)
		}
		}

		func (a *apexBundle) setApexTypeAndSuffix(ctx android.ModuleContext) {
		// Set suffix and primaryApexType depending on the ApexType
		buildFlattenedAsDefault := ctx.Config().FlattenApex()
		switch a.properties.ApexType {
		case imageApex:
		if buildFlattenedAsDefault {
		a.suffix = imageApexSuffix
		} else {
		a.suffix = ""
		a.primaryApexType = true

		if ctx.Config().InstallExtraFlattenedApexes() {
		a.requiredDeps = append(a.requiredDeps, a.Name()+flattenedSuffix)
		}
		}
		case zipApex:
		if proptools.String(a.properties.Payload_type) == "zip" {
		a.suffix = ""
		a.primaryApexType = true
		} else {
		a.suffix = zipApexSuffix
		}
		case flattenedApex:
		if buildFlattenedAsDefault {
		a.suffix = ""
		a.primaryApexType = true
		} else {
		a.suffix = flattenedSuffix
		}
		}
		}

		func (a *apexBundle) commonBuildActions(ctx android.ModuleContext) bool {
		a.checkApexAvailability(ctx)
		a.checkUpdatable(ctx)
		a.CheckMinSdkVersion(ctx)
		a.checkStaticLinkingToStubLibraries(ctx)
		a.checkStaticExecutables(ctx)
		if len(a.properties.Tests) > 0 && !a.testApex {
		ctx.PropertyErrorf("tests", "property allowed only in apex_test module type")
		return false
		}
		return true
		}

		type visitorContext struct {
		// all the files that will be included in this APEX
		filesInfo []apexFile
		@@ -2188,16 +2364,9 @@ func (a apexBundle) depVisitor(vctx visitorContext, ctx android.ModuleContext,
		func (a *apexBundle) GenerateAndroidBuildActions(ctx android.ModuleContext) {
		////////////////////////////////////////////////////////////////////////////////////////////
		// 1) do some validity checks such as apex_available, min_sdk_version, etc.
		a.checkApexAvailability(ctx)
		a.checkUpdatable(ctx)
		a.CheckMinSdkVersion(ctx)
		a.checkStaticLinkingToStubLibraries(ctx)
		a.checkStaticExecutables(ctx)
		if len(a.properties.Tests) > 0 && !a.testApex {
		ctx.PropertyErrorf("tests", "property allowed only in apex_test module type")
		if !a.commonBuildActions(ctx) {
		return
		}

		////////////////////////////////////////////////////////////////////////////////////////////
		// 2) traverse the dependency tree to collect apexFile structs from them.

		@@ -2219,74 +2388,9 @@ func (a *apexBundle) GenerateAndroidBuildActions(ctx android.ModuleContext) {
		a.installDir = android.PathForModuleInstall(ctx, "apex")
		a.filesInfo = vctx.filesInfo

		// Set suffix and primaryApexType depending on the ApexType
		buildFlattenedAsDefault := ctx.Config().FlattenApex()
		switch a.properties.ApexType {
		case imageApex:
		if buildFlattenedAsDefault {
		a.suffix = imageApexSuffix
		} else {
		a.suffix = ""
		a.primaryApexType = true

		if ctx.Config().InstallExtraFlattenedApexes() {
		a.requiredDeps = append(a.requiredDeps, a.Name()+flattenedSuffix)
		}
		}
		case zipApex:
		if proptools.String(a.properties.Payload_type) == "zip" {
		a.suffix = ""
		a.primaryApexType = true
		} else {
		a.suffix = zipApexSuffix
		}
		case flattenedApex:
		if buildFlattenedAsDefault {
		a.suffix = ""
		a.primaryApexType = true
		} else {
		a.suffix = flattenedSuffix
		}
		}

		switch proptools.StringDefault(a.properties.Payload_fs_type, ext4FsType) {
		case ext4FsType:
		a.payloadFsType = ext4
		case f2fsFsType:
		a.payloadFsType = f2fs
		case erofsFsType:
		a.payloadFsType = erofs
		default:
		ctx.PropertyErrorf("payload_fs_type", "%q is not a valid filesystem for apex [ext4, f2fs, erofs]", *a.properties.Payload_fs_type)
		}

		// Optimization. If we are building bundled APEX, for the files that are gathered due to the
		// transitive dependencies, don't place them inside the APEX, but place a symlink pointing
		// the same library in the system partition, thus effectively sharing the same libraries
		// across the APEX boundary. For unbundled APEX, all the gathered files are actually placed
		// in the APEX.
		a.linkToSystemLib = !ctx.Config().UnbundledBuild() && a.installable()

		// APEXes targeting other than system/system_ext partitions use vendor/product variants.
		// So we can't link them to /system/lib libs which are core variants.
		if a.SocSpecific() \|\| a.DeviceSpecific() \|\| (a.ProductSpecific() && ctx.Config().EnforceProductPartitionInterface()) {
		a.linkToSystemLib = false
		}

		forced := ctx.Config().ForceApexSymlinkOptimization()
		updatable := a.Updatable() \|\| a.FutureUpdatable()

		// We don't need the optimization for updatable APEXes, as it might give false signal
		// to the system health when the APEXes are still bundled (b/149805758).
		if !forced && updatable && a.properties.ApexType == imageApex {
		a.linkToSystemLib = false
		}

		// We also don't want the optimization for host APEXes, because it doesn't make sense.
		if ctx.Host() {
		a.linkToSystemLib = false
		}

		a.setApexTypeAndSuffix(ctx)
		a.setPayloadFsType(ctx)
		a.setSystemLibLink(ctx)
		if a.properties.ApexType != zipApex {
		a.compatSymlinks = makeCompatSymlinks(a.BaseModuleName(), ctx, a.primaryApexType)
		}

apex/builder.go

+3 −8

Original line number	Diff line number	Diff line
		@@ -806,8 +806,8 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) {
		return
		}

		if apexType == imageApex && (compressionEnabled \|\| a.testOnlyShouldForceCompression()) {
		a.isCompressed = true
		a.setCompression(ctx)
		if a.isCompressed {
		unsignedCompressedOutputFile := android.PathForModuleOut(ctx, a.Name()+imageCapexSuffix+".unsigned")

		compressRule := android.NewRuleBuilder(pctx, ctx)
		@@ -837,17 +837,12 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) {
		a.outputFile = signedCompressedOutputFile
		}

		installSuffix := suffix
		if a.isCompressed {
		installSuffix = imageCapexSuffix
		}

		if !a.installable() {
		a.SkipInstall()
		}

		// Install to $OUT/soong/{target,host}/.../apex.
		a.installedFile = ctx.InstallFile(a.installDir, a.Name()+installSuffix, a.outputFile,
		a.installedFile = ctx.InstallFile(a.installDir, a.Name()+a.installSuffix(), a.outputFile,
		a.compatSymlinks.Paths()...)

		// installed-files.txt is dist'ed

bazel/aquery.go

+2 −2

Original line number	Diff line number	Diff line
		@@ -574,7 +574,7 @@ func (a *aqueryArtifactHandler) getOutputPaths(actionEntry action) (outputPaths

		// expandTemplateContent substitutes the tokens in a template.
		func expandTemplateContent(actionEntry action) string {
		replacerString := []string{}
		var replacerString []string
		for _, pair := range actionEntry.Substitutions {
		value := pair.Value
		if val, ok := templateActionOverriddenTokens[pair.Key]; ok {
		@@ -647,7 +647,7 @@ func expandPathFragment(id pathFragmentId, pathFragmentsMap map[pathFragmentId]p
		}
		labels = append([]string{currFragment.Label}, labels...)
		if currId == currFragment.ParentId {
		return "", fmt.Errorf("Fragment cannot refer to itself as parent %#v", currFragment)
		return "", fmt.Errorf("fragment cannot refer to itself as parent %#v", currFragment)
		}
		currId = currFragment.ParentId
		}

bazel/cquery/request_type.go

+51 −1

Original line number	Diff line number	Diff line
		package cquery

		import (
		"encoding/json"
		"fmt"
		"strings"
		)
		@@ -9,6 +10,7 @@ var (
		GetOutputFiles = &getOutputFilesRequestType{}
		GetPythonBinary = &getPythonBinaryRequestType{}
		GetCcInfo = &getCcInfoType{}
		GetApexInfo = &getApexInfoType{}
		)

		type CcInfo struct {
		@@ -179,7 +181,7 @@ func (g getCcInfoType) ParseResult(rawString string) (CcInfo, error) {
		const expectedLen = 10
		splitString := strings.Split(rawString, "\|")
		if len(splitString) != expectedLen {
		return CcInfo{}, fmt.Errorf("Expected %d items, got %q", expectedLen, splitString)
		return CcInfo{}, fmt.Errorf("expected %d items, got %q", expectedLen, splitString)
		}
		outputFilesString := splitString[0]
		ccObjectsString := splitString[1]
		@@ -215,6 +217,54 @@ func (g getCcInfoType) ParseResult(rawString string) (CcInfo, error) {
		}, nil
		}

		// Query Bazel for the artifacts generated by the apex modules.
		type getApexInfoType struct{}

		// Name returns a string name for this request type. Such request type names must be unique,
		// and must only consist of alphanumeric characters.
		func (g getApexInfoType) Name() string {
		return "getApexInfo"
		}

		// StarlarkFunctionBody returns a starlark function body to process this request type.
		// The returned string is the body of a Starlark function which obtains
		// all request-relevant information about a target and returns a string containing
		// this information. The function should have the following properties:
		// - `target` is the only parameter to this function (a configured target).
		// - The return value must be a string.
		// - The function body should not be indented outside of its own scope.
		func (g getApexInfoType) StarlarkFunctionBody() string {
		return `info = providers(target)["//build/bazel/rules/apex:apex.bzl%ApexInfo"]
		return "{%s}" % ",".join([
		json_for_file("signed_output", info.signed_output),
		json_for_file("unsigned_output", info.unsigned_output),
		json_for_labels("provides_native_libs", info.provides_native_libs),
		json_for_labels("requires_native_libs", info.requires_native_libs),
		json_for_files("bundle_key_pair", info.bundle_key_pair),
		json_for_files("container_key_pair", info.container_key_pair)
		])`
		}

		type ApexCqueryInfo struct {
		SignedOutput string `json:"signed_output"`
		UnsignedOutput string `json:"unsigned_output"`
		ProvidesLibs []string `json:"provides_native_libs"`
		RequiresLibs []string `json:"requires_native_libs"`
		BundleKeyPair []string `json:"bundle_key_pair"`
		ContainerKeyPair []string `json:"container_key_pair"`
		}

		// ParseResult returns a value obtained by parsing the result of the request's Starlark function.
		// The given rawString must correspond to the string output which was created by evaluating the
		// Starlark given in StarlarkFunctionBody.
		func (g getApexInfoType) ParseResult(rawString string) ApexCqueryInfo {
		var info ApexCqueryInfo
		if err := json.Unmarshal([]byte(rawString), &info); err != nil {
		panic(fmt.Errorf("cannot parse cquery result '%s': %s", rawString, err))
		}
		return info
		}

		// splitOrEmpty is a modification of strings.Split() that returns an empty list
		// if the given string is empty.
		func splitOrEmpty(s string, sep string) []string {