Merge "Implement mixed builds for apex modules, take 2." (88e75125) · Commits · e / os / android_build_soong

android/bazel_handler.go

+49 −11

Original line number	Diff line number	Diff line
		@@ -91,6 +91,10 @@ type configKey struct {
		osType OsType
		}

		func (c configKey) String() string {
		return fmt.Sprintf("%s::%s", c.arch, c.osType)
		}

		// Map key to describe bazel cquery requests.
		type cqueryKey struct {
		label string
		@@ -98,6 +102,11 @@ type cqueryKey struct {
		configKey configKey
		}

		func (c cqueryKey) String() string {
		return fmt.Sprintf("cquery(%s,%s,%s)", c.label, c.requestType.Name(), c.configKey)

		}

		// BazelContext is a context object useful for interacting with Bazel during
		// the course of a build. Use of Bazel to evaluate part of the build graph
		// is referred to as a "mixed build". (Some modules are managed by Soong,
		@@ -123,6 +132,9 @@ type BazelContext interface {
		// TODO(b/232976601): Remove.
		GetPythonBinary(label string, cfgKey configKey) (string, error)

		// Returns the results of the GetApexInfo query (including output files)
		GetApexInfo(label string, cfgkey configKey) (cquery.ApexCqueryInfo, error)

		// ** end Cquery Results Retrieval Functions

		// Issues commands to Bazel to receive results for all cquery requests
		@@ -186,6 +198,7 @@ type MockBazelContext struct {
		LabelToOutputFiles map[string][]string
		LabelToCcInfo map[string]cquery.CcInfo
		LabelToPythonBinary map[string]string
		LabelToApexInfo map[string]cquery.ApexCqueryInfo
		}

		func (m MockBazelContext) QueueBazelRequest(_ string, _ cqueryRequest, _ configKey) {
		@@ -207,6 +220,10 @@ func (m MockBazelContext) GetPythonBinary(label string, _ configKey) (string, er
		return result, nil
		}

		func (n MockBazelContext) GetApexInfo(_ string, _ configKey) (cquery.ApexCqueryInfo, error) {
		panic("unimplemented")
		}

		func (m MockBazelContext) InvokeBazel(_ Config) error {
		panic("unimplemented")
		}
		@@ -261,6 +278,14 @@ func (bazelCtx *bazelContext) GetPythonBinary(label string, cfgKey configKey) (s
		return "", fmt.Errorf("no bazel response found for %v", key)
		}

		func (bazelCtx *bazelContext) GetApexInfo(label string, cfgKey configKey) (cquery.ApexCqueryInfo, error) {
		key := cqueryKey{label, cquery.GetApexInfo, cfgKey}
		if rawString, ok := bazelCtx.results[key]; ok {
		return cquery.GetApexInfo.ParseResult(strings.TrimSpace(rawString)), nil
		}
		return cquery.ApexCqueryInfo{}, fmt.Errorf("no bazel response found for %v", key)
		}

		func (n noopBazelContext) QueueBazelRequest(_ string, _ cqueryRequest, _ configKey) {
		panic("unimplemented")
		}
		@@ -277,6 +302,10 @@ func (n noopBazelContext) GetPythonBinary(_ string, _ configKey) (string, error)
		panic("unimplemented")
		}

		func (n noopBazelContext) GetApexInfo(_ string, _ configKey) (cquery.ApexCqueryInfo, error) {
		panic("unimplemented")
		}

		func (n noopBazelContext) InvokeBazel(_ Config) error {
		panic("unimplemented")
		}
		@@ -401,9 +430,7 @@ func (r builtinBazelRunner) issueBazelCommand(paths bazelPaths, runName bazel.
		cmdFlags := []string{
		"--output_base=" + absolutePath(paths.outputBase),
		command.command,
		}
		cmdFlags = append(cmdFlags, command.expression)
		cmdFlags = append(cmdFlags,
		command.expression,
		// TODO(asmundak): is it needed in every build?
		"--profile=" + shared.BazelMetricsFilename(paths, runName),

		@@ -426,12 +453,12 @@ func (r builtinBazelRunner) issueBazelCommand(paths bazelPaths, runName bazel.

		// Suppress noise
		"--ui_event_filters=-INFO",
		"--noshow_progress")
		"--noshow_progress"}
		cmdFlags = append(cmdFlags, extraFlags...)

		bazelCmd := exec.Command(paths.bazelPath, cmdFlags...)
		bazelCmd.Dir = absolutePath(paths.syntheticWorkspaceDir())
		bazelCmd.Env = append(os.Environ(),
		extraEnv := []string{
		"HOME=" + paths.homeDir,
		pwdPrefix(),
		"BUILD_DIR=" + absolutePath(paths.soongOutDir),
		@@ -440,7 +467,9 @@ func (r builtinBazelRunner) issueBazelCommand(paths bazelPaths, runName bazel.
		"OUT_DIR=" + absolutePath(paths.outDir()),
		// Disables local host detection of gcc; toolchain information is defined
		// explicitly in BUILD files.
		"BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1")
		"BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1",
		}
		bazelCmd.Env = append(os.Environ(), extraEnv...)
		stderr := &bytes.Buffer{}
		bazelCmd.Stderr = stderr

		@@ -651,6 +680,15 @@ def get_arch(target):
		fail("expected platform name of the form 'android_<arch>' or 'linux_<arch>', but was " + str(platforms))
		return "UNKNOWN"

		def json_for_file(key, file):
		return '"' + key + '":"' + file.path + '"'

		def json_for_files(key, files):
		return '"' + key + '":[' + ",".join(['"' + f.path + '"' for f in files]) + ']'

		def json_for_labels(key, ll):
		return '"' + key + '":[' + ",".join(['"' + str(x) + '"' for x in ll]) + ']'

		def format(target):
		id_string = str(target.label) + "\|" + get_arch(target)

		@@ -728,7 +766,7 @@ func (context *bazelContext) InvokeBazel(config Config) error {
		cqueryOutput, cqueryErr, err := context.issueBazelCommand(context.paths, bazel.CqueryBuildRootRunName, cqueryCmd,
		"--output=starlark", "--starlark:file="+absolutePath(cqueryFileRelpath))
		if err != nil {
		err = ioutil.WriteFile(filepath.Join(soongInjectionPath, "cquery.out"), []byte(cqueryOutput), 0666)
		_ = ioutil.WriteFile(filepath.Join(soongInjectionPath, "cquery.out"), []byte(cqueryOutput), 0666)
		}
		if err != nil {
		return err

android/config.go

+2 −2

Original line number	Diff line number	Diff line
		@@ -1446,8 +1446,8 @@ func (c *config) ForceApexSymlinkOptimization() bool {
		return Bool(c.productVariables.ForceApexSymlinkOptimization)
		}

		func (c *config) CompressedApex() bool {
		return Bool(c.productVariables.CompressedApex)
		func (c *config) ApexCompressionEnabled() bool {
		return Bool(c.productVariables.CompressedApex) && !c.UnbundledBuildApps()
		}

		func (c *config) EnforceSystemCertificate() bool {

apex/apex.go

+183 −76

Original line number	Diff line number	Diff line
		@@ -17,6 +17,7 @@
		package apex

		import (
		"android/soong/bazel/cquery"
		"fmt"
		"path/filepath"
		"regexp"
		@@ -1803,6 +1804,184 @@ func (f fsType) string() string {
		}
		}

		var _ android.MixedBuildBuildable = (*apexBundle)(nil)

		func (a *apexBundle) IsMixedBuildSupported(ctx android.BaseModuleContext) bool {
		return ctx.ModuleType() == "apex" && a.properties.ApexType == imageApex
		}

		func (a *apexBundle) QueueBazelCall(ctx android.BaseModuleContext) {
		bazelCtx := ctx.Config().BazelContext
		bazelCtx.QueueBazelRequest(a.GetBazelLabel(ctx, a), cquery.GetApexInfo, android.GetConfigKey(ctx))
		}

		func (a *apexBundle) ProcessBazelQueryResponse(ctx android.ModuleContext) {
		if !a.commonBuildActions(ctx) {
		return
		}

		a.setApexTypeAndSuffix(ctx)
		a.setPayloadFsType(ctx)
		a.setSystemLibLink(ctx)

		if a.properties.ApexType != zipApex {
		a.compatSymlinks = makeCompatSymlinks(a.BaseModuleName(), ctx, a.primaryApexType)
		}

		bazelCtx := ctx.Config().BazelContext
		outputs, err := bazelCtx.GetApexInfo(a.GetBazelLabel(ctx, a), android.GetConfigKey(ctx))
		if err != nil {
		ctx.ModuleErrorf(err.Error())
		return
		}
		a.installDir = android.PathForModuleInstall(ctx, "apex")
		a.outputApexFile = android.PathForBazelOut(ctx, outputs.SignedOutput)
		a.outputFile = a.outputApexFile
		a.setCompression(ctx)

		a.publicKeyFile = android.PathForBazelOut(ctx, outputs.BundleKeyPair[0])
		a.privateKeyFile = android.PathForBazelOut(ctx, outputs.BundleKeyPair[1])
		a.containerCertificateFile = android.PathForBazelOut(ctx, outputs.ContainerKeyPair[0])
		a.containerPrivateKeyFile = android.PathForBazelOut(ctx, outputs.ContainerKeyPair[1])
		apexType := a.properties.ApexType
		switch apexType {
		case imageApex:
		// TODO(asmundak): Bazel does not create these files yet.
		// b/190817312
		a.htmlGzNotice = android.PathForBazelOut(ctx, "NOTICE.html.gz")
		// b/239081457
		a.bundleModuleFile = android.PathForBazelOut(ctx, a.Name()+apexType.suffix()+"-base.zip")
		// b/239081455
		a.nativeApisUsedByModuleFile = android.ModuleOutPath(android.PathForBazelOut(ctx, a.Name()+"_using.txt"))
		// b/239081456
		a.nativeApisBackedByModuleFile = android.ModuleOutPath(android.PathForBazelOut(ctx, a.Name()+"_backing.txt"))
		// b/239084755
		a.javaApisUsedByModuleFile = android.ModuleOutPath(android.PathForBazelOut(ctx, a.Name()+"_using.xml"))
		installSuffix := imageApexSuffix
		if a.isCompressed {
		installSuffix = imageCapexSuffix
		}
		a.installedFile = ctx.InstallFile(a.installDir, a.Name()+installSuffix, a.outputFile,
		a.compatSymlinks.Paths()...)
		default:
		panic(fmt.Errorf("unexpected apex_type for the ProcessBazelQuery: %v", a.properties.ApexType))
		}

		/*
		TODO(asmundak): compared to building an APEX with Soong, building it with Bazel does not
		return filesInfo and requiredDeps fields (in the Soong build the latter is updated).
		Fix this, as these fields are subsequently used in apex/androidmk.go and in apex/builder/go
		To find out what Soong build puts there, run:
		vctx := visitorContext{handleSpecialLibs: !android.Bool(a.properties.Ignore_system_library_special_case)}
		ctx.WalkDepsBlueprint(func(child, parent blueprint.Module) bool {
		return a.depVisitor(&vctx, ctx, child, parent)
		})
		vctx.normalizeFileInfo()
		*/

		}

		func (a *apexBundle) setCompression(ctx android.ModuleContext) {
		if a.properties.ApexType != imageApex {
		a.isCompressed = false
		} else if a.testOnlyShouldForceCompression() {
		a.isCompressed = true
		} else {
		a.isCompressed = ctx.Config().ApexCompressionEnabled() && a.isCompressable()
		}
		}

		func (a *apexBundle) setSystemLibLink(ctx android.ModuleContext) {
		// Optimization. If we are building bundled APEX, for the files that are gathered due to the
		// transitive dependencies, don't place them inside the APEX, but place a symlink pointing
		// the same library in the system partition, thus effectively sharing the same libraries
		// across the APEX boundary. For unbundled APEX, all the gathered files are actually placed
		// in the APEX.
		a.linkToSystemLib = !ctx.Config().UnbundledBuild() && a.installable()

		// APEXes targeting other than system/system_ext partitions use vendor/product variants.
		// So we can't link them to /system/lib libs which are core variants.
		if a.SocSpecific() \|\| a.DeviceSpecific() \|\| (a.ProductSpecific() && ctx.Config().EnforceProductPartitionInterface()) {
		a.linkToSystemLib = false
		}

		forced := ctx.Config().ForceApexSymlinkOptimization()
		updatable := a.Updatable() \|\| a.FutureUpdatable()

		// We don't need the optimization for updatable APEXes, as it might give false signal
		// to the system health when the APEXes are still bundled (b/149805758).
		if !forced && updatable && a.properties.ApexType == imageApex {
		a.linkToSystemLib = false
		}

		// We also don't want the optimization for host APEXes, because it doesn't make sense.
		if ctx.Host() {
		a.linkToSystemLib = false
		}
		}

		func (a *apexBundle) setPayloadFsType(ctx android.ModuleContext) {
		switch proptools.StringDefault(a.properties.Payload_fs_type, ext4FsType) {
		case ext4FsType:
		a.payloadFsType = ext4
		case f2fsFsType:
		a.payloadFsType = f2fs
		case erofsFsType:
		a.payloadFsType = erofs
		default:
		ctx.PropertyErrorf("payload_fs_type", "%q is not a valid filesystem for apex [ext4, f2fs, erofs]", *a.properties.Payload_fs_type)
		}
		}

		func (a *apexBundle) setApexTypeAndSuffix(ctx android.ModuleContext) {
		// Set suffix and primaryApexType depending on the ApexType
		buildFlattenedAsDefault := ctx.Config().FlattenApex()
		switch a.properties.ApexType {
		case imageApex:
		if buildFlattenedAsDefault {
		a.suffix = imageApexSuffix
		} else {
		a.suffix = ""
		a.primaryApexType = true

		if ctx.Config().InstallExtraFlattenedApexes() {
		a.requiredDeps = append(a.requiredDeps, a.Name()+flattenedSuffix)
		}
		}
		case zipApex:
		if proptools.String(a.properties.Payload_type) == "zip" {
		a.suffix = ""
		a.primaryApexType = true
		} else {
		a.suffix = zipApexSuffix
		}
		case flattenedApex:
		if buildFlattenedAsDefault {
		a.suffix = ""
		a.primaryApexType = true
		} else {
		a.suffix = flattenedSuffix
		}
		}
		}

		func (a apexBundle) isCompressable() bool {
		return proptools.BoolDefault(a.overridableProperties.Compressible, false) && !a.testApex
		}

		func (a *apexBundle) commonBuildActions(ctx android.ModuleContext) bool {
		a.checkApexAvailability(ctx)
		a.checkUpdatable(ctx)
		a.CheckMinSdkVersion(ctx)
		a.checkStaticLinkingToStubLibraries(ctx)
		a.checkStaticExecutables(ctx)
		if len(a.properties.Tests) > 0 && !a.testApex {
		ctx.PropertyErrorf("tests", "property allowed only in apex_test module type")
		return false
		}
		return true
		}

		type visitorContext struct {
		// all the files that will be included in this APEX
		filesInfo []apexFile
		@@ -2188,16 +2367,9 @@ func (a apexBundle) depVisitor(vctx visitorContext, ctx android.ModuleContext,
		func (a *apexBundle) GenerateAndroidBuildActions(ctx android.ModuleContext) {
		////////////////////////////////////////////////////////////////////////////////////////////
		// 1) do some validity checks such as apex_available, min_sdk_version, etc.
		a.checkApexAvailability(ctx)
		a.checkUpdatable(ctx)
		a.CheckMinSdkVersion(ctx)
		a.checkStaticLinkingToStubLibraries(ctx)
		a.checkStaticExecutables(ctx)
		if len(a.properties.Tests) > 0 && !a.testApex {
		ctx.PropertyErrorf("tests", "property allowed only in apex_test module type")
		if !a.commonBuildActions(ctx) {
		return
		}

		////////////////////////////////////////////////////////////////////////////////////////////
		// 2) traverse the dependency tree to collect apexFile structs from them.

		@@ -2219,74 +2391,9 @@ func (a *apexBundle) GenerateAndroidBuildActions(ctx android.ModuleContext) {
		a.installDir = android.PathForModuleInstall(ctx, "apex")
		a.filesInfo = vctx.filesInfo

		// Set suffix and primaryApexType depending on the ApexType
		buildFlattenedAsDefault := ctx.Config().FlattenApex()
		switch a.properties.ApexType {
		case imageApex:
		if buildFlattenedAsDefault {
		a.suffix = imageApexSuffix
		} else {
		a.suffix = ""
		a.primaryApexType = true

		if ctx.Config().InstallExtraFlattenedApexes() {
		a.requiredDeps = append(a.requiredDeps, a.Name()+flattenedSuffix)
		}
		}
		case zipApex:
		if proptools.String(a.properties.Payload_type) == "zip" {
		a.suffix = ""
		a.primaryApexType = true
		} else {
		a.suffix = zipApexSuffix
		}
		case flattenedApex:
		if buildFlattenedAsDefault {
		a.suffix = ""
		a.primaryApexType = true
		} else {
		a.suffix = flattenedSuffix
		}
		}

		switch proptools.StringDefault(a.properties.Payload_fs_type, ext4FsType) {
		case ext4FsType:
		a.payloadFsType = ext4
		case f2fsFsType:
		a.payloadFsType = f2fs
		case erofsFsType:
		a.payloadFsType = erofs
		default:
		ctx.PropertyErrorf("payload_fs_type", "%q is not a valid filesystem for apex [ext4, f2fs, erofs]", *a.properties.Payload_fs_type)
		}

		// Optimization. If we are building bundled APEX, for the files that are gathered due to the
		// transitive dependencies, don't place them inside the APEX, but place a symlink pointing
		// the same library in the system partition, thus effectively sharing the same libraries
		// across the APEX boundary. For unbundled APEX, all the gathered files are actually placed
		// in the APEX.
		a.linkToSystemLib = !ctx.Config().UnbundledBuild() && a.installable()

		// APEXes targeting other than system/system_ext partitions use vendor/product variants.
		// So we can't link them to /system/lib libs which are core variants.
		if a.SocSpecific() \|\| a.DeviceSpecific() \|\| (a.ProductSpecific() && ctx.Config().EnforceProductPartitionInterface()) {
		a.linkToSystemLib = false
		}

		forced := ctx.Config().ForceApexSymlinkOptimization()
		updatable := a.Updatable() \|\| a.FutureUpdatable()

		// We don't need the optimization for updatable APEXes, as it might give false signal
		// to the system health when the APEXes are still bundled (b/149805758).
		if !forced && updatable && a.properties.ApexType == imageApex {
		a.linkToSystemLib = false
		}

		// We also don't want the optimization for host APEXes, because it doesn't make sense.
		if ctx.Host() {
		a.linkToSystemLib = false
		}

		a.setApexTypeAndSuffix(ctx)
		a.setPayloadFsType(ctx)
		a.setSystemLibLink(ctx)
		if a.properties.ApexType != zipApex {
		a.compatSymlinks = makeCompatSymlinks(a.BaseModuleName(), ctx, a.primaryApexType)
		}

apex/builder.go

+12 −12

Original line number	Diff line number	Diff line
		@@ -549,8 +549,6 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) {
		outHostBinDir := ctx.Config().HostToolPath(ctx, "").String()
		prebuiltSdkToolsBinDir := filepath.Join("prebuilts", "sdk", "tools", runtime.GOOS, "bin")

		// Figure out if we need to compress the apex.
		compressionEnabled := ctx.Config().CompressedApex() && proptools.BoolDefault(a.overridableProperties.Compressible, false) && !a.testApex && !ctx.Config().UnbundledBuildApps()
		if apexType == imageApex {

		////////////////////////////////////////////////////////////////////////////////////
		@@ -635,10 +633,15 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) {
		implicitInputs = append(implicitInputs, noticeAssetPath)
		optFlags = append(optFlags, "--assets_dir "+filepath.Dir(noticeAssetPath.String()))

		if (moduleMinSdkVersion.GreaterThan(android.SdkVersion_Android10) && !a.shouldGenerateHashtree()) && !compressionEnabled {
		// Apexes which are supposed to be installed in builtin dirs(/system, etc)
		// don't need hashtree for activation. Therefore, by removing hashtree from
		// apex bundle (filesystem image in it, to be specific), we can save storage.
		needHashTree := moduleMinSdkVersion.LessThanOrEqualTo(android.SdkVersion_Android10) \|\|
		a.shouldGenerateHashtree()
		if ctx.Config().ApexCompressionEnabled() && a.isCompressable() {
		needHashTree = true
		}
		if !needHashTree {
		optFlags = append(optFlags, "--no_hashtree")
		}

		@@ -806,8 +809,9 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) {
		return
		}

		if apexType == imageApex && (compressionEnabled \|\| a.testOnlyShouldForceCompression()) {
		a.isCompressed = true
		installSuffix := suffix
		a.setCompression(ctx)
		if a.isCompressed {
		unsignedCompressedOutputFile := android.PathForModuleOut(ctx, a.Name()+imageCapexSuffix+".unsigned")

		compressRule := android.NewRuleBuilder(pctx, ctx)
		@@ -835,10 +839,6 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) {
		Args: args,
		})
		a.outputFile = signedCompressedOutputFile
		}

		installSuffix := suffix
		if a.isCompressed {
		installSuffix = imageCapexSuffix
		}

bazel/aquery.go

+2 −2

Original line number	Diff line number	Diff line
		@@ -574,7 +574,7 @@ func (a *aqueryArtifactHandler) getOutputPaths(actionEntry action) (outputPaths

		// expandTemplateContent substitutes the tokens in a template.
		func expandTemplateContent(actionEntry action) string {
		replacerString := []string{}
		var replacerString []string
		for _, pair := range actionEntry.Substitutions {
		value := pair.Value
		if val, ok := templateActionOverriddenTokens[pair.Key]; ok {
		@@ -647,7 +647,7 @@ func expandPathFragment(id pathFragmentId, pathFragmentsMap map[pathFragmentId]p
		}
		labels = append([]string{currFragment.Label}, labels...)
		if currId == currFragment.ParentId {
		return "", fmt.Errorf("Fragment cannot refer to itself as parent %#v", currFragment)
		return "", fmt.Errorf("fragment cannot refer to itself as parent %#v", currFragment)
		}
		currId = currFragment.ParentId
		}