Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c4a3e6d2 authored by Paul Duffin's avatar Paul Duffin Committed by Automerger Merge Worker
Browse files

Export boot image files from prebuilt_apex/apex_set am: 5466a369

Original change: https://android-review.googlesource.com/c/platform/build/soong/+/1736777

Change-Id: I1cf84ff519ca6865d3a87ae75b1f48fc6ed1e3f4
parents 56a68346 5466a369
Loading
Loading
Loading
Loading
+64 −17
Original line number Diff line number Diff line
@@ -15,31 +15,67 @@
package android

import (
	"fmt"
	"strings"

	"github.com/google/blueprint"
)

// Provides support for interacting with the `deapexer` module to which a `prebuilt_apex` module
// will delegate the work to export files from a prebuilt '.apex` file.
//
// The actual processing that is done is quite convoluted but it is all about combining information
// from multiple different sources in order to allow a prebuilt module to use a file extracted from
// an apex file. As follows:
//
// 1. A prebuilt module, e.g. prebuilt_bootclasspath_fragment or java_import needs to use a file
//    from a prebuilt_apex/apex_set. It knows the path of the file within the apex but does not know
//    where the apex file is or what apex to use.
//
// 2. The connection between the prebuilt module and the prebuilt_apex/apex_set is created through
//    use of an exported_... property on the latter. That causes four things to occur:
//    a. A `deapexer` mopdule is created by the prebuilt_apex/apex_set to extract files from the
//       apex file.
//    b. A dependency is added from the prebuilt_apex/apex_set modules onto the prebuilt modules
//       listed in those properties.
//    c. An APEX variant is created for each of those prebuilt modules.
//    d. A dependency is added from the prebuilt modules to the `deapexer` module.
//
// 3. The prebuilt_apex/apex_set modules do not know which files are available in the apex file.
//    That information could be specified on the prebuilt_apex/apex_set modules but without
//    automated generation of those modules it would be expensive to maintain. So, instead they
//    obtain that information from the prebuilt modules. They do not know what files are actually in
//    the apex file either but they know what files they need from it. So, the
//    prebuilt_apex/apex_set modules obtain the files that should be in the apex file from those
//    modules and then pass those onto the `deapexer` module.
//
// 4. The `deapexer` module's ninja rule extracts all the files from the apex file into an output
//    directory and checks that all the expected files are there. The expected files are declared as
//    the outputs of the ninja rule so they are available to other modules.
//
// 5. The prebuilt modules then retrieve the paths to the files that they needed from the `deapexer`
//    module.
//
// The files that are passed to `deapexer` and those that are passed back have a unique identifier
// that links them together. e.g. If the `deapexer` is passed something like this:
//     core-libart{.dexjar} -> javalib/core-libart.jar
// it will return something like this:
//     core-libart{.dexjar} -> out/soong/.....deapexer.../javalib/core-libart.jar
//
// The reason why the `deapexer` module is separate from the prebuilt_apex/apex_set is to avoid
// cycles. e.g.
//   prebuilt_apex "com.android.art" depends upon java_import "core-libart":
//       This is so it can create an APEX variant of the latter and obtain information about the
//       files that it needs from the apex file.
//   java_import "core-libart" depends upon `deapexer` module:
//       This is so it can retrieve the paths to the files it needs.

// The information exported by the `deapexer` module, access it using `DeapxerInfoProvider`.
type DeapexerInfo struct {
	// map from the name of an exported file from a prebuilt_apex to the path to that file. The
	// exported file name is of the form <module>{<tag>} where <tag> is currently only allowed to be
	// ".dexjar".
	// exported file name is of the form <module>{<tag>}.
	//
	// See Prebuilt.ApexInfoMutator for more information.
	exports map[string]Path
}

// The set of supported prebuilt export tags. Used to verify the tag parameter for
// `PrebuiltExportPath`.
var supportedPrebuiltExportTags = map[string]struct{}{
	".dexjar": {},
}

// PrebuiltExportPath provides the path, or nil if not available, of a file exported from the
// prebuilt_apex that created this ApexInfo.
//
@@ -51,12 +87,6 @@ var supportedPrebuiltExportTags = map[string]struct{}{
//
// See apex/deapexer.go for more information.
func (i DeapexerInfo) PrebuiltExportPath(name, tag string) Path {

	if _, ok := supportedPrebuiltExportTags[tag]; !ok {
		panic(fmt.Errorf("unsupported prebuilt export tag %q, expected one of %s",
			tag, strings.Join(SortedStringKeys(supportedPrebuiltExportTags), ", ")))
	}

	path := i.exports[name+"{"+tag+"}"]
	return path
}
@@ -79,5 +109,22 @@ type deapexerTagStruct struct {
	blueprint.BaseDependencyTag
}

// Mark this tag so dependencies that use it are excluded from APEX contents.
func (t deapexerTagStruct) ExcludeFromApexContents() {}

var _ ExcludeFromApexContentsTag = DeapexerTag

// A tag that is used for dependencies on the `deapexer` module.
var DeapexerTag = deapexerTagStruct{}

// RequiredFilesFromPrebuiltApex must be implemented by modules that require files to be exported
// from a prebuilt_apex/apex_set.
type RequiredFilesFromPrebuiltApex interface {
	// RequiredFilesFromPrebuiltApex returns a map from the key (module name plus tag) to the required
	// path of the file within the prebuilt .apex file.
	//
	// For each key/file pair this will cause the file to be extracted out of the prebuilt .apex file,
	// and the path to the extracted file will be stored in the DeapexerInfo using that key, The path
	// can then be retrieved using the PrebuiltExportPath(name, tag) method.
	RequiredFilesFromPrebuiltApex(ctx BaseModuleContext) map[string]string
}
+1 −0
Original line number Diff line number Diff line
@@ -589,6 +589,7 @@ func TestBootclasspathFragmentInPrebuiltArtApex(t *testing.T) {
	})

	java.CheckModuleDependencies(t, result.TestContext, "mybootclasspathfragment", "android_common_com.android.art", []string{
		`com.android.art.deapexer`,
		`dex2oatd`,
		`prebuilt_bar`,
		`prebuilt_foo`,
+2 −2
Original line number Diff line number Diff line
@@ -42,8 +42,8 @@ import (

// DeapexerExportedFile defines the properties needed to expose a file from the deapexer module.
type DeapexerExportedFile struct {
	// The tag parameter which must be passed to android.OutputFileProducer OutputFiles(tag) method
	// to retrieve the path to the unpacked file.
	// The tag parameter which must be passed to android.DeapexerInfo's PrebuiltExportPath(name, tag)
	// method to retrieve the path to the unpacked file.
	Tag string

	// The path within the APEX that needs to be exported.
+24 −9
Original line number Diff line number Diff line
@@ -549,21 +549,36 @@ func createDeapexerModuleIfNeeded(ctx android.TopDownMutatorContext, deapexerNam
	}

	// Compute the deapexer properties from the transitive dependencies of this module.
	javaModules := []string{}
	exportedFiles := map[string]string{}
	commonModules := []string{}
	exportedFilesByKey := map[string]string{}
	requiringModulesByKey := map[string]android.Module{}
	ctx.WalkDeps(func(child, parent android.Module) bool {
		tag := ctx.OtherModuleDependencyTag(child)

		name := android.RemoveOptionalPrebuiltPrefix(ctx.OtherModuleName(child))
		if java.IsBootclasspathFragmentContentDepTag(tag) || tag == exportedJavaLibTag {
			javaModules = append(javaModules, name)
			commonModules = append(commonModules, name)

			// Add the dex implementation jar to the set of exported files. The path here must match the
			// path of the file in the APEX created by apexFileForJavaModule(...).
			exportedFiles[name+"{.dexjar}"] = filepath.Join("javalib", name+".jar")
			exportedFilesByKey[name+"{.dexjar}"] = filepath.Join("javalib", name+".jar")

		} else if tag == exportedBootclasspathFragmentTag {
			// Only visit the children of the bootclasspath_fragment for now.
			commonModules = append(commonModules, name)

			requiredFiles := child.(android.RequiredFilesFromPrebuiltApex).RequiredFilesFromPrebuiltApex(ctx)
			for k, v := range requiredFiles {
				if f, ok := exportedFilesByKey[k]; ok && f != v {
					otherModule := requiringModulesByKey[k]
					ctx.ModuleErrorf("inconsistent paths have been requested for key %q, %s requires path %s while %s requires path %s",
						k, child, v, otherModule, f)
					continue
				}
				exportedFilesByKey[k] = v
				requiringModulesByKey[k] = child
			}

			// Make sure to visit the children of the bootclasspath_fragment.
			return true
		}

@@ -572,16 +587,16 @@ func createDeapexerModuleIfNeeded(ctx android.TopDownMutatorContext, deapexerNam

	// Create properties for deapexer module.
	deapexerProperties := &DeapexerProperties{
		// Remove any duplicates from the java modules lists as a module may be included via a direct
		// Remove any duplicates from the common modules lists as a module may be included via a direct
		// dependency as well as transitive ones.
		CommonModules: android.SortedUniqueStrings(javaModules),
		CommonModules: android.SortedUniqueStrings(commonModules),
	}

	// Populate the exported files property in a fixed order.
	for _, tag := range android.SortedStringKeys(exportedFiles) {
	for _, tag := range android.SortedStringKeys(exportedFilesByKey) {
		deapexerProperties.ExportedFiles = append(deapexerProperties.ExportedFiles, DeapexerExportedFile{
			Tag:  tag,
			Path: exportedFiles[tag],
			Path: exportedFilesByKey[tag],
		})
	}

+127 −25
Original line number Diff line number Diff line
@@ -144,10 +144,22 @@ type commonBootclasspathFragment interface {
	// module cannot contribute to hidden API processing, e.g. because it is a prebuilt module in a
	// versioned sdk.
	produceHiddenAPIOutput(ctx android.ModuleContext, contents []android.Module, input HiddenAPIFlagInput) *HiddenAPIOutput

	// produceBootImageFiles produces the boot image (i.e. .art, .oat and .vdex) files for each of the
	// required android.ArchType values in the returned map.
	//
	// It must return nil if the boot image files cannot be produced for whatever reason.
	produceBootImageFiles(ctx android.ModuleContext, imageConfig *bootImageConfig, contents []android.Module) bootImageFilesByArch
}

var _ commonBootclasspathFragment = (*BootclasspathFragmentModule)(nil)

// bootImageFilesByArch is a map from android.ArchType to the paths to the boot image files.
//
// The paths include the .art, .oat and .vdex files, one for each of the modules from which the boot
// image is created.
type bootImageFilesByArch map[android.ArchType]android.Paths

func bootclasspathFragmentFactory() android.Module {
	m := &BootclasspathFragmentModule{}
	m.AddProperties(&m.properties)
@@ -285,7 +297,7 @@ type BootclasspathFragmentApexContentInfo struct {
	modules android.ConfiguredJarList

	// Map from arch type to the boot image files.
	bootImageFilesByArch map[android.ArchType]android.OutputPaths
	bootImageFilesByArch bootImageFilesByArch

	// Map from the base module name (without prebuilt_ prefix) of a fragment's contents module to the
	// hidden API encoded dex jar path.
@@ -299,7 +311,7 @@ func (i BootclasspathFragmentApexContentInfo) Modules() android.ConfiguredJarLis
// Get a map from ArchType to the associated boot image's contents for Android.
//
// Extension boot images only return their own files, not the files of the boot images they extend.
func (i BootclasspathFragmentApexContentInfo) AndroidBootImageFilesByArchType() map[android.ArchType]android.OutputPaths {
func (i BootclasspathFragmentApexContentInfo) AndroidBootImageFilesByArchType() bootImageFilesByArch {
	return i.bootImageFilesByArch
}

@@ -409,7 +421,12 @@ func (b *BootclasspathFragmentModule) GenerateAndroidBuildActions(ctx android.Mo
		// Perform hidden API processing.
		hiddenAPIOutput := b.generateHiddenAPIBuildActions(ctx, contents, fragments)

		var bootImageFilesByArch bootImageFilesByArch
		if imageConfig != nil {
			// Delegate the production of the boot image files to a module type specific method.
			common := ctx.Module().(commonBootclasspathFragment)
			bootImageFilesByArch = common.produceBootImageFiles(ctx, imageConfig, contents)

			if shouldCopyBootFilesToPredefinedLocations(ctx, imageConfig) {
				// Copy the dex jars of this fragment's content modules to their predefined locations.
				copyBootJarsToPredefinedLocations(ctx, hiddenAPIOutput.EncodedBootDexFilesByModule, imageConfig.dexPathsByModule)
@@ -419,7 +436,7 @@ func (b *BootclasspathFragmentModule) GenerateAndroidBuildActions(ctx android.Mo
		// A prebuilt fragment cannot contribute to an apex.
		if !android.IsModulePrebuilt(ctx.Module()) {
			// Provide the apex content info.
			b.provideApexContentInfo(ctx, imageConfig, contents, hiddenAPIOutput)
			b.provideApexContentInfo(ctx, imageConfig, hiddenAPIOutput, bootImageFilesByArch)
		}
	}
}
@@ -449,7 +466,7 @@ func shouldCopyBootFilesToPredefinedLocations(ctx android.ModuleContext, imageCo

// provideApexContentInfo creates, initializes and stores the apex content info for use by other
// modules.
func (b *BootclasspathFragmentModule) provideApexContentInfo(ctx android.ModuleContext, imageConfig *bootImageConfig, contents []android.Module, hiddenAPIOutput *HiddenAPIOutput) {
func (b *BootclasspathFragmentModule) provideApexContentInfo(ctx android.ModuleContext, imageConfig *bootImageConfig, hiddenAPIOutput *HiddenAPIOutput, bootImageFilesByArch bootImageFilesByArch) {
	// Construct the apex content info from the config.
	info := BootclasspathFragmentApexContentInfo{
		// Populate the apex content info with paths to the dex jars.
@@ -458,28 +475,10 @@ func (b *BootclasspathFragmentModule) provideApexContentInfo(ctx android.ModuleC

	if imageConfig != nil {
		info.modules = imageConfig.modules

		if !SkipDexpreoptBootJars(ctx) {
			// Force the GlobalSoongConfig to be created and cached for use by the dex_bootjars
			// GenerateSingletonBuildActions method as it cannot create it for itself.
			dexpreopt.GetGlobalSoongConfig(ctx)

			// Only generate the boot image if the configuration does not skip it.
			if b.generateBootImageBuildActions(ctx, contents, imageConfig) {
				// Allow the apex to access the boot image files.
				files := map[android.ArchType]android.OutputPaths{}
				for _, variant := range imageConfig.variants {
					// We also generate boot images for host (for testing), but we don't need those in the apex.
					// TODO(b/177892522) - consider changing this to check Os.OsClass = android.Device
					if variant.target.Os == android.Android {
						files[variant.target.Arch.ArchType] = variant.imagesDeps
					}
				}
				info.bootImageFilesByArch = files
			}
		}
	}

	info.bootImageFilesByArch = bootImageFilesByArch

	// Make the apex content info available for other modules.
	ctx.SetProvider(BootclasspathFragmentApexContentInfoProvider, info)
}
@@ -616,7 +615,6 @@ func retrieveLegacyEncodedBootDexFiles(ctx android.ModuleContext, contents []and
// createHiddenAPIFlagInput creates a HiddenAPIFlagInput struct and initializes it with information derived
// from the properties on this module and its dependencies.
func (b *BootclasspathFragmentModule) createHiddenAPIFlagInput(ctx android.ModuleContext, contents []android.Module, fragments []android.Module) HiddenAPIFlagInput {

	// Merge the HiddenAPIInfo from all the fragment dependencies.
	dependencyHiddenApiInfo := newHiddenAPIInfo()
	dependencyHiddenApiInfo.mergeFromFragmentDeps(ctx, fragments)
@@ -644,6 +642,30 @@ func (b *BootclasspathFragmentModule) produceHiddenAPIOutput(ctx android.ModuleC
	return hiddenAPIRulesForBootclasspathFragment(ctx, contents, input)
}

// produceBootImageFiles builds the boot image files from the source if it is required.
func (b *BootclasspathFragmentModule) produceBootImageFiles(ctx android.ModuleContext, imageConfig *bootImageConfig, contents []android.Module) bootImageFilesByArch {
	if SkipDexpreoptBootJars(ctx) {
		return nil
	}

	// Force the GlobalSoongConfig to be created and cached for use by the dex_bootjars
	// GenerateSingletonBuildActions method as it cannot create it for itself.
	dexpreopt.GetGlobalSoongConfig(ctx)

	// Only generate the boot image if the configuration does not skip it.
	if !b.generateBootImageBuildActions(ctx, contents, imageConfig) {
		return nil
	}

	// Only make the files available to an apex if they were actually generated.
	files := bootImageFilesByArch{}
	for _, variant := range imageConfig.apexVariants() {
		files[variant.target.Arch.ArchType] = variant.imagesDeps.Paths()
	}

	return files
}

// generateBootImageBuildActions generates ninja rules to create the boot image if required for this
// module.
//
@@ -881,8 +903,88 @@ func (module *prebuiltBootclasspathFragmentModule) produceHiddenAPIOutput(ctx an
	return &output
}

// produceBootImageFiles extracts the boot image files from the APEX if available.
func (module *prebuiltBootclasspathFragmentModule) produceBootImageFiles(ctx android.ModuleContext, imageConfig *bootImageConfig, contents []android.Module) bootImageFilesByArch {
	if !shouldCopyBootFilesToPredefinedLocations(ctx, imageConfig) {
		return nil
	}

	var deapexerModule android.Module
	ctx.VisitDirectDeps(func(module android.Module) {
		tag := ctx.OtherModuleDependencyTag(module)
		// Save away the `deapexer` module on which this depends, if any.
		if tag == android.DeapexerTag {
			deapexerModule = module
		}
	})

	if deapexerModule == nil {
		// This should never happen as a variant for a prebuilt_apex is only created if the
		// deapexer module has been configured to export the dex implementation jar for this module.
		ctx.ModuleErrorf("internal error: module does not depend on a `deapexer` module")
		return nil
	}

	di := ctx.OtherModuleProvider(deapexerModule, android.DeapexerProvider).(android.DeapexerInfo)
	name := module.BaseModuleName()
	for _, variant := range imageConfig.apexVariants() {
		arch := variant.target.Arch.ArchType
		for _, toPath := range variant.imagesDeps {
			// Get the path to the file that the deapexer extracted from the prebuilt apex file.
			tag := createBootImageTag(arch, toPath.Base())
			fromPath := di.PrebuiltExportPath(name, tag)

			// Copy the file to the predefined location.
			ctx.Build(pctx, android.BuildParams{
				Rule:   android.Cp,
				Input:  fromPath,
				Output: toPath,
			})
		}
	}

	// The returned files will be made available to APEXes that include a bootclasspath_fragment.
	// However, as a prebuilt_bootclasspath_fragment can never contribute to an APEX there is no point
	// in returning any files.
	return nil
}

var _ commonBootclasspathFragment = (*prebuiltBootclasspathFragmentModule)(nil)

// createBootImageTag creates the tag to uniquely identify the boot image file among all of the
// files that a module requires from the prebuilt .apex file.
func createBootImageTag(arch android.ArchType, baseName string) string {
	tag := fmt.Sprintf(".bootimage-%s-%s", arch, baseName)
	return tag
}

// RequiredFilesFromPrebuiltApex returns the list of all files the prebuilt_bootclasspath_fragment
// requires from a prebuilt .apex file.
//
// If there is no image config associated with this fragment then it returns nil. Otherwise, it
// returns the files that are listed in the image config.
func (module *prebuiltBootclasspathFragmentModule) RequiredFilesFromPrebuiltApex(ctx android.BaseModuleContext) map[string]string {
	imageConfig := module.getImageConfig(ctx)
	if imageConfig != nil {
		// Add the boot image files, e.g. .art, .oat and .vdex files.
		files := map[string]string{}
		name := module.BaseModuleName()
		for _, variant := range imageConfig.apexVariants() {
			arch := variant.target.Arch.ArchType
			for _, path := range variant.imagesDeps.Paths() {
				base := path.Base()
				tag := createBootImageTag(arch, base)
				key := fmt.Sprintf("%s{%s}", name, tag)
				files[key] = filepath.Join("javalib", arch.String(), base)
			}
		}
		return files
	}
	return nil
}

var _ android.RequiredFilesFromPrebuiltApex = (*prebuiltBootclasspathFragmentModule)(nil)

func prebuiltBootclasspathFragmentFactory() android.Module {
	m := &prebuiltBootclasspathFragmentModule{}
	m.AddProperties(&m.properties, &m.prebuiltProperties)
Loading