Documenting java/dexpreopt_bootjars.go

	"github.com/google/blueprint/proptools"

)

// This comment describes:

//   1. ART boot images in general (their types, structure, file layout, etc.)

//   2. build system support for boot images

//

// 1. ART boot images

// ------------------

//

// A boot image in ART is a set of files that contain AOT-compiled native code and a heap snapshot

// of AOT-initialized classes for the bootclasspath Java libraries. A boot image is compiled from a

// set of DEX jars by the dex2oat compiler. A boot image is used for two purposes: 1) it is

// installed on device and loaded at runtime, and 2) other Java libraries and apps are compiled

// against it (compilation may take place either on host, known as "dexpreopt", or on device, known

// as "dexopt").

//

// A boot image is not a single file, but a collection of interrelated files. Each boot image has a

// number of components that correspond to the Java libraries that constitute it. For each component

// there are multiple files:

//   - *.oat or *.odex file with native code (architecture-specific, one per instruction set)

//   - *.art file with pre-initialized Java classes (architecture-specific, one per instruction set)

//   - *.vdex file with verification metadata for the DEX bytecode (architecture independent)

//

// *.vdex files for the boot images do not contain the DEX bytecode itself, because the

// bootclasspath DEX files are stored on disk in uncompressed and aligned form. Consequently a boot

// image is not self-contained and cannot be used without its DEX files. To simplify the management

// of boot image files, ART uses a certain naming scheme and associates the following metadata with

// each boot image:

//   - A stem, which is a symbolic name that is prepended to boot image file names.

//   - A location (on-device path to the boot image files).

//   - A list of boot image locations (on-device paths to dependency boot images).

//   - A set of DEX locations (on-device paths to the DEX files, one location for one DEX file used

//     to compile the boot image).

//

// There are two kinds of boot images:

//   - primary boot images

//   - boot image extensions

//

// 1.1. Primary boot images

// ------------------------

//

// A primary boot image is compiled for a core subset of bootclasspath Java libraries. It does not

// depend on any other images, and other boot images may depend on it.

//

// For example, assuming that the stem is "boot", the location is /apex/com.android.art/javalib/,

// the set of core bootclasspath libraries is A B C, and the boot image is compiled for ARM targets

// (32 and 64 bits), it will have three components with the following files:

//   - /apex/com.android.art/javalib/{arm,arm64}/boot.{art,oat,vdex}

//   - /apex/com.android.art/javalib/{arm,arm64}/boot-B.{art,oat,vdex}

//   - /apex/com.android.art/javalib/{arm,arm64}/boot-C.{art,oat,vdex}

//

// The files of the first component are special: they do not have the component name appended after

// the stem. This naming convention dates back to the times when the boot image was not split into

// components, and there were just boot.oat and boot.art. The decision to split was motivated by

// licensing reasons for one of the bootclasspath libraries.

//

// As of November 2020 the only primary boot image in Android is the image in the ART APEX

// com.android.art. The primary ART boot image contains the Core libraries that are part of the ART

// module. When the ART module gets updated, the primary boot image will be updated with it, and all

// dependent images will get invalidated (the checksum of the primary image stored in dependent

// images will not match), unless they are updated in sync with the ART module.

//

// 1.2. Boot image extensions

// --------------------------

//

// A boot image extension is compiled for a subset of bootclasspath Java libraries (in particular,

// this subset does not include the Core bootclasspath libraries that go into the primary boot

// image). A boot image extension depends on the primary boot image and optionally some other boot

// image extensions. Other images may depend on it. In other words, boot image extensions can form

// acyclic dependency graphs.

//

// The motivation for boot image extensions comes from the Mainline project. Consider a situation

// when the list of bootclasspath libraries is A B C, and both A and B are parts of the Android

// platform, but C is part of an updatable APEX com.android.C. When the APEX is updated, the Java

// code for C might have changed compared to the code that was used to compile the boot image.

// Consequently, the whole boot image is obsolete and invalidated (even though the code for A and B

// that does not depend on C is up to date). To avoid this, the original monolithic boot image is

// split in two parts: the primary boot image that contains A B, and the boot image extension that

// contains C and depends on the primary boot image (extends it).

//

// For example, assuming that the stem is "boot", the location is /system/framework, the set of

// bootclasspath libraries is D E (where D is part of the platform and is located in

// /system/framework, and E is part of a non-updatable APEX com.android.E and is located in

// /apex/com.android.E/javalib), and the boot image is compiled for ARM targets (32 and 64 bits),

// it will have two components with the following files:

//   - /system/framework/{arm,arm64}/boot-D.{art,oat,vdex}

//   - /system/framework/{arm,arm64}/boot-E.{art,oat,vdex}

//

// As of November 2020 the only boot image extension in Android is the Framework boot image

// extension. It extends the primary ART boot image and contains Framework libraries and other

// bootclasspath libraries from the platform and non-updatable APEXes that are not included in the

// ART image. The Framework boot image extension is updated together with the platform. In the

// future other boot image extensions may be added for some updatable modules.

//

//

// 2. Build system support for boot images

// ---------------------------------------

//

// The primary ART boot image needs to be compiled with one dex2oat invocation that depends on DEX

// jars for the core libraries. Framework boot image extension needs to be compiled with one dex2oat

// invocation that depends on the primary ART boot image and all bootclasspath DEX jars except the

// Core libraries.

//

// 2.1. Libraries that go in the boot images

// -----------------------------------------

//

// The contents of each boot image are determined by the PRODUCT variables. The primary ART APEX

// boot image contains libraries listed in the ART_APEX_JARS variable in the AOSP makefiles. The

// Framework boot image extension contains libraries specified in the PRODUCT_BOOT_JARS and

// PRODUCT_BOOT_JARS_EXTRA variables. The AOSP makefiles specify some common Framework libraries,

// but more product-specific libraries can be added in the product makefiles.

//

// Each component of the PRODUCT_BOOT_JARS and PRODUCT_BOOT_JARS_EXTRA variables is either a simple

// name (if the library is a part of the Platform), or a colon-separated pair <apex, name> (if the

// library is a part of a non-updatable APEX).

//

// A related variable PRODUCT_UPDATABLE_BOOT_JARS contains bootclasspath libraries that are in

// updatable APEXes. They are not included in the boot image.

//

// One exception to the above rules are "coverage" builds (a special build flavor which requires

// setting environment variable EMMA_INSTRUMENT_FRAMEWORK=true). In coverage builds the Java code in

// boot image libraries is instrumented, which means that the instrumentation library (jacocoagent)

// needs to be added to the list of bootclasspath DEX jars.

//

// In general, there is a requirement that the source code for a boot image library must be

// available at build time (e.g. it cannot be a stub that has a separate implementation library).

//

// 2.2. Static configs

// -------------------

//

// Because boot images are used to dexpreopt other Java modules, the paths to boot image files must

// be known by the time dexpreopt build rules for the dependent modules are generated. Boot image

// configs are constructed very early during the build, before build rule generation. The configs

// provide predefined paths to boot image files (these paths depend only on static build

// configuration, such as PRODUCT variables, and use hard-coded directory names).

//

// 2.3. Singleton

// --------------

//

// Build rules for the boot images are generated with a Soong singleton. Because a singleton has no

// dependencies on other modules, it has to find the modules for the DEX jars using VisitAllModules.

// Soong loops through all modules and compares each module against a list of bootclasspath library

// names. Then it generates build rules that copy DEX jars from their intermediate module-specific

// locations to the hard-coded locations predefined in the boot image configs.

//

// It would be possible to use a module with proper dependencies instead, but that would require

// changes in the way Soong generates variables for Make: a singleton can use one MakeVars() method

// that writes variables to out/soong/make_vars-*.mk, which is included early by the main makefile,

// but module(s) would have to use out/soong/Android-*.mk which has a group of LOCAL_* variables

// for each module, and is included later.

//

// 2.4. Install rules

// ------------------

//

// The primary boot image and the Framework extension are installed in different ways. The primary

// boot image is part of the ART APEX: it is copied into the APEX intermediate files, packaged

// together with other APEX contents, extracted and mounted on device. The Framework boot image

// extension is installed by the rules defined in makefiles (make/core/dex_preopt_libart.mk). Soong

// writes out a few DEXPREOPT_IMAGE_* variables for Make; these variables contain boot image names,

// paths and so on.

//

// 2.5. JIT-Zygote configuration

// -----------------------------

//

// One special configuration is JIT-Zygote build, when the primary ART image is used for compiling

// apps instead of the Framework boot image extension (see DEXPREOPT_USE_ART_IMAGE and UseArtImage).

//

func init() {

	RegisterDexpreoptBootJarsComponents(android.InitRegistrationContext)

}

// Target-independent description of pre-compiled boot image.

// Target-independent description of a boot image.

type bootImageConfig struct {

	// If this image is an extension, the image that it extends.

	extends *bootImageConfig

	variants []*bootImageVariant

}

// Target-dependent description of pre-compiled boot image.

// Target-dependent description of a boot image.

type bootImageVariant struct {

	*bootImageConfig

	unstrippedInstalls android.RuleBuilderInstalls

}

// Get target-specific boot image variant for the given boot image config and target.

func (image bootImageConfig) getVariant(target android.Target) *bootImageVariant {

	for _, variant := range image.variants {

		if variant.target.Os == target.Os && variant.target.Arch.ArchType == target.Arch.ArchType {

	return nil

}

// Return any (the first) variant which is for the device (as opposed to for the host)

// Return any (the first) variant which is for the device (as opposed to for the host).

func (image bootImageConfig) getAnyAndroidVariant() *bootImageVariant {

	for _, variant := range image.variants {

		if variant.target.Os == android.Android {

	return nil

}

// Return the name of a boot image module given a boot image config and a component (module) index.

// A module name is a combination of the Java library name, and the boot image stem (that is stored

// in the config).

func (image bootImageConfig) moduleName(ctx android.PathContext, idx int) string {

	// Dexpreopt on the boot class path produces multiple files. The first dex file

	// is converted into 'name'.art (to match the legacy assumption that 'name'.art

	// exists), and the rest are converted to 'name'-<jar>.art.

	// The first module of the primary boot image is special: its module name has only the stem, but

	// not the library name. All other module names are of the form <stem>-<library name>

	m := image.modules.Jar(idx)

	name := image.stem

	if idx != 0 || image.extends != nil {

	return name

}

// Return the name of the first boot image module, or stem if the list of modules is empty.

func (image bootImageConfig) firstModuleNameOrStem(ctx android.PathContext) string {

	if image.modules.Len() > 0 {

		return image.moduleName(ctx, 0)

	}

}

// Return filenames for the given boot image component, given the output directory and a list of

// extensions.

func (image bootImageConfig) moduleFiles(ctx android.PathContext, dir android.OutputPath, exts ...string) android.OutputPaths {

	ret := make(android.OutputPaths, 0, image.modules.Len()*len(exts))

	for i := 0; i < image.modules.Len(); i++ {

	return ret

}

// Return boot image locations (as a list of symbolic paths).

//

// The image "location" is a symbolic path that, with multiarchitecture support, doesn't really

// exist on the device. Typically it is /apex/com.android.art/javalib/boot.art and should be the

// same for all supported architectures on the device. The concrete architecture specific files

// actually end up in architecture-specific sub-directory such as arm, arm64, x86, or x86_64.

//

// For example a physical file

// "/apex/com.android.art/javalib/x86/boot.art" has "image location"

// "/apex/com.android.art/javalib/boot.art" (which is not an actual file).

// For example a physical file /apex/com.android.art/javalib/x86/boot.art has "image location"

// /apex/com.android.art/javalib/boot.art (which is not an actual file).

//

// For a primary boot image the list of locations has a single element.

//

// For a boot image extension the list of locations contains a location for all dependency images

// (including the primary image) and the location of the extension itself. For example, for the

// Framework boot image extension that depends on the primary ART boot image the list contains two

// elements.

//

// The location is passed as an argument to the ART tools like dex2oat instead of the real path.

// ART tools will then reconstruct the architecture-specific real path.

//

func (image *bootImageVariant) imageLocations() (imageLocations []string) {

	if image.extends != nil {

		imageLocations = image.extends.getVariant(image.target).imageLocations()

	return append(imageLocations, dexpreopt.PathToLocation(image.images, image.target.Arch.ArchType))

}

func concat(lists ...[]string) []string {

	var size int

	for _, l := range lists {

		size += len(l)

	}

	ret := make([]string, 0, size)

	for _, l := range lists {

		ret = append(ret, l...)

	}

	return ret

}

func dexpreoptBootJarsFactory() android.Singleton {

	return &dexpreoptBootJars{}

}

	return dexpreopt.GetGlobalConfig(ctx).DisablePreopt

}

// Singleton for generating boot image build rules.

type dexpreoptBootJars struct {

	// Default boot image config (currently always the Framework boot image extension). It should be

	// noted that JIT-Zygote builds use ART APEX image instead of the Framework boot image extension,

	// but the switch is handled not here, but in the makefiles (triggered with

	// DEXPREOPT_USE_ART_IMAGE=true).

	defaultBootImage *bootImageConfig

	// Other boot image configs (currently the list contains only the primary ART APEX image. It

	// used to contain an experimental JIT-Zygote image (now replaced with the ART APEX image). In

	// the future other boot image extensions may be added.

	otherImages []*bootImageConfig

	// Build path to a config file that Soong writes for Make (to be used in makefiles that install

	// the default boot image).

	dexpreoptConfigForMake android.WritablePath

}

	return files

}

// dexpreoptBoot singleton rules

// Generate build rules for boot images.

func (d *dexpreoptBootJars) GenerateBuildActions(ctx android.SingletonContext) {

	if skipDexpreoptBootJars(ctx) {

		return

		}

	}

	// The path to bootclasspath dex files needs to be known at module GenerateAndroidBuildAction time, before

	// the bootclasspath modules have been compiled.  Copy the dex jars there so the module rules that have

	// already been set up can find them.

	// The paths to bootclasspath DEX files need to be known at module GenerateAndroidBuildAction

	// time, before the boot images are built (these paths are used in dexpreopt rule generation for

	// Java libraries and apps). Generate rules that copy bootclasspath DEX jars to the predefined

	// paths.

	for i := range bootDexJars {

		ctx.Build(pctx, android.BuildParams{

			Rule:   android.Cp,

	return image

}

// Generate boot image build rules for a specific target.

func buildBootImageVariant(ctx android.SingletonContext, image *bootImageVariant,

	profile android.Path, missingDeps []string) android.WritablePaths {

	}

	if image.extends != nil {

		// It is a boot image extension, so it needs the boot image it depends on (in this case the

		// primary ART APEX image).

		artImage := image.primaryImages

		cmd.

			Flag("--runtime-arg").FlagWithInputList("-Xbootclasspath:", image.dexPathsDeps.Paths(), ":").

			Flag("--runtime-arg").FlagWithList("-Xbootclasspath-locations:", image.dexLocationsDeps, ":").

			FlagWithArg("--boot-image=", dexpreopt.PathToLocation(artImage, arch)).Implicit(artImage)

	} else {

		// It is a primary image, so it needs a base address.

		cmd.FlagWithArg("--base=", ctx.Config().LibartImgDeviceBaseAddress())

	}

	android.WriteFileRule(ctx, path, string(data))

}

// Export paths for default boot image to Make

// Define Make variables for boot image names, paths, etc. These variables are used in makefiles

// (make/core/dex_preopt_libart.mk) to generate install rules that copy boot image files to the

// correct output directories.

func (d *dexpreoptBootJars) MakeVars(ctx android.MakeVarsContext) {

	if d.dexpreoptConfigForMake != nil {

		ctx.Strict("DEX_PREOPT_CONFIG_FOR_MAKE", d.dexpreoptConfigForMake.String())

		ctx.Strict("DEXPREOPT_BOOTCLASSPATH_DEX_LOCATIONS", strings.Join(image.getAnyAndroidVariant().dexLocationsDeps, " "))

		var imageNames []string

		// TODO: the primary ART boot image should not be exposed to Make, as it is installed in a

		// different way as a part of the ART APEX. However, there is a special JIT-Zygote build

		// configuration which uses the primary ART image instead of the Framework boot image

		// extension, and it relies on the ART image being exposed to Make. To fix this, it is

		// necessary to rework the logic in makefiles.

		for _, current := range append(d.otherImages, image) {

			imageNames = append(imageNames, current.name)

			for _, variant := range current.variants {