Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c9b4f6b5 authored by Colin Cross's avatar Colin Cross
Browse files

Use transitive header jars in classpaths 

Skip combining jars into turbine-combined, combined, and withres jars
and instead collect transitive jars to use in the classpath and to
produce the final dexed jar.

This reduces the size of a `m checkbuild` in git_main by 11%, from
1300 KiB to 1154 KiB.  It may also improve caching and reduce uplink
network bandwidth when building with RBE, as now the classpath inputs
to rules are themselves outputs of previous rules and so already in
the RBE CAS.

The downside is that the classpath inputs to each rule are now much
longer, increasing the Soong ninja file size 11%, from 4.6 GiB to
5.1 GiB.  This could be mitigated in the future by supporting something
like depsets in the generated ninja file to reduce duplication.

Bug: 308016794
Test: TestSimple, TestKotlin, TestClasspath
Flag: build.RELEASE_USE_TRANSITIVE_JARS_IN_CLASSPATH
Change-Id: I2b7b4261375494370da70f98597c8719f1d561cf
parent fdaa672a

android/config.go

+4 −0
Original line number Diff line number Diff line
@@ -1962,6 +1962,10 @@ func (c *config) UseResourceProcessorByDefault() bool { 
	return c.productVariables.GetBuildFlagBool("RELEASE_USE_RESOURCE_PROCESSOR_BY_DEFAULT")

}



func (c *config) UseTransitiveJarsInClasspath() bool {

	return c.productVariables.GetBuildFlagBool("RELEASE_USE_TRANSITIVE_JARS_IN_CLASSPATH")

}



var (

	mainlineApexContributionBuildFlagsToApexNames = map[string]string{

		"RELEASE_APEX_CONTRIBUTIONS_ADBD":                    "com.android.adbd",

java/aar.go

+75 −26
Original line number Diff line number Diff line
@@ -1297,6 +1297,10 @@ func (a *AARImport) GenerateAndroidBuildActions(ctx android.ModuleContext) { 
	var staticJars android.Paths

	var staticHeaderJars android.Paths

	var staticResourceJars android.Paths

	var transitiveStaticLibsHeaderJars []*android.DepSet[android.Path]

	var transitiveStaticLibsImplementationJars []*android.DepSet[android.Path]

	var transitiveStaticLibsResourceJars []*android.DepSet[android.Path]



	ctx.VisitDirectDeps(func(module android.Module) {

		if dep, ok := android.OtherModuleProvider(ctx, module, JavaInfoProvider); ok {

			tag := ctx.OtherModuleDependencyTag(module)
@@ -1305,60 +1309,105 @@ func (a *AARImport) GenerateAndroidBuildActions(ctx android.ModuleContext) { 
				staticJars = append(staticJars, dep.ImplementationJars...)

				staticHeaderJars = append(staticHeaderJars, dep.HeaderJars...)

				staticResourceJars = append(staticResourceJars, dep.ResourceJars...)

				if dep.TransitiveStaticLibsHeaderJars != nil {

					transitiveStaticLibsHeaderJars = append(transitiveStaticLibsHeaderJars, dep.TransitiveStaticLibsHeaderJars)

				}

				if dep.TransitiveStaticLibsImplementationJars != nil {

					transitiveStaticLibsImplementationJars = append(transitiveStaticLibsImplementationJars, dep.TransitiveStaticLibsImplementationJars)

				}

				if dep.TransitiveStaticLibsResourceJars != nil {

					transitiveStaticLibsResourceJars = append(transitiveStaticLibsResourceJars, dep.TransitiveStaticLibsResourceJars)

				}

			}

		}

		addCLCFromDep(ctx, module, a.classLoaderContexts)

		addMissingOptionalUsesLibsFromDep(ctx, module, &a.usesLibrary)

	})



	completeStaticLibsHeaderJars := android.NewDepSet(android.PREORDER, android.Paths{classpathFile}, transitiveStaticLibsHeaderJars)

	completeStaticLibsImplementationJars := android.NewDepSet(android.PREORDER, android.Paths{classpathFile}, transitiveStaticLibsImplementationJars)

	completeStaticLibsResourceJars := android.NewDepSet(android.PREORDER, nil, transitiveStaticLibsResourceJars)



	var implementationJarFile android.Path

	if len(staticJars) > 0 {

		combineJars := append(android.Paths{classpathFile}, staticJars...)

		combinedImplementationJar := android.PathForModuleOut(ctx, "combined", jarName).OutputPath

		TransformJarsToJar(ctx, combinedImplementationJar, "combine", combineJars, android.OptionalPath{}, false, nil, nil)

		implementationJarFile = combinedImplementationJar

	var combineJars android.Paths

	if ctx.Config().UseTransitiveJarsInClasspath() {

		combineJars = completeStaticLibsImplementationJars.ToList()

	} else {

		combineJars = append(android.Paths{classpathFile}, staticJars...)

	}



	if len(combineJars) > 1 {

		implementationJarOutputPath := android.PathForModuleOut(ctx, "combined", jarName)

		TransformJarsToJar(ctx, implementationJarOutputPath, "combine", combineJars, android.OptionalPath{}, false, nil, nil)

		implementationJarFile = implementationJarOutputPath

	} else {

		implementationJarFile = classpathFile

	}



	var resourceJarFile android.Path

	if len(staticResourceJars) > 1 {

	var resourceJars android.Paths

	if ctx.Config().UseTransitiveJarsInClasspath() {

		resourceJars = completeStaticLibsResourceJars.ToList()

	} else {

		resourceJars = staticResourceJars

	}

	if len(resourceJars) > 1 {

		combinedJar := android.PathForModuleOut(ctx, "res-combined", jarName)

		TransformJarsToJar(ctx, combinedJar, "for resources", staticResourceJars, android.OptionalPath{},

		TransformJarsToJar(ctx, combinedJar, "for resources", resourceJars, android.OptionalPath{},

			false, nil, nil)

		resourceJarFile = combinedJar

	} else if len(staticResourceJars) == 1 {

		resourceJarFile = staticResourceJars[0]

	} else if len(resourceJars) == 1 {

		resourceJarFile = resourceJars[0]

	}



	// merge implementation jar with resources if necessary

	implementationAndResourcesJar := implementationJarFile

	if resourceJarFile != nil {

		jars := android.Paths{resourceJarFile, implementationAndResourcesJar}

	var implementationAndResourcesJars android.Paths

	if ctx.Config().UseTransitiveJarsInClasspath() {

		implementationAndResourcesJars = append(slices.Clone(resourceJars), combineJars...)

	} else {

		implementationAndResourcesJars = android.PathsIfNonNil(resourceJarFile, implementationJarFile)

	}

	var implementationAndResourcesJar android.Path

	if len(implementationAndResourcesJars) > 1 {

		combinedJar := android.PathForModuleOut(ctx, "withres", jarName)

		TransformJarsToJar(ctx, combinedJar, "for resources", jars, android.OptionalPath{},

		TransformJarsToJar(ctx, combinedJar, "for resources", implementationAndResourcesJars, android.OptionalPath{},

			false, nil, nil)

		implementationAndResourcesJar = combinedJar

	} else {

		implementationAndResourcesJar = implementationAndResourcesJars[0]

	}



	a.implementationJarFile = implementationJarFile

	// Save the output file with no relative path so that it doesn't end up in a subdirectory when used as a resource

	a.implementationAndResourcesJarFile = implementationAndResourcesJar.WithoutRel()



	if len(staticHeaderJars) > 0 {

		combineJars := append(android.Paths{classpathFile}, staticHeaderJars...)

	var headerJars android.Paths

	if ctx.Config().UseTransitiveJarsInClasspath() {

		headerJars = completeStaticLibsHeaderJars.ToList()

	} else {

		headerJars = append(android.Paths{classpathFile}, staticHeaderJars...)

	}

	if len(headerJars) > 1 {

		headerJarFile := android.PathForModuleOut(ctx, "turbine-combined", jarName)

		TransformJarsToJar(ctx, headerJarFile, "combine header jars", combineJars, android.OptionalPath{}, false, nil, nil)

		TransformJarsToJar(ctx, headerJarFile, "combine header jars", headerJars, android.OptionalPath{}, false, nil, nil)

		a.headerJarFile = headerJarFile

	} else {

		a.headerJarFile = classpathFile

		a.headerJarFile = headerJars[0]

	}



	if ctx.Config().UseTransitiveJarsInClasspath() {

		ctx.CheckbuildFile(classpathFile)

	} else {

		ctx.CheckbuildFile(a.headerJarFile)

		ctx.CheckbuildFile(a.implementationJarFile)

	}



	android.SetProvider(ctx, JavaInfoProvider, &JavaInfo{

		HeaderJars:                             android.PathsIfNonNil(a.headerJarFile),

		LocalHeaderJars:                        android.PathsIfNonNil(classpathFile),

		TransitiveStaticLibsHeaderJars:         completeStaticLibsHeaderJars,

		TransitiveStaticLibsImplementationJars: completeStaticLibsImplementationJars,

		TransitiveStaticLibsResourceJars:       completeStaticLibsResourceJars,

		ResourceJars:                           android.PathsIfNonNil(resourceJarFile),

		TransitiveLibsHeaderJarsForR8:          a.transitiveLibsHeaderJarsForR8,

		TransitiveStaticLibsHeaderJarsForR8:    a.transitiveStaticLibsHeaderJarsForR8,

java/base.go

+279 −126

File changed.

Preview size limit exceeded, changes collapsed.

java/device_host_converter.go

+25 −7
Original line number Diff line number Diff line
@@ -96,6 +96,10 @@ func (d *DeviceHostConverter) GenerateAndroidBuildActions(ctx android.ModuleCont 
		ctx.PropertyErrorf("libs", "at least one dependency is required")

	}



	var transitiveHeaderJars []*android.DepSet[android.Path]

	var transitiveImplementationJars []*android.DepSet[android.Path]

	var transitiveResourceJars []*android.DepSet[android.Path]



	ctx.VisitDirectDepsWithTag(deviceHostConverterDepTag, func(m android.Module) {

		if dep, ok := android.OtherModuleProvider(ctx, m, JavaInfoProvider); ok {

			d.headerJars = append(d.headerJars, dep.HeaderJars...)
@@ -105,6 +109,16 @@ func (d *DeviceHostConverter) GenerateAndroidBuildActions(ctx android.ModuleCont 


			d.srcJarArgs = append(d.srcJarArgs, dep.SrcJarArgs...)

			d.srcJarDeps = append(d.srcJarDeps, dep.SrcJarDeps...)



			if dep.TransitiveStaticLibsHeaderJars != nil {

				transitiveHeaderJars = append(transitiveHeaderJars, dep.TransitiveStaticLibsHeaderJars)

			}

			if dep.TransitiveStaticLibsImplementationJars != nil {

				transitiveImplementationJars = append(transitiveImplementationJars, dep.TransitiveStaticLibsImplementationJars)

			}

			if dep.TransitiveStaticLibsResourceJars != nil {

				transitiveResourceJars = append(transitiveResourceJars, dep.TransitiveStaticLibsResourceJars)

			}

		} else {

			ctx.PropertyErrorf("libs", "module %q cannot be used as a dependency", ctx.OtherModuleName(m))

		}
@@ -132,6 +146,10 @@ func (d *DeviceHostConverter) GenerateAndroidBuildActions(ctx android.ModuleCont 


	android.SetProvider(ctx, JavaInfoProvider, &JavaInfo{

		HeaderJars:                             d.headerJars,

		LocalHeaderJars:                        d.headerJars,

		TransitiveStaticLibsHeaderJars:         android.NewDepSet(android.PREORDER, nil, transitiveHeaderJars),

		TransitiveStaticLibsImplementationJars: android.NewDepSet(android.PREORDER, nil, transitiveImplementationJars),

		TransitiveStaticLibsResourceJars:       android.NewDepSet(android.PREORDER, nil, transitiveResourceJars),

		ImplementationAndResourcesJars:         d.implementationAndResourceJars,

		ImplementationJars:                     d.implementationJars,

		ResourceJars:                           d.resourceJars,

java/java.go

+99 −20
Original line number Diff line number Diff line
@@ -254,6 +254,7 @@ var ProguardSpecInfoProvider = blueprint.NewProvider[ProguardSpecInfo]() 
type JavaInfo struct {

	// HeaderJars is a list of jars that can be passed as the javac classpath in order to link

	// against this module.  If empty, ImplementationJars should be used instead.

	// Unlike LocalHeaderJars, HeaderJars includes classes from static dependencies.

	HeaderJars android.Paths



	RepackagedHeaderJars android.Paths
@@ -264,6 +265,15 @@ type JavaInfo struct { 
	// set of header jars for all transitive static libs deps

	TransitiveStaticLibsHeaderJarsForR8 *android.DepSet[android.Path]



	// depset of header jars for this module and all transitive static dependencies

	TransitiveStaticLibsHeaderJars *android.DepSet[android.Path]



	// depset of implementation jars for this module and all transitive static dependencies

	TransitiveStaticLibsImplementationJars *android.DepSet[android.Path]



	// depset of resource jars for this module and all transitive static dependencies

	TransitiveStaticLibsResourceJars *android.DepSet[android.Path]



	// ImplementationAndResourceJars is a list of jars that contain the implementations of classes

	// in the module as well as any resources included in the module.

	ImplementationAndResourcesJars android.Paths
@@ -275,6 +285,9 @@ type JavaInfo struct { 
	// ResourceJars is a list of jars that contain the resources included in the module.

	ResourceJars android.Paths



	// LocalHeaderJars is a list of jars that contain classes from this module, but not from any static dependencies.

	LocalHeaderJars android.Paths



	// AidlIncludeDirs is a list of directories that should be passed to the aidl tool when

	// depending on this module.

	AidlIncludeDirs android.Paths
@@ -568,6 +581,10 @@ type deps struct { 
	aconfigProtoFiles       android.Paths



	disableTurbine bool



	transitiveStaticLibsHeaderJars         []*android.DepSet[android.Path]

	transitiveStaticLibsImplementationJars []*android.DepSet[android.Path]

	transitiveStaticLibsResourceJars       []*android.DepSet[android.Path]

}



func checkProducesJars(ctx android.ModuleContext, dep android.SourceFileProducer) {
@@ -1008,7 +1025,7 @@ func (j *Library) setInstallRules(ctx android.ModuleContext, installModuleName s 
		}

		hostDexNeeded := Bool(j.deviceProperties.Hostdex) && !ctx.Host()

		if hostDexNeeded {

			j.hostdexInstallFile = ctx.InstallFile(

			j.hostdexInstallFile = ctx.InstallFileWithoutCheckbuild(

				android.PathForHostDexInstall(ctx, "framework"),

				j.Stem()+"-hostdex.jar", j.outputFile)

		}
@@ -1022,7 +1039,7 @@ func (j *Library) setInstallRules(ctx android.ModuleContext, installModuleName s 
		} else {

			installDir = android.PathForModuleInstall(ctx, "framework")

		}

		j.installFile = ctx.InstallFile(installDir, j.Stem()+".jar", j.outputFile, extraInstallDeps...)

		j.installFile = ctx.InstallFileWithoutCheckbuild(installDir, j.Stem()+".jar", j.outputFile, extraInstallDeps...)

	}

}
@@ -2360,6 +2377,9 @@ func (al *ApiLibrary) GenerateAndroidBuildActions(ctx android.ModuleContext) { 


	android.SetProvider(ctx, JavaInfoProvider, &JavaInfo{

		HeaderJars:                             android.PathsIfNonNil(al.stubsJar),

		LocalHeaderJars:                        android.PathsIfNonNil(al.stubsJar),

		TransitiveStaticLibsHeaderJars:         android.NewDepSet(android.PREORDER, android.PathsIfNonNil(al.stubsJar), nil),

		TransitiveStaticLibsImplementationJars: android.NewDepSet(android.PREORDER, android.PathsIfNonNil(al.stubsJar), nil),

		ImplementationAndResourcesJars:         android.PathsIfNonNil(al.stubsJar),

		ImplementationJars:                     android.PathsIfNonNil(al.stubsJar),

		AidlIncludeDirs:                        android.Paths{},
@@ -2634,6 +2654,12 @@ func (j *Import) GenerateAndroidBuildActions(ctx android.ModuleContext) { 


	var flags javaBuilderFlags



	var transitiveClasspathHeaderJars []*android.DepSet[android.Path]

	var transitiveBootClasspathHeaderJars []*android.DepSet[android.Path]

	var transitiveStaticLibsHeaderJars []*android.DepSet[android.Path]

	var transitiveStaticLibsImplementationJars []*android.DepSet[android.Path]

	var transitiveStaticLibsResourceJars []*android.DepSet[android.Path]



	j.collectTransitiveHeaderJarsForR8(ctx)

	var staticJars android.Paths

	var staticResourceJars android.Paths
@@ -2645,32 +2671,67 @@ func (j *Import) GenerateAndroidBuildActions(ctx android.ModuleContext) { 
			case libTag, sdkLibTag:

				flags.classpath = append(flags.classpath, dep.HeaderJars...)

				flags.dexClasspath = append(flags.dexClasspath, dep.HeaderJars...)

				if dep.TransitiveStaticLibsHeaderJars != nil {

					transitiveClasspathHeaderJars = append(transitiveClasspathHeaderJars, dep.TransitiveStaticLibsHeaderJars)

				}

			case staticLibTag:

				flags.classpath = append(flags.classpath, dep.HeaderJars...)

				staticJars = append(staticJars, dep.ImplementationJars...)

				staticResourceJars = append(staticResourceJars, dep.ResourceJars...)

				staticHeaderJars = append(staticHeaderJars, dep.HeaderJars...)

				if dep.TransitiveStaticLibsHeaderJars != nil {

					transitiveClasspathHeaderJars = append(transitiveClasspathHeaderJars, dep.TransitiveStaticLibsHeaderJars)

					transitiveStaticLibsHeaderJars = append(transitiveStaticLibsHeaderJars, dep.TransitiveStaticLibsHeaderJars)

				}

				if dep.TransitiveStaticLibsImplementationJars != nil {

					transitiveStaticLibsImplementationJars = append(transitiveStaticLibsImplementationJars, dep.TransitiveStaticLibsImplementationJars)

				}

				if dep.TransitiveStaticLibsResourceJars != nil {

					transitiveStaticLibsResourceJars = append(transitiveStaticLibsResourceJars, dep.TransitiveStaticLibsResourceJars)

				}

			case bootClasspathTag:

				flags.bootClasspath = append(flags.bootClasspath, dep.HeaderJars...)

				if dep.TransitiveStaticLibsHeaderJars != nil {

					transitiveBootClasspathHeaderJars = append(transitiveBootClasspathHeaderJars, dep.TransitiveStaticLibsHeaderJars)

				}

			}

		} else if dep, ok := module.(SdkLibraryDependency); ok {

			switch tag {

			case libTag, sdkLibTag:

				flags.classpath = append(flags.classpath, dep.SdkHeaderJars(ctx, j.SdkVersion(ctx))...)

				depHeaderJars := dep.SdkHeaderJars(ctx, j.SdkVersion(ctx))

				flags.classpath = append(flags.classpath, depHeaderJars...)

				transitiveClasspathHeaderJars = append(transitiveClasspathHeaderJars,

					android.NewDepSet(android.PREORDER, depHeaderJars, nil))

			}

		}



		addCLCFromDep(ctx, module, j.classLoaderContexts)

	})



	jars := android.PathsForModuleSrc(ctx, j.properties.Jars)

	localJars := android.PathsForModuleSrc(ctx, j.properties.Jars)

	jarName := j.Stem() + ".jar"



	// Combine only the local jars together for use in transitive classpaths.

	// Always pass input jar through TransformJarsToJar to strip module-info.class from prebuilts.

	localCombinedHeaderJar := android.PathForModuleOut(ctx, "local-combined", jarName)

	TransformJarsToJar(ctx, localCombinedHeaderJar, "combine local prebuilt implementation jars", localJars, android.OptionalPath{},

		false, j.properties.Exclude_files, j.properties.Exclude_dirs)

	localStrippedJars := android.Paths{localCombinedHeaderJar}



	completeStaticLibsHeaderJars := android.NewDepSet(android.PREORDER, localStrippedJars, transitiveStaticLibsHeaderJars)

	completeStaticLibsImplementationJars := android.NewDepSet(android.PREORDER, localStrippedJars, transitiveStaticLibsImplementationJars)

	completeStaticLibsResourceJars := android.NewDepSet(android.PREORDER, nil, transitiveStaticLibsResourceJars)



	// Always pass the input jars to TransformJarsToJar, even if there is only a single jar, we need the output

	// file of the module to be named jarName.

	var outputFile android.Path

	combinedImplementationJar := android.PathForModuleOut(ctx, "combined", jarName)

	implementationJars := append(slices.Clone(jars), staticJars...)

	var implementationJars android.Paths

	if ctx.Config().UseTransitiveJarsInClasspath() {

		implementationJars = completeStaticLibsImplementationJars.ToList()

	} else {

		implementationJars = append(slices.Clone(localJars), staticJars...)

	}

	TransformJarsToJar(ctx, combinedImplementationJar, "combine prebuilt implementation jars", implementationJars, android.OptionalPath{},

		false, j.properties.Exclude_files, j.properties.Exclude_dirs)

	outputFile = combinedImplementationJar
@@ -2693,7 +2754,12 @@ func (j *Import) GenerateAndroidBuildActions(ctx android.ModuleContext) { 
	if reuseImplementationJarAsHeaderJar {

		headerJar = outputFile

	} else {

		headerJars := append(slices.Clone(jars), staticHeaderJars...)

		var headerJars android.Paths

		if ctx.Config().UseTransitiveJarsInClasspath() {

			headerJars = completeStaticLibsHeaderJars.ToList()

		} else {

			headerJars = append(slices.Clone(localJars), staticHeaderJars...)

		}

		headerOutputFile := android.PathForModuleOut(ctx, "turbine-combined", jarName)

		TransformJarsToJar(ctx, headerOutputFile, "combine prebuilt header jars", headerJars, android.OptionalPath{},

			false, j.properties.Exclude_files, j.properties.Exclude_dirs)
@@ -2712,6 +2778,11 @@ func (j *Import) GenerateAndroidBuildActions(ctx android.ModuleContext) { 
		} else {

			headerJar = outputFile

		}



		// Enabling jetifier requires modifying classes from transitive dependencies, disable transitive

		// classpath and use the combined header jar instead.

		completeStaticLibsHeaderJars = android.NewDepSet(android.PREORDER, android.Paths{headerJar}, nil)

		completeStaticLibsImplementationJars = android.NewDepSet(android.PREORDER, android.Paths{outputFile}, nil)

	}



	implementationJarFile := outputFile
@@ -2735,7 +2806,11 @@ func (j *Import) GenerateAndroidBuildActions(ctx android.ModuleContext) { 


	j.exportAidlIncludeDirs = android.PathsForModuleSrc(ctx, j.properties.Aidl.Export_include_dirs)



	if ctx.Config().UseTransitiveJarsInClasspath() {

		ctx.CheckbuildFile(localJars...)

	} else {

		ctx.CheckbuildFile(outputFile)

	}



	if ctx.Device() {

		// If this is a variant created for a prebuilt_apex then use the dex implementation jar
@@ -2818,8 +2893,12 @@ func (j *Import) GenerateAndroidBuildActions(ctx android.ModuleContext) { 


	android.SetProvider(ctx, JavaInfoProvider, &JavaInfo{

		HeaderJars:                             android.PathsIfNonNil(j.combinedHeaderFile),

		LocalHeaderJars:                        android.PathsIfNonNil(j.combinedHeaderFile),

		TransitiveLibsHeaderJarsForR8:          j.transitiveLibsHeaderJarsForR8,

		TransitiveStaticLibsHeaderJarsForR8:    j.transitiveStaticLibsHeaderJarsForR8,

		TransitiveStaticLibsHeaderJars:         completeStaticLibsHeaderJars,

		TransitiveStaticLibsImplementationJars: completeStaticLibsImplementationJars,

		TransitiveStaticLibsResourceJars:       completeStaticLibsResourceJars,

		ImplementationAndResourcesJars:         android.PathsIfNonNil(j.combinedImplementationFile),

		ImplementationJars:                     android.PathsIfNonNil(implementationJarFile.WithoutRel()),

		ResourceJars:                           android.PathsIfNonNil(resourceJarFile),