Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 366afc50 authored by Kousik Kumar's avatar Kousik Kumar
Browse files

Add support for remote-execution / caching of jar/zip actions 

Test: Ran a build with `RBE_JAR="true" RBE_JAR_EXEC_STRATEGY="remote"
RBE_ZIP="true" RBE_ZIP_EXEC_STRATEGY="remote" ... use_rbe m` and that
succeeded.

Bug: b/156765207
Change-Id: Ia17b532fbb282be414adf879914870082dd0841b
parent 8f37ec82

java/app_builder.go

+10 −4
Original line number Original line Diff line number Diff line
@@ -232,14 +232,20 @@ func TransformJniLibsToJar(ctx android.ModuleContext, outputFile android.Writabl 
			"-f", j.path.String())

			"-f", j.path.String())

	}

	}





	rule := zip

	args := map[string]string{

		"jarArgs": strings.Join(proptools.NinjaAndShellEscapeList(jarArgs), " "),

	}

	if ctx.Config().IsEnvTrue("RBE_ZIP") {

		rule = zipRE

		args["implicits"] = strings.Join(deps.Strings(), ",")

	}

	ctx.Build(pctx, android.BuildParams{

	ctx.Build(pctx, android.BuildParams{

		Rule:        zip,

		Rule:        rule,

		Description: "zip jni libs",

		Description: "zip jni libs",

		Output:      outputFile,

		Output:      outputFile,

		Implicits:   deps,

		Implicits:   deps,

		Args: map[string]string{

		Args:        args,

			"jarArgs": strings.Join(proptools.NinjaAndShellEscapeList(jarArgs), " "),

		},

	})

	})

}

}

java/builder.go

+38 −13
Original line number Original line Diff line number Diff line
@@ -40,17 +40,17 @@ var ( 
	// (if the rule produces .class files) or a .srcjar file (if the rule produces .java files).

	// (if the rule produces .class files) or a .srcjar file (if the rule produces .java files).

	// .srcjar files are unzipped into a temporary directory when compiled with javac.

	// .srcjar files are unzipped into a temporary directory when compiled with javac.

	// TODO(b/143658984): goma can't handle the --system argument to javac.

	// TODO(b/143658984): goma can't handle the --system argument to javac.

	javac, javacRE = remoteexec.StaticRules(pctx, "javac",

	javac, javacRE = remoteexec.MultiCommandStaticRules(pctx, "javac",

		blueprint.RuleParams{

		blueprint.RuleParams{

			Command: `rm -rf "$outDir" "$annoDir" "$srcJarDir" && mkdir -p "$outDir" "$annoDir" "$srcJarDir" && ` +

			Command: `rm -rf "$outDir" "$annoDir" "$srcJarDir" && mkdir -p "$outDir" "$annoDir" "$srcJarDir" && ` +

				`${config.ZipSyncCmd} -d $srcJarDir -l $srcJarDir/list -f "*.java" $srcJars && ` +

				`${config.ZipSyncCmd} -d $srcJarDir -l $srcJarDir/list -f "*.java" $srcJars && ` +

				`(if [ -s $srcJarDir/list ] || [ -s $out.rsp ] ; then ` +

				`(if [ -s $srcJarDir/list ] || [ -s $out.rsp ] ; then ` +

				`${config.SoongJavacWrapper} $reTemplate${config.JavacCmd} ` +

				`${config.SoongJavacWrapper} $javaTemplate${config.JavacCmd} ` +

				`${config.JavacHeapFlags} ${config.JavacVmFlags} ${config.CommonJdkFlags} ` +

				`${config.JavacHeapFlags} ${config.JavacVmFlags} ${config.CommonJdkFlags} ` +

				`$processorpath $processor $javacFlags $bootClasspath $classpath ` +

				`$processorpath $processor $javacFlags $bootClasspath $classpath ` +

				`-source $javaVersion -target $javaVersion ` +

				`-source $javaVersion -target $javaVersion ` +

				`-d $outDir -s $annoDir @$out.rsp @$srcJarDir/list ; fi ) && ` +

				`-d $outDir -s $annoDir @$out.rsp @$srcJarDir/list ; fi ) && ` +

				`${config.SoongZipCmd} -jar -o $out -C $outDir -D $outDir && ` +

				`$zipTemplate${config.SoongZipCmd} -jar -o $out -C $outDir -D $outDir && ` +

				`rm -rf "$srcJarDir"`,

				`rm -rf "$srcJarDir"`,

			CommandDeps: []string{

			CommandDeps: []string{

				"${config.JavacCmd}",

				"${config.JavacCmd}",
@@ -60,10 +60,19 @@ var ( 
			CommandOrderOnly: []string{"${config.SoongJavacWrapper}"},

			CommandOrderOnly: []string{"${config.SoongJavacWrapper}"},

			Rspfile:          "$out.rsp",

			Rspfile:          "$out.rsp",

			RspfileContent:   "$in",

			RspfileContent:   "$in",

		}, &remoteexec.REParams{

		}, map[string]*remoteexec.REParams{

			"$javaTemplate": &remoteexec.REParams{

				Labels:       map[string]string{"type": "compile", "lang": "java", "compiler": "javac"},

				Labels:       map[string]string{"type": "compile", "lang": "java", "compiler": "javac"},

				ExecStrategy: "${config.REJavacExecStrategy}",

				ExecStrategy: "${config.REJavacExecStrategy}",

				Platform:     map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},

				Platform:     map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},

			},

			"$zipTemplate": &remoteexec.REParams{

				Labels:       map[string]string{"type": "tool", "name": "soong_zip"},

				Inputs:       []string{"${config.SoongZipCmd}", "$outDir"},

				OutputFiles:  []string{"$out"},

				ExecStrategy: "${config.REJavacExecStrategy}",

				Platform:     map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},

			},

		}, []string{"javacFlags", "bootClasspath", "classpath", "processorpath", "processor", "srcJars", "srcJarDir",

		}, []string{"javacFlags", "bootClasspath", "classpath", "processorpath", "processor", "srcJars", "srcJarDir",

			"outDir", "annoDir", "javaVersion"}, nil)

			"outDir", "annoDir", "javaVersion"}, nil)
@@ -144,23 +153,35 @@ var ( 
			Platform:          map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},

			Platform:          map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},

		}, []string{"javacFlags", "bootClasspath", "classpath", "srcJars", "outDir", "javaVersion"}, []string{"implicits"})

		}, []string{"javacFlags", "bootClasspath", "classpath", "srcJars", "outDir", "javaVersion"}, []string{"implicits"})





	jar = pctx.AndroidStaticRule("jar",

	jar, jarRE = remoteexec.StaticRules(pctx, "jar",

		blueprint.RuleParams{

		blueprint.RuleParams{

			Command:        `${config.SoongZipCmd} -jar -o $out @$out.rsp`,

			Command:        `$reTemplate${config.SoongZipCmd} -jar -o $out @$out.rsp`,

			CommandDeps:    []string{"${config.SoongZipCmd}"},

			CommandDeps:    []string{"${config.SoongZipCmd}"},

			Rspfile:        "$out.rsp",

			Rspfile:        "$out.rsp",

			RspfileContent: "$jarArgs",

			RspfileContent: "$jarArgs",

		},

		},

		"jarArgs")

		&remoteexec.REParams{

			ExecStrategy: "${config.REJarExecStrategy}",

			Inputs:       []string{"${config.SoongZipCmd}", "${out}.rsp"},

			RSPFile:      "${out}.rsp",

			OutputFiles:  []string{"$out"},

			Platform:     map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},

		}, []string{"jarArgs"}, nil)





	zip = pctx.AndroidStaticRule("zip",

	zip, zipRE = remoteexec.StaticRules(pctx, "zip",

		blueprint.RuleParams{

		blueprint.RuleParams{

			Command:        `${config.SoongZipCmd} -o $out @$out.rsp`,

			Command:        `${config.SoongZipCmd} -o $out @$out.rsp`,

			CommandDeps:    []string{"${config.SoongZipCmd}"},

			CommandDeps:    []string{"${config.SoongZipCmd}"},

			Rspfile:        "$out.rsp",

			Rspfile:        "$out.rsp",

			RspfileContent: "$jarArgs",

			RspfileContent: "$jarArgs",

		},

		},

		"jarArgs")

		&remoteexec.REParams{

			ExecStrategy: "${config.REZipExecStrategy}",

			Inputs:       []string{"${config.SoongZipCmd}", "${out}.rsp", "$implicits"},

			RSPFile:      "${out}.rsp",

			OutputFiles:  []string{"$out"},

			Platform:     map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},

		}, []string{"jarArgs"}, []string{"implicits"})





	combineJar = pctx.AndroidStaticRule("combineJar",

	combineJar = pctx.AndroidStaticRule("combineJar",

		blueprint.RuleParams{

		blueprint.RuleParams{
@@ -457,8 +478,12 @@ func transformJavaToClasses(ctx android.ModuleContext, outputFile android.Writab 
func TransformResourcesToJar(ctx android.ModuleContext, outputFile android.WritablePath,

func TransformResourcesToJar(ctx android.ModuleContext, outputFile android.WritablePath,

	jarArgs []string, deps android.Paths) {

	jarArgs []string, deps android.Paths) {





	rule := jar

	if ctx.Config().IsEnvTrue("RBE_JAR") {

		rule = jarRE

	}

	ctx.Build(pctx, android.BuildParams{

	ctx.Build(pctx, android.BuildParams{

		Rule:        jar,

		Rule:        rule,

		Description: "jar",

		Description: "jar",

		Output:      outputFile,

		Output:      outputFile,

		Implicits:   deps,

		Implicits:   deps,

java/config/config.go

+2 −0
Original line number Original line Diff line number Diff line
@@ -150,6 +150,8 @@ func init() { 
	pctx.VariableFunc("RER8ExecStrategy", remoteexec.EnvOverrideFunc("RBE_R8_EXEC_STRATEGY", remoteexec.LocalExecStrategy))

	pctx.VariableFunc("RER8ExecStrategy", remoteexec.EnvOverrideFunc("RBE_R8_EXEC_STRATEGY", remoteexec.LocalExecStrategy))

	pctx.VariableFunc("RETurbineExecStrategy", remoteexec.EnvOverrideFunc("RBE_TURBINE_EXEC_STRATEGY", remoteexec.LocalExecStrategy))

	pctx.VariableFunc("RETurbineExecStrategy", remoteexec.EnvOverrideFunc("RBE_TURBINE_EXEC_STRATEGY", remoteexec.LocalExecStrategy))

	pctx.VariableFunc("RESignApkExecStrategy", remoteexec.EnvOverrideFunc("RBE_SIGNAPK_EXEC_STRATEGY", remoteexec.LocalExecStrategy))

	pctx.VariableFunc("RESignApkExecStrategy", remoteexec.EnvOverrideFunc("RBE_SIGNAPK_EXEC_STRATEGY", remoteexec.LocalExecStrategy))

	pctx.VariableFunc("REJarExecStrategy", remoteexec.EnvOverrideFunc("RBE_JAR_EXEC_STRATEGY", remoteexec.LocalExecStrategy))

	pctx.VariableFunc("REZipExecStrategy", remoteexec.EnvOverrideFunc("RBE_ZIP_EXEC_STRATEGY", remoteexec.LocalExecStrategy))





	pctx.HostJavaToolVariable("JacocoCLIJar", "jacoco-cli.jar")

	pctx.HostJavaToolVariable("JacocoCLIJar", "jacoco-cli.jar")

java/dex.go

+36 −18
Original line number Original line Diff line number Diff line
@@ -24,48 +24,66 @@ import ( 
	"android/soong/remoteexec"

	"android/soong/remoteexec"

)

)





var d8, d8RE = remoteexec.StaticRules(pctx, "d8",

var d8, d8RE = remoteexec.MultiCommandStaticRules(pctx, "d8",

	blueprint.RuleParams{

	blueprint.RuleParams{

		Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` +

		Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` +

			`$reTemplate${config.D8Cmd} ${config.DexFlags} --output $outDir $d8Flags $in && ` +

			`$d8Template${config.D8Cmd} ${config.DexFlags} --output $outDir $d8Flags $in && ` +

			`${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` +

			`$zipTemplate${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` +

			`${config.MergeZipsCmd} -D -stripFile "**/*.class" $out $outDir/classes.dex.jar $in`,

			`${config.MergeZipsCmd} -D -stripFile "**/*.class" $out $outDir/classes.dex.jar $in`,

		CommandDeps: []string{

		CommandDeps: []string{

			"${config.D8Cmd}",

			"${config.D8Cmd}",

			"${config.SoongZipCmd}",

			"${config.SoongZipCmd}",

			"${config.MergeZipsCmd}",

			"${config.MergeZipsCmd}",

		},

		},

	}, &remoteexec.REParams{

	}, map[string]*remoteexec.REParams{

		"$d8Template": &remoteexec.REParams{

			Labels:          map[string]string{"type": "compile", "compiler": "d8"},

			Labels:          map[string]string{"type": "compile", "compiler": "d8"},

			Inputs:          []string{"${config.D8Jar}"},

			Inputs:          []string{"${config.D8Jar}"},

			ExecStrategy:    "${config.RED8ExecStrategy}",

			ExecStrategy:    "${config.RED8ExecStrategy}",

			ToolchainInputs: []string{"${config.JavaCmd}"},

			ToolchainInputs: []string{"${config.JavaCmd}"},

			Platform:        map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},

			Platform:        map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},

		},

		"$zipTemplate": &remoteexec.REParams{

			Labels:       map[string]string{"type": "tool", "name": "soong_zip"},

			Inputs:       []string{"${config.SoongZipCmd}", "$outDir"},

			OutputFiles:  []string{"$outDir/classes.dex.jar"},

			ExecStrategy: "${config.RED8ExecStrategy}",

			Platform:     map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},

		},

	}, []string{"outDir", "d8Flags", "zipFlags"}, nil)

	}, []string{"outDir", "d8Flags", "zipFlags"}, nil)





var r8, r8RE = remoteexec.StaticRules(pctx, "r8",

var r8, r8RE = remoteexec.MultiCommandStaticRules(pctx, "r8",

	blueprint.RuleParams{

	blueprint.RuleParams{

		Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` +

		Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` +

			`rm -f "$outDict" && ` +

			`rm -f "$outDict" && ` +

			`$reTemplate${config.R8Cmd} ${config.DexFlags} -injars $in --output $outDir ` +

			`$r8Template${config.R8Cmd} ${config.DexFlags} -injars $in --output $outDir ` +

			`--force-proguard-compatibility ` +

			`--force-proguard-compatibility ` +

			`--no-data-resources ` +

			`--no-data-resources ` +

			`-printmapping $outDict ` +

			`-printmapping $outDict ` +

			`$r8Flags && ` +

			`$r8Flags && ` +

			`touch "$outDict" && ` +

			`touch "$outDict" && ` +

			`${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` +

			`$zipTemplate${config.SoongZipCmd} $zipFlags -o $outDir/classes.dex.jar -C $outDir -f "$outDir/classes*.dex" && ` +

			`${config.MergeZipsCmd} -D -stripFile "**/*.class" $out $outDir/classes.dex.jar $in`,

			`${config.MergeZipsCmd} -D -stripFile "**/*.class" $out $outDir/classes.dex.jar $in`,

		CommandDeps: []string{

		CommandDeps: []string{

			"${config.R8Cmd}",

			"${config.R8Cmd}",

			"${config.SoongZipCmd}",

			"${config.SoongZipCmd}",

			"${config.MergeZipsCmd}",

			"${config.MergeZipsCmd}",

		},

		},

	}, &remoteexec.REParams{

	}, map[string]*remoteexec.REParams{

		"$r8Template": &remoteexec.REParams{

			Labels:          map[string]string{"type": "compile", "compiler": "r8"},

			Labels:          map[string]string{"type": "compile", "compiler": "r8"},

			Inputs:          []string{"$implicits", "${config.R8Jar}"},

			Inputs:          []string{"$implicits", "${config.R8Jar}"},

			ExecStrategy:    "${config.RER8ExecStrategy}",

			ExecStrategy:    "${config.RER8ExecStrategy}",

			ToolchainInputs: []string{"${config.JavaCmd}"},

			ToolchainInputs: []string{"${config.JavaCmd}"},

			Platform:        map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},

			Platform:        map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},

		},

		"$zipTemplate": &remoteexec.REParams{

			Labels:       map[string]string{"type": "tool", "name": "soong_zip"},

			Inputs:       []string{"${config.SoongZipCmd}", "$outDir"},

			OutputFiles:  []string{"$outDir/classes.dex.jar"},

			ExecStrategy: "${config.RER8ExecStrategy}",

			Platform:     map[string]string{remoteexec.PoolKey: "${config.REJavaPool}"},

		},

	}, []string{"outDir", "outDict", "r8Flags", "zipFlags"}, []string{"implicits"})

	}, []string{"outDir", "outDict", "r8Flags", "zipFlags"}, []string{"implicits"})





func (j *Module) dexCommonFlags(ctx android.ModuleContext) []string {

func (j *Module) dexCommonFlags(ctx android.ModuleContext) []string {

java/java.go

+10 −4
Original line number Original line Diff line number Diff line
@@ -1456,13 +1456,19 @@ func (j *Module) compile(ctx android.ModuleContext, aaptSrcJar android.Path) { 
			serviceFile := file.String()

			serviceFile := file.String()

			zipargs = append(zipargs, "-C", filepath.Dir(serviceFile), "-f", serviceFile)

			zipargs = append(zipargs, "-C", filepath.Dir(serviceFile), "-f", serviceFile)

		}

		}

		rule := zip

		args := map[string]string{

			"jarArgs": "-P META-INF/services/ " + strings.Join(proptools.NinjaAndShellEscapeList(zipargs), " "),

		}

		if ctx.Config().IsEnvTrue("RBE_ZIP") {

			rule = zipRE

			args["implicits"] = strings.Join(services.Strings(), ",")

		}

		ctx.Build(pctx, android.BuildParams{

		ctx.Build(pctx, android.BuildParams{

			Rule:      zip,

			Rule:      rule,

			Output:    servicesJar,

			Output:    servicesJar,

			Implicits: services,

			Implicits: services,

			Args: map[string]string{

			Args:      args,

				"jarArgs": "-P META-INF/services/ " + strings.Join(proptools.NinjaAndShellEscapeList(zipargs), " "),

			},

		})

		})

		jars = append(jars, servicesJar)

		jars = append(jars, servicesJar)

	}

	}