Merge "[HostStubGen] Introduce sharding for class processing" into main (4cf0bad8) · Commits · e / os / android_frameworks_base

ravenwood/Framework.bp

+18 −104

Original line number	Diff line number	Diff line
		@@ -38,12 +38,6 @@ genrule_defaults {
		// framework-minus-apex
		/////////////////////////

		// Process framework-minus-apex with hoststubgen for Ravenwood.
		// This step takes several tens of seconds, so we manually shard it to multiple modules.
		// All the copies have to be kept in sync.
		// TODO: Do the sharding better, either by making hostsubgen support sharding natively, or
		// making a better build rule.

		genrule_defaults {
		name: "framework-minus-apex.ravenwood-base_defaults",
		defaults: ["ravenwood_hsg_defaults"],
		@@ -60,98 +54,14 @@ framework_minus_apex_options = ravenwood_common_options +
		"--policy-override-file $(location :ravenwood-framework-policies) " +
		"--annotation-allowed-classes-file $(location :ravenwood-annotation-allowed-classes) "

		genrule_defaults {
		name: "framework-minus-apex.ravenwood-shard_defaults",
		java_genrule {
		name: "framework-minus-apex.ravenwood-base",
		defaults: ["framework-minus-apex.ravenwood-base_defaults"],
		tools: ["hoststubgen"],
		out: ["ravenwood.jar"],
		}

		framework_minus_apex_cmd = "$(location hoststubgen) " +
		cmd: "$(location hoststubgen) " +
		framework_minus_apex_options +
		"--out-jar $(location ravenwood.jar) "

		java_genrule {
		name: "framework-minus-apex.ravenwood-base_X0",
		defaults: ["framework-minus-apex.ravenwood-shard_defaults"],
		cmd: framework_minus_apex_cmd + " --num-shards 10 --shard-index 0",
		}

		java_genrule {
		name: "framework-minus-apex.ravenwood-base_X1",
		defaults: ["framework-minus-apex.ravenwood-shard_defaults"],
		cmd: framework_minus_apex_cmd + " --num-shards 10 --shard-index 1",
		}

		java_genrule {
		name: "framework-minus-apex.ravenwood-base_X2",
		defaults: ["framework-minus-apex.ravenwood-shard_defaults"],
		cmd: framework_minus_apex_cmd + " --num-shards 10 --shard-index 2",
		}

		java_genrule {
		name: "framework-minus-apex.ravenwood-base_X3",
		defaults: ["framework-minus-apex.ravenwood-shard_defaults"],
		cmd: framework_minus_apex_cmd + " --num-shards 10 --shard-index 3",
		}

		java_genrule {
		name: "framework-minus-apex.ravenwood-base_X4",
		defaults: ["framework-minus-apex.ravenwood-shard_defaults"],
		cmd: framework_minus_apex_cmd + " --num-shards 10 --shard-index 4",
		}

		java_genrule {
		name: "framework-minus-apex.ravenwood-base_X5",
		defaults: ["framework-minus-apex.ravenwood-shard_defaults"],
		cmd: framework_minus_apex_cmd + " --num-shards 10 --shard-index 5",
		}

		java_genrule {
		name: "framework-minus-apex.ravenwood-base_X6",
		defaults: ["framework-minus-apex.ravenwood-shard_defaults"],
		cmd: framework_minus_apex_cmd + " --num-shards 10 --shard-index 6",
		}

		java_genrule {
		name: "framework-minus-apex.ravenwood-base_X7",
		defaults: ["framework-minus-apex.ravenwood-shard_defaults"],
		cmd: framework_minus_apex_cmd + " --num-shards 10 --shard-index 7",
		}

		java_genrule {
		name: "framework-minus-apex.ravenwood-base_X8",
		defaults: ["framework-minus-apex.ravenwood-shard_defaults"],
		cmd: framework_minus_apex_cmd + " --num-shards 10 --shard-index 8",
		}

		java_genrule {
		name: "framework-minus-apex.ravenwood-base_X9",
		defaults: ["framework-minus-apex.ravenwood-shard_defaults"],
		cmd: framework_minus_apex_cmd + " --num-shards 10 --shard-index 9",
		}

		// Merge all the sharded jars
		java_genrule {
		name: "framework-minus-apex.ravenwood",
		defaults: ["ravenwood-internal-only-visibility-java"],
		cmd: "$(location merge_zips) $(out) $(in)",
		tools: ["merge_zips"],
		srcs: [
		":framework-minus-apex.ravenwood-base_X0{ravenwood.jar}",
		":framework-minus-apex.ravenwood-base_X1{ravenwood.jar}",
		":framework-minus-apex.ravenwood-base_X2{ravenwood.jar}",
		":framework-minus-apex.ravenwood-base_X3{ravenwood.jar}",
		":framework-minus-apex.ravenwood-base_X4{ravenwood.jar}",
		":framework-minus-apex.ravenwood-base_X5{ravenwood.jar}",
		":framework-minus-apex.ravenwood-base_X6{ravenwood.jar}",
		":framework-minus-apex.ravenwood-base_X7{ravenwood.jar}",
		":framework-minus-apex.ravenwood-base_X8{ravenwood.jar}",
		":framework-minus-apex.ravenwood-base_X9{ravenwood.jar}",
		],
		out: [
		"framework-minus-apex.ravenwood.jar",
		],
		"--out-jar $(location ravenwood.jar) ",
		out: ["ravenwood.jar"],
		}

		java_genrule {
		@@ -170,10 +80,22 @@ java_genrule {
		],
		}

		java_import {
		name: "framework-minus-apex.ravenwood",
		defaults: ["ravenwood-internal-only-visibility-java"],
		jars: [":framework-minus-apex.ravenwood-base{ravenwood.jar}"],
		}

		//////////////////
		// services.core
		//////////////////

		services_core_options = ravenwood_common_options +
		"--debug-log $(location hoststubgen_services.core.log) " +
		"--in-jar $(location :services.core-for-host) " +
		"--policy-override-file $(location :ravenwood-services-policies) " +
		"--annotation-allowed-classes-file $(location :ravenwood-annotation-allowed-classes) "

		java_library {
		name: "services.core-for-host",
		installable: false, // host only jar.
		@@ -194,12 +116,6 @@ genrule_defaults {
		out: ["hoststubgen_services.core.log"],
		}

		services_core_options = ravenwood_common_options +
		"--debug-log $(location hoststubgen_services.core.log) " +
		"--in-jar $(location :services.core-for-host) " +
		"--policy-override-file $(location :ravenwood-services-policies) " +
		"--annotation-allowed-classes-file $(location :ravenwood-annotation-allowed-classes) "

		java_genrule {
		name: "services.core.ravenwood-base",
		defaults: ["services.core.ravenwood-base_defaults"],
		@@ -207,9 +123,7 @@ java_genrule {
		cmd: "$(location hoststubgen) " +
		services_core_options +
		"--out-jar $(location ravenwood.jar) ",
		out: [
		"ravenwood.jar",
		],
		out: ["ravenwood.jar"],
		}

		java_genrule {

ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/HostStubGenClassProcessorOptions.kt

+22 −22

Original line number	Diff line number	Diff line
		@@ -34,37 +34,37 @@ private fun parsePackageRedirect(fromColonTo: String): Pair<String, String> {
		* Options to configure [HostStubGenClassProcessor].
		*/
		open class HostStubGenClassProcessorOptions(
		var keepAnnotations: MutableSet<String> = mutableSetOf(),
		var throwAnnotations: MutableSet<String> = mutableSetOf(),
		var removeAnnotations: MutableSet<String> = mutableSetOf(),
		var ignoreAnnotations: MutableSet<String> = mutableSetOf(),
		var keepClassAnnotations: MutableSet<String> = mutableSetOf(),
		var partiallyAllowedAnnotations: MutableSet<String> = mutableSetOf(),
		var redirectAnnotations: MutableSet<String> = mutableSetOf(),
		val keepAnnotations: MutableSet<String> = mutableSetOf(),
		val throwAnnotations: MutableSet<String> = mutableSetOf(),
		val removeAnnotations: MutableSet<String> = mutableSetOf(),
		val ignoreAnnotations: MutableSet<String> = mutableSetOf(),
		val keepClassAnnotations: MutableSet<String> = mutableSetOf(),
		val partiallyAllowedAnnotations: MutableSet<String> = mutableSetOf(),
		val redirectAnnotations: MutableSet<String> = mutableSetOf(),

		var substituteAnnotations: MutableSet<String> = mutableSetOf(),
		var redirectionClassAnnotations: MutableSet<String> = mutableSetOf(),
		var classLoadHookAnnotations: MutableSet<String> = mutableSetOf(),
		var keepStaticInitializerAnnotations: MutableSet<String> = mutableSetOf(),
		val substituteAnnotations: MutableSet<String> = mutableSetOf(),
		val redirectionClassAnnotations: MutableSet<String> = mutableSetOf(),
		val classLoadHookAnnotations: MutableSet<String> = mutableSetOf(),
		val keepStaticInitializerAnnotations: MutableSet<String> = mutableSetOf(),

		var packageRedirects: MutableList<Pair<String, String>> = mutableListOf(),
		val packageRedirects: MutableList<Pair<String, String>> = mutableListOf(),

		var annotationAllowedClassesFile: SetOnce<String?> = SetOnce(null),
		val annotationAllowedClassesFile: SetOnce<String?> = SetOnce(null),

		var defaultClassLoadHook: SetOnce<String?> = SetOnce(null),
		var defaultMethodCallHook: SetOnce<String?> = SetOnce(null),
		val defaultClassLoadHook: SetOnce<String?> = SetOnce(null),
		val defaultMethodCallHook: SetOnce<String?> = SetOnce(null),

		var policyOverrideFiles: MutableList<FileOrResource> = mutableListOf(),
		val policyOverrideFiles: MutableList<FileOrResource> = mutableListOf(),

		var defaultPolicy: SetOnce<FilterPolicy> = SetOnce(FilterPolicy.Remove),
		val defaultPolicy: SetOnce<FilterPolicy> = SetOnce(FilterPolicy.Remove),

		var deleteFinals: SetOnce<Boolean> = SetOnce(false),
		val deleteFinals: SetOnce<Boolean> = SetOnce(false),

		var throwExceptionType: SetOnce<String> = SetOnce("java.lang.UnsupportedOperationException"),
		val throwExceptionType: SetOnce<String> = SetOnce("java.lang.UnsupportedOperationException"),

		var enableClassChecker: SetOnce<Boolean> = SetOnce(false),
		var enablePreTrace: SetOnce<Boolean> = SetOnce(false),
		var enablePostTrace: SetOnce<Boolean> = SetOnce(false),
		val enableClassChecker: SetOnce<Boolean> = SetOnce(false),
		val enablePreTrace: SetOnce<Boolean> = SetOnce(false),
		val enablePostTrace: SetOnce<Boolean> = SetOnce(false),
		) : BaseOptions() {

		private val allAnnotations = mutableSetOf<String>()

ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/HostStubGenLogger.kt

+121 −85

Original line number	Diff line number	Diff line
		@@ -19,8 +19,24 @@ import java.io.BufferedOutputStream
		import java.io.FileOutputStream
		import java.io.PrintWriter
		import java.io.Writer
		import java.util.Collections

		val log: HostStubGenLogger = HostStubGenLogger().setConsoleLogLevel(LogLevel.Info)
		val log: HostStubGenLogger
		get() {
		var log = threadLocalLogger.get()
		if (log == null) {
		log = BufferedLogger(mainLogger)
		threadLocalLogger.set(log)
		allLoggers.add(log)
		}
		return log
		}
		val logOptions = HostStubGenLoggerOptions().setConsoleLogLevel(LogLevel.Info)

		private val mainLogger = HostStubGenLogger(logOptions)
		private val threadLocalLogger = ThreadLocal<HostStubGenLogger>().apply { set(mainLogger) }
		val allLoggers: MutableList<HostStubGenLogger> =
		Collections.synchronizedList(ArrayList<HostStubGenLogger>()).apply { add(mainLogger) }

		/** Logging level */
		enum class LogLevel {
		@@ -38,65 +54,24 @@ enum class LogLevel {
		* By default, it has no printers set. Use [setConsoleLogLevel] or [addFilePrinter] to actually
		* write log.
		*/
		class HostStubGenLogger {
		private var indentLevel: Int = 0
		get() = field
		set(value) {
		field = value
		indent = " ".repeat(value)
		}
		private var indent: String = ""

		private val printers: MutableList<LogPrinter> = mutableListOf()

		private var consolePrinter: LogPrinter? = null

		private var maxLogLevel = LogLevel.None

		private fun updateMaxLogLevel() {
		maxLogLevel = LogLevel.None

		printers.forEach {
		if (maxLogLevel < it.logLevel) {
		maxLogLevel = it.logLevel
		}
		}
		}
		open class HostStubGenLogger(val options: HostStubGenLoggerOptions) {
		protected var indentLevel: Int = 0

		private fun addPrinter(printer: LogPrinter) {
		printers.add(printer)
		updateMaxLogLevel()
		constructor(other: HostStubGenLogger) : this(other.options) {
		indentLevel = other.indentLevel
		}

		private fun removePrinter(printer: LogPrinter) {
		printers.remove(printer)
		updateMaxLogLevel()
		}

		fun setConsoleLogLevel(level: LogLevel): HostStubGenLogger {
		// If there's already a console log printer set, remove it, and then add a new one
		consolePrinter?.let {
		removePrinter(it)
		protected inline fun forPrinters(callback: (LogPrinter) -> Unit) {
		synchronized(options.printers) {
		options.printers.forEach {
		callback(it)
		}
		val cp = StreamPrinter(level, PrintWriter(System.out))
		addPrinter(cp)
		consolePrinter = cp

		return this
		}

		fun addFilePrinter(level: LogLevel, logFilename: String): HostStubGenLogger {
		addPrinter(StreamPrinter(level, PrintWriter(BufferedOutputStream(
		FileOutputStream(logFilename)))))

		log.i("Log file set: $logFilename for $level")

		return this
		}

		/** Flush all the printers */
		fun flush() {
		printers.forEach { it.flush() }
		open fun flush() {
		forPrinters { it.flush() }
		}

		fun indent() {
		@@ -119,24 +94,20 @@ class HostStubGenLogger {
		}
		}

		fun isEnabled(level: LogLevel): Boolean {
		return level.ordinal <= maxLogLevel.ordinal
		}

		fun println(level: LogLevel, message: String) {
		open fun println(level: LogLevel, message: String) {
		if (message.isEmpty()) {
		return // Don't print an empty message.
		}
		printers.forEach {
		forPrinters {
		if (it.logLevel.ordinal >= level.ordinal) {
		it.println(level, indent, message)
		it.println(indentLevel, message)
		}
		}
		}

		fun println(level: LogLevel, format: String, vararg args: Any?) {
		if (isEnabled(level)) {
		println(level, String.format(format, *args))
		if (options.isEnabled(level)) {
		println(level, format.format(*args))
		}
		}

		@@ -198,9 +169,8 @@ class HostStubGenLogger {
		} finally {
		val end = System.currentTimeMillis()
		ret = (end - start) / 1000.0
		if (isEnabled(level)) {
		println(level,
		String.format("%s: took %.1f second(s).", message, (end - start) / 1000.0))
		if (options.isEnabled(level)) {
		println(level, "%s: took %.1f second(s).".format(message, (end - start) / 1000.0))
		}
		}
		return ret
		@@ -230,13 +200,13 @@ class HostStubGenLogger {
		}

		inline fun forVerbose(block: () -> Unit) {
		if (isEnabled(LogLevel.Verbose)) {
		if (options.isEnabled(LogLevel.Verbose)) {
		block()
		}
		}

		inline fun forDebug(block: () -> Unit) {
		if (isEnabled(LogLevel.Debug)) {
		if (options.isEnabled(LogLevel.Debug)) {
		block()
		}
		}
		@@ -246,31 +216,97 @@ class HostStubGenLogger {
		return MultiplexingWriter(level)
		}

		private inner class MultiplexingWriter(val level: LogLevel) : Writer() {
		private inline fun forPrinters(callback: (LogPrinter) -> Unit) {
		printers.forEach {
		if (it.logLevel.ordinal >= level.ordinal) {
		callback(it)
		}
		private inner class MultiplexingWriter(private val level: LogLevel) : Writer() {
		override fun close() {
		flush()
		}

		override fun flush() {
		this@HostStubGenLogger.flush()
		}

		override fun close() {
		flush()
		override fun write(cbuf: CharArray, off: Int, len: Int) {
		println(level, String(cbuf, off, len))
		}
		}
		}

		/**
		* A logging class that only write outputs when [flush] is called.
		*/
		private class BufferedLogger(base: HostStubGenLogger) : HostStubGenLogger(base) {
		val output = mutableListOf<Triple<Int, LogLevel, String>>()

		override fun flush() {
		forPrinters {
		output.forEach { (indent, level, message) ->
		if (it.logLevel.ordinal >= level.ordinal) {
		it.println(indent, message)
		}
		}
		output.clear()
		it.flush()
		}
		}

		override fun write(cbuf: CharArray, off: Int, len: Int) {
		// TODO Apply indent
		forPrinters {
		it.write(cbuf, off, len)
		override fun println(level: LogLevel, message: String) {
		if (message.isEmpty()) {
		return // Don't print an empty message.
		}
		output.add(Triple(indentLevel, level, message))
		}
		}

		class HostStubGenLoggerOptions {
		val printers = mutableListOf<LogPrinter>()

		private var consolePrinter: LogPrinter? = null

		private var maxLogLevel = LogLevel.None

		private fun updateMaxLogLevel() {
		maxLogLevel = LogLevel.None

		printers.forEach {
		if (maxLogLevel < it.logLevel) {
		maxLogLevel = it.logLevel
		}
		}
		}

		private fun addPrinter(printer: LogPrinter) {
		printers.add(printer)
		updateMaxLogLevel()
		}

		private fun removePrinter(printer: LogPrinter) {
		printers.remove(printer)
		updateMaxLogLevel()
		}

		fun setConsoleLogLevel(level: LogLevel): HostStubGenLoggerOptions {
		// If there's already a console log printer set, remove it, and then add a new one
		consolePrinter?.let {
		removePrinter(it)
		}
		val cp = StreamPrinter(level, PrintWriter(System.out))
		addPrinter(cp)
		consolePrinter = cp

		return this
		}

		fun addFilePrinter(level: LogLevel, logFilename: String): HostStubGenLoggerOptions {
		addPrinter(StreamPrinter(level, PrintWriter(BufferedOutputStream(
		FileOutputStream(logFilename)))))

		log.i("Log file set: $logFilename for $level")

		return this
		}

		fun isEnabled(level: LogLevel): Boolean {
		return level.ordinal <= maxLogLevel.ordinal
		}

		/**
		@@ -289,10 +325,10 @@ class HostStubGenLogger {
		}
		}

		private interface LogPrinter {
		interface LogPrinter {
		val logLevel: LogLevel

		fun println(logLevel: LogLevel, indent: String, message: String)
		fun println(indent: Int, message: String)

		// TODO: This should be removed once MultiplexingWriter starts applying indent, at which point
		// println() should be used instead.
		@@ -305,8 +341,8 @@ private class StreamPrinter(
		override val logLevel: LogLevel,
		val out: PrintWriter,
		) : LogPrinter {
		override fun println(logLevel: LogLevel, indent: String, message: String) {
		out.print(indent)
		override fun println(indent: Int, message: String) {
		out.print(" ".repeat(indent))
		out.println(message)
		}

ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/HostStubGenStats.kt

0 → 100644

+55 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2025 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/
		package com.android.hoststubgen

		import com.android.hoststubgen.asm.ClassNodes
		import java.util.concurrent.atomic.AtomicInteger

		/**
		* Various stats of HostStubGen processing.
		*/
		open class HostStubGenStats(
		/** Total end-to-end time. */
		var totalTime: Double = .0,

		/** Time took to build [ClassNodes] */
		var loadStructureTime: Double = .0,

		/** Total real time spent for processing bytecode */
		var totalProcessTime: Double = .0,

		/** Total real time spent on writing class files into zip. */
		var totalWriteTime: Double = .0,

		/** # of entries in the input jar file */
		var totalEntries: AtomicInteger = AtomicInteger(),

		/** # of .class files in the input jar file /
		var totalClasses: AtomicInteger = AtomicInteger(),
		) {
		override fun toString(): String {
		return """
		HostStubGenStats {
		totalTime=$totalTime,
		loadStructureTime=$loadStructureTime,
		totalProcessTime=$totalProcessTime,
		totalWriteTime=$totalWriteTime,
		totalEntries=$totalEntries,
		totalClasses=$totalClasses,
		}
		""".trimIndent()
		}
		}

ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/Utils.kt

+3 −37

Original line number	Diff line number	Diff line
		@@ -16,15 +16,7 @@
		package com.android.hoststubgen

		import java.io.PrintWriter
		import java.util.zip.CRC32
		import org.apache.commons.compress.archivers.zip.ZipArchiveEntry
		import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream
		import org.apache.commons.compress.archivers.zip.ZipFile

		/**
		* Whether to skip compression when adding processed entries back to a zip file.
		*/
		private const val SKIP_COMPRESSION = false
		import kotlin.system.exitProcess

		/**
		* Name of this executable. Set it in the main method.
		@@ -122,34 +114,8 @@ inline fun runMainWithBoilerplate(realMain: () -> Unit) {
		}
		} finally {
		log.i("$executableName finished")
		log.flush()
		allLoggers.forEach { it.flush() }
		}

		System.exit(if (success) 0 else 1 )
		}

		/**
		* Copy a single ZIP entry to the output.
		*/
		fun copyZipEntry(
		inZip: ZipFile,
		entry: ZipArchiveEntry,
		out: ZipArchiveOutputStream,
		) {
		inZip.getRawInputStream(entry).use { out.addRawArchiveEntry(entry, it) }
		}

		/**
		* Add a single ZIP entry with data.
		*/
		fun ZipArchiveOutputStream.addBytesEntry(name: String, data: ByteArray) {
		val newEntry = ZipArchiveEntry(name)
		if (SKIP_COMPRESSION) {
		newEntry.method = 0
		newEntry.size = data.size.toLong()
		newEntry.crc = CRC32().apply { update(data) }.value
		}
		putArchiveEntry(newEntry)
		write(data)
		closeArchiveEntry()
		exitProcess(if (success) 0 else 1)
		}