Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 5361a4fa authored by John Wu's avatar John Wu
Browse files

[HostStubGen] Introduce sharding for class processing 

Allow hoststubgen and ravenizer to process jar files with multiple
threads to reduce the total processing time.

Bug: 397498134
Flag: EXEMPT host side change only
Test: f/b/r/scripts/run-ravenwood-tests.sh
Change-Id: I228a1ec69fab49eb855246151899b700c0df972c
parent 0a8f7f56

ravenwood/Framework.bp

+18 −104
Original line number Diff line number Diff line
@@ -38,12 +38,6 @@ genrule_defaults { 
// framework-minus-apex

/////////////////////////



// Process framework-minus-apex with hoststubgen for Ravenwood.

// This step takes several tens of seconds, so we manually shard it to multiple modules.

// All the copies have to be kept in sync.

// TODO: Do the sharding better, either by making hostsubgen support sharding natively, or

// making a better build rule.



genrule_defaults {

    name: "framework-minus-apex.ravenwood-base_defaults",

    defaults: ["ravenwood_hsg_defaults"],
@@ -60,98 +54,14 @@ framework_minus_apex_options = ravenwood_common_options + 
    "--policy-override-file $(location :ravenwood-framework-policies) " +

    "--annotation-allowed-classes-file $(location :ravenwood-annotation-allowed-classes) "



genrule_defaults {

    name: "framework-minus-apex.ravenwood-shard_defaults",

java_genrule {

    name: "framework-minus-apex.ravenwood-base",

    defaults: ["framework-minus-apex.ravenwood-base_defaults"],

    tools: ["hoststubgen"],

    out: ["ravenwood.jar"],

}



framework_minus_apex_cmd = "$(location hoststubgen) " +

    cmd: "$(location hoststubgen) " +

        framework_minus_apex_options +

    "--out-jar $(location ravenwood.jar) "



java_genrule {

    name: "framework-minus-apex.ravenwood-base_X0",

    defaults: ["framework-minus-apex.ravenwood-shard_defaults"],

    cmd: framework_minus_apex_cmd + " --num-shards 10 --shard-index 0",

}



java_genrule {

    name: "framework-minus-apex.ravenwood-base_X1",

    defaults: ["framework-minus-apex.ravenwood-shard_defaults"],

    cmd: framework_minus_apex_cmd + " --num-shards 10 --shard-index 1",

}



java_genrule {

    name: "framework-minus-apex.ravenwood-base_X2",

    defaults: ["framework-minus-apex.ravenwood-shard_defaults"],

    cmd: framework_minus_apex_cmd + " --num-shards 10 --shard-index 2",

}



java_genrule {

    name: "framework-minus-apex.ravenwood-base_X3",

    defaults: ["framework-minus-apex.ravenwood-shard_defaults"],

    cmd: framework_minus_apex_cmd + " --num-shards 10 --shard-index 3",

}



java_genrule {

    name: "framework-minus-apex.ravenwood-base_X4",

    defaults: ["framework-minus-apex.ravenwood-shard_defaults"],

    cmd: framework_minus_apex_cmd + " --num-shards 10 --shard-index 4",

}



java_genrule {

    name: "framework-minus-apex.ravenwood-base_X5",

    defaults: ["framework-minus-apex.ravenwood-shard_defaults"],

    cmd: framework_minus_apex_cmd + " --num-shards 10 --shard-index 5",

}



java_genrule {

    name: "framework-minus-apex.ravenwood-base_X6",

    defaults: ["framework-minus-apex.ravenwood-shard_defaults"],

    cmd: framework_minus_apex_cmd + " --num-shards 10 --shard-index 6",

}



java_genrule {

    name: "framework-minus-apex.ravenwood-base_X7",

    defaults: ["framework-minus-apex.ravenwood-shard_defaults"],

    cmd: framework_minus_apex_cmd + " --num-shards 10 --shard-index 7",

}



java_genrule {

    name: "framework-minus-apex.ravenwood-base_X8",

    defaults: ["framework-minus-apex.ravenwood-shard_defaults"],

    cmd: framework_minus_apex_cmd + " --num-shards 10 --shard-index 8",

}



java_genrule {

    name: "framework-minus-apex.ravenwood-base_X9",

    defaults: ["framework-minus-apex.ravenwood-shard_defaults"],

    cmd: framework_minus_apex_cmd + " --num-shards 10 --shard-index 9",

}



// Merge all the sharded jars

java_genrule {

    name: "framework-minus-apex.ravenwood",

    defaults: ["ravenwood-internal-only-visibility-java"],

    cmd: "$(location merge_zips) $(out) $(in)",

    tools: ["merge_zips"],

    srcs: [

        ":framework-minus-apex.ravenwood-base_X0{ravenwood.jar}",

        ":framework-minus-apex.ravenwood-base_X1{ravenwood.jar}",

        ":framework-minus-apex.ravenwood-base_X2{ravenwood.jar}",

        ":framework-minus-apex.ravenwood-base_X3{ravenwood.jar}",

        ":framework-minus-apex.ravenwood-base_X4{ravenwood.jar}",

        ":framework-minus-apex.ravenwood-base_X5{ravenwood.jar}",

        ":framework-minus-apex.ravenwood-base_X6{ravenwood.jar}",

        ":framework-minus-apex.ravenwood-base_X7{ravenwood.jar}",

        ":framework-minus-apex.ravenwood-base_X8{ravenwood.jar}",

        ":framework-minus-apex.ravenwood-base_X9{ravenwood.jar}",

    ],

    out: [

        "framework-minus-apex.ravenwood.jar",

    ],

        "--out-jar $(location ravenwood.jar) ",

    out: ["ravenwood.jar"],

}



java_genrule {
@@ -170,10 +80,22 @@ java_genrule { 
    ],

}



java_import {

    name: "framework-minus-apex.ravenwood",

    defaults: ["ravenwood-internal-only-visibility-java"],

    jars: [":framework-minus-apex.ravenwood-base{ravenwood.jar}"],

}



//////////////////

// services.core

//////////////////



services_core_options = ravenwood_common_options +

    "--debug-log $(location hoststubgen_services.core.log) " +

    "--in-jar $(location :services.core-for-host) " +

    "--policy-override-file $(location :ravenwood-services-policies) " +

    "--annotation-allowed-classes-file $(location :ravenwood-annotation-allowed-classes) "



java_library {

    name: "services.core-for-host",

    installable: false, // host only jar.
@@ -194,12 +116,6 @@ genrule_defaults { 
    out: ["hoststubgen_services.core.log"],

}



services_core_options = ravenwood_common_options +

    "--debug-log $(location hoststubgen_services.core.log) " +

    "--in-jar $(location :services.core-for-host) " +

    "--policy-override-file $(location :ravenwood-services-policies) " +

    "--annotation-allowed-classes-file $(location :ravenwood-annotation-allowed-classes) "



java_genrule {

    name: "services.core.ravenwood-base",

    defaults: ["services.core.ravenwood-base_defaults"],
@@ -207,9 +123,7 @@ java_genrule { 
    cmd: "$(location hoststubgen) " +

        services_core_options +

        "--out-jar $(location ravenwood.jar) ",

    out: [

        "ravenwood.jar",

    ],

    out: ["ravenwood.jar"],

}



java_genrule {

ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/HostStubGenClassProcessorOptions.kt

+22 −22
Original line number Diff line number Diff line
@@ -34,37 +34,37 @@ private fun parsePackageRedirect(fromColonTo: String): Pair<String, String> { 
 * Options to configure [HostStubGenClassProcessor].

 */

open class HostStubGenClassProcessorOptions(

    var keepAnnotations: MutableSet<String> = mutableSetOf(),

    var throwAnnotations: MutableSet<String> = mutableSetOf(),

    var removeAnnotations: MutableSet<String> = mutableSetOf(),

    var ignoreAnnotations: MutableSet<String> = mutableSetOf(),

    var keepClassAnnotations: MutableSet<String> = mutableSetOf(),

    var partiallyAllowedAnnotations: MutableSet<String> = mutableSetOf(),

    var redirectAnnotations: MutableSet<String> = mutableSetOf(),

    val keepAnnotations: MutableSet<String> = mutableSetOf(),

    val throwAnnotations: MutableSet<String> = mutableSetOf(),

    val removeAnnotations: MutableSet<String> = mutableSetOf(),

    val ignoreAnnotations: MutableSet<String> = mutableSetOf(),

    val keepClassAnnotations: MutableSet<String> = mutableSetOf(),

    val partiallyAllowedAnnotations: MutableSet<String> = mutableSetOf(),

    val redirectAnnotations: MutableSet<String> = mutableSetOf(),



    var substituteAnnotations: MutableSet<String> = mutableSetOf(),

    var redirectionClassAnnotations: MutableSet<String> = mutableSetOf(),

    var classLoadHookAnnotations: MutableSet<String> = mutableSetOf(),

    var keepStaticInitializerAnnotations: MutableSet<String> = mutableSetOf(),

    val substituteAnnotations: MutableSet<String> = mutableSetOf(),

    val redirectionClassAnnotations: MutableSet<String> = mutableSetOf(),

    val classLoadHookAnnotations: MutableSet<String> = mutableSetOf(),

    val keepStaticInitializerAnnotations: MutableSet<String> = mutableSetOf(),



    var packageRedirects: MutableList<Pair<String, String>> = mutableListOf(),

    val packageRedirects: MutableList<Pair<String, String>> = mutableListOf(),



    var annotationAllowedClassesFile: SetOnce<String?> = SetOnce(null),

    val annotationAllowedClassesFile: SetOnce<String?> = SetOnce(null),



    var defaultClassLoadHook: SetOnce<String?> = SetOnce(null),

    var defaultMethodCallHook: SetOnce<String?> = SetOnce(null),

    val defaultClassLoadHook: SetOnce<String?> = SetOnce(null),

    val defaultMethodCallHook: SetOnce<String?> = SetOnce(null),



    var policyOverrideFiles: MutableList<FileOrResource> = mutableListOf(),

    val policyOverrideFiles: MutableList<FileOrResource> = mutableListOf(),



    var defaultPolicy: SetOnce<FilterPolicy> = SetOnce(FilterPolicy.Remove),

    val defaultPolicy: SetOnce<FilterPolicy> = SetOnce(FilterPolicy.Remove),



    var deleteFinals: SetOnce<Boolean> = SetOnce(false),

    val deleteFinals: SetOnce<Boolean> = SetOnce(false),



    var throwExceptionType: SetOnce<String> = SetOnce("java.lang.UnsupportedOperationException"),

    val throwExceptionType: SetOnce<String> = SetOnce("java.lang.UnsupportedOperationException"),



    var enableClassChecker: SetOnce<Boolean> = SetOnce(false),

    var enablePreTrace: SetOnce<Boolean> = SetOnce(false),

    var enablePostTrace: SetOnce<Boolean> = SetOnce(false),

    val enableClassChecker: SetOnce<Boolean> = SetOnce(false),

    val enablePreTrace: SetOnce<Boolean> = SetOnce(false),

    val enablePostTrace: SetOnce<Boolean> = SetOnce(false),

) : BaseOptions() {



    private val allAnnotations = mutableSetOf<String>()

ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/HostStubGenLogger.kt

+121 −85
Original line number Diff line number Diff line
@@ -19,8 +19,24 @@ import java.io.BufferedOutputStream 
import java.io.FileOutputStream

import java.io.PrintWriter

import java.io.Writer

import java.util.Collections



val log: HostStubGenLogger = HostStubGenLogger().setConsoleLogLevel(LogLevel.Info)

val log: HostStubGenLogger

    get() {

        var log = threadLocalLogger.get()

        if (log == null) {

            log = BufferedLogger(mainLogger)

            threadLocalLogger.set(log)

            allLoggers.add(log)

        }

        return log

    }

val logOptions = HostStubGenLoggerOptions().setConsoleLogLevel(LogLevel.Info)



private val mainLogger = HostStubGenLogger(logOptions)

private val threadLocalLogger = ThreadLocal<HostStubGenLogger>().apply { set(mainLogger) }

val allLoggers: MutableList<HostStubGenLogger> =

    Collections.synchronizedList(ArrayList<HostStubGenLogger>()).apply { add(mainLogger) }



/** Logging level */

enum class LogLevel {
@@ -38,65 +54,24 @@ enum class LogLevel { 
 * By default, it has no printers set. Use [setConsoleLogLevel] or [addFilePrinter] to actually

 * write log.

 */

class HostStubGenLogger {

    private var indentLevel: Int = 0

        get() = field

        set(value) {

            field = value

            indent = "  ".repeat(value)

        }

    private var indent: String = ""



    private val printers: MutableList<LogPrinter> = mutableListOf()



    private var consolePrinter: LogPrinter? = null



    private var maxLogLevel = LogLevel.None



    private fun updateMaxLogLevel() {

        maxLogLevel = LogLevel.None



        printers.forEach {

            if (maxLogLevel < it.logLevel) {

                maxLogLevel = it.logLevel

            }

        }

    }

open class HostStubGenLogger(val options: HostStubGenLoggerOptions) {

    protected var indentLevel: Int = 0



    private fun addPrinter(printer: LogPrinter) {

        printers.add(printer)

        updateMaxLogLevel()

    constructor(other: HostStubGenLogger) : this(other.options) {

        indentLevel = other.indentLevel

    }



    private fun removePrinter(printer: LogPrinter) {

        printers.remove(printer)

        updateMaxLogLevel()

    }



    fun setConsoleLogLevel(level: LogLevel): HostStubGenLogger {

        // If there's already a console log printer set, remove it, and then add a new one

        consolePrinter?.let {

            removePrinter(it)

    protected inline fun forPrinters(callback: (LogPrinter) -> Unit) {

        synchronized(options.printers) {

            options.printers.forEach {

                callback(it)

            }

        val cp = StreamPrinter(level, PrintWriter(System.out))

        addPrinter(cp)

        consolePrinter = cp



        return this

        }



    fun addFilePrinter(level: LogLevel, logFilename: String): HostStubGenLogger {

        addPrinter(StreamPrinter(level, PrintWriter(BufferedOutputStream(

            FileOutputStream(logFilename)))))



        log.i("Log file set: $logFilename for $level")



        return this

    }



    /** Flush all the printers */

    fun flush() {

        printers.forEach { it.flush() }

    open fun flush() {

        forPrinters { it.flush() }

    }



    fun indent() {
@@ -119,24 +94,20 @@ class HostStubGenLogger { 
        }

    }



    fun isEnabled(level: LogLevel): Boolean {

        return level.ordinal <= maxLogLevel.ordinal

    }



    fun println(level: LogLevel, message: String) {

    open fun println(level: LogLevel, message: String) {

        if (message.isEmpty()) {

            return // Don't print an empty message.

        }

        printers.forEach {

        forPrinters {

            if (it.logLevel.ordinal >= level.ordinal) {

                it.println(level, indent, message)

                it.println(indentLevel, message)

            }

        }

    }



    fun println(level: LogLevel, format: String, vararg args: Any?) {

        if (isEnabled(level)) {

            println(level, String.format(format, *args))

        if (options.isEnabled(level)) {

            println(level, format.format(*args))

        }

    }
@@ -198,9 +169,8 @@ class HostStubGenLogger { 
        } finally {

            val end = System.currentTimeMillis()

            ret = (end - start) / 1000.0

            if (isEnabled(level)) {

                println(level,

                    String.format("%s: took %.1f second(s).", message, (end - start) / 1000.0))

            if (options.isEnabled(level)) {

                println(level, "%s: took %.1f second(s).".format(message, (end - start) / 1000.0))

            }

        }

        return ret
@@ -230,13 +200,13 @@ class HostStubGenLogger { 
    }



    inline fun forVerbose(block: () -> Unit) {

        if (isEnabled(LogLevel.Verbose)) {

        if (options.isEnabled(LogLevel.Verbose)) {

            block()

        }

    }



    inline fun forDebug(block: () -> Unit) {

        if (isEnabled(LogLevel.Debug)) {

        if (options.isEnabled(LogLevel.Debug)) {

            block()

        }

    }
@@ -246,31 +216,97 @@ class HostStubGenLogger { 
        return MultiplexingWriter(level)

    }



    private inner class MultiplexingWriter(val level: LogLevel) : Writer() {

        private inline fun forPrinters(callback: (LogPrinter) -> Unit) {

            printers.forEach {

                if (it.logLevel.ordinal >= level.ordinal) {

                    callback(it)

                }

    private inner class MultiplexingWriter(private val level: LogLevel) : Writer() {

        override fun close() {

            flush()

        }



        override fun flush() {

            this@HostStubGenLogger.flush()

        }



        override fun close() {

            flush()

        override fun write(cbuf: CharArray, off: Int, len: Int) {

            println(level, String(cbuf, off, len))

        }

    }

}



/**

 * A logging class that only write outputs when [flush] is called.

 */

private class BufferedLogger(base: HostStubGenLogger) : HostStubGenLogger(base) {

    val output = mutableListOf<Triple<Int, LogLevel, String>>()



    override fun flush() {

        forPrinters {

            output.forEach { (indent, level, message) ->

                if (it.logLevel.ordinal >= level.ordinal) {

                    it.println(indent, message)

                }

            }

            output.clear()

            it.flush()

        }

    }



        override fun write(cbuf: CharArray, off: Int, len: Int) {

            // TODO Apply indent

            forPrinters {

                it.write(cbuf, off, len)

    override fun println(level: LogLevel, message: String) {

        if (message.isEmpty()) {

            return // Don't print an empty message.

        }

        output.add(Triple(indentLevel, level, message))

    }

}



class HostStubGenLoggerOptions {

    val printers = mutableListOf<LogPrinter>()



    private var consolePrinter: LogPrinter? = null



    private var maxLogLevel = LogLevel.None



    private fun updateMaxLogLevel() {

        maxLogLevel = LogLevel.None



        printers.forEach {

            if (maxLogLevel < it.logLevel) {

                maxLogLevel = it.logLevel

            }

        }

    }



    private fun addPrinter(printer: LogPrinter) {

        printers.add(printer)

        updateMaxLogLevel()

    }



    private fun removePrinter(printer: LogPrinter) {

        printers.remove(printer)

        updateMaxLogLevel()

    }



    fun setConsoleLogLevel(level: LogLevel): HostStubGenLoggerOptions {

        // If there's already a console log printer set, remove it, and then add a new one

        consolePrinter?.let {

            removePrinter(it)

        }

        val cp = StreamPrinter(level, PrintWriter(System.out))

        addPrinter(cp)

        consolePrinter = cp



        return this

    }



    fun addFilePrinter(level: LogLevel, logFilename: String): HostStubGenLoggerOptions {

        addPrinter(StreamPrinter(level, PrintWriter(BufferedOutputStream(

            FileOutputStream(logFilename)))))



        log.i("Log file set: $logFilename for $level")



        return this

    }



    fun isEnabled(level: LogLevel): Boolean {

        return level.ordinal <= maxLogLevel.ordinal

    }



    /**
@@ -289,10 +325,10 @@ class HostStubGenLogger { 
    }

}



private interface LogPrinter {

interface LogPrinter {

    val logLevel: LogLevel



    fun println(logLevel: LogLevel, indent: String, message: String)

    fun println(indent: Int, message: String)



    // TODO: This should be removed once MultiplexingWriter starts applying indent, at which point

    // println() should be used instead.
@@ -305,8 +341,8 @@ private class StreamPrinter( 
    override val logLevel: LogLevel,

    val out: PrintWriter,

) : LogPrinter {

    override fun println(logLevel: LogLevel, indent: String, message: String) {

        out.print(indent)

    override fun println(indent: Int, message: String) {

        out.print("  ".repeat(indent))

        out.println(message)

    }

ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/HostStubGenStats.kt

0 → 100644
+55 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2025 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

package com.android.hoststubgen



import com.android.hoststubgen.asm.ClassNodes

import java.util.concurrent.atomic.AtomicInteger



/**

 * Various stats of HostStubGen processing.

 */

open class HostStubGenStats(

    /** Total end-to-end time. */

    var totalTime: Double = .0,



    /** Time took to build [ClassNodes] */

    var loadStructureTime: Double = .0,



    /** Total real time spent for processing bytecode */

    var totalProcessTime: Double = .0,



    /** Total real time spent on writing class files into zip. */

    var totalWriteTime: Double = .0,



    /** # of entries in the input jar file */

    var totalEntries: AtomicInteger = AtomicInteger(),



    /** # of *.class files in the input jar file */

    var totalClasses: AtomicInteger = AtomicInteger(),

) {

    override fun toString(): String {

        return """

            HostStubGenStats {

              totalTime=$totalTime,

              loadStructureTime=$loadStructureTime,

              totalProcessTime=$totalProcessTime,

              totalWriteTime=$totalWriteTime,

              totalEntries=$totalEntries,

              totalClasses=$totalClasses,

            }

            """.trimIndent()

    }

}

ravenwood/tools/hoststubgen/lib/com/android/hoststubgen/Utils.kt

+3 −37
Original line number Diff line number Diff line
@@ -16,15 +16,7 @@ 
package com.android.hoststubgen



import java.io.PrintWriter

import java.util.zip.CRC32

import org.apache.commons.compress.archivers.zip.ZipArchiveEntry

import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream

import org.apache.commons.compress.archivers.zip.ZipFile



/**

 * Whether to skip compression when adding processed entries back to a zip file.

 */

private const val SKIP_COMPRESSION = false

import kotlin.system.exitProcess



/**

 * Name of this executable. Set it in the main method.
@@ -122,34 +114,8 @@ inline fun runMainWithBoilerplate(realMain: () -> Unit) { 
        }

    } finally {

        log.i("$executableName finished")

        log.flush()

        allLoggers.forEach { it.flush() }

    }



    System.exit(if (success) 0 else 1 )

}



/**

 * Copy a single ZIP entry to the output.

 */

fun copyZipEntry(

    inZip: ZipFile,

    entry: ZipArchiveEntry,

    out: ZipArchiveOutputStream,

) {

    inZip.getRawInputStream(entry).use { out.addRawArchiveEntry(entry, it) }

}



/**

 * Add a single ZIP entry with data.

 */

fun ZipArchiveOutputStream.addBytesEntry(name: String, data: ByteArray) {

    val newEntry = ZipArchiveEntry(name)

    if (SKIP_COMPRESSION) {

        newEntry.method = 0

        newEntry.size = data.size.toLong()

        newEntry.crc = CRC32().apply { update(data) }.value

    }

    putArchiveEntry(newEntry)

    write(data)

    closeArchiveEntry()

    exitProcess(if (success) 0 else 1)

}