Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit a0ad8aa1 authored by Makoto Onuki's avatar Makoto Onuki
Browse files

[Ravenwood] Poorman's sharding 

Framework-minus-apex is rather slow to process.

Ideally HostStubGen should be internally parallelize the processing,
but that means now we'll have to carefully synchronize concurrent access
to the common data structure.

So instead, let's just run multiple HSG instances in parallel.

Now the build trace looks like this: http://screen/9B5NUtPjaaQ8jfA

Bug: 313930116
Test: $ANDROID_BUILD_TOP/frameworks/base/ravenwood/scripts/run-ravenwood-tests.sh
Flag: EXEMPT host side test change only

Change-Id: I50fd932a8e61196eda66c074eb2f8089e455b515
parent 2c399676

Ravenwood.bp

+209 −23
Original line number Diff line number Diff line
@@ -26,10 +26,105 @@ java_library { 
}



// Generate the stub/impl from framework-all, with hidden APIs.

java_genrule {

    name: "framework-minus-apex.ravenwood-base",

// This step takes several tens of seconds, so we manually shard it to multiple modules.

// All the copies have to be kept in sync.

// TODO: Do the sharding better.



genrule_defaults {

    name: "framework-minus-apex.ravenwood-base_defaults",

    tools: ["hoststubgen"],

    srcs: [

        ":framework-minus-apex-for-hoststubgen",

        ":ravenwood-framework-policies",

        ":ravenwood-standard-options",

        ":ravenwood-annotation-allowed-classes",

    ],

    out: [

        "ravenwood.jar",



        // Following files are created just as FYI.

        "hoststubgen_framework-minus-apex_keep_all.txt",

        "hoststubgen_framework-minus-apex_dump.txt",



        "hoststubgen_framework-minus-apex.log",

        "hoststubgen_framework-minus-apex_stats.csv",

        "hoststubgen_framework-minus-apex_apis.csv",

    ],

    visibility: ["//visibility:private"],

}



java_genrule {

    name: "framework-minus-apex.ravenwood-base_X0",

    defaults: ["framework-minus-apex.ravenwood-base_defaults"],

    cmd: "$(location hoststubgen) " +

        "--num-shards 6 --shard-index 0 " + // Only this line differs



        "@$(location :ravenwood-standard-options) " +



        "--debug-log $(location hoststubgen_framework-minus-apex.log) " +

        "--stats-file $(location hoststubgen_framework-minus-apex_stats.csv) " +

        "--supported-api-list-file $(location hoststubgen_framework-minus-apex_apis.csv) " +



        "--out-impl-jar $(location ravenwood.jar) " +



        "--gen-keep-all-file $(location hoststubgen_framework-minus-apex_keep_all.txt) " +

        "--gen-input-dump-file $(location hoststubgen_framework-minus-apex_dump.txt) " +



        "--in-jar $(location :framework-minus-apex-for-hoststubgen) " +

        "--policy-override-file $(location :ravenwood-framework-policies) " +

        "--annotation-allowed-classes-file $(location :ravenwood-annotation-allowed-classes) ",

}



java_genrule {

    name: "framework-minus-apex.ravenwood-base_X1",

    defaults: ["framework-minus-apex.ravenwood-base_defaults"],

    cmd: "$(location hoststubgen) " +

        "--num-shards 6 --shard-index 1 " + // Only this line differs



        "@$(location :ravenwood-standard-options) " +



        "--debug-log $(location hoststubgen_framework-minus-apex.log) " +

        "--stats-file $(location hoststubgen_framework-minus-apex_stats.csv) " +

        "--supported-api-list-file $(location hoststubgen_framework-minus-apex_apis.csv) " +



        "--out-impl-jar $(location ravenwood.jar) " +



        "--gen-keep-all-file $(location hoststubgen_framework-minus-apex_keep_all.txt) " +

        "--gen-input-dump-file $(location hoststubgen_framework-minus-apex_dump.txt) " +



        "--in-jar $(location :framework-minus-apex-for-hoststubgen) " +

        "--policy-override-file $(location :ravenwood-framework-policies) " +

        "--annotation-allowed-classes-file $(location :ravenwood-annotation-allowed-classes) ",

}



java_genrule {

    name: "framework-minus-apex.ravenwood-base_X2",

    defaults: ["framework-minus-apex.ravenwood-base_defaults"],

    cmd: "$(location hoststubgen) " +

        "--num-shards 6 --shard-index 2 " + // Only this line differs



        "@$(location :ravenwood-standard-options) " +



        "--debug-log $(location hoststubgen_framework-minus-apex.log) " +

        "--stats-file $(location hoststubgen_framework-minus-apex_stats.csv) " +

        "--supported-api-list-file $(location hoststubgen_framework-minus-apex_apis.csv) " +



        "--out-impl-jar $(location ravenwood.jar) " +



        "--gen-keep-all-file $(location hoststubgen_framework-minus-apex_keep_all.txt) " +

        "--gen-input-dump-file $(location hoststubgen_framework-minus-apex_dump.txt) " +



        "--in-jar $(location :framework-minus-apex-for-hoststubgen) " +

        "--policy-override-file $(location :ravenwood-framework-policies) " +

        "--annotation-allowed-classes-file $(location :ravenwood-annotation-allowed-classes) ",

}



java_genrule {

    name: "framework-minus-apex.ravenwood-base_X3",

    defaults: ["framework-minus-apex.ravenwood-base_defaults"],

    cmd: "$(location hoststubgen) " +

        "--num-shards 6 --shard-index 3 " + // Only this line differs



        "@$(location :ravenwood-standard-options) " +



        "--debug-log $(location hoststubgen_framework-minus-apex.log) " +
@@ -44,39 +139,130 @@ java_genrule { 
        "--in-jar $(location :framework-minus-apex-for-hoststubgen) " +

        "--policy-override-file $(location :ravenwood-framework-policies) " +

        "--annotation-allowed-classes-file $(location :ravenwood-annotation-allowed-classes) ",

}



java_genrule {

    name: "framework-minus-apex.ravenwood-base_X4",

    defaults: ["framework-minus-apex.ravenwood-base_defaults"],

    cmd: "$(location hoststubgen) " +

        "--num-shards 6 --shard-index 4 " + // Only this line differs



        "@$(location :ravenwood-standard-options) " +



        "--debug-log $(location hoststubgen_framework-minus-apex.log) " +

        "--stats-file $(location hoststubgen_framework-minus-apex_stats.csv) " +

        "--supported-api-list-file $(location hoststubgen_framework-minus-apex_apis.csv) " +



        "--out-impl-jar $(location ravenwood.jar) " +



        "--gen-keep-all-file $(location hoststubgen_framework-minus-apex_keep_all.txt) " +

        "--gen-input-dump-file $(location hoststubgen_framework-minus-apex_dump.txt) " +



        "--in-jar $(location :framework-minus-apex-for-hoststubgen) " +

        "--policy-override-file $(location :ravenwood-framework-policies) " +

        "--annotation-allowed-classes-file $(location :ravenwood-annotation-allowed-classes) ",

}



java_genrule {

    name: "framework-minus-apex.ravenwood-base_X5",

    defaults: ["framework-minus-apex.ravenwood-base_defaults"],

    cmd: "$(location hoststubgen) " +

        "--num-shards 6 --shard-index 5 " + // Only this line differs



        "@$(location :ravenwood-standard-options) " +



        "--debug-log $(location hoststubgen_framework-minus-apex.log) " +

        "--stats-file $(location hoststubgen_framework-minus-apex_stats.csv) " +

        "--supported-api-list-file $(location hoststubgen_framework-minus-apex_apis.csv) " +



        "--out-impl-jar $(location ravenwood.jar) " +



        "--gen-keep-all-file $(location hoststubgen_framework-minus-apex_keep_all.txt) " +

        "--gen-input-dump-file $(location hoststubgen_framework-minus-apex_dump.txt) " +



        "--in-jar $(location :framework-minus-apex-for-hoststubgen) " +

        "--policy-override-file $(location :ravenwood-framework-policies) " +

        "--annotation-allowed-classes-file $(location :ravenwood-annotation-allowed-classes) ",

}



// Marge all the sharded jars

java_genrule {

    name: "framework-minus-apex.ravenwood",

    defaults: ["ravenwood-internal-only-visibility-java"],

    cmd: "$(location merge_zips) $(out) $(in)",

    tools: ["merge_zips"],

    srcs: [

        ":framework-minus-apex-for-hoststubgen",

        ":ravenwood-framework-policies",

        ":ravenwood-standard-options",

        ":ravenwood-annotation-allowed-classes",

        ":framework-minus-apex.ravenwood-base_X0{ravenwood.jar}",

        ":framework-minus-apex.ravenwood-base_X1{ravenwood.jar}",

        ":framework-minus-apex.ravenwood-base_X2{ravenwood.jar}",

        ":framework-minus-apex.ravenwood-base_X3{ravenwood.jar}",

        ":framework-minus-apex.ravenwood-base_X4{ravenwood.jar}",

        ":framework-minus-apex.ravenwood-base_X5{ravenwood.jar}",

    ],

    out: [

        "ravenwood.jar",



        // Following files are created just as FYI.

        "hoststubgen_framework-minus-apex_keep_all.txt",

        "hoststubgen_framework-minus-apex_dump.txt",

        "framework-minus-apex.ravenwood.jar",

    ],

}



        "hoststubgen_framework-minus-apex.log",

        "hoststubgen_framework-minus-apex_stats.csv",

        "hoststubgen_framework-minus-apex_apis.csv",

// Merge the sharded text files

genrule {

    name: "hoststubgen_framework-minus-apex_stats.csv",

    defaults: ["ravenwood-internal-only-visibility-genrule"],

    cmd: "cat $(in) > $(out)",

    srcs: [

        ":framework-minus-apex.ravenwood-base_X0{hoststubgen_framework-minus-apex_stats.csv}",

        ":framework-minus-apex.ravenwood-base_X1{hoststubgen_framework-minus-apex_stats.csv}",

        ":framework-minus-apex.ravenwood-base_X2{hoststubgen_framework-minus-apex_stats.csv}",

        ":framework-minus-apex.ravenwood-base_X3{hoststubgen_framework-minus-apex_stats.csv}",

        ":framework-minus-apex.ravenwood-base_X4{hoststubgen_framework-minus-apex_stats.csv}",

        ":framework-minus-apex.ravenwood-base_X5{hoststubgen_framework-minus-apex_stats.csv}",

    ],

    out: ["hoststubgen_framework-minus-apex_stats.csv"],

}



genrule {

    name: "hoststubgen_framework-minus-apex_apis.csv",

    defaults: ["ravenwood-internal-only-visibility-genrule"],

    cmd: "cat $(in) > $(out)",

    srcs: [

        ":framework-minus-apex.ravenwood-base_X0{hoststubgen_framework-minus-apex_apis.csv}",

        ":framework-minus-apex.ravenwood-base_X1{hoststubgen_framework-minus-apex_apis.csv}",

        ":framework-minus-apex.ravenwood-base_X2{hoststubgen_framework-minus-apex_apis.csv}",

        ":framework-minus-apex.ravenwood-base_X3{hoststubgen_framework-minus-apex_apis.csv}",

        ":framework-minus-apex.ravenwood-base_X4{hoststubgen_framework-minus-apex_apis.csv}",

        ":framework-minus-apex.ravenwood-base_X5{hoststubgen_framework-minus-apex_apis.csv}",

    ],

    out: ["hoststubgen_framework-minus-apex_apis.csv"],

}



// Extract the impl jar from "framework-minus-apex.ravenwood-base" for subsequent build rules.

// Note this emits a "device side" output, so that ravenwood tests can (implicitly)

// depend on it.

java_genrule {

    name: "framework-minus-apex.ravenwood",

genrule {

    name: "hoststubgen_framework-minus-apex_keep_all.txt",

    defaults: ["ravenwood-internal-only-visibility-genrule"],

    cmd: "cp $(in) $(out)",

    cmd: "cat $(in) > $(out)",

    srcs: [

        ":framework-minus-apex.ravenwood-base{ravenwood.jar}",

        ":framework-minus-apex.ravenwood-base_X0{hoststubgen_framework-minus-apex_keep_all.txt}",

        ":framework-minus-apex.ravenwood-base_X1{hoststubgen_framework-minus-apex_keep_all.txt}",

        ":framework-minus-apex.ravenwood-base_X2{hoststubgen_framework-minus-apex_keep_all.txt}",

        ":framework-minus-apex.ravenwood-base_X3{hoststubgen_framework-minus-apex_keep_all.txt}",

        ":framework-minus-apex.ravenwood-base_X4{hoststubgen_framework-minus-apex_keep_all.txt}",

        ":framework-minus-apex.ravenwood-base_X5{hoststubgen_framework-minus-apex_keep_all.txt}",

    ],

    out: [

        "framework-minus-apex.ravenwood.jar",

    out: ["hoststubgen_framework-minus-apex_keep_all.txt"],

}



genrule {

    name: "hoststubgen_framework-minus-apex_dump.txt",

    defaults: ["ravenwood-internal-only-visibility-genrule"],

    cmd: "cat $(in) > $(out)",

    srcs: [

        ":framework-minus-apex.ravenwood-base_X0{hoststubgen_framework-minus-apex_dump.txt}",

        ":framework-minus-apex.ravenwood-base_X1{hoststubgen_framework-minus-apex_dump.txt}",

        ":framework-minus-apex.ravenwood-base_X2{hoststubgen_framework-minus-apex_dump.txt}",

        ":framework-minus-apex.ravenwood-base_X3{hoststubgen_framework-minus-apex_dump.txt}",

        ":framework-minus-apex.ravenwood-base_X4{hoststubgen_framework-minus-apex_dump.txt}",

        ":framework-minus-apex.ravenwood-base_X5{hoststubgen_framework-minus-apex_dump.txt}",

    ],

    out: ["hoststubgen_framework-minus-apex_dump.txt"],

}



java_library {

ravenwood/Android.bp

+4 −4
Original line number Diff line number Diff line
@@ -280,10 +280,10 @@ sh_test_host { 
    src: "scripts/ravenwood-stats-checker.sh",

    test_suites: ["general-tests"],

    data: [

        ":framework-minus-apex.ravenwood-base{hoststubgen_framework-minus-apex_stats.csv}",

        ":framework-minus-apex.ravenwood-base{hoststubgen_framework-minus-apex_apis.csv}",

        ":framework-minus-apex.ravenwood-base{hoststubgen_framework-minus-apex_keep_all.txt}",

        ":framework-minus-apex.ravenwood-base{hoststubgen_framework-minus-apex_dump.txt}",

        ":hoststubgen_framework-minus-apex_stats.csv",

        ":hoststubgen_framework-minus-apex_apis.csv",

        ":hoststubgen_framework-minus-apex_keep_all.txt",

        ":hoststubgen_framework-minus-apex_dump.txt",

        ":services.core.ravenwood-base{hoststubgen_services.core_stats.csv}",

        ":services.core.ravenwood-base{hoststubgen_services.core_apis.csv}",

        ":services.core.ravenwood-base{hoststubgen_services.core_keep_all.txt}",

tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGen.kt

+21 −1
Original line number Diff line number Diff line
@@ -85,6 +85,8 @@ class HostStubGen(val options: HostStubGenOptions) { 
                errors,

                stats,

                policyFileRemapper,

                options.numShards.get,

                options.shard.get,

        )



        // Dump statistics, if specified.
@@ -213,6 +215,8 @@ class HostStubGen(val options: HostStubGenOptions) { 
            errors: HostStubGenErrors,

            stats: HostStubGenStats,

            remapper: Remapper?,

            numShards: Int,

            shard: Int,

            ) {

        log.i("Converting %s into [stub: %s, impl: %s] ...", inJar, outStubJar, outImplJar)

        log.i("ASM CheckClassAdapter is %s", if (enableChecker) "enabled" else "disabled")
@@ -221,17 +225,32 @@ class HostStubGen(val options: HostStubGenOptions) { 


        val packageRedirector = PackageRedirectRemapper(options.packageRedirects)



        var itemIndex = 0

        var numItemsProcessed = 0

        var numItems = -1 // == Unknown



        log.withIndent {

            // Open the input jar file and process each entry.

            ZipFile(inJar).use { inZip ->



                numItems = inZip.size()

                val shardStart = numItems * shard / numShards

                val shardNextStart = numItems * (shard + 1) / numShards



                maybeWithZipOutputStream(outStubJar) { stubOutStream ->

                    maybeWithZipOutputStream(outImplJar) { implOutStream ->

                        val inEntries = inZip.entries()

                        while (inEntries.hasMoreElements()) {

                            val entry = inEntries.nextElement()

                            val inShard = (shardStart <= itemIndex) && (itemIndex < shardNextStart)

                            itemIndex++

                            if (!inShard) {

                                continue

                            }

                            convertSingleEntry(inZip, entry, stubOutStream, implOutStream,

                                    filter, packageRedirector, remapper,

                                    enableChecker, classes, errors, stats)

                            numItemsProcessed++

                        }

                        log.i("Converted all entries.")

                    }
@@ -241,7 +260,8 @@ class HostStubGen(val options: HostStubGenOptions) { 
            }

        }

        val end = System.currentTimeMillis()

        log.i("Done transforming the jar in %.1f second(s).", (end - start) / 1000.0)

        log.i("Done transforming the jar in %.1f second(s). %d / %d item(s) processed.",

            (end - start) / 1000.0, numItemsProcessed, numItems)

    }



    private fun <T> maybeWithZipOutputStream(filename: String?, block: (ZipOutputStream?) -> T): T {

tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenOptions.kt

+15 −0
Original line number Diff line number Diff line
@@ -112,6 +112,9 @@ class HostStubGenOptions( 
        var statsFile: SetOnce<String?> = SetOnce(null),



        var apiListFile: SetOnce<String?> = SetOnce(null),



        var numShards: SetOnce<Int> = SetOnce(1),

        var shard: SetOnce<Int> = SetOnce(0),

) {

    companion object {
@@ -162,6 +165,13 @@ class HostStubGenOptions( 
                fun SetOnce<String?>.setNextStringArg(): String = nextArg().also { this.set(it) }

                fun MutableSet<String>.addUniqueAnnotationArg(): String =

                        nextArg().also { this += ensureUniqueAnnotation(it) }

                fun SetOnce<Int>.setNextIntArg(): String = nextArg().also {

                    try {

                        this.set(it.toInt())

                    } catch (e: NumberFormatException) {

                        throw ArgumentsException("Invalid integer for $arg: $it")

                    }

                }



                try {

                    when (arg) {
@@ -259,6 +269,9 @@ class HostStubGenOptions( 
                        "--stats-file" -> ret.statsFile.setNextStringArg()

                        "--supported-api-list-file" -> ret.apiListFile.setNextStringArg()



                        "--num-shards" -> ret.numShards.setNextIntArg()

                        "--shard-index" -> ret.shard.setNextIntArg()



                        else -> throw ArgumentsException("Unknown option: $arg")

                    }

                } catch (e: SetOnce.SetMoreThanOnceException) {
@@ -396,6 +409,8 @@ class HostStubGenOptions( 
              enableNonStubMethodCallDetection=$enableNonStubMethodCallDetection,

              statsFile=$statsFile,

              apiListFile=$apiListFile,

              numShards=$numShards,

              shard=$shard,

            }

            """.trimIndent()

    }