Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 91f66c16 authored by Treehugger Robot's avatar Treehugger Robot Committed by Android (Google) Code Review
Browse files

Merge "[Ravenwood] Poorman's sharding" into main

parents af9edb8f a0ad8aa1

Ravenwood.bp

+209 −23
Original line number Diff line number Diff line
@@ -26,10 +26,105 @@ java_library { 
}



// Generate the stub/impl from framework-all, with hidden APIs.

java_genrule {

    name: "framework-minus-apex.ravenwood-base",

// This step takes several tens of seconds, so we manually shard it to multiple modules.

// All the copies have to be kept in sync.

// TODO: Do the sharding better.



genrule_defaults {

    name: "framework-minus-apex.ravenwood-base_defaults",

    tools: ["hoststubgen"],

    srcs: [

        ":framework-minus-apex-for-hoststubgen",

        ":ravenwood-framework-policies",

        ":ravenwood-standard-options",

        ":ravenwood-annotation-allowed-classes",

    ],

    out: [

        "ravenwood.jar",



        // Following files are created just as FYI.

        "hoststubgen_framework-minus-apex_keep_all.txt",

        "hoststubgen_framework-minus-apex_dump.txt",



        "hoststubgen_framework-minus-apex.log",

        "hoststubgen_framework-minus-apex_stats.csv",

        "hoststubgen_framework-minus-apex_apis.csv",

    ],

    visibility: ["//visibility:private"],

}



java_genrule {

    name: "framework-minus-apex.ravenwood-base_X0",

    defaults: ["framework-minus-apex.ravenwood-base_defaults"],

    cmd: "$(location hoststubgen) " +

        "--num-shards 6 --shard-index 0 " + // Only this line differs



        "@$(location :ravenwood-standard-options) " +



        "--debug-log $(location hoststubgen_framework-minus-apex.log) " +

        "--stats-file $(location hoststubgen_framework-minus-apex_stats.csv) " +

        "--supported-api-list-file $(location hoststubgen_framework-minus-apex_apis.csv) " +



        "--out-impl-jar $(location ravenwood.jar) " +



        "--gen-keep-all-file $(location hoststubgen_framework-minus-apex_keep_all.txt) " +

        "--gen-input-dump-file $(location hoststubgen_framework-minus-apex_dump.txt) " +



        "--in-jar $(location :framework-minus-apex-for-hoststubgen) " +

        "--policy-override-file $(location :ravenwood-framework-policies) " +

        "--annotation-allowed-classes-file $(location :ravenwood-annotation-allowed-classes) ",

}



java_genrule {

    name: "framework-minus-apex.ravenwood-base_X1",

    defaults: ["framework-minus-apex.ravenwood-base_defaults"],

    cmd: "$(location hoststubgen) " +

        "--num-shards 6 --shard-index 1 " + // Only this line differs



        "@$(location :ravenwood-standard-options) " +



        "--debug-log $(location hoststubgen_framework-minus-apex.log) " +

        "--stats-file $(location hoststubgen_framework-minus-apex_stats.csv) " +

        "--supported-api-list-file $(location hoststubgen_framework-minus-apex_apis.csv) " +



        "--out-impl-jar $(location ravenwood.jar) " +



        "--gen-keep-all-file $(location hoststubgen_framework-minus-apex_keep_all.txt) " +

        "--gen-input-dump-file $(location hoststubgen_framework-minus-apex_dump.txt) " +



        "--in-jar $(location :framework-minus-apex-for-hoststubgen) " +

        "--policy-override-file $(location :ravenwood-framework-policies) " +

        "--annotation-allowed-classes-file $(location :ravenwood-annotation-allowed-classes) ",

}



java_genrule {

    name: "framework-minus-apex.ravenwood-base_X2",

    defaults: ["framework-minus-apex.ravenwood-base_defaults"],

    cmd: "$(location hoststubgen) " +

        "--num-shards 6 --shard-index 2 " + // Only this line differs



        "@$(location :ravenwood-standard-options) " +



        "--debug-log $(location hoststubgen_framework-minus-apex.log) " +

        "--stats-file $(location hoststubgen_framework-minus-apex_stats.csv) " +

        "--supported-api-list-file $(location hoststubgen_framework-minus-apex_apis.csv) " +



        "--out-impl-jar $(location ravenwood.jar) " +



        "--gen-keep-all-file $(location hoststubgen_framework-minus-apex_keep_all.txt) " +

        "--gen-input-dump-file $(location hoststubgen_framework-minus-apex_dump.txt) " +



        "--in-jar $(location :framework-minus-apex-for-hoststubgen) " +

        "--policy-override-file $(location :ravenwood-framework-policies) " +

        "--annotation-allowed-classes-file $(location :ravenwood-annotation-allowed-classes) ",

}



java_genrule {

    name: "framework-minus-apex.ravenwood-base_X3",

    defaults: ["framework-minus-apex.ravenwood-base_defaults"],

    cmd: "$(location hoststubgen) " +

        "--num-shards 6 --shard-index 3 " + // Only this line differs



        "@$(location :ravenwood-standard-options) " +



        "--debug-log $(location hoststubgen_framework-minus-apex.log) " +
@@ -44,39 +139,130 @@ java_genrule { 
        "--in-jar $(location :framework-minus-apex-for-hoststubgen) " +

        "--policy-override-file $(location :ravenwood-framework-policies) " +

        "--annotation-allowed-classes-file $(location :ravenwood-annotation-allowed-classes) ",

}



java_genrule {

    name: "framework-minus-apex.ravenwood-base_X4",

    defaults: ["framework-minus-apex.ravenwood-base_defaults"],

    cmd: "$(location hoststubgen) " +

        "--num-shards 6 --shard-index 4 " + // Only this line differs



        "@$(location :ravenwood-standard-options) " +



        "--debug-log $(location hoststubgen_framework-minus-apex.log) " +

        "--stats-file $(location hoststubgen_framework-minus-apex_stats.csv) " +

        "--supported-api-list-file $(location hoststubgen_framework-minus-apex_apis.csv) " +



        "--out-impl-jar $(location ravenwood.jar) " +



        "--gen-keep-all-file $(location hoststubgen_framework-minus-apex_keep_all.txt) " +

        "--gen-input-dump-file $(location hoststubgen_framework-minus-apex_dump.txt) " +



        "--in-jar $(location :framework-minus-apex-for-hoststubgen) " +

        "--policy-override-file $(location :ravenwood-framework-policies) " +

        "--annotation-allowed-classes-file $(location :ravenwood-annotation-allowed-classes) ",

}



java_genrule {

    name: "framework-minus-apex.ravenwood-base_X5",

    defaults: ["framework-minus-apex.ravenwood-base_defaults"],

    cmd: "$(location hoststubgen) " +

        "--num-shards 6 --shard-index 5 " + // Only this line differs



        "@$(location :ravenwood-standard-options) " +



        "--debug-log $(location hoststubgen_framework-minus-apex.log) " +

        "--stats-file $(location hoststubgen_framework-minus-apex_stats.csv) " +

        "--supported-api-list-file $(location hoststubgen_framework-minus-apex_apis.csv) " +



        "--out-impl-jar $(location ravenwood.jar) " +



        "--gen-keep-all-file $(location hoststubgen_framework-minus-apex_keep_all.txt) " +

        "--gen-input-dump-file $(location hoststubgen_framework-minus-apex_dump.txt) " +



        "--in-jar $(location :framework-minus-apex-for-hoststubgen) " +

        "--policy-override-file $(location :ravenwood-framework-policies) " +

        "--annotation-allowed-classes-file $(location :ravenwood-annotation-allowed-classes) ",

}



// Marge all the sharded jars

java_genrule {

    name: "framework-minus-apex.ravenwood",

    defaults: ["ravenwood-internal-only-visibility-java"],

    cmd: "$(location merge_zips) $(out) $(in)",

    tools: ["merge_zips"],

    srcs: [

        ":framework-minus-apex-for-hoststubgen",

        ":ravenwood-framework-policies",

        ":ravenwood-standard-options",

        ":ravenwood-annotation-allowed-classes",

        ":framework-minus-apex.ravenwood-base_X0{ravenwood.jar}",

        ":framework-minus-apex.ravenwood-base_X1{ravenwood.jar}",

        ":framework-minus-apex.ravenwood-base_X2{ravenwood.jar}",

        ":framework-minus-apex.ravenwood-base_X3{ravenwood.jar}",

        ":framework-minus-apex.ravenwood-base_X4{ravenwood.jar}",

        ":framework-minus-apex.ravenwood-base_X5{ravenwood.jar}",

    ],

    out: [

        "ravenwood.jar",



        // Following files are created just as FYI.

        "hoststubgen_framework-minus-apex_keep_all.txt",

        "hoststubgen_framework-minus-apex_dump.txt",

        "framework-minus-apex.ravenwood.jar",

    ],

}



        "hoststubgen_framework-minus-apex.log",

        "hoststubgen_framework-minus-apex_stats.csv",

        "hoststubgen_framework-minus-apex_apis.csv",

// Merge the sharded text files

genrule {

    name: "hoststubgen_framework-minus-apex_stats.csv",

    defaults: ["ravenwood-internal-only-visibility-genrule"],

    cmd: "cat $(in) > $(out)",

    srcs: [

        ":framework-minus-apex.ravenwood-base_X0{hoststubgen_framework-minus-apex_stats.csv}",

        ":framework-minus-apex.ravenwood-base_X1{hoststubgen_framework-minus-apex_stats.csv}",

        ":framework-minus-apex.ravenwood-base_X2{hoststubgen_framework-minus-apex_stats.csv}",

        ":framework-minus-apex.ravenwood-base_X3{hoststubgen_framework-minus-apex_stats.csv}",

        ":framework-minus-apex.ravenwood-base_X4{hoststubgen_framework-minus-apex_stats.csv}",

        ":framework-minus-apex.ravenwood-base_X5{hoststubgen_framework-minus-apex_stats.csv}",

    ],

    out: ["hoststubgen_framework-minus-apex_stats.csv"],

}



genrule {

    name: "hoststubgen_framework-minus-apex_apis.csv",

    defaults: ["ravenwood-internal-only-visibility-genrule"],

    cmd: "cat $(in) > $(out)",

    srcs: [

        ":framework-minus-apex.ravenwood-base_X0{hoststubgen_framework-minus-apex_apis.csv}",

        ":framework-minus-apex.ravenwood-base_X1{hoststubgen_framework-minus-apex_apis.csv}",

        ":framework-minus-apex.ravenwood-base_X2{hoststubgen_framework-minus-apex_apis.csv}",

        ":framework-minus-apex.ravenwood-base_X3{hoststubgen_framework-minus-apex_apis.csv}",

        ":framework-minus-apex.ravenwood-base_X4{hoststubgen_framework-minus-apex_apis.csv}",

        ":framework-minus-apex.ravenwood-base_X5{hoststubgen_framework-minus-apex_apis.csv}",

    ],

    out: ["hoststubgen_framework-minus-apex_apis.csv"],

}



// Extract the impl jar from "framework-minus-apex.ravenwood-base" for subsequent build rules.

// Note this emits a "device side" output, so that ravenwood tests can (implicitly)

// depend on it.

java_genrule {

    name: "framework-minus-apex.ravenwood",

genrule {

    name: "hoststubgen_framework-minus-apex_keep_all.txt",

    defaults: ["ravenwood-internal-only-visibility-genrule"],

    cmd: "cp $(in) $(out)",

    cmd: "cat $(in) > $(out)",

    srcs: [

        ":framework-minus-apex.ravenwood-base{ravenwood.jar}",

        ":framework-minus-apex.ravenwood-base_X0{hoststubgen_framework-minus-apex_keep_all.txt}",

        ":framework-minus-apex.ravenwood-base_X1{hoststubgen_framework-minus-apex_keep_all.txt}",

        ":framework-minus-apex.ravenwood-base_X2{hoststubgen_framework-minus-apex_keep_all.txt}",

        ":framework-minus-apex.ravenwood-base_X3{hoststubgen_framework-minus-apex_keep_all.txt}",

        ":framework-minus-apex.ravenwood-base_X4{hoststubgen_framework-minus-apex_keep_all.txt}",

        ":framework-minus-apex.ravenwood-base_X5{hoststubgen_framework-minus-apex_keep_all.txt}",

    ],

    out: [

        "framework-minus-apex.ravenwood.jar",

    out: ["hoststubgen_framework-minus-apex_keep_all.txt"],

}



genrule {

    name: "hoststubgen_framework-minus-apex_dump.txt",

    defaults: ["ravenwood-internal-only-visibility-genrule"],

    cmd: "cat $(in) > $(out)",

    srcs: [

        ":framework-minus-apex.ravenwood-base_X0{hoststubgen_framework-minus-apex_dump.txt}",

        ":framework-minus-apex.ravenwood-base_X1{hoststubgen_framework-minus-apex_dump.txt}",

        ":framework-minus-apex.ravenwood-base_X2{hoststubgen_framework-minus-apex_dump.txt}",

        ":framework-minus-apex.ravenwood-base_X3{hoststubgen_framework-minus-apex_dump.txt}",

        ":framework-minus-apex.ravenwood-base_X4{hoststubgen_framework-minus-apex_dump.txt}",

        ":framework-minus-apex.ravenwood-base_X5{hoststubgen_framework-minus-apex_dump.txt}",

    ],

    out: ["hoststubgen_framework-minus-apex_dump.txt"],

}



java_library {

ravenwood/Android.bp

+4 −4
Original line number Diff line number Diff line
@@ -280,10 +280,10 @@ sh_test_host { 
    src: "scripts/ravenwood-stats-checker.sh",

    test_suites: ["general-tests"],

    data: [

        ":framework-minus-apex.ravenwood-base{hoststubgen_framework-minus-apex_stats.csv}",

        ":framework-minus-apex.ravenwood-base{hoststubgen_framework-minus-apex_apis.csv}",

        ":framework-minus-apex.ravenwood-base{hoststubgen_framework-minus-apex_keep_all.txt}",

        ":framework-minus-apex.ravenwood-base{hoststubgen_framework-minus-apex_dump.txt}",

        ":hoststubgen_framework-minus-apex_stats.csv",

        ":hoststubgen_framework-minus-apex_apis.csv",

        ":hoststubgen_framework-minus-apex_keep_all.txt",

        ":hoststubgen_framework-minus-apex_dump.txt",

        ":services.core.ravenwood-base{hoststubgen_services.core_stats.csv}",

        ":services.core.ravenwood-base{hoststubgen_services.core_apis.csv}",

        ":services.core.ravenwood-base{hoststubgen_services.core_keep_all.txt}",

tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGen.kt

+21 −1
Original line number Diff line number Diff line
@@ -85,6 +85,8 @@ class HostStubGen(val options: HostStubGenOptions) { 
                errors,

                stats,

                policyFileRemapper,

                options.numShards.get,

                options.shard.get,

        )



        // Dump statistics, if specified.
@@ -213,6 +215,8 @@ class HostStubGen(val options: HostStubGenOptions) { 
            errors: HostStubGenErrors,

            stats: HostStubGenStats,

            remapper: Remapper?,

            numShards: Int,

            shard: Int,

            ) {

        log.i("Converting %s into [stub: %s, impl: %s] ...", inJar, outStubJar, outImplJar)

        log.i("ASM CheckClassAdapter is %s", if (enableChecker) "enabled" else "disabled")
@@ -221,17 +225,32 @@ class HostStubGen(val options: HostStubGenOptions) { 


        val packageRedirector = PackageRedirectRemapper(options.packageRedirects)



        var itemIndex = 0

        var numItemsProcessed = 0

        var numItems = -1 // == Unknown



        log.withIndent {

            // Open the input jar file and process each entry.

            ZipFile(inJar).use { inZip ->



                numItems = inZip.size()

                val shardStart = numItems * shard / numShards

                val shardNextStart = numItems * (shard + 1) / numShards



                maybeWithZipOutputStream(outStubJar) { stubOutStream ->

                    maybeWithZipOutputStream(outImplJar) { implOutStream ->

                        val inEntries = inZip.entries()

                        while (inEntries.hasMoreElements()) {

                            val entry = inEntries.nextElement()

                            val inShard = (shardStart <= itemIndex) && (itemIndex < shardNextStart)

                            itemIndex++

                            if (!inShard) {

                                continue

                            }

                            convertSingleEntry(inZip, entry, stubOutStream, implOutStream,

                                    filter, packageRedirector, remapper,

                                    enableChecker, classes, errors, stats)

                            numItemsProcessed++

                        }

                        log.i("Converted all entries.")

                    }
@@ -241,7 +260,8 @@ class HostStubGen(val options: HostStubGenOptions) { 
            }

        }

        val end = System.currentTimeMillis()

        log.i("Done transforming the jar in %.1f second(s).", (end - start) / 1000.0)

        log.i("Done transforming the jar in %.1f second(s). %d / %d item(s) processed.",

            (end - start) / 1000.0, numItemsProcessed, numItems)

    }



    private fun <T> maybeWithZipOutputStream(filename: String?, block: (ZipOutputStream?) -> T): T {

tools/hoststubgen/hoststubgen/src/com/android/hoststubgen/HostStubGenOptions.kt

+15 −0
Original line number Diff line number Diff line
@@ -112,6 +112,9 @@ class HostStubGenOptions( 
        var statsFile: SetOnce<String?> = SetOnce(null),



        var apiListFile: SetOnce<String?> = SetOnce(null),



        var numShards: SetOnce<Int> = SetOnce(1),

        var shard: SetOnce<Int> = SetOnce(0),

) {

    companion object {
@@ -162,6 +165,13 @@ class HostStubGenOptions( 
                fun SetOnce<String?>.setNextStringArg(): String = nextArg().also { this.set(it) }

                fun MutableSet<String>.addUniqueAnnotationArg(): String =

                        nextArg().also { this += ensureUniqueAnnotation(it) }

                fun SetOnce<Int>.setNextIntArg(): String = nextArg().also {

                    try {

                        this.set(it.toInt())

                    } catch (e: NumberFormatException) {

                        throw ArgumentsException("Invalid integer for $arg: $it")

                    }

                }



                try {

                    when (arg) {
@@ -259,6 +269,9 @@ class HostStubGenOptions( 
                        "--stats-file" -> ret.statsFile.setNextStringArg()

                        "--supported-api-list-file" -> ret.apiListFile.setNextStringArg()



                        "--num-shards" -> ret.numShards.setNextIntArg()

                        "--shard-index" -> ret.shard.setNextIntArg()



                        else -> throw ArgumentsException("Unknown option: $arg")

                    }

                } catch (e: SetOnce.SetMoreThanOnceException) {
@@ -396,6 +409,8 @@ class HostStubGenOptions( 
              enableNonStubMethodCallDetection=$enableNonStubMethodCallDetection,

              statsFile=$statsFile,

              apiListFile=$apiListFile,

              numShards=$numShards,

              shard=$shard,

            }

            """.trimIndent()

    }