Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 74c708ca authored by Remi NGUYEN VAN's avatar Remi NGUYEN VAN
Browse files

Add reporting and command args to deflake tests 

Allow tests extending DeflakeHostTestBase to be used as command-line
tools to find flakes:
 - Add support for deflake_run_count, deflake_single_run_timeout,
   and deflake_test command-line args to customize runs of the test
   package.
 - Add method-level reporting of the number of flakes for each test and
   the last stacktrace.

Passing arguments to atest is still inconvenient but works; see example
in the class javadoc.

Test: made tests flaky, run with atest, see failure report
Bug: 134730673
Change-Id: I23f8c2db939ab49e0a2f314573de6460cd110318
parent 670a869a

tests/hostlib/src/com/android/testutils/host/DeflakeHostTestBase.kt

+88 −11
Original line number Diff line number Diff line
@@ -17,15 +17,35 @@ 
package com.android.testutils.host



import com.android.tests.util.ModuleTestUtils

import com.android.tradefed.config.Option

import com.android.tradefed.testtype.DeviceJUnit4ClassRunner

import com.android.tradefed.testtype.junit4.BaseHostJUnit4Test

import com.android.tradefed.testtype.junit4.DeviceTestRunOptions

import com.android.tradefed.util.AaptParser

import org.junit.Before

import org.junit.Test

import org.junit.runner.RunWith

import kotlin.test.assertTrue

import kotlin.test.fail



private data class TestFailure(val description: String, val stacktrace: String)



/**

 * Base class for host-driven tests to deflake a test package.

 *

 * <p>Classes implementing this base class must define a test APK to be run, and default run

 * count, timeout and test classes. In manual runs, the run count, timeout and test classes can be

 * overridden via command-line parameters, such as:

 *

 * <pre>

 * atest TestName -- \

 *   --test-arg com.android.tradefed.testtype.HostTest:set-option:deflake_run_count:10 \

 *   --test-arg com.android.tradefed.testtype.HostTest:set-option:deflake_single_run_timeout:10s \

 *   --test-arg \

 *      com.android.tradefed.testtype.HostTest:set-option:deflake_test:one.test.Class \

 *   --test-arg \

 *      com.android.tradefed.testtype.HostTest:set-option:deflake_test:another.test.Class

 * </pre>

 */

@RunWith(DeviceJUnit4ClassRunner::class)

abstract class DeflakeHostTestBase : BaseHostJUnit4Test() {
@@ -34,6 +54,11 @@ abstract class DeflakeHostTestBase : BaseHostJUnit4Test() { 
     */

    protected abstract val runCount: Int



    @Option(name = "deflake_run_count",

            description = "How many times to run each test case.",

            importance = Option.Importance.ALWAYS)

    private var mRunCountOption: Int? = null



    /**

     * Filename of the APK to run as part of the test.

     *
@@ -48,11 +73,24 @@ abstract class DeflakeHostTestBase : BaseHostJUnit4Test() { 
     */

    protected open val singleRunTimeoutMs = 5 * 60_000L



    @Option(name = "deflake_single_run_timeout",

            description = "Timeout for each single run.",

            importance = Option.Importance.ALWAYS,

            isTimeVal = true)

    private var mSingleRunTimeoutMsOption: Long? = null



    /**

     * List of classes to run in the test package. If empty, all classes in the package will be run.

     */

    protected open val testClasses: List<String> = emptyList()



    // TODO: also support single methods, not just whole classes

    @Option(name = "deflake_test",

            description = "Test class to deflake. Can be repeated. " +

                    "Default classes configured for the test are run if omitted.",

            importance = Option.Importance.ALWAYS)

    private var mTestClassesOption: ArrayList<String?> = ArrayList()



    @Before

    fun setUp() {

        // APK will be auto-cleaned
@@ -62,16 +100,55 @@ abstract class DeflakeHostTestBase : BaseHostJUnit4Test() { 
    @Test

    fun testDeflake() {

        val apkFile = ModuleTestUtils(this).getTestFile(testApkFilename)

        val pkgName = AaptParser.parse(apkFile)?.packageName ?:

                fail("Could not parse test package name")

        // null class name runs all classes in the package

        val tc = if (testClasses.isEmpty()) listOf(null) else testClasses



        repeat(runCount) {

            // TODO: improve reporting by always running all tests and counting flakes

            tc.forEach {

                assertTrue(runDeviceTests(pkgName, it, singleRunTimeoutMs))

        val pkgName = AaptParser.parse(apkFile)?.packageName

                ?: fail("Could not parse test package name")



        val classes = mTestClassesOption.filterNotNull().ifEmpty { testClasses }

                .ifEmpty { listOf(null) } // null class name runs all classes in the package

        val runOptions = DeviceTestRunOptions(pkgName)

                .setDevice(device)

                .setTestTimeoutMs(mSingleRunTimeoutMsOption ?: singleRunTimeoutMs)

                .setCheckResults(false)

        // Pair is (test identifier, last stacktrace)

        val failures = ArrayList<TestFailure>()

        val count = mRunCountOption ?: runCount

        repeat(count) {

            classes.forEach { testClass ->

                runDeviceTests(runOptions.setTestClassName(testClass))

                failures += getLastRunFailures()

            }

        }

        if (failures.isEmpty()) return

        val failuresByTest = failures.groupBy(TestFailure::description)

        val failMessage = failuresByTest.toList().fold("") { msg, (testDescription, failures) ->

            val stacktraces = formatStacktraces(failures)

            msg + "\n$testDescription: ${failures.count()}/$count failures. " +

                    "Stacktraces:\n$stacktraces"

        }

        fail("Some tests failed:$failMessage")

    }



    private fun getLastRunFailures(): List<TestFailure> {

        with(lastDeviceRunResults) {

            if (isRunFailure) {

                return listOf(TestFailure("All tests in run", runFailureMessage))

            }



            return failedTests.map {

                val stackTrace = testResults[it]?.stackTrace

                        ?: fail("Missing stacktrace for failed test $it")

                TestFailure(it.toString(), stackTrace)

            }

        }

    }



    private fun formatStacktraces(failures: List<TestFailure>): String {

        // Calculate list of (stacktrace, frequency) pairs ordered from most to least frequent

        val frequencies = failures.groupingBy(TestFailure::stacktrace).eachCount().toList()

                .sortedByDescending { it.second }

        // Print each stacktrace with its frequency

        return frequencies.fold("") { msg, (stacktrace, numFailures) ->

            "$msg\n$numFailures failures:\n$stacktrace"

        }

    }

}