Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit b58f28e5 authored by John Wu's avatar John Wu
Browse files

Allow using a text file to toggle Ravenwood tests 

In addition to using annotations to explicitly disable/enable test
classes and methods, introduce a new mechanism to toggle tests.
A policy text file can be provided through the environment variable
RAVENWOOD_TEST_ENABLEMENT_POLICY, and the Ravenwood runtime internally
reads from it and enable/disable tests accordingly.

The benefit for enabling/disabling tests through this mechanism is to
allow us to quickly iterate massive test modules like SysUi tests
without the need to rebuild the entire module.

Bug: 292141694
Flag: EXEMPT host side change only
Test: atest RavenwoodCoreTest
Change-Id: Ia5002aa7545c65ed139b63ebe4a8997b1e736f37
parent 89fccca8

ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodEnablementChecker.java

+115 −2
Original line number Diff line number Diff line
@@ -23,9 +23,15 @@ import android.platform.test.annotations.DisabledOnRavenwood; 
import android.platform.test.annotations.EnabledOnRavenwood;



import com.android.ravenwood.common.RavenwoodInternalUtils;

import com.android.ravenwood.common.SneakyThrow;



import org.junit.runner.Description;



import java.io.IOException;

import java.nio.file.Files;

import java.nio.file.Path;

import java.util.HashMap;

import java.util.Map;

import java.util.Objects;

import java.util.regex.Pattern;
@@ -75,6 +81,89 @@ public class RavenwoodEnablementChecker { 
    public static volatile Pattern REALLY_DISABLED_PATTERN = Pattern.compile(

            Objects.requireNonNullElse(System.getenv("RAVENWOOD_REALLY_DISABLED"), ""));



    /**

     * When using RAVENWOOD_TEST_ENABLEMENT_POLICY, you can provide an external policy text file

     * to change whether each test classes or methods are enabled in Ravenwood without the need

     * to use {@link DisabledOnRavenwood} or {@link EnabledOnRavenwood} annotations.

     *

     * The policy file are lines, each with 2 space delimited fields in the following format:

     *

     * [signature: string] [enabled: boolean]

     *

     * The "signature" field has 2 subcomponents: a class name and method name, separated with the

     * '#' symbol (e.g. com.example.TestClass#testMethod). Method name can be omitted if the

     * policy in question applies to the entire class, not a specific method.

     *

     * When the "signature" field is the special value "*", the "enable" field sets the

     * global default enablement status.

     */

    private static final String RAVENWOOD_TEST_ENABLEMENT_POLICY

            = System.getenv("RAVENWOOD_TEST_ENABLEMENT_POLICY");



    private static final EnablementPolicy sEnablementPolicy = new EnablementPolicy();



    private static class ClassEnablementPolicy {

        Boolean mEnabled;

        Map<String, Boolean> mMethods;

    }



    private static class EnablementPolicy {

        boolean mEnabled = true;

        final Map<String, ClassEnablementPolicy> mClasses = new HashMap<>();



        boolean shouldEnableClass(String className) {

            if (mClasses.isEmpty()) {

                return mEnabled;

            }

            var clazz = mClasses.get(className);

            if (clazz == null) {

                return mEnabled;

            }

            return clazz.mEnabled != null ? clazz.mEnabled : mEnabled;

        }



        Boolean shouldEnableMethod(String className, String methodName) {

            if (mClasses.isEmpty()) {

                return null;

            }

            var clazz = mClasses.get(className);

            if (clazz == null) {

                return null;

            }

            if (clazz.mMethods == null) {

                return null;

            }

            return clazz.mMethods.get(methodName);

        }



        void parseLine(String line) {

            var columns = line.split("\\s", 2);

            if (columns.length != 2) return;

            var signature = columns[0];

            boolean enable = Boolean.parseBoolean(columns[1]);

            if (signature.equals("*")) {

                // Setting the global default policy

                mEnabled = enable;

            } else {

                var s = signature.split("\\#");

                var clazz = s[0];

                var method = s.length > 1 ? s[1] : null;

                var policy = mClasses.computeIfAbsent(clazz, k -> new ClassEnablementPolicy());

                if (method != null) {

                    if (policy.mMethods == null) policy.mMethods = new HashMap<>();

                    policy.mMethods.put(method, enable);

                } else {

                    policy.mEnabled = enable;

                }

            }

        }



        void clear() {

            mEnabled = true;

            mClasses.clear();

        }

    }



    static {

        if (RUN_DISABLED_TESTS) {

            log(TAG, "$RAVENWOOD_RUN_DISABLED_TESTS enabled: running only disabled tests");
@@ -82,11 +171,30 @@ public class RavenwoodEnablementChecker { 
                log(TAG, "$RAVENWOOD_REALLY_DISABLED=" + REALLY_DISABLED_PATTERN.pattern());

            }

        }



        if (RAVENWOOD_TEST_ENABLEMENT_POLICY != null) {

            try {

                var policy = Files.readString(Path.of(RAVENWOOD_TEST_ENABLEMENT_POLICY));

                setTestEnablementPolicy(policy);

            } catch (IOException e) {

                SneakyThrow.sneakyThrow(e);

            }

        }

    }



    private RavenwoodEnablementChecker() {

    }



    public static void setTestEnablementPolicy(String policy) {

        sEnablementPolicy.clear();

        policy.lines().map(String::strip)

                // Remove inline comments

                .map(line -> line.replaceAll("\\s+\\#.*$", "").strip())

                // Ignore empty lines and full line comments

                .filter(line -> !line.isEmpty() && !line.startsWith("#"))

                .forEach(sEnablementPolicy::parseLine);

    }



    /**

     * Determine if the given {@link Description} should be enabled when running on the

     * Ravenwood test environment.
@@ -105,6 +213,9 @@ public class RavenwoodEnablementChecker { 
                result = true;

            } else if (description.getAnnotation(DisabledOnRavenwood.class) != null) {

                result = false;

            } else {

                result = sEnablementPolicy.shouldEnableMethod(

                        description.getClassName(), description.getMethodName());

            }

            if (result != null) {

                if (checkRunDisabledTestsFlag && RUN_DISABLED_TESTS) {
@@ -125,11 +236,13 @@ public class RavenwoodEnablementChecker { 


    public static boolean shouldRunClassOnRavenwood(@NonNull Class<?> testClass,

            boolean checkRunDisabledTestsFlag) {

        boolean result = true;

        boolean result;

        if (testClass.getAnnotation(EnabledOnRavenwood.class) != null) {

            result = true;

        } else if (testClass.getAnnotation(DisabledOnRavenwood.class) != null) {

            result = false;

        } else {

            result = sEnablementPolicy.shouldEnableClass(testClass.getName());

        }

        if (checkRunDisabledTestsFlag && RUN_DISABLED_TESTS) {

            // Invert the result + check the really disable pattern
@@ -144,7 +257,7 @@ public class RavenwoodEnablementChecker { 
     *

     * This only works on tests, not on classes.

     */

    static boolean shouldReallyDisableTest(@NonNull Class<?> testClass,

    private static boolean shouldReallyDisableTest(@NonNull Class<?> testClass,

            @Nullable String methodName) {

        if (REALLY_DISABLED_PATTERN.pattern().isEmpty()) {

            return false;

ravenwood/junit-src/android/platform/test/annotations/DisabledOnRavenwood.java

+2 −4
Original line number Diff line number Diff line
@@ -29,10 +29,8 @@ import java.lang.annotation.Target; 
 * annotation, and an {@link EnabledOnRavenwood} annotation always takes precedence over

 * an {@link DisabledOnRavenwood} annotation.

 *

 * This annotation only takes effect when the containing class has a {@code

 * RavenwoodRule} or {@code RavenwoodClassRule} configured. Ignoring is accomplished by

 * throwing an {@code org.junit.AssumptionViolatedException} which test infrastructure treats as

 * being ignored.

 * Ignoring is accomplished by throwing an {@code org.junit.AssumptionViolatedException} which

 * test infrastructure treats as being ignored.

 *

 * This annotation has no effect on any other non-Ravenwood test environments.

 *

ravenwood/junit-src/android/platform/test/annotations/EnabledOnRavenwood.java

+2 −4
Original line number Diff line number Diff line
@@ -29,10 +29,8 @@ import java.lang.annotation.Target; 
 * annotation, and an {@link EnabledOnRavenwood} annotation always takes precedence over

 * an {@link DisabledOnRavenwood} annotation.

 *

 * This annotation only takes effect when the containing class has a {@code

 * RavenwoodRule} or {@code RavenwoodClassRule} configured. Ignoring is accomplished by

 * throwing an {@code org.junit.AssumptionViolatedException} which test infrastructure treats as

 * being ignored.

 * Ignoring is accomplished by throwing an {@code org.junit.AssumptionViolatedException} which

 * test infrastructure treats as being ignored.

 *

 * This annotation has no effect on any other non-Ravenwood test environments.

 *

ravenwood/tests/coretest/test/com/android/ravenwoodtest/coretest/RavenwoodEnablementTest.java

+83 −0
Original line number Diff line number Diff line
@@ -23,6 +23,7 @@ import static org.junit.Assert.fail; 
import android.platform.test.annotations.DisabledOnRavenwood;

import android.platform.test.annotations.EnabledOnRavenwood;

import android.platform.test.annotations.NoRavenizer;

import android.platform.test.ravenwood.RavenwoodEnablementChecker;



import androidx.test.ext.junit.runners.AndroidJUnit4;
@@ -38,6 +39,28 @@ import java.util.regex.Pattern; 
@NoRavenizer

public class RavenwoodEnablementTest extends RavenwoodRunnerTestBase {



    private static final String ENABLEMENT_POLICY = """

            # Enable all tests by default

            * true  # inline comments should work



            # Disable only the method TestPolicy#testDisabled

            com.android.ravenwoodtest.coretest.RavenwoodEnablementTest$TestPolicy#testDisabled false



            # Disable the entire TestPolicyDisableClass class

            com.android.ravenwoodtest.coretest.RavenwoodEnablementTest$TestPolicyDisableClass false

            """;



    @BeforeClass

    public static void beforeClass() {

        RavenwoodEnablementChecker.setTestEnablementPolicy(ENABLEMENT_POLICY);

    }



    @AfterClass

    public static void afterClass() {

        // Clear the test enablement policy

        RavenwoodEnablementChecker.setTestEnablementPolicy("");

    }



    @RunWith(AndroidJUnit4.class)

    // CHECKSTYLE:OFF

    @Expected("""
@@ -224,4 +247,64 @@ public class RavenwoodEnablementTest extends RavenwoodRunnerTestBase { 
            fail("This should not run");

        }

    }



    @RunWith(AndroidJUnit4.class)

    // CHECKSTYLE:OFF

    @Expected("""

    testRunStarted: classes

    testSuiteStarted: classes

    testSuiteStarted: com.android.ravenwoodtest.coretest.RavenwoodEnablementTest$TestPolicy

    testStarted: testEnabled(com.android.ravenwoodtest.coretest.RavenwoodEnablementTest$TestPolicy)

    testFinished: testEnabled(com.android.ravenwoodtest.coretest.RavenwoodEnablementTest$TestPolicy)

    testStarted: testDisabled(com.android.ravenwoodtest.coretest.RavenwoodEnablementTest$TestPolicy)

    testAssumptionFailure: got: <false>, expected: is <true>

    testFinished: testDisabled(com.android.ravenwoodtest.coretest.RavenwoodEnablementTest$TestPolicy)

    testStarted: testDisabledByAnnotation(com.android.ravenwoodtest.coretest.RavenwoodEnablementTest$TestPolicy)

    testAssumptionFailure: got: <false>, expected: is <true>

    testFinished: testDisabledByAnnotation(com.android.ravenwoodtest.coretest.RavenwoodEnablementTest$TestPolicy)

    testSuiteFinished: com.android.ravenwoodtest.coretest.RavenwoodEnablementTest$TestPolicy

    testSuiteFinished: classes

    testRunFinished: 3,0,2,0

    """)

    // CHECKSTYLE:ON

    public static class TestPolicy {

        @Test

        public void testDisabled() {

            fail("This should be disabled by policy file");

        }



        @Test

        @DisabledOnRavenwood

        public void testDisabledByAnnotation() {

            fail("This should be disabled by policy file");

        }



        @Test

        public void testEnabled() {

        }

    }



    @RunWith(AndroidJUnit4.class)

    //CHECKSTYLE:OFF

    @Expected("""

    testRunStarted: classes

    testSuiteStarted: classes

    testSuiteStarted: TestPolicyDisableClass(com.android.ravenwoodtest.coretest.RavenwoodEnablementTest$TestPolicyDisableClass)

    testIgnored: TestPolicyDisableClass(com.android.ravenwoodtest.coretest.RavenwoodEnablementTest$TestPolicyDisableClass)

    testSuiteFinished: TestPolicyDisableClass(com.android.ravenwoodtest.coretest.RavenwoodEnablementTest$TestPolicyDisableClass)

    testSuiteFinished: classes

    testRunFinished: 0,0,0,1

    """)

    //CHECKSTYLE:ON

    public static class TestPolicyDisableClass {

        @Test

        public void testDisabled() {

            fail("This should be disabled by policy file");

        }



        @Test

        public void testEnabled() {

            fail("This should be disabled by policy file");

        }

    }

}