Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 4f414bd7 authored by Jack Wang's avatar Jack Wang
Browse files

Make perf metrics collection & output via instrumentation simpler 

- Added includeDetailedStats metadata to TimedTest for collecting additional
  metrics like memory usage and binder transactions
- Added PerformanceCollectorTestCase interface for hard-typing test class as
  performance test while still able to inherit from instrumenation test classes
- Reverted previous changes to PerformanceTestBase, will deprecate
  PerformanceTestBase/Case in a future CL
- Removed 'performance.' prefix from keys written to instrumentation output
parent 075997f1

test-runner/android/test/AndroidTestRunner.java

+5 −5
Original line number Diff line number Diff line
@@ -165,7 +165,7 @@ public class AndroidTestRunner extends BaseTestRunner { 
        for (TestCase testCase : mTestCases) {

            setContextIfAndroidTestCase(testCase, mContext, testContext);

            setInstrumentationIfInstrumentationTestCase(testCase, mInstrumentation);

            setPerformanceWriterIfPerformanceTestCase(testCase, mPerfWriter);

            setPerformanceWriterIfPerformanceCollectorTestCase(testCase, mPerfWriter);

            testCase.run(mTestResult);

        }

    }
@@ -188,10 +188,10 @@ public class AndroidTestRunner extends BaseTestRunner { 
        }

    }



    private void setPerformanceWriterIfPerformanceTestCase(

    private void setPerformanceWriterIfPerformanceCollectorTestCase(

            Test test, PerformanceResultsWriter writer) {

        if (PerformanceTestBase.class.isAssignableFrom(test.getClass())) {

            ((PerformanceTestBase) test).setPerformanceResultsWriter(writer);

        if (PerformanceCollectorTestCase.class.isAssignableFrom(test.getClass())) {

            ((PerformanceCollectorTestCase) test).setPerformanceResultsWriter(writer);

        }

    }

test-runner/android/test/InstrumentationTestRunner.java

+25 −37
Original line number Diff line number Diff line
@@ -27,8 +27,6 @@ import android.os.Debug; 
import android.os.Looper;

import android.os.Parcelable;

import android.os.PerformanceCollector;

import android.os.Process;

import android.os.SystemClock;

import android.os.PerformanceCollector.PerformanceResultsWriter;

import android.test.suitebuilder.TestMethod;

import android.test.suitebuilder.TestPredicates;
@@ -226,23 +224,6 @@ public class InstrumentationTestRunner extends Instrumentation implements TestSu 
     * identifies the path to the generated code coverage file.

     */

    private static final String REPORT_KEY_COVERAGE_PATH = "coverageFilePath";

    /**

     * If included at the start of reporting keys, this prefix marks the key as a performance

     * metric.

     */

    private static final String REPORT_KEY_PREFIX = "performance.";

    /**

     * If included in the status or final bundle sent to an IInstrumentationWatcher, this key

     * reports the cpu time in milliseconds of the current test.

     */

    private static final String REPORT_KEY_PERF_CPU_TIME =

        REPORT_KEY_PREFIX + PerformanceCollector.METRIC_KEY_CPU_TIME;

    /**

     * If included in the status or final bundle sent to an IInstrumentationWatcher, this key

     * reports the run time in milliseconds of the current test.

     */

    private static final String REPORT_KEY_PERF_EXECUTION_TIME =

        REPORT_KEY_PREFIX + PerformanceCollector.METRIC_KEY_EXECUTION_TIME;



    /**

     * The test is starting.
@@ -630,9 +611,9 @@ public class InstrumentationTestRunner extends Instrumentation implements TestSu 
        int mTestNum = 0;

        int mTestResultCode = 0;

        String mTestClass = null;

        PerformanceCollector mPerfCollector = new PerformanceCollector();

        boolean mIsTimedTest = false;

        long mCpuTime = 0;

        long mExecTime = 0;

        boolean mIncludeDetailedStats = false;



        public WatcherResultPrinter(int numTests) {

            mResultTemplate = new Bundle();
@@ -675,20 +656,28 @@ public class InstrumentationTestRunner extends Instrumentation implements TestSu 
            mTestResultCode = 0;



            mIsTimedTest = false;

            mIncludeDetailedStats = false;

            try {

                // Look for TimedTest annotation on both test class and test

                // method

                mIsTimedTest = test.getClass().isAnnotationPresent(TimedTest.class) ||

                    test.getClass().getMethod(testName).isAnnotationPresent(TimedTest.class);

                // Look for TimedTest annotation on both test class and test method

                if (test.getClass().getMethod(testName).isAnnotationPresent(TimedTest.class)) {

                    mIsTimedTest = true;

                    mIncludeDetailedStats = test.getClass().getMethod(testName).getAnnotation(

                            TimedTest.class).includeDetailedStats();

                } else if (test.getClass().isAnnotationPresent(TimedTest.class)) {

                    mIsTimedTest = true;

                    mIncludeDetailedStats = test.getClass().getAnnotation(

                            TimedTest.class).includeDetailedStats();

                }

            } catch (SecurityException e) {

                throw new IllegalStateException(e);

            } catch (NoSuchMethodException e) {

                throw new IllegalStateException(e);

            }



            if (mIsTimedTest) {

                mExecTime = SystemClock.uptimeMillis();

                mCpuTime = Process.getElapsedCpuTime();

            if (mIsTimedTest && mIncludeDetailedStats) {

                mPerfCollector.beginSnapshot("");

            } else if (mIsTimedTest) {

                mPerfCollector.startTiming("");

            }

        }
@@ -720,11 +709,10 @@ public class InstrumentationTestRunner extends Instrumentation implements TestSu 
         * @see junit.framework.TestListener#endTest(Test)

         */

        public void endTest(Test test) {

            if (mIsTimedTest) {

                mCpuTime = Process.getElapsedCpuTime() - mCpuTime;

                mExecTime = SystemClock.uptimeMillis() - mExecTime;

                mTestResult.putLong(REPORT_KEY_PERF_CPU_TIME, mCpuTime);

                mTestResult.putLong(REPORT_KEY_PERF_EXECUTION_TIME, mExecTime);

            if (mIsTimedTest && mIncludeDetailedStats) {

                mTestResult.putAll(mPerfCollector.endSnapshot());

            } else if (mIsTimedTest) {

                writeStopTiming(mPerfCollector.stopTiming(""));

            }



            if (mTestResultCode == 0) {
@@ -760,7 +748,7 @@ public class InstrumentationTestRunner extends Instrumentation implements TestSu 
            for (Parcelable p :

                    results.getParcelableArrayList(PerformanceCollector.METRIC_KEY_ITERATIONS)) {

                Bundle iteration = (Bundle)p;

                String index = "performance.iteration" + i + ".";

                String index = "iteration" + i + ".";

                mTestResult.putString(index + PerformanceCollector.METRIC_KEY_LABEL,

                        iteration.getString(PerformanceCollector.METRIC_KEY_LABEL));

                mTestResult.putLong(index + PerformanceCollector.METRIC_KEY_CPU_TIME,
@@ -772,15 +760,15 @@ public class InstrumentationTestRunner extends Instrumentation implements TestSu 
        }



        public void writeMeasurement(String label, long value) {

            mTestResult.putLong(REPORT_KEY_PREFIX + label, value);

            mTestResult.putLong(label, value);

        }



        public void writeMeasurement(String label, float value) {

            mTestResult.putFloat(REPORT_KEY_PREFIX + label, value);

            mTestResult.putFloat(label, value);

        }



        public void writeMeasurement(String label, String value) {

            mTestResult.putString(REPORT_KEY_PREFIX + label, value);

            mTestResult.putString(label, value);

        }



        // TODO report the end of the cycle

test-runner/android/test/PerformanceCollectorTestCase.java

0 → 100644
+37 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2009 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



package android.test;



import android.os.PerformanceCollector;

import android.os.PerformanceCollector.PerformanceResultsWriter;



/**

 * A simple interface for passing in a PerformanceResultsWriter instance to be used with

 * PerformanceCollector.

 * <p/>

 * A one line implementation of {@link #setPerformanceResultsWriter(PerformanceResultsWriter)}

 * is sufficient in most cases:

 * <p/>

 * <code>mPerfCollector.setPerformanceResultsWriter(writer);</code>

 *

 * {@hide} Not needed for SDK.

 */

public interface PerformanceCollectorTestCase {

    public PerformanceCollector mPerfCollector = new PerformanceCollector();



    public void setPerformanceResultsWriter(PerformanceResultsWriter writer);

}

test-runner/android/test/PerformanceTestBase.java

+3 −86
Original line number Diff line number Diff line
@@ -16,95 +16,12 @@ 


package android.test;



import android.os.Bundle;

import android.os.PerformanceCollector;

import android.os.PerformanceCollector.PerformanceResultsWriter;



import java.lang.reflect.Method;



/**

 * Provides hooks and wrappers to automatically and manually collect and report

 * performance data in tests.

 *

 * {@hide} Pending approval for public API.

 */

public class PerformanceTestBase extends InstrumentationTestCase implements PerformanceTestCase {



    private static PerformanceCollector sPerfCollector = new PerformanceCollector();

    private static int sNumTestMethods = 0;

    private static int sNumTestMethodsLeft = 0;



    // Count number of tests, used to emulate beforeClass and afterClass from JUnit4

    public PerformanceTestBase() {

        if (sNumTestMethods == 0) {

            Method methods[] = getClass().getMethods();

            for (Method m : methods) {

                if (m.getName().startsWith("test")) {

                    sNumTestMethods ++;

                    sNumTestMethodsLeft ++;

                }

            }

        }

    }



    @Override

    protected void setUp() throws Exception {

        super.setUp();

        // @beforeClass

        // Will skew timing measured by TestRunner, but not by PerformanceCollector

        if (sNumTestMethodsLeft == sNumTestMethods) {

            sPerfCollector.beginSnapshot(this.getClass().getName());

        }

    }



    @Override

    protected void tearDown() throws Exception {

        // @afterClass

        // Will skew timing measured by TestRunner, but not by PerformanceCollector

        if (--sNumTestMethodsLeft == 0) {

            sPerfCollector.endSnapshot();

        }

        super.tearDown();

    }



    public void setPerformanceResultsWriter(PerformanceResultsWriter writer) {

        sPerfCollector.setPerformanceResultsWriter(writer);

    }



    /**

     * @see PerformanceCollector#beginSnapshot(String)

     */

    protected void beginSnapshot(String label) {

        sPerfCollector.beginSnapshot(label);

    }



    /**

     * @see PerformanceCollector#endSnapshot()

     */

    protected Bundle endSnapshot() {

        return sPerfCollector.endSnapshot();

    }

import junit.framework.TestCase;



/**

     * @see PerformanceCollector#startTiming(String)

 * {@hide} Not needed for SDK.

 */

    protected void startTiming(String label) {

        sPerfCollector.startTiming(label);

    }



    /**

     * @see PerformanceCollector#addIteration(String)

     */

    protected Bundle addIteration(String label) {

        return sPerfCollector.addIteration(label);

    }



    /**

     * @see PerformanceCollector#stopTiming(String)

     */

    protected Bundle stopTiming(String label) {

        return sPerfCollector.stopTiming(label);

    }

public class PerformanceTestBase extends TestCase implements PerformanceTestCase {



    public int startPerformance(PerformanceTestCase.Intermediates intermediates) {

        return 0;

core/java/android/test/TimedTest.java→test-runner/android/test/TimedTest.java

+10 −6
Original line number Diff line number Diff line
@@ -20,13 +20,17 @@ import java.lang.annotation.Retention; 
import java.lang.annotation.RetentionPolicy;



/**

 * This annotation can be used on an {@link junit.framework.TestCase}'s test

 * methods. When the annotation is present, the test method is timed and the

 * results written through instrumentation output. It can also be used on the

 * class itself, which is equivalent to tagging all test methods with this

 * annotation.

 * This annotation can be used on an {@link junit.framework.TestCase}'s test methods. When the

 * annotation is present, the test method is timed and the results written through instrumentation

 * output. It can also be used on the class itself, which is equivalent to tagging all test methods

 * with this annotation.

 * <p/>

 * Setting {@link #includeDetailedStats()} to true causes additional metrics such as memory usage

 * and binder transactions to be gathered and written through instrumentation output.

 *

 * {@hide} Pending approval for public API.

 */

@Retention(RetentionPolicy.RUNTIME)

public @interface TimedTest { }
 
 No newline at end of file
 
public @interface TimedTest {

    boolean includeDetailedStats() default false;

}