Merge "Add performance test for BroadcastReceiver" (143de945) · Commits · e / os / android_frameworks_base

apct-tests/perftests/utils/src/android/perftests/utils/BenchmarkState.java

+6 −40

Original line number	Diff line number	Diff line
		@@ -25,7 +25,6 @@ import android.util.Log;

		import java.io.File;
		import java.util.ArrayList;
		import java.util.Collections;
		import java.util.concurrent.TimeUnit;

		/**
		@@ -78,10 +77,7 @@ public final class BenchmarkState {

		// Statistics. These values will be filled when the benchmark has finished.
		// The computation needs double precision, but long int is fine for final reporting.
		private long mMedian = 0;
		private double mMean = 0.0;
		private double mStandardDeviation = 0.0;
		private long mMin = 0;
		private Stats mStats;

		// Individual duration in nano seconds.
		private ArrayList<Long> mResults = new ArrayList<>();
		@@ -90,36 +86,6 @@ public final class BenchmarkState {
		return TimeUnit.MILLISECONDS.toNanos(ms);
		}

		/**
		* Calculates statistics.
		*/
		private void calculateSatistics() {
		final int size = mResults.size();
		if (size <= 1) {
		throw new IllegalStateException("At least two results are necessary.");
		}

		Collections.sort(mResults);
		mMedian = size % 2 == 0 ? (mResults.get(size / 2) + mResults.get(size / 2 + 1)) / 2 :
		mResults.get(size / 2);

		mMin = mResults.get(0);
		for (int i = 0; i < size; ++i) {
		long result = mResults.get(i);
		mMean += result;
		if (result < mMin) {
		mMin = result;
		}
		}
		mMean /= (double) size;

		for (int i = 0; i < size; ++i) {
		final double tmp = mResults.get(i) - mMean;
		mStandardDeviation += tmp * tmp;
		}
		mStandardDeviation = Math.sqrt(mStandardDeviation / (double) (size - 1));
		}

		// Stops the benchmark timer.
		// This method can be called only when the timer is running.
		public void pauseTiming() {
		@@ -173,7 +139,7 @@ public final class BenchmarkState {
		if (ENABLE_PROFILING) {
		Debug.stopMethodTracing();
		}
		calculateSatistics();
		mStats = new Stats(mResults);
		mState = FINISHED;
		return false;
		}
		@@ -224,28 +190,28 @@ public final class BenchmarkState {
		if (mState != FINISHED) {
		throw new IllegalStateException("The benchmark hasn't finished");
		}
		return (long) mMean;
		return (long) mStats.getMean();
		}

		private long median() {
		if (mState != FINISHED) {
		throw new IllegalStateException("The benchmark hasn't finished");
		}
		return mMedian;
		return mStats.getMedian();
		}

		private long min() {
		if (mState != FINISHED) {
		throw new IllegalStateException("The benchmark hasn't finished");
		}
		return mMin;
		return mStats.getMin();
		}

		private long standardDeviation() {
		if (mState != FINISHED) {
		throw new IllegalStateException("The benchmark hasn't finished");
		}
		return (long) mStandardDeviation;
		return (long) mStats.getStandardDeviation();
		}

		private String summaryLine() {

apct-tests/perftests/utils/src/android/perftests/utils/ManualBenchmarkState.java

0 → 100644

+157 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2018 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package android.perftests.utils;

		import android.app.Activity;
		import android.app.Instrumentation;
		import android.os.Bundle;
		import android.util.Log;

		import java.util.ArrayList;
		import java.util.concurrent.TimeUnit;

		/**
		* Provides a benchmark framework.
		*
		* This differs from BenchmarkState in that rather than the class measuring the the elapsed time,
		* the test passes in the elapsed time.
		*
		* Example usage:
		*
		* public void sampleMethod() {
		* ManualBenchmarkState state = new ManualBenchmarkState();
		*
		* int[] src = new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
		* long elapsedTime = 0;
		* while (state.keepRunning(elapsedTime)) {
		* long startTime = System.nanoTime();
		* int[] dest = new int[src.length];
		* System.arraycopy(src, 0, dest, 0, src.length);
		* elapsedTime = System.nanoTime() - startTime;
		* }
		* System.out.println(state.summaryLine());
		* }
		*
		* Or use the PerfManualStatusReporter TestRule.
		*
		* Make sure that the overhead of checking the clock does not noticeably affect the results.
		*/
		public final class ManualBenchmarkState {
		private static final String TAG = ManualBenchmarkState.class.getSimpleName();

		// TODO: Tune these values.
		// warm-up for duration
		private static final long WARMUP_DURATION_NS = TimeUnit.SECONDS.toNanos(5);
		// minimum iterations to warm-up for
		private static final int WARMUP_MIN_ITERATIONS = 8;

		// target testing for duration
		private static final long TARGET_TEST_DURATION_NS = TimeUnit.SECONDS.toNanos(16);
		private static final int MAX_TEST_ITERATIONS = 1000000;
		private static final int MIN_TEST_ITERATIONS = 10;

		private static final int NOT_STARTED = 0; // The benchmark has not started yet.
		private static final int WARMUP = 1; // The benchmark is warming up.
		private static final int RUNNING = 2; // The benchmark is running.
		private static final int FINISHED = 3; // The benchmark has stopped.

		private int mState = NOT_STARTED; // Current benchmark state.

		private long mWarmupStartTime = 0;
		private int mWarmupIterations = 0;

		private int mMaxIterations = 0;

		// Individual duration in nano seconds.
		private ArrayList<Long> mResults = new ArrayList<>();

		// Statistics. These values will be filled when the benchmark has finished.
		// The computation needs double precision, but long int is fine for final reporting.
		private Stats mStats;

		private void beginBenchmark(long warmupDuration, int iterations) {
		mMaxIterations = (int) (TARGET_TEST_DURATION_NS / (warmupDuration / iterations));
		mMaxIterations = Math.min(MAX_TEST_ITERATIONS,
		Math.max(mMaxIterations, MIN_TEST_ITERATIONS));
		mState = RUNNING;
		}

		/**
		* Judges whether the benchmark needs more samples.
		*
		* For the usage, see class comment.
		*/
		public boolean keepRunning(long duration) {
		if (duration < 0) {
		throw new RuntimeException("duration is negative: " + duration);
		}
		switch (mState) {
		case NOT_STARTED:
		mState = WARMUP;
		mWarmupStartTime = System.nanoTime();
		return true;
		case WARMUP: {
		final long timeSinceStartingWarmup = System.nanoTime() - mWarmupStartTime;
		++mWarmupIterations;
		if (mWarmupIterations >= WARMUP_MIN_ITERATIONS
		&& timeSinceStartingWarmup >= WARMUP_DURATION_NS) {
		beginBenchmark(timeSinceStartingWarmup, mWarmupIterations);
		}
		return true;
		}
		case RUNNING: {
		mResults.add(duration);
		final boolean keepRunning = mResults.size() < mMaxIterations;
		if (!keepRunning) {
		mStats = new Stats(mResults);
		mState = FINISHED;
		}
		return keepRunning;
		}
		case FINISHED:
		throw new IllegalStateException("The benchmark has finished.");
		default:
		throw new IllegalStateException("The benchmark is in an unknown state.");
		}
		}

		private String summaryLine() {
		final StringBuilder sb = new StringBuilder();
		sb.append("Summary: ");
		sb.append("median=").append(mStats.getMedian()).append("ns, ");
		sb.append("mean=").append(mStats.getMean()).append("ns, ");
		sb.append("min=").append(mStats.getMin()).append("ns, ");
		sb.append("max=").append(mStats.getMax()).append("ns, ");
		sb.append("sigma=").append(mStats.getStandardDeviation()).append(", ");
		sb.append("iteration=").append(mResults.size()).append(", ");
		sb.append("values=").append(mResults.toString());
		return sb.toString();
		}

		public void sendFullStatusReport(Instrumentation instrumentation, String key) {
		if (mState != FINISHED) {
		throw new IllegalStateException("The benchmark hasn't finished");
		}
		Log.i(TAG, key + summaryLine());
		final Bundle status = new Bundle();
		status.putLong(key + "_median", mStats.getMedian());
		status.putLong(key + "_mean", (long) mStats.getMean());
		status.putLong(key + "_stddev", (long) mStats.getStandardDeviation());
		instrumentation.sendStatus(Activity.RESULT_OK, status);
		}
		}

apct-tests/perftests/utils/src/android/perftests/utils/PerfManualStatusReporter.java

0 → 100644

+73 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2018 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package android.perftests.utils;

		import android.support.test.InstrumentationRegistry;

		import org.junit.rules.TestRule;
		import org.junit.runner.Description;
		import org.junit.runners.model.Statement;

		/**
		* Use this rule to make sure we report the status after the test success.
		*
		* <code>
		*
		* @Rule public PerfManualStatusReporter mPerfStatusReporter = new PerfManualStatusReporter();
		* @Test public void functionName() {
		* ManualBenchmarkState state = mPerfStatusReporter.getBenchmarkState();
		*
		* int[] src = new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
		* long elapsedTime = 0;
		* while (state.keepRunning(elapsedTime)) {
		* long startTime = System.nanoTime();
		* int[] dest = new int[src.length];
		* System.arraycopy(src, 0, dest, 0, src.length);
		* elapsedTime = System.nanoTime() - startTime;
		* }
		* }
		* </code>
		*
		* When test succeeded, the status report will use the key as
		* "functionName_*"
		*/

		public class PerfManualStatusReporter implements TestRule {
		private final ManualBenchmarkState mState;

		public PerfManualStatusReporter() {
		mState = new ManualBenchmarkState();
		}

		public ManualBenchmarkState getBenchmarkState() {
		return mState;
		}

		@Override
		public Statement apply(Statement base, Description description) {
		return new Statement() {
		@Override
		public void evaluate() throws Throwable {
		base.evaluate();

		mState.sendFullStatusReport(InstrumentationRegistry.getInstrumentation(),
		description.getMethodName());
		}
		};
		}
		}

apct-tests/perftests/utils/src/android/perftests/utils/Stats.java

0 → 100644

+76 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2018 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package android.perftests.utils;

		import java.util.ArrayList;
		import java.util.Collections;
		import java.util.List;

		public class Stats {
		private long mMedian, mMin, mMax;
		private double mMean, mStandardDeviation;

		/* Calculate stats in constructor. */
		public Stats(List<Long> values) {
		// make a copy since we're modifying it
		values = new ArrayList<>(values);
		final int size = values.size();
		if (size < 2) {
		throw new IllegalArgumentException("At least two results are necessary.");
		}

		Collections.sort(values);

		mMedian = size % 2 == 0 ? (values.get(size / 2) + values.get(size / 2 - 1)) / 2 :
		values.get(size / 2);

		mMin = values.get(0);
		mMax = values.get(values.size() - 1);

		for (int i = 0; i < size; ++i) {
		long result = values.get(i);
		mMean += result;
		}
		mMean /= (double) size;

		for (int i = 0; i < size; ++i) {
		final double tmp = values.get(i) - mMean;
		mStandardDeviation += tmp * tmp;
		}
		mStandardDeviation = Math.sqrt(mStandardDeviation / (double) (size - 1));
		}

		public double getMean() {
		return mMean;
		}

		public long getMedian() {
		return mMedian;
		}

		public long getMax() {
		return mMax;
		}

		public long getMin() {
		return mMin;
		}

		public double getStandardDeviation() {
		return mStandardDeviation;
		}
		}

tests/ActivityManagerPerfTests/README.txt

0 → 100644

+34 −0

Original line number	Diff line number	Diff line
		ActivityManagerPerfTests

		Performance tests for various ActivityManager components, e.g. Services, Broadcasts

		Command to run tests (not working yet, atest seems buggy)
		* atest .../frameworks/base/tests/ActivityManagerPerfTests
		* m ActivityManagerPerfTests ActivityManagerPerfTestsTestApp && \
		adb install $OUT/data/app/ActivityManagerPerfTests/ActivityManagerPerfTests.apk && \
		adb install $OUT/data/app/ActivityManagerPerfTestsTestApp/ActivityManagerPerfTestsTestApp.apk && \
		adb shell am instrument -w \
		com.android.frameworks.perftests.amtests/android.support.test.runner.AndroidJUnitRunner

		Overview
		* The numbers we are trying to measure are end-to-end numbers
		* For example, the time it takes from sending an Intent to start a Service
		to the time the Service runs its callbacks
		* System.nanoTime() is monotonic and consistent between processes, so we use that for measuring time
		* To make sure the test app is running, we start an Activity
		* If the test app is involved, it will measure the time and send it back to the instrumentation test
		* The time is sent back through a Binder interface in the Intent
		* Each sent time is tagged with an id since there can be multiple events that send back a time
		* For example, one is sent when the Activity is started, and another could be sent when a
		Broadcast is received

		Structure
		* tests
		* Instrumentation test which runs the various performance tests and reports the results

		* test-app
		* Target package which contains the Services, BroadcastReceivers, etc. to test against
		* Sends the time it measures back to the test package

		* utils
		* Utilities that both the instrumentation test and test app can use