am 9b9fe68e: am 18455c30: Merge "Add input/output JSON data for baseline... (181e3cb3) · Commits · e / os / android_frameworks_base

tests/CanvasCompare/AndroidManifest.xml

+2 −1

Original line number	Diff line number	Diff line
		@@ -15,8 +15,9 @@
		<manifest xmlns:android="http://schemas.android.com/apk/res/android"
		package="com.android.test.hwuicompare" >

		<!-- for perfhud -->
		<uses-permission android:name="android.permission.INTERNET" />
		<uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE"/>
		<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE"/>

		<application
		android:label="@string/app_name"

tests/CanvasCompare/src/com/android/test/hwuicompare/AutomaticActivity.java

+209 −35

Original line number	Diff line number	Diff line
		@@ -16,11 +16,20 @@

		package com.android.test.hwuicompare;

		import java.io.File;
		import java.io.FileInputStream;
		import java.io.FileOutputStream;
		import java.io.IOException;
		import java.util.ArrayList;
		import java.util.Comparator;
		import java.util.HashMap;
		import java.util.TreeSet;

		import com.android.test.hwuicompare.R;
		import org.json.JSONException;
		import org.json.JSONObject;

		import android.os.Bundle;
		import android.os.Environment;
		import android.os.Trace;
		import android.util.Log;
		import android.widget.ImageView;
		@@ -31,20 +40,31 @@ public class AutomaticActivity extends CompareActivity {
		private static final float ERROR_DISPLAY_THRESHOLD = 0.01f;
		protected static final boolean DRAW_BITMAPS = false;

		private ImageView mSoftwareImageView = null;
		private ImageView mHardwareImageView = null;
		/**
		* Threshold of error change required to consider a test regressed/improved
		*/
		private static final float ERROR_CHANGE_THRESHOLD = 0.001f;

		private static final float[] ERROR_CUTOFFS = {0, 0.005f, 0.01f, 0.02f, 0.05f, 0.1f, 0.25f, 0.5f, 1f, 2f};
		private float[] mErrorRates = new float[ERROR_CUTOFFS.length];
		private static final float[] ERROR_CUTOFFS = {
		0, 0.005f, 0.01f, 0.02f, 0.05f, 0.1f, 0.25f, 0.5f, 1f, 2f
		};

		private final float[] mErrorRates = new float[ERROR_CUTOFFS.length];
		private float mTotalTests = 0;
		private float mTotalError = 0;
		private int mTestsRegressed = 0;
		private int mTestsImproved = 0;

		private ImageView mSoftwareImageView = null;
		private ImageView mHardwareImageView = null;


		public abstract static class TestCallback {
		public abstract static class FinalCallback {
		abstract void report(String name, float value);
		void complete() {}
		void complete() {};
		}

		private ArrayList<TestCallback> mTestCallbacks = new ArrayList<TestCallback>();
		private final ArrayList<FinalCallback> mFinalCallbacks = new ArrayList<FinalCallback>();

		Runnable mRunnable = new Runnable() {
		@Override
		@@ -64,32 +84,11 @@ public class AutomaticActivity extends CompareActivity {
		float error = mErrorCalculator.calcErrorRS(mSoftwareBitmap, mHardwareBitmap);
		Trace.traceEnd(Trace.TRACE_TAG_ALWAYS);

		if (error > ERROR_DISPLAY_THRESHOLD) {
		String modname = "";
		for (String s : DisplayModifier.getLastAppliedModifications()) {
		modname = modname.concat(s + ".");
		}
		Log.d(LOG_TAG, String.format("error for %s was %2.9f", modname, error));
		}
		for (int i = 0; i < ERROR_CUTOFFS.length; i++) {
		if (error <= ERROR_CUTOFFS[i]) break;
		mErrorRates[i]++;
		}
		mTotalError += error;
		mTotalTests++;
		final String[] modifierNames = DisplayModifier.getLastAppliedModifications();
		handleError(modifierNames, error);

		if (DisplayModifier.step()) {
		for (TestCallback c : mTestCallbacks) {
		c.report("averageError", (mTotalError / mTotalTests));
		for (int i = 1; i < ERROR_CUTOFFS.length; i++) {
		c.report(String.format("error over %1.3f", ERROR_CUTOFFS[i]),
		mErrorRates[i]/mTotalTests);
		}
		c.complete();
		}

		Toast.makeText(getApplicationContext(), "done!", Toast.LENGTH_SHORT).show();
		finish();
		finishTest();
		} else {
		mHardwareView.invalidate();
		if (DRAW_BITMAPS) {
		@@ -116,11 +115,186 @@ public class AutomaticActivity extends CompareActivity {
		mHardwareImageView = (ImageView) findViewById(R.id.hardware_image_view);

		onCreateCommon(mRunnable);
		mTestCallbacks.add(new TestCallback() {
		beginTest();
		}

		private static class TestResult {
		TestResult(String label, float error) {
		mLabel = label;
		mTotalError = error;
		mCount = 1;
		}
		public void addInto(float error) {
		mTotalError += error;
		mCount++;
		}
		public float getAverage() {
		return mTotalError / mCount;
		}
		final String mLabel;
		float mTotalError;
		int mCount;
		}

		JSONObject mOutputJson = null;
		JSONObject mInputJson = null;
		final HashMap<String, TestResult> mModifierResults = new HashMap<String, TestResult>();
		final HashMap<String, TestResult> mIndividualResults = new HashMap<String, TestResult>();
		final HashMap<String, TestResult> mModifierDiffResults = new HashMap<String, TestResult>();
		final HashMap<String, TestResult> mIndividualDiffResults = new HashMap<String, TestResult>();
		private void beginTest() {
		mFinalCallbacks.add(new FinalCallback() {
		@Override
		void report(String name, float value) {
		Log.d(LOG_TAG, name + " " + value);
		};
		});

		File inputFile = new File(Environment.getExternalStorageDirectory(),
		"CanvasCompareInput.json");
		if (inputFile.exists() && inputFile.canRead() && inputFile.length() > 0) {
		try {
		FileInputStream inputStream = new FileInputStream(inputFile);
		Log.d(LOG_TAG, "Parsing input file...");
		StringBuffer content = new StringBuffer((int)inputFile.length());
		byte[] buffer = new byte[1024];
		while (inputStream.read(buffer) != -1) {
		content.append(new String(buffer));
		}
		mInputJson = new JSONObject(content.toString());
		inputStream.close();
		Log.d(LOG_TAG, "Parsed input file with " + mInputJson.length() + " entries");
		} catch (JSONException e) {
		Log.e(LOG_TAG, "error parsing input json", e);
		} catch (IOException e) {
		Log.e(LOG_TAG, "error reading input json from sd", e);
		}
		}

		mOutputJson = new JSONObject();
		}

		private static void logTestResultHash(String label, HashMap<String, TestResult> map) {
		Log.d(LOG_TAG, "---------------");
		Log.d(LOG_TAG, label + ":");
		Log.d(LOG_TAG, "---------------");
		TreeSet<TestResult> set = new TreeSet<TestResult>(new Comparator<TestResult>() {
		@Override
		public int compare(TestResult lhs, TestResult rhs) {
		if (lhs == rhs) return 0; // don't need to worry about complex equality

		int cmp = Float.compare(lhs.getAverage(), rhs.getAverage());
		if (cmp != 0) {
		return cmp;
		}
		return lhs.mLabel.compareTo(rhs.mLabel);
		}
		});

		for (TestResult t : map.values()) {
		set.add(t);
		}

		for (TestResult t : set.descendingSet()) {
		if (Math.abs(t.getAverage()) > ERROR_DISPLAY_THRESHOLD) {
		Log.d(LOG_TAG, String.format("%2.4f : %s", t.getAverage(), t.mLabel));
		}
		}
		Log.d(LOG_TAG, "");
		}

		private void finishTest() {
		for (FinalCallback c : mFinalCallbacks) {
		c.report("averageError", (mTotalError / mTotalTests));
		for (int i = 1; i < ERROR_CUTOFFS.length; i++) {
		c.report(String.format("tests with error over %1.3f", ERROR_CUTOFFS[i]),
		mErrorRates[i]);
		}
		if (mInputJson != null) {
		c.report("tests regressed", mTestsRegressed);
		c.report("tests improved", mTestsImproved);
		}
		c.complete();
		}

		try {
		if (mOutputJson != null) {
		String outputString = mOutputJson.toString(4);
		File outputFile = new File(Environment.getExternalStorageDirectory(),
		"CanvasCompareOutput.json");
		FileOutputStream outputStream = new FileOutputStream(outputFile);
		outputStream.write(outputString.getBytes());
		outputStream.close();
		Log.d(LOG_TAG, "Saved output file with " + mOutputJson.length() + " entries");
		}
		} catch (JSONException e) {
		Log.e(LOG_TAG, "error during JSON stringify", e);
		} catch (IOException e) {
		Log.e(LOG_TAG, "error storing JSON output on sd", e);
		}

		logTestResultHash("Modifier change vs previous", mModifierDiffResults);
		logTestResultHash("Invidual test change vs previous", mIndividualDiffResults);
		logTestResultHash("Modifier average test results", mModifierResults);
		logTestResultHash("Individual test results", mIndividualResults);

		Toast.makeText(getApplicationContext(), "done!", Toast.LENGTH_SHORT).show();
		finish();
		}

		/**
		* Inserts the error value into all TestResult objects, associated with each of its modifiers
		*/
		private static void addForAllModifiers(String fullName, float error, String[] modifierNames,
		HashMap<String, TestResult> modifierResults) {
		for (String modifierName : modifierNames) {
		TestResult r = modifierResults.get(fullName);
		if (r == null) {
		modifierResults.put(modifierName, new TestResult(modifierName, error));
		} else {
		r.addInto(error);
		}
		}
		}

		private void handleError(final String[] modifierNames, final float error) {
		String fullName = "";
		for (String s : modifierNames) {
		fullName = fullName.concat("." + s);
		}
		fullName = fullName.substring(1);

		float deltaError = 0;
		if (mInputJson != null) {
		try {
		deltaError = error - (float)mInputJson.getDouble(fullName);
		} catch (JSONException e) {
		Log.w(LOG_TAG, "Warning: unable to read from input json", e);
		}
		if (deltaError > ERROR_CHANGE_THRESHOLD) mTestsRegressed++;
		if (deltaError < -ERROR_CHANGE_THRESHOLD) mTestsImproved++;
		mIndividualDiffResults.put(fullName, new TestResult(fullName, deltaError));
		addForAllModifiers(fullName, deltaError, modifierNames, mModifierDiffResults);
		}

		mIndividualResults.put(fullName, new TestResult(fullName, error));
		addForAllModifiers(fullName, error, modifierNames, mModifierResults);

		try {
		if (mOutputJson != null) {
		mOutputJson.put(fullName, error);
		}
		} catch (JSONException e) {
		Log.e(LOG_TAG, "exception during JSON recording", e);
		mOutputJson = null;
		}

		for (int i = 0; i < ERROR_CUTOFFS.length; i++) {
		if (error <= ERROR_CUTOFFS[i]) break;
		mErrorRates[i]++;
		}
		mTotalError += error;
		mTotalTests++;
		}

		@Override
		@@ -130,7 +304,7 @@ public class AutomaticActivity extends CompareActivity {
		}

		// FOR TESTING
		public void setCallback(TestCallback c) {
		mTestCallbacks.add(c);
		public void setFinalCallback(FinalCallback c) {
		mFinalCallbacks.add(c);
		}
		}

tests/CanvasCompare/src/com/android/test/hwuicompare/Test.java

+2 −2

Original line number	Diff line number	Diff line
		package com.android.test.hwuicompare;

		import com.android.test.hwuicompare.AutomaticActivity.TestCallback;
		import com.android.test.hwuicompare.AutomaticActivity.FinalCallback;

		import android.os.Bundle;
		import android.test.ActivityInstrumentationTestCase2;
		@@ -18,7 +18,7 @@ public class Test extends ActivityInstrumentationTestCase2<AutomaticActivity> {
		super.setUp();
		mBundle = new Bundle();
		mActivity = getActivity();
		mActivity.setCallback(new TestCallback() {
		mActivity.setFinalCallback(new FinalCallback() {

		@Override
		void report(String key, float value) {