Merge "Update DumpRenderTree2 to handle failing tests where no meaningful diff is available" (b6a2364e) · Commits · e / os / android_frameworks_base

tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java

+44 −8

Original line number	Diff line number	Diff line
		@@ -46,12 +46,14 @@ public abstract class AbstractResult implements Comparable<AbstractResult> {
		public abstract AbstractResult createResult(Bundle bundle);
		}

		/**
		* A code representing the result of comparing actual and expected results.
		*/
		public enum ResultCode {
		PASS("Passed"),
		FAIL_RESULT_DIFFERS("Result differs"),
		FAIL_NO_EXPECTED_RESULT("No expected result"),
		FAIL_TIMED_OUT("Timed out"),
		FAIL_CRASHED("Crashed");
		RESULTS_MATCH("Results match"),
		RESULTS_DIFFER("Results differ"),
		NO_EXPECTED_RESULT("No expected result"),
		NO_ACTUAL_RESULT("No actual result");

		private String mTitle;

		@@ -123,13 +125,47 @@ public abstract class AbstractResult implements Comparable<AbstractResult> {
		public abstract String getActualTextResult();

		/**
		* Returns the code of this result.
		* Returns the status code representing the result of comparing actual and expected results.
		*
		* @return
		* the code of this result
		* the status code from comparing actual and expected results
		*/
		public abstract ResultCode getResultCode();

		/**
		* Returns whether this test crashed.
		*
		* @return
		* whether this test crashed
		*/
		public abstract boolean didCrash();

		/**
		* Returns whether this test timed out.
		*
		* @return
		* whether this test timed out
		*/
		public abstract boolean didTimeOut();

		/**
		* Sets that this test timed out.
		*/
		public abstract void setDidTimeOut();

		/**
		* Returns whether the test passed.
		*
		* @return
		* whether the test passed
		*/
		public boolean didPass() {
		// Tests that crash can't have timed out or have an actual result.
		assert !(didCrash() && didTimeOut());
		assert !(didCrash() && getResultCode() != ResultCode.NO_ACTUAL_RESULT);
		return !didCrash() && !didTimeOut() && getResultCode() == ResultCode.RESULTS_MATCH;
		}

		/**
		* Return the type of the result data.
		*

tests/DumpRenderTree2/src/com/android/dumprendertree2/CrashedDummyResult.java

+18 −2

Original line number	Diff line number	Diff line
		@@ -61,7 +61,23 @@ public class CrashedDummyResult extends AbstractResult {

		@Override
		public ResultCode getResultCode() {
		return ResultCode.FAIL_CRASHED;
		return ResultCode.NO_ACTUAL_RESULT;
		}

		@Override
		public boolean didCrash() {
		return true;
		}

		@Override
		public boolean didTimeOut() {
		return false;
		}

		@Override
		public void setDidTimeOut() {
		/** This method is not applicable for this type of result */
		assert false;
		}

		@Override

tests/DumpRenderTree2/src/com/android/dumprendertree2/LayoutTestsExecutor.java

+4 −4

Original line number	Diff line number	Diff line
		@@ -426,6 +426,7 @@ public class LayoutTestsExecutor extends Activity {
		assert mCurrentState.isRunningState() : "mCurrentState = " + mCurrentState.name();

		Log.i(LOG_TAG, "onTestFinished(): " + mCurrentTestRelativePath);
		mResultHandler.removeMessages(MSG_TEST_TIMED_OUT);
		obtainActualResultsFromWebView();
		}

		@@ -441,6 +442,9 @@ public class LayoutTestsExecutor extends Activity {

		mCurrentState = CurrentState.OBTAINING_RESULT;

		if (mCurrentTestTimedOut) {
		mCurrentResult.setDidTimeOut();
		}
		mCurrentResult.obtainActualResults(mCurrentWebView,
		mResultHandler.obtainMessage(MSG_ACTUAL_RESULT_OBTAINED));
		}
		@@ -452,7 +456,6 @@ public class LayoutTestsExecutor extends Activity {
		Log.i(LOG_TAG, "onActualResultsObtained(): " + mCurrentTestRelativePath);
		mCurrentState = CurrentState.IDLE;

		mResultHandler.removeMessages(MSG_TEST_TIMED_OUT);
		reportResultToService();
		mCurrentTestIndex++;
		updateProgressBar();
		@@ -470,9 +473,6 @@ public class LayoutTestsExecutor extends Activity {

		Bundle bundle = mCurrentResult.getBundle();
		bundle.putInt("testIndex", mCurrentTestIndex);
		if (mCurrentTestTimedOut) {
		bundle.putString("resultCode", AbstractResult.ResultCode.FAIL_TIMED_OUT.name());
		}
		if (!mTestsList.isEmpty()) {
		bundle.putString("nextTest", mTestsList.get(0));
		}

tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java

+73 −62

Original line number	Diff line number	Diff line
		@@ -75,9 +75,12 @@ public class Summarizer {
		" width: 20px;}" +
		"h3 span.sqr {" +
		" text-decoration: none;" +
		" color: #8ee100;" +
		" float: left;" +
		" width: 20px;}" +
		"h3 span.sqr_pass {" +
		" color: #8ee100;}" +
		"h3 span.sqr_fail {" +
		" color: #c30000;}" +
		"span.source {" +
		" display: block;" +
		" font-size: 10px;" +
		@@ -145,16 +148,16 @@ public class Summarizer {
		" text-transform: uppercase;" +
		" padding: 3px;" +
		" -webkit-border-radius: 4px;}" +
		"span." + AbstractResult.ResultCode.FAIL_RESULT_DIFFERS.name() + "{" +
		"span." + AbstractResult.ResultCode.RESULTS_DIFFER.name() + "{" +
		" background-color: #ccc;" +
		" color: black;}" +
		"span." + AbstractResult.ResultCode.FAIL_NO_EXPECTED_RESULT.name() + "{" +
		"span." + AbstractResult.ResultCode.NO_EXPECTED_RESULT.name() + "{" +
		" background-color: #a700e4;" +
		" color: #fff;}" +
		"span." + AbstractResult.ResultCode.FAIL_TIMED_OUT.name() + "{" +
		"span.timed_out {" +
		" background-color: #f3cb00;" +
		" color: black;}" +
		"span." + AbstractResult.ResultCode.FAIL_CRASHED.name() + "{" +
		"span.crashed {" +
		" background-color: #c30000;" +
		" color: #fff;}" +
		"span.noLtc {" +
		@@ -208,11 +211,11 @@ public class Summarizer {
		public void appendTest(AbstractResult result) {
		String relativePath = result.getRelativePath();

		if (result.getResultCode() == AbstractResult.ResultCode.FAIL_CRASHED) {
		if (result.didCrash()) {
		mCrashedTestsCount++;
		}

		if (result.getResultCode() == AbstractResult.ResultCode.PASS) {
		if (result.didPass()) {
		if (mFileFilter.isFail(relativePath)) {
		mUnexpectedPasses.add(result);
		} else {
		@@ -276,13 +279,10 @@ public class Summarizer {

		createTopSummaryTable(webKitRevision, html);

		createResultsListWithDiff(html, "Unexpected failures", mUnexpectedFailures);

		createResultsListNoDiff(html, "Unexpected passes", mUnexpectedPasses);

		createResultsListWithDiff(html, "Expected failures", mExpectedFailures);

		createResultsListNoDiff(html, "Expected passes", mExpectedPasses);
		createResultsList(html, "Unexpected failures", mUnexpectedFailures);
		createResultsList(html, "Unexpected passes", mUnexpectedPasses);
		createResultsList(html, "Expected failures", mExpectedFailures);
		createResultsList(html, "Expected passes", mExpectedPasses);

		html.append("</body></html>");

		@@ -357,8 +357,8 @@ public class Summarizer {
		html.append("</tr>");
		}

		private void createResultsListWithDiff(StringBuilder html, String title,
		List<AbstractResult> resultsList) {
		private void createResultsList(
		StringBuilder html, String title, List<AbstractResult> resultsList) {
		String relativePath;
		String id = "";
		AbstractResult.ResultCode resultCode;
		@@ -368,7 +368,6 @@ public class Summarizer {
		for (AbstractResult result : resultsList) {
		relativePath = result.getRelativePath();
		resultCode = result.getResultCode();
		assert resultCode != AbstractResult.ResultCode.PASS : "resultCode=" + resultCode;

		html.append("<h3>");

		@@ -378,33 +377,31 @@ public class Summarizer {
		* to cause any problems in this case
		*/
		id = relativePath.replace(File.separator, ":");

		/** Write the test name */
		if (resultCode == AbstractResult.ResultCode.RESULTS_DIFFER) {
		html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');");
		html.append("return false;\">");
		html.append("<span class=\"tri\" id=\"tri." + id + "\">▶ </span>");
		html.append("<span class=\"path\">" + relativePath + "</span>");
		html.append("</a>");

		html.append(" <span class=\"listItem " + resultCode.name() + "\">");
		html.append(resultCode.toString());
		html.append("</span>");

		/** Detect missing LTC function */
		String additionalTextOutputString = result.getAdditionalTextOutputString();
		if (additionalTextOutputString != null &&
		additionalTextOutputString.contains("com.android.dumprendertree") &&
		additionalTextOutputString.contains("has no method")) {
		if (additionalTextOutputString.contains("LayoutTestController")) {
		html.append(" <span class=\"listItem noLtc\">LTC function missing</span>");
		}
		if (additionalTextOutputString.contains("EventSender")) {
		html.append(" <span class=\"listItem noEventSender\">");
		html.append("ES function missing</span>");
		} else {
		html.append("<a href=\"" + getViewSourceUrl(result.getRelativePath()).toString() + "\"");
		html.append(" target=\"_blank\">");
		html.append("<span class=\"sqr sqr_" + (result.didPass() ? "pass" : "fail"));
		html.append("\">■ </span>");
		html.append("<span class=\"path\">" + result.getRelativePath() + "</span>");
		html.append("</a>");
		}

		if (!result.didPass()) {
		appendTags(html, result);
		}

		html.append("</h3>");
		appendExpectedResultsSources(result, html);

		if (resultCode == AbstractResult.ResultCode.RESULTS_DIFFER) {
		html.append("<div class=\"diff\" style=\"display: none;\" id=\"" + id + "\">");
		html.append(result.getDiffAsHtml());
		html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');");
		@@ -413,26 +410,40 @@ public class Summarizer {
		html.append("<a href=\"" + getViewSourceUrl(relativePath).toString() + "\"");
		html.append(" target=\"_blank\">Show source</a>");
		html.append("</div>");
		}

		html.append("<div class=\"space\"></div>");
		}
		}

		private void createResultsListNoDiff(StringBuilder html, String title,
		List<AbstractResult> resultsList) {
		Collections.sort(resultsList);
		html.append("<h2>" + title + " [" + resultsList.size() + "]</h2>");
		for (AbstractResult result : resultsList) {
		html.append("<h3>");
		html.append("<a href=\"" + getViewSourceUrl(result.getRelativePath()).toString() +
		"\"");
		html.append(" target=\"_blank\">");
		html.append("<span class=\"sqr\">■ </span>");
		html.append("<span class=\"path\">" + result.getRelativePath() + "</span>");
		html.append("</a>");
		html.append("</h3>");
		appendExpectedResultsSources(result, html);
		html.append("<div class=\"space\"></div>");
		private void appendTags(StringBuilder html, AbstractResult result) {
		/** Tag tests which crash, time out or where results don't match */
		if (result.didCrash()) {
		html.append(" <span class=\"listItem crashed\">Crashed</span>");
		} else {
		if (result.didTimeOut()) {
		html.append(" <span class=\"listItem timed_out\">Timed out</span>");
		}
		AbstractResult.ResultCode resultCode = result.getResultCode();
		if (resultCode != AbstractResult.ResultCode.RESULTS_MATCH) {
		html.append(" <span class=\"listItem " + resultCode.name() + "\">");
		html.append(resultCode.toString());
		html.append("</span>");
		}
		}

		/** Detect missing LTC function */
		String additionalTextOutputString = result.getAdditionalTextOutputString();
		if (additionalTextOutputString != null &&
		additionalTextOutputString.contains("com.android.dumprendertree") &&
		additionalTextOutputString.contains("has no method")) {
		if (additionalTextOutputString.contains("LayoutTestController")) {
		html.append(" <span class=\"listItem noLtc\">LTC function missing</span>");
		}
		if (additionalTextOutputString.contains("EventSender")) {
		html.append(" <span class=\"listItem noEventSender\">");
		html.append("ES function missing</span>");
		}
		}
		}

tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java

+22 −12

Original line number	Diff line number	Diff line
		@@ -39,6 +39,7 @@ public class TextResult extends AbstractResult {
		private String mExpectedResultPath;
		private String mActualResult;
		private String mRelativePath;
		private boolean mDidTimeOut;
		private ResultCode mResultCode;
		private Message mResultObtainedMsg;

		@@ -74,10 +75,7 @@ public class TextResult extends AbstractResult {
		mActualResult = bundle.getString("actualTextualResult");
		setAdditionalTextOutputString(bundle.getString("additionalTextOutputString"));
		mRelativePath = bundle.getString("relativePath");
		String resultCode = bundle.getString("resultCode");
		if (resultCode != null) {
		mResultCode = ResultCode.valueOf(resultCode);
		}
		mDidTimeOut = bundle.getBoolean("didTimeOut");
		}

		@Override
		@@ -87,16 +85,30 @@ public class TextResult extends AbstractResult {
		}

		if (mExpectedResult == null) {
		mResultCode = AbstractResult.ResultCode.FAIL_NO_EXPECTED_RESULT;
		mResultCode = AbstractResult.ResultCode.NO_EXPECTED_RESULT;
		} else if (!mExpectedResult.equals(mActualResult)) {
		mResultCode = AbstractResult.ResultCode.FAIL_RESULT_DIFFERS;
		mResultCode = AbstractResult.ResultCode.RESULTS_DIFFER;
		} else {
		mResultCode = AbstractResult.ResultCode.PASS;
		mResultCode = AbstractResult.ResultCode.RESULTS_MATCH;
		}

		return mResultCode;
		}

		@Override
		public boolean didCrash() {
		return false;
		}

		@Override
		public boolean didTimeOut() {
		return mDidTimeOut;
		}

		@Override
		public void setDidTimeOut() {
		mDidTimeOut = true;
		}

		@Override
		public byte[] getActualImageResult() {
		return null;
		@@ -239,9 +251,7 @@ public class TextResult extends AbstractResult {
		bundle.putString("actualTextualResult", getActualTextResult());
		bundle.putString("additionalTextOutputString", getAdditionalTextOutputString());
		bundle.putString("relativePath", mRelativePath);
		if (mResultCode != null) {
		bundle.putString("resultCode", mResultCode.name());
		}
		bundle.putBoolean("didTimeOut", mDidTimeOut);
		bundle.putString("type", getType().name());
		return bundle;
		}