Adds to DumpRenderTree the ability to ignore the results of tests (cf0fd789) · Commits · e / os / android_frameworks_base

tests/DumpRenderTree/assets/run_layout_tests.py

+4 −0

Original line number	Diff line number	Diff line
		@@ -246,6 +246,7 @@ def main(options, args):

		result_files = ["/sdcard/layout_tests_passed.txt",
		"/sdcard/layout_tests_failed.txt",
		"/sdcard/layout_tests_ignored.txt",
		"/sdcard/layout_tests_nontext.txt"]
		for file in result_files:
		shell_cmd_str = adb_cmd + " pull " + file + " " + results_dir
		@@ -263,10 +264,13 @@ def main(options, args):
		logging.info(str(passed_tests) + " passed")
		failed_tests = CountLineNumber(results_dir + "/layout_tests_failed.txt")
		logging.info(str(failed_tests) + " failed")
		ignored_tests = CountLineNumber(results_dir + "/layout_tests_ignored.txt")
		logging.info(str(ignored_tests) + " ignored results")
		crashed_tests = CountLineNumber(results_dir + "/layout_tests_crashed.txt")
		logging.info(str(crashed_tests) + " crashed")
		nontext_tests = CountLineNumber(results_dir + "/layout_tests_nontext.txt")
		logging.info(str(nontext_tests) + " no dumpAsText")
		logging.info(str(passed_tests + failed_tests + ignored_tests + crashed_tests + nontext_tests) + " TOTAL")

		logging.info("Results are stored under: " + results_dir + "\n")

tests/DumpRenderTree/src/com/android/dumprendertree/FileFilter.java

+28 −20

Original line number	Diff line number	Diff line
		@@ -23,27 +23,33 @@ public class FileFilter {

		private static final String LOGTAG = "FileFilter";

		// Returns whether we should ignore this test and skip running it.
		// Currently we use this only for tests that crash the browser.
		// TODO: Once these crashes are fixed, we should probably eliminate this
		// method, as no test should crash.
		public static boolean ignoreTest(String file) {
		// treat files like directories for the time being.
		for (int i = 0; i < ignoreTestList.length; i++) {
		if (file.endsWith(ignoreTestList[i])) {
		Log.v(LOGTAG, "File path in list of ignored tests: " + file);
		return true;
		}
		}
		for (int i = 0; i < ignoreTestDirs.length; i++) {
		if (file.endsWith(ignoreTestDirs[i])) {
		Log.v(LOGTAG, "File path in list of ignored directories: " + file);
		return false;
		}

		// Returns whether a directory does not contain layout tests and so can be
		// ignored.
		public static boolean isNonTestDir(String file) {
		for (int i = 0; i < nonTestDirs.length; i++) {
		if (file.endsWith(nonTestDirs[i])) {
		return true;
		}
		}
		// We should run tests for which the expected result is wrong, as there is
		// value in checking that they don't cause crashes.
		// TODO: Run these tests but ignore the result.
		return ignoreResults(file);
		return false;
		}

		public static boolean ignoreResults(String file) {
		// Returns whether we should ignore the result of this test.
		public static boolean ignoreResult(String file) {
		for (int i = 0; i < ignoreResultList.size(); i++) {
		if (file.endsWith(ignoreResultList.get(i))) {
		Log.v(LOGTAG, "File path in list of ignored results: " + file);
		@@ -59,21 +65,16 @@ public class FileFilter {
		fillIgnoreResultList();
		}

		static final String[] ignoreTestDirs = {
		static final String[] nonTestDirs = {
		".", // ignore hidden directories and files
		"resources", // ignore resource directories
		".svn", // don't run anything under .svn folder
		"profiler", // profiler is not supported
		"svg", // svg is not supported
		"platform", // platform specific
		"platform" // No-Android specific tests
		};

		static final String[] ignoreTestList = {
		"editing/selection/move-left-right.html",
		"fast/js/regexp-charclass-crash.html", // RegExp is too large, causing OOM
		"storage/domstorage/localstorage/private-browsing-affects-storage.html", // No notion of private browsing.
		"storage/domstorage/sessionstorage/private-browsing-affects-storage.html", // No notion of private browsing.
		"storage/private-browsing-readonly.html", // No notion of private browsing.
		"editing/selection/move-left-right.html" // Causes DumpRenderTree to hang
		};

		static void fillIgnoreResultList() {
		@@ -113,6 +114,7 @@ public class FileFilter {
		ignoreResultList.add("fast/workers/shared-worker-shared.html"); // shared workers not supported
		ignoreResultList.add("fast/workers/shared-worker-simple.html"); // shared workers not supported

		// TODO: These need to be triaged
		ignoreResultList.add("fast/css/case-transform.html"); // will not fix #619707
		ignoreResultList.add("fast/dom/Element/offsetLeft-offsetTop-body-quirk.html"); // different screen size result in extra spaces in Apple compared to us
		ignoreResultList.add("fast/dom/Window/Plug-ins.html"); // need test plugin
		@@ -183,6 +185,12 @@ public class FileFilter {
		ignoreResultList.add("fast/parser/script-tag-with-trailing-slash.html"); // not capturing the console messages
		ignoreResultList.add("fast/replaced/image-map.html"); // requires eventSender.mouseDown(),mouseUp()
		ignoreResultList.add("fast/text/plain-text-line-breaks.html"); // extra spacing because iFrames rendered next to each other on Apple
		ignoreResultList.add("profiler"); // profiler is not supported
		ignoreResultList.add("storage/domstorage/localstorage/private-browsing-affects-storage.html"); // No notion of private browsing.
		ignoreResultList.add("storage/domstorage/sessionstorage/private-browsing-affects-storage.html"); // No notion of private browsing.
		ignoreResultList.add("storage/private-browsing-readonly.html"); // No notion of private browsing.
		ignoreResultList.add("svg"); // svg is not supported

		}

		}

tests/DumpRenderTree/src/com/android/dumprendertree/FsUtils.java

+21 −14

Original line number	Diff line number	Diff line
		@@ -43,38 +43,45 @@ public class FsUtils {
		}

		public static void findLayoutTestsRecursively(BufferedOutputStream bos,
		String dir) throws IOException {
		String dir, boolean ignoreResultsInDir) throws IOException {
		Log.v(LOGTAG, "Searching tests under " + dir);

		File d = new File(dir);
		if (!d.isDirectory()) {
		throw new AssertionError("A directory expected, but got " + dir);
		}
		ignoreResultsInDir \|= FileFilter.ignoreResult(dir);

		String[] files = d.list();
		for (int i = 0; i < files.length; i++) {
		String s = dir + "/" + files[i];
		if (s.endsWith("TEMPLATE.html")) {

		File f = new File(s);
		if (f.isDirectory()) {
		// If this is not a test directory, we don't recurse into it.
		if (!FileFilter.isNonTestDir(s)) {
		Log.v(LOGTAG, "Recursing on " + s);
		findLayoutTestsRecursively(bos, s, ignoreResultsInDir);
		}
		continue;
		}

		// If this test should be ignored, we skip it completely.
		if (FileFilter.ignoreTest(s)) {
		Log.v(LOGTAG, "Ignoring: " + s);
		continue;
		}
		if (s.toLowerCase().endsWith(".html")
		\|\| s.toLowerCase().endsWith(".xml")) {

		if ((s.toLowerCase().endsWith(".html") \|\| s.toLowerCase().endsWith(".xml"))
		&& !s.endsWith("TEMPLATE.html")) {
		Log.v(LOGTAG, "Recording " + s);
		bos.write(s.getBytes());
		bos.write('\n');
		continue;
		// If the result of this test should be ignored, we still run the test.
		if (ignoreResultsInDir \|\| FileFilter.ignoreResult(s)) {
		bos.write((" IGNORE_RESULT").getBytes());
		}

		File f = new File(s);
		if (f.isDirectory()) {
		findLayoutTestsRecursively(bos, s);
		continue;
		bos.write('\n');
		}

		Log.v(LOGTAG, "Skipping " + s);
		}
		}

tests/DumpRenderTree/src/com/android/dumprendertree/LayoutTestsAutoTest.java

+42 −8

Original line number	Diff line number	Diff line
		@@ -38,14 +38,16 @@ import java.io.InputStream;
		import java.io.OutputStream;
		import java.util.Vector;

		// TestRecorder creates three files ...
		// TestRecorder creates four files ...
		// - passing tests
		// - failing tests
		// - tests for which results are ignored
		// - tests with no text results available
		// TestRecorder does not have the ability to clear the results.
		class MyTestRecorder {
		private BufferedOutputStream mBufferedOutputPassedStream;
		private BufferedOutputStream mBufferedOutputFailedStream;
		private BufferedOutputStream mBufferedOutputIgnoreResultStream;
		private BufferedOutputStream mBufferedOutputNoResultStream;

		public void passed(String layout_file) {
		@@ -68,6 +70,16 @@ class MyTestRecorder {
		}
		}

		public void ignoreResult(String layout_file) {
		try {
		mBufferedOutputIgnoreResultStream.write(layout_file.getBytes());
		mBufferedOutputIgnoreResultStream.write('\n');
		mBufferedOutputIgnoreResultStream.flush();
		} catch(Exception e) {
		e.printStackTrace();
		}
		}

		public void noResult(String layout_file) {
		try {
		mBufferedOutputNoResultStream.write(layout_file.getBytes());
		@@ -82,12 +94,15 @@ class MyTestRecorder {
		try {
		File resultsPassedFile = new File("/sdcard/layout_tests_passed.txt");
		File resultsFailedFile = new File("/sdcard/layout_tests_failed.txt");
		File resultsIgnoreResultFile = new File("/sdcard/layout_tests_ignored.txt");
		File noExpectedResultFile = new File("/sdcard/layout_tests_nontext.txt");

		mBufferedOutputPassedStream =
		new BufferedOutputStream(new FileOutputStream(resultsPassedFile, resume));
		mBufferedOutputFailedStream =
		new BufferedOutputStream(new FileOutputStream(resultsFailedFile, resume));
		mBufferedOutputIgnoreResultStream =
		new BufferedOutputStream(new FileOutputStream(resultsIgnoreResultFile, resume));
		mBufferedOutputNoResultStream =
		new BufferedOutputStream(new FileOutputStream(noExpectedResultFile, resume));
		} catch (Exception e) {
		@@ -99,6 +114,7 @@ class MyTestRecorder {
		try {
		mBufferedOutputPassedStream.close();
		mBufferedOutputFailedStream.close();
		mBufferedOutputIgnoreResultStream.close();
		mBufferedOutputNoResultStream.close();
		} catch (Exception e) {
		e.printStackTrace();
		@@ -132,6 +148,8 @@ public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestSh

		private MyTestRecorder mResultRecorder;
		private Vector<String> mTestList;
		// Whether we should ignore the result for the corresponding test. Ordered same as mTestList.
		private Vector<Boolean> mTestListIgnoreResult;
		private boolean mRebaselineResults;
		// The JavaScript engine currently in use. This determines which set of Android-specific
		// expected test results we use.
		@@ -158,8 +176,11 @@ public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestSh
		BufferedReader inReader = new BufferedReader(new FileReader(LAYOUT_TESTS_LIST_FILE));
		String line = inReader.readLine();
		while (line != null) {
		if (line.startsWith(mTestPathPrefix))
		mTestList.add(line);
		if (line.startsWith(mTestPathPrefix)) {
		String[] components = line.split(" ");
		mTestList.add(components[0]);
		mTestListIgnoreResult.add(components.length > 1 && components[1].equals("IGNORE_RESULT"));
		}
		line = inReader.readLine();
		}
		inReader.close();
		@@ -176,6 +197,7 @@ public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestSh
		for (int i = 0; i < mTestList.size(); i++) {
		if (mTestList.elementAt(i).equals(line)) {
		mTestList = new Vector<String>(mTestList.subList(i+1, mTestList.size()));
		mTestListIgnoreResult = new Vector<Boolean>(mTestListIgnoreResult.subList(i+1, mTestListIgnoreResult.size()));
		break;
		}
		}
		@@ -236,14 +258,24 @@ public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestSh
		mResultRecorder.passed(file);
		}

		private void ignoreResultCase(String file) {
		Log.v("Layout test:", file + " ignore result");
		mResultRecorder.ignoreResult(file);
		}

		private void noResultCase(String file) {
		Log.v("Layout test:", file + " no expected result");
		mResultRecorder.noResult(file);
		}

		private void processResult(String testFile, String actualResultFile, String expectedResultFile) {
		private void processResult(String testFile, String actualResultFile, String expectedResultFile, boolean ignoreResult) {
		Log.v(LOGTAG, " Processing result: " + testFile);

		if (ignoreResult) {
		ignoreResultCase(testFile);
		return;
		}

		File actual = new File(actualResultFile);
		File expected = new File(expectedResultFile);
		if (actual.exists() && expected.exists()) {
		@@ -266,7 +298,7 @@ public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestSh
		}
		}

		private void runTestAndWaitUntilDone(TestShellActivity activity, String test, int timeout) {
		private void runTestAndWaitUntilDone(TestShellActivity activity, String test, int timeout, boolean ignoreResult) {
		activity.setCallback(new TestShellCallback() {
		public void finished() {
		synchronized (LayoutTestsAutoTest.this) {
		@@ -320,7 +352,7 @@ public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestSh
		expectedResultFile = getAndroidExpectedResultFile(expectedResultFile);
		}

		processResult(test, resultFile, expectedResultFile);
		processResult(test, resultFile, expectedResultFile, ignoreResult);
		}
		}

		@@ -336,6 +368,7 @@ public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestSh
		}

		this.mTestList = new Vector<String>();
		this.mTestListIgnoreResult = new Vector<Boolean>();

		// Read settings
		mTestPathPrefix = (new File(LAYOUT_TESTS_ROOT + runner.mTestPath)).getAbsolutePath();
		@@ -372,9 +405,10 @@ public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestSh
		}
		for (int i = 0; i < mTestList.size(); i++) {
		String s = mTestList.elementAt(i);
		boolean ignoreResult = mTestListIgnoreResult.elementAt(i);
		FsUtils.updateTestStatus(TEST_STATUS_FILE, s);
		// Run tests
		runTestAndWaitUntilDone(activity, s, runner.mTimeoutInMillis);
		runTestAndWaitUntilDone(activity, s, runner.mTimeoutInMillis, ignoreResult);
		}

		FsUtils.updateTestStatus(TEST_STATUS_FILE, "#DONE");
		@@ -399,7 +433,7 @@ public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestSh
		try {
		File tests_list = new File(LAYOUT_TESTS_LIST_FILE);
		BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(tests_list, false));
		FsUtils.findLayoutTestsRecursively(bos, getTestPath());
		FsUtils.findLayoutTestsRecursively(bos, getTestPath(), false); // Don't ignore results
		bos.flush();
		bos.close();
		} catch (Exception e) {

tests/DumpRenderTree/src/com/android/dumprendertree/Menu.java

+1 −2

Original line number	Diff line number	Diff line
		@@ -68,7 +68,7 @@ public class Menu extends FileList {
		try {
		File tests_list = new File(LAYOUT_TESTS_LIST_FILE);
		BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(tests_list, false));
		FsUtils.findLayoutTestsRecursively(bos, path);
		FsUtils.findLayoutTestsRecursively(bos, path, false); // Don't ignore results
		bos.flush();
		bos.close();
		} catch (Exception e) {
		@@ -77,4 +77,3 @@ public class Menu extends FileList {
		}

		}