Merge change 1012 (7a98dae9) · Commits · e / os / android_frameworks_base

tests/DumpRenderTree/AndroidManifest.xml

+1 −0

Original line number	Diff line number	Diff line
		@@ -31,4 +31,5 @@
		android:targetPackage="com.android.dumprendertree"
		android:label="Layout test automation runner"
		/>
		<uses-permission android:name="android.permission.INTERNET"></uses-permission>
		</manifest>

tests/DumpRenderTree/assets/run_reliability_tests.py

0 → 100755

+214 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/python2.4

		"""Run reliability tests using Android instrumentation.

		A test file consists of list web sites to test is needed as a parameter

		Usage:
		run_reliability_tests.py path/to/url/list
		"""

		import logging
		import optparse
		import random
		import subprocess
		import sys
		import time

		TEST_LIST_FILE = "/sdcard/android/reliability_tests_list.txt"
		TEST_STATUS_FILE = "/sdcard/android/reliability_running_test.txt"
		TEST_TIMEOUT_FILE = "/sdcard/android/reliability_timeout_test.txt"
		HTTP_URL_FILE = "urllist_http"
		HTTPS_URL_FILE = "urllist_https"
		NUM_URLS = 25


		def DumpRenderTreeFinished(adb_cmd):
		"""Check if DumpRenderTree finished running.

		Args:
		adb_cmd: adb command string

		Returns:
		True if DumpRenderTree has finished, False otherwise
		"""

		# pull test status file and look for "#DONE"
		shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE
		adb_output = subprocess.Popen(shell_cmd_str,
		shell=True, stdout=subprocess.PIPE,
		stderr=subprocess.PIPE).communicate()[0]
		return adb_output.strip() == "#DONE"


		def RandomPick(file_name, approx_size, num_needed):
		"""Randomly pick lines from the text file specifed.

		Args:
		file_name: the text file where lines should be picked from
		approx_size: an approximate size of the text file
		num_needed: how many lines are needed from the file

		Returns:
		an array of string
		"""
		p = float(num_needed) / approx_size
		num_picked = 0
		lines = []
		random.seed()

		while num_picked < num_needed:
		file_handle = open(file_name, "r")
		for line in file_handle:
		line = line.strip()
		if float(random.randint(0, approx_size)) / approx_size < p:
		lines.append(line)
		num_picked += 1
		if num_picked == num_needed:
		break
		file_handle.close()
		return lines


		def main(options, args):
		"""Send the url list to device and start testing, restart if crashed."""

		generate_url = False

		# Set up logging format.
		log_level = logging.INFO
		if options.verbose:
		log_level = logging.DEBUG
		logging.basicConfig(level=log_level,
		format="%(message)s")

		# Include all tests if none are specified.
		if not args:
		path = "/tmp/url_list_%d.txt" % time.time()
		generate_url = True
		logging.info("A URL list is not provided, will be automatically generated.")
		else:
		path = args[0]

		if not options.crash_file:
		print "missing crash file name, use --crash-file to specify"
		sys.exit(1)
		else:
		crashed_file = options.crash_file

		if not options.timeout_file:
		print "missing timeout file, use --timeout-file to specify"
		sys.exit(1)
		else:
		timedout_file = options.timeout_file

		http = RandomPick(HTTP_URL_FILE, 500000, NUM_URLS)
		https = RandomPick(HTTPS_URL_FILE, 45000, NUM_URLS)

		if generate_url:
		file_handle = open(path, "w")
		for i in range(0, NUM_URLS):
		file_handle.write(http[i] + "\n")
		file_handle.write(https[i] + "\n")
		file_handle.close()

		adb_cmd = "adb "
		if options.adb_options:
		adb_cmd += options.adb_options + " "

		# push url list to device
		test_cmd = adb_cmd + " push \"" + path + "\" \"" + TEST_LIST_FILE + "\""
		proc = subprocess.Popen(test_cmd, shell=True,
		stdout=subprocess.PIPE,
		stderr=subprocess.PIPE)
		(adb_output, adb_error) = proc.communicate()
		if proc.returncode != 0:
		logging.error("failed to push url list to device.")
		logging.error(adb_output)
		logging.error(adb_error)
		sys.exit(1)

		logging.info("Running the test ...")

		# Count crashed tests.
		crashed_tests = []

		if options.time_out_ms:
		timeout_ms = options.time_out_ms

		# Run test until it's done
		test_cmd_prefix = adb_cmd + " shell am instrument"
		test_cmd_postfix = " -w com.android.dumprendertree/.LayoutTestsAutoRunner"

		# Call ReliabilityTestsAutoTest#startReliabilityTests
		test_cmd = (test_cmd_prefix + " -e class "
		"com.android.dumprendertree.ReliabilityTestsAutoTest#"
		"startReliabilityTests -e timeout " + timeout_ms
		+ test_cmd_postfix)

		time_start = time.time()
		adb_output = subprocess.Popen(test_cmd, shell=True,
		stdout=subprocess.PIPE,
		stderr=subprocess.PIPE).communicate()[0]
		while not DumpRenderTreeFinished(adb_cmd):
		logging.error("DumpRenderTree exited before all URLs are visited.")
		shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE
		crashed_test = subprocess.Popen(shell_cmd_str, shell=True,
		stdout=subprocess.PIPE).communicate()[0]
		logging.info(crashed_test + " CRASHED")
		crashed_tests.append(crashed_test)
		logging.info("Resuming reliability test runner...")

		test_cmd = (test_cmd_prefix + " -e class "
		"com.android.dumprendertree.ReliabilityTestsAutoTest#"
		"resumeReliabilityTests -e timeout " + timeout_ms
		+ test_cmd_postfix)
		adb_output = subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
		stderr=subprocess.PIPE).communicate()[0]

		time_end = time.time()
		fp = open("time_stat", "a")
		fp.writelines("%.2f\n" % ((time_end - time_start) / NUM_URLS / 2))
		fp.close()
		if (adb_output.find("INSTRUMENTATION_FAILED") != -1 or
		adb_output.find("Process crashed.") != -1):
		logging.error("Error happened : " + adb_output)
		sys.exit(1)

		logging.info(adb_output)
		logging.info("Done\n")

		if crashed_tests:
		file_handle = open(crashed_file, "w")
		file_handle.writelines("\n".join(crashed_tests))
		logging.info("Crashed URL list stored in: " + crashed_file)
		file_handle.close()
		else:
		logging.info("No crash found.")

		test_cmd = (adb_cmd + "pull \"" + TEST_TIMEOUT_FILE + "\" \""
		+ timedout_file + "\"")

		subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
		stderr=subprocess.PIPE).communicate()


		if "__main__" == __name__:
		option_parser = optparse.OptionParser()
		option_parser.add_option("", "--time-out-ms",
		default=60000,
		help="set the timeout for each test")
		option_parser.add_option("", "--verbose", action="store_true",
		default=False,
		help="include debug-level logging")
		option_parser.add_option("", "--adb-options",
		default=None,
		help="pass options to adb, such as -d -e, etc")
		option_parser.add_option("", "--crash-file",
		default="reliability_crashed_sites.txt",
		help="the list of sites that cause browser to crash")
		option_parser.add_option("", "--timeout-file",
		default="reliability_timedout_sites.txt",
		help="the list of sites that timedout during test.")
		opts, arguments = option_parser.parse_args()
		main(opts, arguments)

tests/DumpRenderTree/src/com/android/dumprendertree/LayoutTestsAutoTest.java

+19 −1

Original line number	Diff line number	Diff line
		@@ -41,6 +41,7 @@ class MyTestRecorder {
		private BufferedOutputStream mBufferedOutputPassedStream;
		private BufferedOutputStream mBufferedOutputFailedStream;
		private BufferedOutputStream mBufferedOutputNoresultStream;
		private BufferedOutputStream mBufferedOutputTimedoutStream;

		public void passed(String layout_file) {
		try {
		@@ -72,11 +73,22 @@ class MyTestRecorder {
		}
		}

		public void timedout(String url) {
		try {
		mBufferedOutputTimedoutStream.write(url.getBytes());
		mBufferedOutputTimedoutStream.write('\n');
		mBufferedOutputTimedoutStream.flush();
		} catch (Exception e) {
		e.printStackTrace();
		}
		}

		public MyTestRecorder(boolean resume) {
		try {
		File resultsPassedFile = new File("/sdcard/layout_tests_passed.txt");
		File resultsFailedFile = new File("/sdcard/layout_tests_failed.txt");
		File noExpectedResultFile = new File("/sdcard/layout_tests_nontext.txt");
		File resultTimedoutFile = new File("/sdcard/layout_tests_timedout.txt");

		mBufferedOutputPassedStream =
		new BufferedOutputStream(new FileOutputStream(resultsPassedFile, resume));
		@@ -84,6 +96,8 @@ class MyTestRecorder {
		new BufferedOutputStream(new FileOutputStream(resultsFailedFile, resume));
		mBufferedOutputNoresultStream =
		new BufferedOutputStream(new FileOutputStream(noExpectedResultFile, resume));
		mBufferedOutputTimedoutStream =
		new BufferedOutputStream(new FileOutputStream(resultTimedoutFile, resume));
		} catch (Exception e) {
		e.printStackTrace();
		}
		@@ -94,6 +108,7 @@ class MyTestRecorder {
		mBufferedOutputPassedStream.close();
		mBufferedOutputFailedStream.close();
		mBufferedOutputNoresultStream.close();
		mBufferedOutputTimedoutStream.close();
		} catch (Exception e) {
		e.printStackTrace();
		}
		@@ -282,6 +297,9 @@ public class LayoutTestsAutoTest extends ActivityInstrumentationTestCase2<TestSh
		LayoutTestsAutoTest.this.notifyAll();
		}
		}

		public void timedOut(String url) {
		}
		});

		String resultFile = getResultFile(test);

tests/DumpRenderTree/src/com/android/dumprendertree/LoadTestsAutoTest.java

+4 −1

Original line number	Diff line number	Diff line
		@@ -134,6 +134,9 @@ public class LoadTestsAutoTest extends ActivityInstrumentationTestCase2<TestShel
		LoadTestsAutoTest.this.notifyAll();
		}
		}

		public void timedOut(String url) {
		}
		});

		mFinished = false;

tests/DumpRenderTree/src/com/android/dumprendertree/ReliabilityTestsAutoTest.java

0 → 100644

+209 −0

Original line number	Diff line number	Diff line
		package com.android.dumprendertree;

		import com.android.dumprendertree.TestShellActivity.DumpDataType;

		import android.content.Intent;
		import android.test.ActivityInstrumentationTestCase2;
		import android.util.Log;

		import java.io.BufferedOutputStream;
		import java.io.BufferedReader;
		import java.io.File;
		import java.io.FileOutputStream;
		import java.io.FileReader;
		import java.io.IOException;
		import java.io.InputStream;
		import java.io.OutputStream;
		import java.util.List;
		import java.util.Vector;

		public class ReliabilityTestsAutoTest extends ActivityInstrumentationTestCase2<TestShellActivity> {

		private static final String LOGTAG = "ReliabilityTests";
		private static final String TEST_LIST_FILE = "/sdcard/android/reliability_tests_list.txt";
		private static final String TEST_STATUS_FILE = "/sdcard/android/reliability_running_test.txt";
		private static final String TEST_TIMEOUT_FILE = "/sdcard/android/reliability_timeout_test.txt";
		static final String RELIABILITY_TEST_RUNNER_FILES[] = {
		"run_reliability_tests.py"
		};

		private boolean finished;
		private List<String> testList;

		public ReliabilityTestsAutoTest() {
		super("com.android.dumprendertree", TestShellActivity.class);
		}

		private void getTestList() {
		// Read test list.
		testList = new Vector<String>();
		try {
		BufferedReader inReader = new BufferedReader(new FileReader(TEST_LIST_FILE));
		String line;
		while ((line = inReader.readLine()) != null) {
		testList.add(line);
		}
		inReader.close();
		Log.v(LOGTAG, "Test list has " + testList.size() + " test(s).");
		} catch (Exception e) {
		Log.e(LOGTAG, "Error while reading test list : " + e.getMessage());
		}
		}

		private void resumeTestList() {
		// read out the test name it stopped last time.
		try {
		BufferedReader inReader = new BufferedReader(new FileReader(TEST_STATUS_FILE));
		String line = inReader.readLine();
		for (int i = 0; i < testList.size(); i++) {
		if (testList.get(i).equals(line)) {
		testList = new Vector<String>(testList.subList(i+1, testList.size()));
		break;
		}
		}
		inReader.close();
		} catch (Exception e) {
		Log.e(LOGTAG, "Error reading " + TEST_STATUS_FILE);
		}
		}

		private void clearTestStatus() {
		// Delete TEST_STATUS_FILE
		try {
		File f = new File(TEST_STATUS_FILE);
		if (f.delete())
		Log.v(LOGTAG, "Deleted " + TEST_STATUS_FILE);
		else
		Log.e(LOGTAG, "Fail to delete " + TEST_STATUS_FILE);
		} catch (Exception e) {
		Log.e(LOGTAG, "Fail to delete " + TEST_STATUS_FILE + " : " + e.getMessage());
		}
		}

		private void clearTestTimeout() {
		// Delete TEST_TIMEOUT_FILE
		try {
		File f = new File(TEST_TIMEOUT_FILE);
		if (f.delete())
		Log.v(LOGTAG, "Deleted " + TEST_TIMEOUT_FILE);
		else
		Log.e(LOGTAG, "Fail to delete " + TEST_TIMEOUT_FILE);
		} catch (Exception e) {
		Log.e(LOGTAG, "Fail to delete " + TEST_TIMEOUT_FILE + " : " + e.getMessage());
		}
		}

		private void updateTestStatus(String s) {
		// Write TEST_STATUS_FILE
		try {
		BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(TEST_STATUS_FILE));
		bos.write(s.getBytes());
		bos.close();
		} catch (Exception e) {
		Log.e(LOGTAG, "Cannot update file " + TEST_STATUS_FILE);
		}
		}

		private void writeTimeoutFile(String s) {
		// Write TEST_TIMEOUT_FILE
		try {
		BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(TEST_TIMEOUT_FILE, true));
		bos.write(s.getBytes());
		bos.write('\n');
		bos.close();
		} catch (Exception e) {
		Log.e(LOGTAG, "Cannot update file " + TEST_TIMEOUT_FILE);
		}
		}

		private void runReliabilityTest(boolean resume) {
		LayoutTestsAutoRunner runner = (LayoutTestsAutoRunner) getInstrumentation();

		getTestList();
		if(!resume)
		clearTestStatus();
		else
		resumeTestList();

		TestShellActivity activity = getActivity();
		activity.setDefaultDumpDataType(DumpDataType.NO_OP);
		// Run tests.
		for (int i = 0; i < testList.size(); i++) {
		String s = testList.get(i);
		updateTestStatus(s);
		// Run tests
		runTestAndWaitUntilDone(activity, s, runner.mTimeoutInMillis);
		}

		updateTestStatus("#DONE");

		activity.finish();
		}

		private void runTestAndWaitUntilDone(TestShellActivity activity, String url, int timeout) {
		activity.setCallback(new TestShellCallback() {
		public void finished() {
		synchronized (ReliabilityTestsAutoTest.this) {
		finished = true;
		ReliabilityTestsAutoTest.this.notifyAll();
		}
		}

		public void timedOut(String url) {
		writeTimeoutFile(url);
		}
		});

		finished = false;
		Intent intent = new Intent(Intent.ACTION_VIEW);
		intent.setClass(activity, TestShellActivity.class);
		intent.addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP);
		intent.putExtra(TestShellActivity.TEST_URL, url);
		intent.putExtra(TestShellActivity.TIMEOUT_IN_MILLIS, timeout);
		activity.startActivity(intent);

		// Wait until done.
		synchronized (this) {
		while(!finished){
		try {
		this.wait();
		} catch (InterruptedException e) { }
		}
		}
		}

		public void startReliabilityTests() {
		clearTestTimeout();
		runReliabilityTest(false);
		}

		public void resumeReliabilityTests() {
		runReliabilityTest(true);
		}

		public void copyRunnerAssetsToCache() {
		try {
		String out_dir = getActivity().getApplicationContext()
		.getCacheDir().getPath() + "/";

		for( int i=0; i< RELIABILITY_TEST_RUNNER_FILES.length; i++) {
		InputStream in = getActivity().getAssets().open(
		RELIABILITY_TEST_RUNNER_FILES[i]);
		OutputStream out = new FileOutputStream(
		out_dir + RELIABILITY_TEST_RUNNER_FILES[i]);

		byte[] buf = new byte[2048];
		int len;

		while ((len = in.read(buf)) >= 0 ) {
		out.write(buf, 0, len);
		}
		out.close();
		in.close();
		}
		}catch (IOException e) {
		e.printStackTrace();
		}

		}
		}