Update reliability test code. Use a separate activity with simpler calback mechanism. (65455a1a) · Commits · e / os / android_frameworks_base

tests/DumpRenderTree/AndroidManifest.xml

+3 −3

Original line number	Diff line number	Diff line
		@@ -23,8 +23,8 @@
		<category android:name="android.intent.category.TEST" />
		</intent-filter>
		</activity>
		<activity android:name="TestShellActivity" android:launchMode="singleTop">
		</activity>
		<activity android:name="TestShellActivity" android:launchMode="singleTop" />
		<activity android:name="ReliabilityTestActivity" />
		</application>

		<instrumentation android:name=".LayoutTestsAutoRunner"
		@@ -32,5 +32,5 @@
		android:label="Layout test automation runner"
		/>
		<uses-permission android:name="android.permission.INTERNET" />
		<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
		<uses-permission android:name="android.permission.WRITE_SDCARD" />
		</manifest>

tests/DumpRenderTree/assets/run_reliability_tests.py

+19 −57

Original line number	Diff line number	Diff line
		@@ -10,10 +10,8 @@

		import logging
		import optparse
		import random
		import subprocess
		import sys
		import time

		TEST_LIST_FILE = "/sdcard/android/reliability_tests_list.txt"
		TEST_STATUS_FILE = "/sdcard/android/reliability_running_test.txt"
		@@ -41,40 +39,16 @@ def DumpRenderTreeFinished(adb_cmd):
		return adb_output.strip() == "#DONE"


		def RandomPick(file_name, approx_size, num_needed):
		"""Randomly pick lines from the text file specifed.

		Args:
		file_name: the text file where lines should be picked from
		approx_size: an approximate size of the text file
		num_needed: how many lines are needed from the file

		Returns:
		an array of string
		"""
		p = float(num_needed) / approx_size
		num_picked = 0
		lines = []
		random.seed()

		while num_picked < num_needed:
		file_handle = open(file_name, "r")
		for line in file_handle:
		line = line.strip()
		if float(random.randint(0, approx_size)) / approx_size < p:
		lines.append(line)
		num_picked += 1
		if num_picked == num_needed:
		break
		file_handle.close()
		return lines
		def RemoveDeviceFile(adb_cmd, file_name):
		shell_cmd_str = adb_cmd + " shell rm " + file_name
		subprocess.Popen(shell_cmd_str,
		shell=True, stdout=subprocess.PIPE,
		stderr=subprocess.PIPE).communicate()


		def main(options, args):
		"""Send the url list to device and start testing, restart if crashed."""

		generate_url = False

		# Set up logging format.
		log_level = logging.INFO
		if options.verbose:
		@@ -84,34 +58,23 @@ def main(options, args):

		# Include all tests if none are specified.
		if not args:
		path = "/tmp/url_list_%d.txt" % time.time()
		generate_url = True
		logging.info("A URL list is not provided, will be automatically generated.")
		print "Missing URL list file"
		sys.exit(1)
		else:
		path = args[0]

		if not options.crash_file:
		print "missing crash file name, use --crash-file to specify"
		print "Missing crash file name, use --crash-file to specify"
		sys.exit(1)
		else:
		crashed_file = options.crash_file

		if not options.timeout_file:
		print "missing timeout file, use --timeout-file to specify"
		print "Missing timeout file, use --timeout-file to specify"
		sys.exit(1)
		else:
		timedout_file = options.timeout_file

		http = RandomPick(HTTP_URL_FILE, 500000, NUM_URLS)
		https = RandomPick(HTTPS_URL_FILE, 45000, NUM_URLS)

		if generate_url:
		file_handle = open(path, "w")
		for i in range(0, NUM_URLS):
		file_handle.write(http[i] + "\n")
		file_handle.write(https[i] + "\n")
		file_handle.close()

		adb_cmd = "adb "
		if options.adb_options:
		adb_cmd += options.adb_options + " "
		@@ -128,6 +91,10 @@ def main(options, args):
		logging.error(adb_error)
		sys.exit(1)

		# clean up previous results
		RemoveDeviceFile(adb_cmd, TEST_STATUS_FILE)
		RemoveDeviceFile(adb_cmd, TEST_TIMEOUT_FILE)

		logging.info("Running the test ...")

		# Count crashed tests.
		@@ -142,11 +109,10 @@ def main(options, args):

		# Call ReliabilityTestsAutoTest#startReliabilityTests
		test_cmd = (test_cmd_prefix + " -e class "
		"com.android.dumprendertree.ReliabilityTestsAutoTest#"
		"startReliabilityTests -e timeout " + timeout_ms
		+ test_cmd_postfix)
		"com.android.dumprendertree.ReliabilityTest#"
		"runTest -e timeout %d %s" %
		(timeout_ms, test_cmd_postfix))

		time_start = time.time()
		adb_output = subprocess.Popen(test_cmd, shell=True,
		stdout=subprocess.PIPE,
		stderr=subprocess.PIPE).communicate()[0]
		@@ -160,16 +126,12 @@ def main(options, args):
		logging.info("Resuming reliability test runner...")

		test_cmd = (test_cmd_prefix + " -e class "
		"com.android.dumprendertree.ReliabilityTestsAutoTest#"
		"resumeReliabilityTests -e timeout " + timeout_ms
		+ test_cmd_postfix)
		"com.android.dumprendertree.ReliabilityTest#"
		"runTest -e timeout %d %s" %
		(timeout_ms, test_cmd_postfix))
		adb_output = subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
		stderr=subprocess.PIPE).communicate()[0]

		time_end = time.time()
		fp = open("time_stat", "a")
		fp.writelines("%.2f\n" % ((time_end - time_start) / NUM_URLS / 2))
		fp.close()
		if (adb_output.find("INSTRUMENTATION_FAILED") != -1 or
		adb_output.find("Process crashed.") != -1):
		logging.error("Error happened : " + adb_output)

tests/DumpRenderTree/src/com/android/dumprendertree/ReliabilityTestsAutoTest.java→tests/DumpRenderTree/src/com/android/dumprendertree/ReliabilityTest.java

+170 −0

Original line number	Diff line number	Diff line
		package com.android.dumprendertree;

		import com.android.dumprendertree.TestShellActivity.DumpDataType;

		import android.content.Intent;
		import android.os.Handler;
		import android.test.ActivityInstrumentationTestCase2;
		import android.util.Log;

		import java.io.BufferedOutputStream;
		import java.io.BufferedReader;
		import java.io.File;
		import java.io.FileNotFoundException;
		import java.io.FileOutputStream;
		import java.io.FileReader;
		import java.io.IOException;
		import java.io.InputStream;
		import java.io.OutputStream;
		import java.util.List;
		import java.util.Vector;

		public class ReliabilityTestsAutoTest extends ActivityInstrumentationTestCase2<TestShellActivity> {
		public class ReliabilityTest extends ActivityInstrumentationTestCase2<ReliabilityTestActivity> {

		private static final String LOGTAG = "ReliabilityTests";
		private static final String LOGTAG = "ReliabilityTest";
		private static final String PKG_NAME = "com.android.dumprendertree";
		private static final String TEST_LIST_FILE = "/sdcard/android/reliability_tests_list.txt";
		private static final String TEST_STATUS_FILE = "/sdcard/android/reliability_running_test.txt";
		private static final String TEST_TIMEOUT_FILE = "/sdcard/android/reliability_timeout_test.txt";
		private static final String TEST_DONE = "#DONE";
		static final String RELIABILITY_TEST_RUNNER_FILES[] = {
		"run_reliability_tests.py"
		};

		private boolean finished;
		private List<String> testList;

		public ReliabilityTestsAutoTest() {
		super("com.android.dumprendertree", TestShellActivity.class);
		public ReliabilityTest() {
		super(PKG_NAME, ReliabilityTestActivity.class);
		}

		private void getTestList() {
		// Read test list.
		testList = new Vector<String>();
		try {
		BufferedReader inReader = new BufferedReader(new FileReader(TEST_LIST_FILE));
		String line;
		while ((line = inReader.readLine()) != null) {
		testList.add(line);
		}
		inReader.close();
		Log.v(LOGTAG, "Test list has " + testList.size() + " test(s).");
		} catch (Exception e) {
		Log.e(LOGTAG, "Error while reading test list : " + e.getMessage());
		}
		}
		@Override
		protected void runTest() throws Throwable {
		ReliabilityTestActivity activity = getActivity();
		LayoutTestsAutoRunner runner = (LayoutTestsAutoRunner)getInstrumentation();

		private void resumeTestList() {
		// read out the test name it stopped last time.
		try {
		BufferedReader inReader = new BufferedReader(new FileReader(TEST_STATUS_FILE));
		String line = inReader.readLine();
		for (int i = 0; i < testList.size(); i++) {
		if (testList.get(i).equals(line)) {
		testList = new Vector<String>(testList.subList(i+1, testList.size()));
		break;
		}
		File testListFile = new File(TEST_LIST_FILE);
		if(!testListFile.exists())
		throw new FileNotFoundException("test list file not found.");

		BufferedReader listReader = new BufferedReader(
		new FileReader(testListFile));

		//always try to resume first, hence cleaning up status will be the
		//responsibility of driver scripts
		String lastUrl = readTestStatus();
		if(lastUrl != null && !TEST_DONE.equals(lastUrl))
		fastForward(listReader, lastUrl);

		String url = null;
		Handler handler = null;
		boolean timeoutFlag = false;
		long start, elapsed;
		//read from BufferedReader instead of populating a list in advance,
		//this will avoid excessive memory usage in case of a large list
		while((url = listReader.readLine()) != null) {
		start = System.currentTimeMillis();
		Log.v(LOGTAG, "Testing URL: " + url);
		updateTestStatus(url);
		activity.reset();
		//use message to send new URL to avoid interacting with
		//WebView in non-UI thread
		handler = activity.getHandler();
		handler.sendMessage(handler.obtainMessage(
		ReliabilityTestActivity.MSG_NAVIGATE,
		runner.mTimeoutInMillis, 0, url));
		timeoutFlag = activity.waitUntilDone();
		elapsed = System.currentTimeMillis() - start;
		if(elapsed < 1000) {
		Log.w(LOGTAG, "Page load finished in " + elapsed
		+ "ms, too soon?");
		} else {
		Log.v(LOGTAG, "Page load finished in " + elapsed + "ms");
		}
		if(timeoutFlag) {
		writeTimeoutFile(url);
		}
		inReader.close();
		} catch (Exception e) {
		Log.e(LOGTAG, "Error reading " + TEST_STATUS_FILE);
		System.runFinalization();
		System.gc();
		System.gc();
		}
		activity.finish();
		listReader.close();
		}

		private void clearTestStatus() {
		// Delete TEST_STATUS_FILE
		public void copyRunnerAssetsToCache() {
		try {
		File f = new File(TEST_STATUS_FILE);
		if (f.delete())
		Log.v(LOGTAG, "Deleted " + TEST_STATUS_FILE);
		else
		Log.e(LOGTAG, "Fail to delete " + TEST_STATUS_FILE);
		} catch (Exception e) {
		Log.e(LOGTAG, "Fail to delete " + TEST_STATUS_FILE + " : " + e.getMessage());
		String out_dir = getActivity().getApplicationContext()
		.getCacheDir().getPath() + "/";

		for( int i=0; i< RELIABILITY_TEST_RUNNER_FILES.length; i++) {
		InputStream in = getActivity().getAssets().open(
		RELIABILITY_TEST_RUNNER_FILES[i]);
		OutputStream out = new FileOutputStream(
		out_dir + RELIABILITY_TEST_RUNNER_FILES[i]);

		byte[] buf = new byte[2048];
		int len;

		while ((len = in.read(buf)) >= 0 ) {
		out.write(buf, 0, len);
		}
		out.close();
		in.close();
		}

		private void clearTestTimeout() {
		// Delete TEST_TIMEOUT_FILE
		try {
		File f = new File(TEST_TIMEOUT_FILE);
		if (f.delete())
		Log.v(LOGTAG, "Deleted " + TEST_TIMEOUT_FILE);
		else
		Log.e(LOGTAG, "Fail to delete " + TEST_TIMEOUT_FILE);
		} catch (Exception e) {
		Log.e(LOGTAG, "Fail to delete " + TEST_TIMEOUT_FILE + " : " + e.getMessage());
		}catch (IOException e) {
		Log.e(LOGTAG, "Cannot extract scripts for testing.", e);
		}
		}

		private void updateTestStatus(String s) {
		// Write TEST_STATUS_FILE
		// write last tested url into status file
		try {
		BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(TEST_STATUS_FILE));
		BufferedOutputStream bos = new BufferedOutputStream(
		new FileOutputStream(TEST_STATUS_FILE));
		bos.write(s.getBytes());
		bos.close();
		} catch (Exception e) {
		Log.e(LOGTAG, "Cannot update file " + TEST_STATUS_FILE);
		} catch (IOException e) {
		Log.e(LOGTAG, "Cannot update file " + TEST_STATUS_FILE, e);
		}
		}

		private void writeTimeoutFile(String s) {
		// Write TEST_TIMEOUT_FILE
		private String readTestStatus() {
		// read out the test name it stopped last time.
		String status = null;
		File testStatusFile = new File(TEST_STATUS_FILE);
		if(testStatusFile.exists()) {
		try {
		BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(TEST_TIMEOUT_FILE, true));
		bos.write(s.getBytes());
		bos.write('\n');
		bos.close();
		} catch (Exception e) {
		Log.e(LOGTAG, "Cannot update file " + TEST_TIMEOUT_FILE);
		}
		}

		private void runReliabilityTest(boolean resume) {
		LayoutTestsAutoRunner runner = (LayoutTestsAutoRunner) getInstrumentation();

		getTestList();
		if(!resume)
		clearTestStatus();
		else
		resumeTestList();

		TestShellActivity activity = getActivity();
		activity.setDefaultDumpDataType(DumpDataType.NO_OP);
		// Run tests.
		for (int i = 0; i < testList.size(); i++) {
		String s = testList.get(i);
		updateTestStatus(s);
		// Run tests
		runTestAndWaitUntilDone(activity, s, runner.mTimeoutInMillis);
		}

		updateTestStatus("#DONE");

		activity.finish();
		}

		private void runTestAndWaitUntilDone(TestShellActivity activity, String url, int timeout) {
		activity.setCallback(new TestShellCallback() {
		public void finished() {
		synchronized (ReliabilityTestsAutoTest.this) {
		finished = true;
		ReliabilityTestsAutoTest.this.notifyAll();
		BufferedReader inReader = new BufferedReader(
		new FileReader(testStatusFile));
		status = inReader.readLine();
		inReader.close();
		} catch (IOException e) {
		Log.e(LOGTAG, "Error reading test status.", e);
		}
		}

		public void timedOut(String url) {
		writeTimeoutFile(url);
		return status;
		}
		});

		finished = false;
		Intent intent = new Intent(Intent.ACTION_VIEW);
		intent.setClass(activity, TestShellActivity.class);
		intent.addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP);
		intent.putExtra(TestShellActivity.TEST_URL, url);
		intent.putExtra(TestShellActivity.TIMEOUT_IN_MILLIS, timeout);
		activity.startActivity(intent);
		private void fastForward(BufferedReader testListReader, String lastUrl) {
		//fastforward the BufferedReader to the position right after last url
		if(lastUrl == null)
		return;

		// Wait until done.
		synchronized (this) {
		while(!finished){
		String line = null;
		try {
		this.wait();
		} catch (InterruptedException e) { }
		}
		while((line = testListReader.readLine()) != null) {
		if(lastUrl.equals(line))
		return;
		}
		} catch (IOException ioe) {
		Log.e(LOGTAG, "Error while reading test list.", ioe);
		return;
		}

		public void startReliabilityTests() {
		clearTestTimeout();
		runReliabilityTest(false);
		}

		public void resumeReliabilityTests() {
		runReliabilityTest(true);
		}

		public void copyRunnerAssetsToCache() {
		private void writeTimeoutFile(String s) {
		//append to the file containing the list of timeout urls
		try {
		String out_dir = getActivity().getApplicationContext()
		.getCacheDir().getPath() + "/";

		for( int i=0; i< RELIABILITY_TEST_RUNNER_FILES.length; i++) {
		InputStream in = getActivity().getAssets().open(
		RELIABILITY_TEST_RUNNER_FILES[i]);
		OutputStream out = new FileOutputStream(
		out_dir + RELIABILITY_TEST_RUNNER_FILES[i]);

		byte[] buf = new byte[2048];
		int len;

		while ((len = in.read(buf)) >= 0 ) {
		out.write(buf, 0, len);
		}
		out.close();
		in.close();
		}
		}catch (IOException e) {
		e.printStackTrace();
		BufferedOutputStream bos = new BufferedOutputStream(
		new FileOutputStream(TEST_TIMEOUT_FILE, true));
		bos.write(s.getBytes());
		bos.write('\n');
		bos.close();
		} catch (Exception e) {
		Log.e(LOGTAG, "Cannot update file " + TEST_TIMEOUT_FILE, e);
		}

		}
		}

tests/DumpRenderTree/src/com/android/dumprendertree/ReliabilityTestActivity.java

0 → 100644

+257 −0

File added.

Preview size limit exceeded, changes collapsed.