Merge "[Ravenwood] Add test failure reason to stats CSV" into main (7f550079) · Commits · e / os / android_frameworks_base

ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodBasePackageManager.java

+2 −1

Original line number	Original line	Diff line number	Diff line
	@@ -60,7 +60,8 @@ public class RavenwoodBasePackageManager extends PackageManager {

	private static RavenwoodUnsupportedApiException notSupported() {		private static RavenwoodUnsupportedApiException notSupported() {
	return new RavenwoodUnsupportedApiException("This PackageManager API is not yet supported "		return new RavenwoodUnsupportedApiException("This PackageManager API is not yet supported "
	+ "under the Ravenwood deviceless testing environment. Contact g/ravenwood");		+ "under the Ravenwood deviceless testing environment. Contact g/ravenwood")
			.skipStackTraces(1);
	}		}

ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodTestStats.java

+85 −77

Original line number	Original line	Diff line number	Diff line
	@@ -38,6 +38,7 @@ import java.time.LocalDateTime;
	import java.time.format.DateTimeFormatter;		import java.time.format.DateTimeFormatter;
	import java.util.LinkedHashMap;		import java.util.LinkedHashMap;
	import java.util.Map;		import java.util.Map;
			import java.util.TreeMap;

	/**		/**
	* Collect test result stats and write them into a CSV file containing the test results.		* Collect test result stats and write them into a CSV file containing the test results.
	@@ -47,12 +48,12 @@ import java.util.Map;
	* `/tmp/Ravenwood-stats_[TEST-MODULE=NAME]_latest.csv`.		* `/tmp/Ravenwood-stats_[TEST-MODULE=NAME]_latest.csv`.
	*		*
	* Also responsible for dumping all called methods in the form of policy file, by calling		* Also responsible for dumping all called methods in the form of policy file, by calling
	* {@link RavenwoodMethodCallLogger#dumpAllCalledMethodsInner()}, if the method call log is enabled.		* {@link RavenwoodMethodCallLogger#dumpAllCalledMethodsInner}, if the method call log is enabled.
	*/		*/
	public class RavenwoodTestStats {		public class RavenwoodTestStats {
	private static final String TAG = RavenwoodInternalUtils.TAG;		private static final String TAG = RavenwoodInternalUtils.TAG;
	private static final String HEADER =		private static final String HEADER =
	"ClassOrMethod,Module,Class,OuterClass,Method,Passed,Failed,Skipped,DurationMillis";		"ClassOrMethod,Class,Method,Reason,Passed,Failed,Skipped,DurationMillis";

	private static RavenwoodTestStats sInstance;		private static RavenwoodTestStats sInstance;

	@@ -69,21 +70,18 @@ public class RavenwoodTestStats {
	/**		/**
	* Represents a test result.		* Represents a test result.
	*/		*/
	public enum Result {		enum Result {
	Passed,		Passed,
	Failed,		Failed,
	Skipped,		Skipped,
	}		}

	public static class Outcome {		private static String getCaller(Throwable throwable) {
	public final Result result;		var caller = throwable.getStackTrace()[0];
	public final Duration duration;		return caller.getClassName() + "#" + caller.getMethodName();

	public Outcome(Result result, Duration duration) {
	this.result = result;
	this.duration = duration;
	}		}

			record Outcome(Result result, Duration duration, Failure failure) {
	/** @return 1 if {@link #result} is "passed". */		/** @return 1 if {@link #result} is "passed". */
	public int passedCount() {		public int passedCount() {
	return result == Result.Passed ? 1 : 0;		return result == Result.Passed ? 1 : 0;
	@@ -98,42 +96,74 @@ public class RavenwoodTestStats {
	public int skippedCount() {		public int skippedCount() {
	return result == Result.Skipped ? 1 : 0;		return result == Result.Skipped ? 1 : 0;
	}		}

			/**
			* Try to extract the real reason behind a test failure.
			* The logic here is just some heuristic to generate human-readable information.
			*/
			public String reason() {
			if (failure != null) {
			var ex = failure.getException();
			// Keep unwrapping the exception until we found
			// unsupported API exception or the deepest cause.
			for (;;) {
			if (ex instanceof RavenwoodUnsupportedApiException) {
			// The test hit a Ravenwood unsupported API
			return getCaller(ex);
			}
			var cause = ex.getCause();
			if (cause == null) {
			if (ex instanceof ExceptionInInitializerError
			&& ex.getMessage().contains("RavenwoodUnsupportedApiException")) {
			// A static initializer hit a Ravenwood unsupported API
			return getCaller(ex);
			}
			if ("Stub!".equals(ex.getMessage())) {
			// The test hit a stub API
			return getCaller(ex);
			}
			// We don't actually know what's up, just report the exception class name.
			return ex.getClass().getName();
			} else {
			ex = cause;
			}
			}
			}
			return "-";
			}
	}		}

	private final File mOutputFile;
	private final File mOutputSymlinkFile;		private final File mOutputSymlinkFile;
	private final PrintWriter mOutputWriter;		private final PrintWriter mOutputWriter;
	private final String mTestModuleName;		private final Map<String, Map<String, Outcome>> mStats = new LinkedHashMap<>();

	public final Map<String, Map<String, Outcome>> mStats = new LinkedHashMap<>();

	/** Ctor */		/** Ctor */
	public RavenwoodTestStats() {		public RavenwoodTestStats() {
	mTestModuleName = guessTestModuleName();		String testModuleName = guessTestModuleName();

	var basename = "Ravenwood-stats_" + mTestModuleName + "_";		var basename = "Ravenwood-stats_" + testModuleName + "_";

	// Get the current time		// Get the current time
	LocalDateTime now = LocalDateTime.now();		LocalDateTime now = LocalDateTime.now();
	DateTimeFormatter fmt = DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss");		DateTimeFormatter fmt = DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss");

	var tmpdir = System.getProperty("java.io.tmpdir");		var tmpdir = System.getProperty("java.io.tmpdir");
	mOutputFile = new File(tmpdir, basename + now.format(fmt) + ".csv");		File outputFile = new File(tmpdir, basename + now.format(fmt) + ".csv");

	try {		try {
	mOutputWriter = new PrintWriter(mOutputFile);		mOutputWriter = new PrintWriter(outputFile);
	} catch (IOException e) {		} catch (IOException e) {
	throw new RuntimeException("Failed to create logfile. File=" + mOutputFile, e);		throw new RuntimeException("Failed to create logfile. File=" + outputFile, e);
	}		}

	// Create the "latest" symlink.		// Create the "latest" symlink.
	Path symlink = Paths.get(tmpdir, basename + "latest.csv");		Path symlink = Paths.get(tmpdir, basename + "latest.csv");
	try {		try {
	Files.deleteIfExists(symlink);		Files.deleteIfExists(symlink);
	Files.createSymbolicLink(symlink, Paths.get(mOutputFile.getName()));		Files.createSymbolicLink(symlink, Paths.get(outputFile.getName()));

	} catch (IOException e) {		} catch (IOException e) {
	throw new RuntimeException("Failed to create logfile. File=" + mOutputFile, e);		throw new RuntimeException("Failed to create logfile. File=" + outputFile, e);
	}		}
	mOutputSymlinkFile = symlink.toFile();		mOutputSymlinkFile = symlink.toFile();

	@@ -155,81 +185,52 @@ public class RavenwoodTestStats {
	return cwd.getName();		return cwd.getName();
	}		}

	private void addResult(String className, String methodName,		private void addResult(String className, String methodName, Outcome outcome) {
	Result result, Duration duration) {		mStats.computeIfAbsent(className, k -> new TreeMap<>()).putIfAbsent(methodName, outcome);
	mStats.compute(className, (className_, value) -> {
	if (value == null) {
	value = new LinkedHashMap<>();
	}
	// If the result is already set, don't overwrite it.
	if (!value.containsKey(methodName)) {
	value.put(methodName, new Outcome(result, duration));
	}
	return value;
	});
	}		}

	/**		/**
	* Call it when a test method is finished.		* Make sure the string properly escapes commas for CSV fields.
	*/		*/
	private void onTestFinished(String className,		private static String normalize(String s) {
	String testName,		return '"' + s.replace("\"", "\"\"") + '"';
	Result result,
	Duration duration) {
	addResult(className, testName, result, duration);
	}		}

	/**		/**
	* Dump all the results and clear it.		* Dump all the results and clear it.
	*/		*/
	private void dumpAllAndClear() {		private void dumpAllAndClear() {
	for (var entry : mStats.entrySet()) {		mStats.forEach((className, outcomes) -> {
	var className = entry.getKey();
	var outcomes = entry.getValue();

	int passed = 0;		int passed = 0;
	int skipped = 0;		int skipped = 0;
	int failed = 0;		int failed = 0;
	Duration totalDuration = Duration.ZERO;		Duration totalDuration = Duration.ZERO;

	var methods = outcomes.keySet().stream().sorted().toList();		for (var entry : outcomes.entrySet()) {
			var method = entry.getKey();
	for (var method : methods) {		var outcome = entry.getValue();
	var outcome = outcomes.get(method);

	// Per-method status, with "m".		// Per-method status, with "m".
	mOutputWriter.printf("m,%s,%s,%s,%s,%d,%d,%d,%d\n",		mOutputWriter.printf("m,%s,%s,%s,%d,%d,%d,%d\n",
	mTestModuleName, className, getOuterClassName(className), method,		className, normalize(method), normalize(outcome.reason()),
	outcome.passedCount(), outcome.failedCount(), outcome.skippedCount(),		outcome.passedCount(), outcome.failedCount(), outcome.skippedCount(),
	outcome.duration.toMillis());		outcome.duration.toMillis());

	passed += outcome.passedCount();		passed += outcome.passedCount();
	skipped += outcome.skippedCount();		skipped += outcome.skippedCount();
	failed += outcome.failedCount();		failed += outcome.failedCount();

	totalDuration = totalDuration.plus(outcome.duration);		totalDuration = totalDuration.plus(outcome.duration);
	}		}

	// Per-class status, with "c".		// Per-class status, with "c".
	mOutputWriter.printf("c,%s,%s,%s,%s,%d,%d,%d,%d\n",		mOutputWriter.printf("c,%s,-,-,%d,%d,%d,%d\n", className,
	mTestModuleName, className, getOuterClassName(className), "-",
	passed, failed, skipped, totalDuration.toMillis());		passed, failed, skipped, totalDuration.toMillis());
	}		});
	mOutputWriter.flush();		mOutputWriter.flush();
	mStats.clear();		mStats.clear();
	Log.i(TAG, "Added result to stats file: " + mOutputSymlinkFile);		Log.i(TAG, "Added result to stats file: " + mOutputSymlinkFile);
	}		}

	private static String getOuterClassName(String className) {
	// Just delete the '$', because I'm not sure if the className we get here is actaully a
	// valid class name that does exist. (it might have a parameter name, etc?)
	int p = className.indexOf('$');
	if (p < 0) {
	return className;
	}
	return className.substring(0, p);
	}

	private static void createCalledMethodPolicyFile() {		private static void createCalledMethodPolicyFile() {
	// Ideally we want to call it only once, when the very last test class finishes,		// Ideally we want to call it only once, when the very last test class finishes,
	// but we don't know which test class is last or not, so let's just do the dump		// but we don't know which test class is last or not, so let's just do the dump
	@@ -283,18 +284,25 @@ public class RavenwoodTestStats {
	mStartTime = Instant.now();		mStartTime = Instant.now();
	}		}

	private void addResult(		private Outcome createOutcome(Result result, Failure failure) {
			var endTime = Instant.now();
			return new Outcome(result, Duration.between(mStartTime, endTime), failure);
			}

			private Outcome createOutcome(Result result) {
			return createOutcome(result, null);
			}

			private void addResultWithLogging(
	String className,		String className,
	String methodName,		String methodName,
	Result result,		Outcome outcome,
	String logMessage,		String logMessage,
	Object messageExtra) {		Object messageExtra) {
	var endTime = Instant.now();
	if (RAVENWOOD_VERBOSE_LOGGING) {		if (RAVENWOOD_VERBOSE_LOGGING) {
	Log.d(TAG, logMessage + messageExtra);		Log.d(TAG, logMessage + messageExtra);
	}		}
			addResult(className, methodName, outcome);
	onTestFinished(className, methodName, result, Duration.between(mStartTime, endTime));
	}		}

	@Override		@Override
	@@ -302,9 +310,9 @@ public class RavenwoodTestStats {
	// Note: testFinished() is always called, even in failure cases and another callback		// Note: testFinished() is always called, even in failure cases and another callback
	// (e.g. testFailure) has already called. But we just call it anyway because if		// (e.g. testFailure) has already called. But we just call it anyway because if
	// we already recorded a result to the same metho, we won't overwrite it.		// we already recorded a result to the same metho, we won't overwrite it.
	addResult(description.getClassName(),		addResultWithLogging(description.getClassName(),
	description.getMethodName(),		description.getMethodName(),
	Result.Passed,		createOutcome(Result.Passed),
	" testFinished: ",		" testFinished: ",
	description);		description);
	}		}
	@@ -312,9 +320,9 @@ public class RavenwoodTestStats {
	@Override		@Override
	public void testFailure(Failure failure) {		public void testFailure(Failure failure) {
	var description = failure.getDescription();		var description = failure.getDescription();
	addResult(description.getClassName(),		addResultWithLogging(description.getClassName(),
	description.getMethodName(),		description.getMethodName(),
	Result.Failed,		createOutcome(Result.Failed, failure),
	" testFailure: ",		" testFailure: ",
	failure);		failure);
	}		}
	@@ -322,18 +330,18 @@ public class RavenwoodTestStats {
	@Override		@Override
	public void testAssumptionFailure(Failure failure) {		public void testAssumptionFailure(Failure failure) {
	var description = failure.getDescription();		var description = failure.getDescription();
	addResult(description.getClassName(),		addResultWithLogging(description.getClassName(),
	description.getMethodName(),		description.getMethodName(),
	Result.Skipped,		createOutcome(Result.Skipped),
	" testAssumptionFailure: ",		" testAssumptionFailure: ",
	failure);		failure);
	}		}

	@Override		@Override
	public void testIgnored(Description description) {		public void testIgnored(Description description) {
	addResult(description.getClassName(),		addResultWithLogging(description.getClassName(),
	description.getMethodName(),		description.getMethodName(),
	Result.Skipped,		createOutcome(Result.Skipped),
	" testIgnored: ",		" testIgnored: ",
	description);		description);
	}		}

ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodUnsupportedApiException.java

+22 −0

Original line number	Original line	Diff line number	Diff line
	@@ -15,7 +15,12 @@
	*/		*/
	package android.platform.test.ravenwood;		package android.platform.test.ravenwood;

			import java.util.Arrays;

	public class RavenwoodUnsupportedApiException extends UnsupportedOperationException {		public class RavenwoodUnsupportedApiException extends UnsupportedOperationException {

			private int mSkipStackTraces = 0;

	public RavenwoodUnsupportedApiException(String message) {		public RavenwoodUnsupportedApiException(String message) {
	super(message);		super(message);
	}		}
	@@ -25,4 +30,21 @@ public class RavenwoodUnsupportedApiException extends UnsupportedOperationExcept
	+ "environment; consider requesting support from the API owner or "		+ "environment; consider requesting support from the API owner or "
	+ "consider using Mockito; more details at go/ravenwood");		+ "consider using Mockito; more details at go/ravenwood");
	}		}

			/**
			* Sets the number of stack frames to skip when calling {@link #getStackTrace()}.
			*/
			public RavenwoodUnsupportedApiException skipStackTraces(int number) {
			mSkipStackTraces = number;
			return this;
			}

			@Override
			public StackTraceElement[] getStackTrace() {
			var traces = super.getStackTrace();
			if (mSkipStackTraces > 0) {
			return Arrays.copyOfRange(traces, mSkipStackTraces, traces.length);
			}
			return traces;
			}
	}		}

ravenwood/scripts/run-sysui-tests.sh

+1 −0

Original line number	Original line	Diff line number	Diff line
	@@ -18,5 +18,6 @@ set -e
	# Move to the script's directory		# Move to the script's directory
	cd "${0%/*}"		cd "${0%/*}"

			export ROLLING_TF_SUBPROCESS_OUTPUT=0
	export RAVENWOOD_TEST_ENABLEMENT_POLICY=$(readlink -f ../texts/sysui-enablement-policy.txt)		export RAVENWOOD_TEST_ENABLEMENT_POLICY=$(readlink -f ../texts/sysui-enablement-policy.txt)
	${ATEST:-atest} --class-level-report SystemUiRavenTests "$@"		${ATEST:-atest} --class-level-report SystemUiRavenTests "$@"