Merge "Implementing detector of view flashes" into udc-qpr-dev (7ed637b0) · Commits · e / os / android_packages_apps_Trebuchet

tests/src/com/android/launcher3/util/viewcapture_analysis/AlphaJumpDetector.java

+5 −41

Original line number	Diff line number	Diff line
		@@ -16,11 +16,8 @@
		package com.android.launcher3.util.viewcapture_analysis;

		import com.android.launcher3.util.viewcapture_analysis.ViewCaptureAnalyzer.AnalysisNode;
		import com.android.launcher3.util.viewcapture_analysis.ViewCaptureAnalyzer.AnomalyDetector;

		import java.util.HashMap;
		import java.util.List;
		import java.util.Map;

		/**
		* Anomaly detector that triggers an error when alpha of a view changes too rapidly.
		@@ -34,8 +31,7 @@ final class AlphaJumpDetector extends AnomalyDetector {
		private static final String RECENTS_DRAG_LAYER =
		CONTENT + "LauncherRootView:id/launcher\|RecentsDragLayer:id/drag_layer\|";

		// Paths of nodes that are excluded from analysis.
		private static final Iterable<String> PATHS_TO_IGNORE = List.of(
		private static final IgnoreNode IGNORED_NODES_ROOT = buildIgnoreNodesTree(List.of(
		CONTENT
		+ "AddItemDragLayer:id/add_item_drag_layer\|AddItemWidgetsBottomSheet:id"
		+ "/add_item_bottom_sheet\|LinearLayout:id/add_item_bottom_sheet_content"
		@@ -135,38 +131,7 @@ final class AlphaJumpDetector extends AnomalyDetector {
		+ "NexusOverviewActionsView:id/overview_actions_view"
		+ "\|LinearLayout:id/action_buttons\|Button:id/action_split",
		DRAG_LAYER + "IconView"
		);

		/**
		* Element of the tree of ignored nodes.
		* If the "children" map is empty, then this node should be ignored, i.e. alpha jumps analysis
		* shouldn't run for it.
		* I.e. ignored nodes correspond to the leaves in the ignored nodes tree.
		*/
		private static class IgnoreNode {
		// Map from child node identities to ignore-nodes for these children.
		public final Map<String, IgnoreNode> children = new HashMap<>();
		}

		private static final IgnoreNode IGNORED_NODES_ROOT = buildIgnoreNodesTree();

		// Converts the list of full paths of nodes to ignore to a more efficient tree of ignore-nodes.
		private static IgnoreNode buildIgnoreNodesTree() {
		final IgnoreNode root = new IgnoreNode();
		for (String pathToIgnore : PATHS_TO_IGNORE) {
		// Scan the diag path of an ignored node and add its elements into the tree.
		IgnoreNode currentIgnoreNode = root;
		for (String part : pathToIgnore.split("\\\|")) {
		// Ensure that the child of the node is added to the tree.
		IgnoreNode child = currentIgnoreNode.children.get(part);
		if (child == null) {
		currentIgnoreNode.children.put(part, child = new IgnoreNode());
		}
		currentIgnoreNode = child;
		}
		}
		return root;
		}
		));

		// Minimal increase or decrease of view's alpha between frames that triggers the error.
		private static final float ALPHA_JUMP_THRESHOLD = 1f;
		@@ -213,7 +178,7 @@ final class AlphaJumpDetector extends AnomalyDetector {
		}

		@Override
		String detectAnomalies(AnalysisNode oldInfo, AnalysisNode newInfo, int frameN) {
		String detectAnomalies(AnalysisNode oldInfo, AnalysisNode newInfo, int frameN, long timestamp) {
		// If the view was previously seen, proceed with analysis only if it was present in the
		// view hierarchy in the previous frame.
		if (oldInfo != null && oldInfo.frameN != frameN) return null;
		@@ -229,9 +194,8 @@ final class AlphaJumpDetector extends AnomalyDetector {
		if (alphaDeltaAbs >= ALPHA_JUMP_THRESHOLD) {
		nodeData.ignoreAlphaJumps = true; // No need to report alpha jump in children.
		return String.format(
		"Alpha jump detected in ViewCapture data: alpha change: %s (%s -> %s)"
		+ ", threshold: %s, %s", // ----------- no need to include view?
		alphaDeltaAbs, oldAlpha, newAlpha, ALPHA_JUMP_THRESHOLD, latestInfo);
		"Alpha jump detected: alpha change: %s (%s -> %s), threshold: %s",
		alphaDeltaAbs, oldAlpha, newAlpha, ALPHA_JUMP_THRESHOLD);
		}
		return null;
		}

tests/src/com/android/launcher3/util/viewcapture_analysis/AnomalyDetector.java

0 → 100644

+84 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2023 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/
		package com.android.launcher3.util.viewcapture_analysis;

		import androidx.annotation.NonNull;
		import androidx.annotation.Nullable;

		import java.util.HashMap;
		import java.util.Map;

		/**
		* Detector of one kind of anomaly.
		*/
		abstract class AnomalyDetector {
		// Index of this detector in ViewCaptureAnalyzer.ANOMALY_DETECTORS
		public int detectorOrdinal;

		/**
		* Element of the tree of ignored nodes.
		* If the "children" map is empty, then this node should be ignored, i.e. the analysis shouldn't
		* run for it.
		* I.e. ignored nodes correspond to the leaves in the ignored nodes tree.
		*/
		protected static class IgnoreNode {
		// Map from child node identities to ignore-nodes for these children.
		public final Map<String, IgnoreNode> children = new HashMap<>();
		}

		// Converts the list of full paths of nodes to ignore to a more efficient tree of ignore-nodes.
		protected static IgnoreNode buildIgnoreNodesTree(Iterable<String> pathsToIgnore) {
		final IgnoreNode root = new IgnoreNode();
		for (String pathToIgnore : pathsToIgnore) {
		// Scan the diag path of an ignored node and add its elements into the tree.
		IgnoreNode currentIgnoreNode = root;
		for (String part : pathToIgnore.split("\\\|")) {
		// Ensure that the child of the node is added to the tree.
		IgnoreNode child = currentIgnoreNode.children.get(part);
		if (child == null) {
		currentIgnoreNode.children.put(part, child = new IgnoreNode());
		}
		currentIgnoreNode = child;
		}
		}
		return root;
		}

		/**
		* Initializes fields of the node that are specific to the anomaly detected by this
		* detector.
		*/
		abstract void initializeNode(@NonNull ViewCaptureAnalyzer.AnalysisNode info);

		/**
		* Detects anomalies by looking at the last occurrence of a view, and the current one.
		* null value means that the view. 'oldInfo' and 'newInfo' cannot be both null.
		* If an anomaly is detected, an exception will be thrown.
		*
		* @param oldInfo the view, as seen in the last frame that contained it in the view
		* hierarchy before 'currentFrame'. 'null' means that the view is first seen
		* in the 'currentFrame'.
		* @param newInfo the view in the view hierarchy of the 'currentFrame'. 'null' means that
		* the view is not present in the 'currentFrame', but was present in the previous
		* frame.
		* @param frameN number of the current frame.
		* @return Anomaly diagnostic message if an anomaly has been detected; null otherwise.
		*/
		abstract String detectAnomalies(
		@Nullable ViewCaptureAnalyzer.AnalysisNode oldInfo,
		@Nullable ViewCaptureAnalyzer.AnalysisNode newInfo, int frameN,
		long frameTimeNs);
		}

tests/src/com/android/launcher3/util/viewcapture_analysis/FlashDetector.java

0 → 100644

+173 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2023 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/
		package com.android.launcher3.util.viewcapture_analysis;

		import static org.junit.Assert.assertTrue;

		import com.android.launcher3.util.viewcapture_analysis.ViewCaptureAnalyzer.AnalysisNode;

		import java.util.List;

		/**
		* Anomaly detector that triggers an error when a view flashes, i.e. appears or disappears for a too
		* short period of time.
		*/
		final class FlashDetector extends AnomalyDetector {
		// Maximum time period of a view visibility or invisibility that is recognized as a flash.
		private static final int FLASH_DURATION_MS = 300;

		// Commonly used parts of the paths to ignore.
		private static final String CONTENT = "DecorView\|LinearLayout\|FrameLayout:id/content\|";
		private static final String DRAG_LAYER =
		CONTENT + "LauncherRootView:id/launcher\|DragLayer:id/drag_layer\|";
		private static final String RECENTS_DRAG_LAYER =
		CONTENT + "LauncherRootView:id/launcher\|RecentsDragLayer:id/drag_layer\|";

		private static final IgnoreNode IGNORED_NODES_ROOT = buildIgnoreNodesTree(List.of(
		CONTENT + "LauncherRootView:id/launcher\|FloatingIconView",
		DRAG_LAYER
		+ "SearchContainerView:id/apps_view\|AllAppsRecyclerView:id/apps_list_view"
		+ "\|BubbleTextView:id/icon",
		DRAG_LAYER + "LauncherDragView\|ImageView",
		DRAG_LAYER + "LauncherRecentsView:id/overview_panel\|TaskView\|TextView",
		DRAG_LAYER
		+ "LauncherAllAppsContainerView:id/apps_view\|AllAppsRecyclerView:id"
		+ "/apps_list_view\|BubbleTextView:id/icon",
		DRAG_LAYER + "LauncherDragView\|View",
		CONTENT
		+ "AddItemDragLayer:id/add_item_drag_layer\|AddItemWidgetsBottomSheet:id"
		+ "/add_item_bottom_sheet\|LinearLayout:id/add_item_bottom_sheet_content"
		+ "\|ScrollView:id/widget_preview_scroll_view\|WidgetCell:id/widget_cell"
		+ "\|WidgetCellPreview:id/widget_preview_container\|WidgetImageView:id"
		+ "/widget_preview",
		CONTENT
		+ "AddItemDragLayer:id/add_item_drag_layer\|AddItemWidgetsBottomSheet:id"
		+ "/add_item_bottom_sheet\|LinearLayout:id/add_item_bottom_sheet_content"
		+ "\|ScrollView:id/widget_preview_scroll_view\|WidgetCell:id/widget_cell"
		+ "\|WidgetCellPreview:id/widget_preview_container\|ImageView:id/widget_badge",
		RECENTS_DRAG_LAYER + "FallbackRecentsView:id/overview_panel\|TaskView\|IconView:id/icon",
		DRAG_LAYER
		+ "SearchContainerView:id/apps_view\|UniversalSearchInputView:id"
		+ "/search_container_all_apps\|View:id/ripple"
		));

		// Per-AnalysisNode data that's specific to this detector.
		private static class NodeData {
		public boolean ignoreFlashes;

		// If ignoreNode is null, then this AnalysisNode node will be ignored if its parent is
		// ignored.
		// Otherwise, this AnalysisNode will be ignored if ignoreNode is a leaf i.e. has no
		// children.
		public IgnoreNode ignoreNode;
		}

		private NodeData getNodeData(AnalysisNode info) {
		return (NodeData) info.detectorsData[detectorOrdinal];
		}

		@Override
		void initializeNode(AnalysisNode info) {
		final NodeData nodeData = new NodeData();
		info.detectorsData[detectorOrdinal] = nodeData;

		// If the parent view ignores flashes, its descendants will too.
		final boolean parentIgnoresFlashes = info.parent != null && getNodeData(
		info.parent).ignoreFlashes;
		if (parentIgnoresFlashes) {
		nodeData.ignoreFlashes = true;
		return;
		}

		// Parent view doesn't ignore flashes.
		// Initialize this AnalysisNode's ignore-node with the corresponding child of the
		// ignore-node of the parent, if present.
		final IgnoreNode parentIgnoreNode = info.parent != null
		? getNodeData(info.parent).ignoreNode
		: IGNORED_NODES_ROOT;
		nodeData.ignoreNode = parentIgnoreNode != null
		? parentIgnoreNode.children.get(info.nodeIdentity) : null;
		// AnalysisNode will be ignored if the corresponding ignore-node is a leaf.
		nodeData.ignoreFlashes =
		nodeData.ignoreNode != null && nodeData.ignoreNode.children.isEmpty();
		}

		@Override
		String detectAnomalies(AnalysisNode oldInfo, AnalysisNode newInfo, int frameN,
		long frameTimeNs) {
		// Should we check when a view was visible for a short period, then its alpha became 0?
		// Then 'lastVisible' time should be the last one still visible?
		// Check only transitions of alpha between 0 and 1?

		// If this is the first time ever when we see the view, there have been no flashes yet.
		if (oldInfo == null) return null;

		// A flash requires a view to go from the full visibility to no-visibility and then back,
		// or vice versa.
		// If the last time the view was seen before the current frame, it didn't have full
		// visibility; no flash can possibly be detected at the current frame.
		if (oldInfo.alpha < 1) return null;

		final AnalysisNode latestInfo = newInfo != null ? newInfo : oldInfo;
		final NodeData nodeData = getNodeData(latestInfo);
		if (nodeData.ignoreFlashes) return null;

		// Once the view becomes invisible, see for how long it was visible prior to that. If it
		// was visible only for a short interval of time, it's a flash.
		if (
		// View is invisible in the current frame
		newInfo == null
		// When the view became visible last time, it was a transition from
		// no-visibility to full visibility.
		&& oldInfo.timeBecameVisibleNs != -1) {
		final long wasVisibleTimeMs = (frameTimeNs - oldInfo.timeBecameVisibleNs) / 1000000;

		if (wasVisibleTimeMs <= FLASH_DURATION_MS) {
		nodeData.ignoreFlashes = true; // No need to report flashes in children.
		return
		String.format(
		"View was visible for a too short period of time %dms, which is a"
		+ " flash",
		wasVisibleTimeMs
		);
		}
		}

		// Once a view becomes visible, see for how long it was invisible prior to that. If it
		// was invisible only for a short interval of time, it's a flash.
		if (
		// The view is fully visible now
		newInfo != null && newInfo.alpha >= 1
		// The view wasn't visible in the previous frame
		&& frameN != oldInfo.frameN + 1) {
		// We can assert the below condition because at this point, we know that
		// oldInfo.alpha >= 1, i.e. it disappeared abruptly.
		assertTrue("oldInfo.timeBecameInvisibleNs must not be -1",
		oldInfo.timeBecameInvisibleNs != -1);

		final long wasInvisibleTimeMs = (frameTimeNs - oldInfo.timeBecameInvisibleNs) / 1000000;
		if (wasInvisibleTimeMs <= FLASH_DURATION_MS) {
		nodeData.ignoreFlashes = true; // No need to report flashes in children.
		return
		String.format(
		"View was invisible for a too short period of time %dms, which "
		+ "is a flash",
		wasInvisibleTimeMs);
		}
		}
		return null;
		}
		}

tests/src/com/android/launcher3/util/viewcapture_analysis/ViewCaptureAnalyzer.java

+47 −46

Original line number	Diff line number	Diff line
		@@ -17,9 +17,6 @@ package com.android.launcher3.util.viewcapture_analysis;

		import static android.view.View.VISIBLE;

		import androidx.annotation.NonNull;
		import androidx.annotation.Nullable;

		import com.android.app.viewcapture.data.ExportedData;
		import com.android.app.viewcapture.data.FrameData;
		import com.android.app.viewcapture.data.ViewNode;
		@@ -36,40 +33,10 @@ import java.util.Map;
		public class ViewCaptureAnalyzer {
		private static final String SCRIM_VIEW_CLASS = "com.android.launcher3.views.ScrimView";

		/**
		* Detector of one kind of anomaly.
		*/
		abstract static class AnomalyDetector {
		// Index of this detector in ViewCaptureAnalyzer.ANOMALY_DETECTORS
		public int detectorOrdinal;

		/**
		* Initializes fields of the node that are specific to the anomaly detected by this
		* detector.
		*/
		abstract void initializeNode(@NonNull AnalysisNode info);

		/**
		* Detects anomalies by looking at the last occurrence of a view, and the current one.
		* null value means that the view. 'oldInfo' and 'newInfo' cannot be both null.
		* If an anomaly is detected, an exception will be thrown.
		*
		* @param oldInfo the view, as seen in the last frame that contained it in the view
		* hierarchy before 'currentFrame'. 'null' means that the view is first seen
		* in the 'currentFrame'.
		* @param newInfo the view in the view hierarchy of the 'currentFrame'. 'null' means that
		* the view is not present in the 'currentFrame', but was present in earlier
		* frames.
		* @param frameN number of the current frame.
		* @return Anomaly diagnostic message if an anomaly has been detected; null otherwise.
		*/
		abstract String detectAnomalies(
		@Nullable AnalysisNode oldInfo, @Nullable AnalysisNode newInfo, int frameN);
		}

		// All detectors. They will be invoked in the order listed here.
		private static final AnomalyDetector[] ANOMALY_DETECTORS = {
		new AlphaJumpDetector()
		new AlphaJumpDetector(),
		new FlashDetector()
		};

		static {
		@@ -89,9 +56,21 @@ public class ViewCaptureAnalyzer {
		// Visible scale and alpha, build recursively from the ancestor list.
		public float scaleX;
		public float scaleY;
		public float alpha;
		public float alpha; // Always > 0

		public int frameN;

		// Timestamp of the frame when this view became abruptly visible, i.e. its alpha became 1
		// the next frame after it was 0 or the view wasn't visible.
		// If the view is currently invisible or the last appearance wasn't abrupt, the value is -1.
		public long timeBecameVisibleNs;

		// Timestamp of the frame when this view became abruptly invisible last time, i.e. its
		// alpha became 0, or view disappeared, after being 1 in the previous frame.
		// If the view is currently visible or the last disappearance wasn't abrupt, the value is
		// -1.
		public long timeBecameInvisibleNs;

		public ViewNode viewCaptureNode;

		// Class name + resource id
		@@ -143,7 +122,9 @@ public class ViewCaptureAnalyzer {
		Map<Integer, AnalysisNode> lastSeenNodes, int scrimClassIndex,
		Map<String, String> anomalies) {
		// Analyze the node tree starting from the root.
		long frameTimeNs = frame.getTimestamp();
		analyzeView(
		frameTimeNs,
		frame.getNode(),
		/* parent = */ null,
		frameN,
		@@ -154,7 +135,7 @@ public class ViewCaptureAnalyzer {
		scrimClassIndex,
		anomalies);

		// Analyze transitions when a view visible in the last frame become invisible in the
		// Analyze transitions when a view visible in the previous frame became invisible in the
		// current one.
		for (AnalysisNode info : lastSeenNodes.values()) {
		if (info.frameN == frameN - 1) {
		@@ -166,14 +147,18 @@ public class ViewCaptureAnalyzer {
		frameN,
		/* oldInfo = */ info,
		/* newInfo = */ null,
		anomalies)
		anomalies,
		frameTimeNs)
		);
		}
		info.timeBecameInvisibleNs = info.alpha == 1 ? frameTimeNs : -1;
		info.timeBecameVisibleNs = -1;
		}
		}
		}

		private static void analyzeView(ViewNode viewCaptureNode, AnalysisNode parent, int frameN,
		private static void analyzeView(long frameTimeNs, ViewNode viewCaptureNode, AnalysisNode parent,
		int frameN,
		float leftShift, float topShift, ExportedData viewCaptureData,
		Map<Integer, AnalysisNode> lastSeenNodes, int scrimClassIndex,
		Map<String, String> anomalies) {
		@@ -211,17 +196,31 @@ public class ViewCaptureAnalyzer {
		newAnalysisNode.scaleY = scaleY;
		newAnalysisNode.alpha = alpha;
		newAnalysisNode.frameN = frameN;
		newAnalysisNode.timeBecameInvisibleNs = -1;
		newAnalysisNode.viewCaptureNode = viewCaptureNode;
		Arrays.stream(ANOMALY_DETECTORS).forEach(
		detector -> detector.initializeNode(newAnalysisNode));

		// Detect anomalies for the view
		final AnalysisNode oldAnalysisNode = lastSeenNodes.get(hashcode); // may be null

		if (oldAnalysisNode != null && oldAnalysisNode.frameN + 1 == frameN) {
		// If this view was present in the previous frame, keep the time when it became visible.
		newAnalysisNode.timeBecameVisibleNs = oldAnalysisNode.timeBecameVisibleNs;
		} else {
		// If the view is becoming visible after being invisible, initialize the time when it
		// became visible with a new value.
		// If the view became visible abruptly, i.e. alpha jumped from 0 to 1 between the
		// previous and the current frames, then initialize with the time of the current
		// frame. Otherwise, use -1.
		newAnalysisNode.timeBecameVisibleNs = newAnalysisNode.alpha >= 1 ? frameTimeNs : -1;
		}

		// Detect anomalies for the view.
		if (frameN != 0 && !viewCaptureNode.getWillNotDraw()) {
		Arrays.stream(ANOMALY_DETECTORS).forEach(
		detector ->
		detectAnomaly(detector, frameN, oldAnalysisNode, newAnalysisNode,
		anomalies)
		anomalies, frameTimeNs)
		);
		}
		lastSeenNodes.put(hashcode, newAnalysisNode);
		@@ -236,20 +235,22 @@ public class ViewCaptureAnalyzer {
		// transparent.
		if (child.getClassnameIndex() == scrimClassIndex) break;

		analyzeView(child, newAnalysisNode, frameN, leftShiftForChildren, topShiftForChildren,
		analyzeView(frameTimeNs, child, newAnalysisNode, frameN, leftShiftForChildren,
		topShiftForChildren,
		viewCaptureData, lastSeenNodes, scrimClassIndex, anomalies);
		}
		}

		private static void detectAnomaly(AnomalyDetector detector, int frameN,
		AnalysisNode oldAnalysisNode, AnalysisNode newAnalysisNode,
		Map<String, String> anomalies) {
		Map<String, String> anomalies, long frameTimeNs) {
		final String maybeAnomaly =
		detector.detectAnomalies(oldAnalysisNode, newAnalysisNode, frameN);
		detector.detectAnomalies(oldAnalysisNode, newAnalysisNode, frameN, frameTimeNs);
		if (maybeAnomaly != null) {
		final String viewDiagPath = diagPathFromRoot(newAnalysisNode);
		AnalysisNode latestInfo = newAnalysisNode != null ? newAnalysisNode : oldAnalysisNode;
		final String viewDiagPath = diagPathFromRoot(latestInfo);
		if (!anomalies.containsKey(viewDiagPath)) {
		anomalies.put(viewDiagPath, maybeAnomaly);
		anomalies.put(viewDiagPath, String.format("%s, %s", maybeAnomaly, latestInfo));
		}
		}
		}