Merge "Move a11y window bounds computation into a11y package" into main (ee646bd8) · Commits · e / os / android_frameworks_base

services/accessibility/accessibility.aconfig

+7 −0

Original line number	Diff line number	Diff line
		@@ -23,6 +23,13 @@ flag {
		bug: "271490102"
		}

		flag {
		name: "compute_window_changes_on_a11y"
		namespace: "accessibility"
		description: "Computes accessibility window changes in accessibility instead of wm package."
		bug: "322444245"
		}

		flag {
		name: "deprecate_package_list_observer"
		namespace: "accessibility"

services/accessibility/java/com/android/server/accessibility/AccessibilityWindowManager.java

+165 −3

Original line number	Diff line number	Diff line
		@@ -19,6 +19,8 @@ package com.android.server.accessibility;
		import static android.accessibilityservice.AccessibilityTrace.FLAGS_ACCESSIBILITY_INTERACTION_CONNECTION;
		import static android.accessibilityservice.AccessibilityTrace.FLAGS_WINDOW_MANAGER_INTERNAL;
		import static android.view.WindowManager.LayoutParams.TYPE_ACCESSIBILITY_OVERLAY;
		import static android.view.WindowManager.LayoutParams.TYPE_DOCK_DIVIDER;
		import static android.view.WindowManager.LayoutParams.TYPE_MAGNIFICATION_OVERLAY;
		import static android.view.accessibility.AccessibilityEvent.WINDOWS_CHANGE_ACCESSIBILITY_FOCUSED;

		import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage;
		@@ -27,6 +29,7 @@ import static com.android.server.accessibility.AbstractAccessibilityServiceConne

		import android.annotation.NonNull;
		import android.annotation.Nullable;
		import android.graphics.Point;
		import android.graphics.Region;
		import android.os.Binder;
		import android.os.Handler;
		@@ -37,6 +40,7 @@ import android.os.SystemClock;
		import android.os.UserHandle;
		import android.text.TextUtils;
		import android.util.ArrayMap;
		import android.util.ArraySet;
		import android.util.Slog;
		import android.util.SparseArray;
		import android.view.Display;
		@@ -52,6 +56,7 @@ import android.view.accessibility.IAccessibilityInteractionConnection;
		import com.android.internal.annotations.VisibleForTesting;
		import com.android.server.accessibility.AccessibilitySecurityPolicy.AccessibilityUserManager;
		import com.android.server.utils.Slogf;
		import com.android.server.wm.AccessibilityWindowsPopulator.AccessibilityWindow;
		import com.android.server.wm.WindowManagerInternal;

		import java.io.FileDescriptor;
		@@ -60,6 +65,7 @@ import java.util.ArrayList;
		import java.util.Arrays;
		import java.util.Collections;
		import java.util.List;
		import java.util.Set;
		import java.util.stream.Collectors;

		/**
		@@ -478,6 +484,162 @@ public class AccessibilityWindowManager {
		}
		}

		/**
		* Called when the windows for accessibility changed. This is called if
		* {@link com.android.server.accessibility.Flags.FLAG_COMPUTE_WINDOW_CHANGES_ON_A11Y} is
		* true.
		*
		* @param forceSend Send the windows for accessibility even if they haven't
		* changed.
		* @param topFocusedDisplayId The display Id which has the top focused window.
		* @param topFocusedWindowToken The window token of top focused window.
		* @param screenSize The size of the display that the change happened.
		* @param windows The windows for accessibility.
		*/
		@Override
		public void onAccessibilityWindowsChanged(boolean forceSend, int topFocusedDisplayId,
		@NonNull IBinder topFocusedWindowToken, @NonNull Point screenSize,
		@NonNull List<AccessibilityWindow> windows) {
		// TODO(b/322444245): Get a screenSize from DisplayManager#getDisplay(int)
		// .getRealSize().
		final List<WindowInfo> windowInfoList = createWindowInfoList(screenSize, windows);
		onWindowsForAccessibilityChanged(forceSend, topFocusedDisplayId,
		topFocusedWindowToken, windowInfoList);
		}

		private static List<WindowInfo> createWindowInfoList(@NonNull Point screenSize,
		@NonNull List<AccessibilityWindow> visibleWindows) {
		final Set<IBinder> addedWindows = new ArraySet<>();
		final List<WindowInfo> windows = new ArrayList<>();

		// Avoid allocating Region for each window.
		final Region regionInWindow = new Region();
		final Region touchableRegionInScreen = new Region();

		// Iterate until we figure out what is touchable for the entire screen.
		boolean focusedWindowAdded = false;
		final Region unaccountedSpace = new Region(0, 0, screenSize.x, screenSize.y);
		for (final AccessibilityWindow a11yWindow : visibleWindows) {
		a11yWindow.getTouchableRegionInWindow(regionInWindow);
		if (windowMattersToAccessibility(a11yWindow, regionInWindow, unaccountedSpace)) {
		final WindowInfo window = a11yWindow.getWindowInfo();
		if (window.token != null) {
		// Even if token is null, the window will be used in calculating visible
		// windows, but is excluded from the accessibility window list.
		// TODO(b/322444245): We can call #updateWindowWithWindowAttributes() here.
		window.regionInScreen.set(regionInWindow);
		window.layer = addedWindows.size();
		windows.add(window);
		addedWindows.add(window.token);
		}

		if (windowMattersToUnaccountedSpaceComputation(a11yWindow)) {
		// Account for the space this window takes.
		a11yWindow.getTouchableRegionInScreen(touchableRegionInScreen);
		unaccountedSpace.op(touchableRegionInScreen, unaccountedSpace,
		Region.Op.REVERSE_DIFFERENCE);
		}

		focusedWindowAdded \|= a11yWindow.isFocused();
		} else if (a11yWindow.isUntouchableNavigationBar()
		&& a11yWindow.getSystemBarInsetsFrame() != null) {
		// If this widow is navigation bar without touchable region, accounting the
		// region of navigation bar inset because all touch events from this region
		// would be received by launcher, i.e. this region is a un-touchable one
		// for the application.
		unaccountedSpace.op(
		a11yWindow.getSystemBarInsetsFrame(),
		unaccountedSpace,
		Region.Op.REVERSE_DIFFERENCE);
		}

		if (unaccountedSpace.isEmpty() && focusedWindowAdded) {
		break;
		}
		}

		// Remove child/parent references to windows that were not added.
		for (final WindowInfo window : windows) {
		if (!addedWindows.contains(window.parentToken)) {
		window.parentToken = null;
		}
		if (window.childTokens != null) {
		final int childTokenCount = window.childTokens.size();
		for (int j = childTokenCount - 1; j >= 0; j--) {
		if (!addedWindows.contains(window.childTokens.get(j))) {
		window.childTokens.remove(j);
		}
		}
		// Leave the child token list if empty.
		}
		}

		return windows;
		}

		private static boolean windowMattersToAccessibility(AccessibilityWindow a11yWindow,
		Region regionInScreen, Region unaccountedSpace) {
		if (a11yWindow.ignoreRecentsAnimationForAccessibility()) {
		return false;
		}

		if (a11yWindow.isFocused()) {
		return true;
		}

		// Ignore non-touchable windows, except the split-screen divider, which is
		// occasionally non-touchable but still useful for identifying split-screen
		// mode and the PIP menu.
		if (!a11yWindow.isTouchable()
		&& a11yWindow.getType() != TYPE_DOCK_DIVIDER && !a11yWindow.isPIPMenu()) {
		return false;
		}

		// If the window is completely covered by other windows - ignore.
		if (unaccountedSpace.quickReject(regionInScreen)) {
		return false;
		}

		// Add windows of certain types not covered by modal windows.
		if (isReportedWindowType(a11yWindow.getType())) {
		return true;
		}

		return false;
		}

		private static boolean isReportedWindowType(int windowType) {
		return (windowType != WindowManager.LayoutParams.TYPE_WALLPAPER
		&& windowType != WindowManager.LayoutParams.TYPE_BOOT_PROGRESS
		&& windowType != WindowManager.LayoutParams.TYPE_DISPLAY_OVERLAY
		&& windowType != WindowManager.LayoutParams.TYPE_DRAG
		&& windowType != WindowManager.LayoutParams.TYPE_INPUT_CONSUMER
		&& windowType != WindowManager.LayoutParams.TYPE_POINTER
		&& windowType != TYPE_MAGNIFICATION_OVERLAY
		&& windowType != WindowManager.LayoutParams.TYPE_APPLICATION_MEDIA_OVERLAY
		&& windowType != WindowManager.LayoutParams.TYPE_SECURE_SYSTEM_OVERLAY
		&& windowType != WindowManager.LayoutParams.TYPE_PRIVATE_PRESENTATION);
		}

		// Some windows should be excluded from unaccounted space computation, though they still
		// should be reported
		private static boolean windowMattersToUnaccountedSpaceComputation(
		AccessibilityWindow a11yWindow) {
		// Do not account space of trusted non-touchable windows, except the split-screen
		// divider.
		// If it's not trusted, touch events are not sent to the windows behind it.
		if (!a11yWindow.isTouchable()
		&& (a11yWindow.getType() != TYPE_DOCK_DIVIDER)
		&& a11yWindow.isTrustedOverlay()) {
		return false;
		}

		if (a11yWindow.getType() == WindowManager.LayoutParams.TYPE_ACCESSIBILITY_OVERLAY) {
		return false;
		}
		return true;
		}

		private void updateWindowsByWindowAttributesLocked(List<WindowInfo> windows) {
		for (int i = windows.size() - 1; i >= 0; i--) {
		final WindowInfo windowInfo = windows.get(i);

services/core/Android.bp

+1 −0

Original line number	Diff line number	Diff line
		@@ -207,6 +207,7 @@ java_library_static {
		"notification_flags_lib",
		"biometrics_flags_lib",
		"am_flags_lib",
		"com_android_server_accessibility_flags_lib",
		"com_android_systemui_shared_flags_lib",
		"com_android_wm_shell_flags_lib",
		"com.android.server.utils_aconfig-java",

services/core/java/com/android/server/wm/AccessibilityController.java

+19 −7

Original line number	Diff line number	Diff line
		@@ -1543,8 +1543,6 @@ final class AccessibilityController {

		private final Set<IBinder> mTempBinderSet = new ArraySet<>();

		private final Point mTempPoint = new Point();

		private final Region mTempRegion = new Region();

		private final Region mTempRegion2 = new Region();
		@@ -1614,8 +1612,9 @@ final class AccessibilityController {
		Slog.i(LOG_TAG, "computeChangedWindows()");
		}

		final List<WindowInfo> windows;
		List<WindowInfo> windows = null;
		final List<AccessibilityWindow> visibleWindows = new ArrayList<>();
		final Point screenSize = new Point();
		final int topFocusedDisplayId;
		IBinder topFocusedWindowToken = null;

		@@ -1643,19 +1642,27 @@ final class AccessibilityController {
		return;
		}
		final Display display = dc.getDisplay();
		display.getRealSize(mTempPoint);
		display.getRealSize(screenSize);

		mA11yWindowsPopulator.populateVisibleWindowsOnScreenLocked(
		mDisplayId, visibleWindows);

		windows = buildWindowInfoListLocked(visibleWindows, mTempPoint);
		if (!com.android.server.accessibility.Flags.computeWindowChangesOnA11y()) {
		windows = buildWindowInfoListLocked(visibleWindows, screenSize);
		}

		// Gets the top focused display Id and window token for supporting multi-display.
		topFocusedDisplayId = mService.mRoot.getTopFocusedDisplayContent().getDisplayId();
		topFocusedWindowToken = topFocusedWindowState.mClient.asBinder();
		}

		if (com.android.server.accessibility.Flags.computeWindowChangesOnA11y()) {
		mCallback.onAccessibilityWindowsChanged(forceSend, topFocusedDisplayId,
		topFocusedWindowToken, screenSize, visibleWindows);
		} else {
		mCallback.onWindowsForAccessibilityChanged(forceSend, topFocusedDisplayId,
		topFocusedWindowToken, windows);
		}

		// Recycle the windows as we do not need them.
		for (final AccessibilityWindowsPopulator.AccessibilityWindow window : visibleWindows) {
		@@ -1664,6 +1671,9 @@ final class AccessibilityController {
		mInitialized = true;
		}

		// Here are old code paths, called when computeWindowChangesOnA11y flag is disabled.
		// LINT.IfChange

		/**
		* From a list of windows, decides windows to be exposed to accessibility based on touchable
		* region in the screen.
		@@ -1823,6 +1833,8 @@ final class AccessibilityController {
		&& windowType != WindowManager.LayoutParams.TYPE_PRIVATE_PRESENTATION);
		}

		// LINT.ThenChange(/services/accessibility/java/com/android/server/accessibility/AccessibilityWindowManager.java)

		private WindowState getTopFocusWindow() {
		return mService.mRoot.getTopFocusedDisplayContent().mCurrentFocus;
		}

services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java

+21 −0

Original line number	Diff line number	Diff line
		@@ -22,6 +22,7 @@ import static com.android.internal.util.DumpUtils.dumpSparseArray;
		import static com.android.server.wm.utils.RegionUtils.forEachRect;

		import android.annotation.NonNull;
		import android.annotation.Nullable;
		import android.graphics.Matrix;
		import android.graphics.Rect;
		import android.graphics.RectF;
		@@ -655,6 +656,7 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener {
		private final Region mTouchableRegionInScreen = new Region();
		private final Region mTouchableRegionInWindow = new Region();
		private WindowInfo mWindowInfo;
		private Rect mSystemBarInsetFrame = null;


		/**
		@@ -718,6 +720,16 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener {
		Slog.w(TAG, "can't find spec");
		}
		}

		// Compute system bar insets frame if needed.
		if (com.android.server.accessibility.Flags.computeWindowChangesOnA11y()
		&& windowState != null && instance.isUntouchableNavigationBar()) {
		final InsetsSourceProvider provider =
		windowState.getControllableInsetProvider();
		if (provider != null) {
		instance.mSystemBarInsetFrame = provider.getSource().getFrame();
		}
		}
		return instance;
		}

		@@ -812,6 +824,15 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener {
		return mIsPIPMenu;
		}

		/**
		* @return the system inset frame size if the window is untouchable navigation bar.
		* Returns null otherwise.
		*/
		@Nullable
		public Rect getSystemBarInsetsFrame() {
		return mSystemBarInsetFrame;
		}

		private static void getTouchableRegionInWindow(boolean shouldMagnify, Region inRegion,
		Region outRegion, Rect frame, Matrix inverseMatrix, Matrix displayMatrix) {
		// Some modal windows, like the activity with Theme.dialog, has the full screen