Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 707c3dd3 authored by Hiroki Sato's avatar Hiroki Sato
Browse files

Move a11y window bounds computation into a11y package 

This moves accessibility windows' touchable region computation and
exposed window selection from the wm package to the a11y package.
This shouldn't change any behavior. Our final goal is to eliminate the
dependency to WM package, but this just moves the code so that future
work will be smoother.

This change is guarded by a new flag.

Bug: 322444245
Test: CtsAccessibilityServiceTestCases CtsAccessibilityTestCases AccessibilityWindowManagerTest
Change-Id: Ib37cb934686d7677f2da74d6ed9ad6e7fcb5efd1
parent 1d10ffef

services/accessibility/accessibility.aconfig

+7 −0
Original line number Diff line number Diff line
@@ -16,6 +16,13 @@ flag { 
    bug: "271490102"

}



flag {

    name: "compute_window_changes_on_a11y"

    namespace: "accessibility"

    description: "Computes accessibility window changes in accessibility instead of wm package."

    bug: "322444245"

}



flag {

    name: "deprecate_package_list_observer"

    namespace: "accessibility"

services/accessibility/java/com/android/server/accessibility/AccessibilityWindowManager.java

+165 −3
Original line number Diff line number Diff line
@@ -19,6 +19,8 @@ package com.android.server.accessibility; 
import static android.accessibilityservice.AccessibilityTrace.FLAGS_ACCESSIBILITY_INTERACTION_CONNECTION;

import static android.accessibilityservice.AccessibilityTrace.FLAGS_WINDOW_MANAGER_INTERNAL;

import static android.view.WindowManager.LayoutParams.TYPE_ACCESSIBILITY_OVERLAY;

import static android.view.WindowManager.LayoutParams.TYPE_DOCK_DIVIDER;

import static android.view.WindowManager.LayoutParams.TYPE_MAGNIFICATION_OVERLAY;

import static android.view.accessibility.AccessibilityEvent.WINDOWS_CHANGE_ACCESSIBILITY_FOCUSED;



import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage;
@@ -27,6 +29,7 @@ import static com.android.server.accessibility.AbstractAccessibilityServiceConne 


import android.annotation.NonNull;

import android.annotation.Nullable;

import android.graphics.Point;

import android.graphics.Region;

import android.os.Binder;

import android.os.Handler;
@@ -37,6 +40,7 @@ import android.os.SystemClock; 
import android.os.UserHandle;

import android.text.TextUtils;

import android.util.ArrayMap;

import android.util.ArraySet;

import android.util.Slog;

import android.util.SparseArray;

import android.view.Display;
@@ -52,6 +56,7 @@ import android.view.accessibility.IAccessibilityInteractionConnection; 
import com.android.internal.annotations.VisibleForTesting;

import com.android.server.accessibility.AccessibilitySecurityPolicy.AccessibilityUserManager;

import com.android.server.utils.Slogf;

import com.android.server.wm.AccessibilityWindowsPopulator.AccessibilityWindow;

import com.android.server.wm.WindowManagerInternal;



import java.io.FileDescriptor;
@@ -60,6 +65,7 @@ import java.util.ArrayList; 
import java.util.Arrays;

import java.util.Collections;

import java.util.List;

import java.util.Set;

import java.util.stream.Collectors;



/**
@@ -478,6 +484,162 @@ public class AccessibilityWindowManager { 
            }

        }



        /**

         * Called when the windows for accessibility changed. This is called if

         * {@link com.android.server.accessibility.Flags.FLAG_COMPUTE_WINDOW_CHANGES_ON_A11Y} is

         * true.

         *

         * @param forceSend             Send the windows for accessibility even if they haven't

         *                              changed.

         * @param topFocusedDisplayId   The display Id which has the top focused window.

         * @param topFocusedWindowToken The window token of top focused window.

         * @param screenSize            The size of the display that the change happened.

         * @param windows               The windows for accessibility.

         */

        @Override

        public void onAccessibilityWindowsChanged(boolean forceSend, int topFocusedDisplayId,

                @NonNull IBinder topFocusedWindowToken, @NonNull Point screenSize,

                @NonNull List<AccessibilityWindow> windows) {

            // TODO(b/322444245): Get a screenSize from DisplayManager#getDisplay(int)

            //  .getRealSize().

            final List<WindowInfo> windowInfoList = createWindowInfoList(screenSize, windows);

            onWindowsForAccessibilityChanged(forceSend, topFocusedDisplayId,

                    topFocusedWindowToken, windowInfoList);

        }



        private static List<WindowInfo> createWindowInfoList(@NonNull Point screenSize,

                @NonNull List<AccessibilityWindow> visibleWindows) {

            final Set<IBinder> addedWindows = new ArraySet<>();

            final List<WindowInfo> windows = new ArrayList<>();



            // Avoid allocating Region for each window.

            final Region regionInWindow = new Region();

            final Region touchableRegionInScreen = new Region();



            // Iterate until we figure out what is touchable for the entire screen.

            boolean focusedWindowAdded = false;

            final Region unaccountedSpace = new Region(0, 0, screenSize.x, screenSize.y);

            for (final AccessibilityWindow a11yWindow : visibleWindows) {

                a11yWindow.getTouchableRegionInWindow(regionInWindow);

                if (windowMattersToAccessibility(a11yWindow, regionInWindow, unaccountedSpace)) {

                    final WindowInfo window = a11yWindow.getWindowInfo();

                    if (window.token != null) {

                        // Even if token is null, the window will be used in calculating visible

                        // windows, but is excluded from the accessibility window list.

                        // TODO(b/322444245): We can call #updateWindowWithWindowAttributes() here.

                        window.regionInScreen.set(regionInWindow);

                        window.layer = addedWindows.size();

                        windows.add(window);

                        addedWindows.add(window.token);

                    }



                    if (windowMattersToUnaccountedSpaceComputation(a11yWindow)) {

                        // Account for the space this window takes.

                        a11yWindow.getTouchableRegionInScreen(touchableRegionInScreen);

                        unaccountedSpace.op(touchableRegionInScreen, unaccountedSpace,

                                Region.Op.REVERSE_DIFFERENCE);

                    }



                    focusedWindowAdded |= a11yWindow.isFocused();

                } else if (a11yWindow.isUntouchableNavigationBar()

                        && a11yWindow.getSystemBarInsetsFrame() != null) {

                    // If this widow is navigation bar without touchable region, accounting the

                    // region of navigation bar inset because all touch events from this region

                    // would be received by launcher, i.e. this region is a un-touchable one

                    // for the application.

                    unaccountedSpace.op(

                            a11yWindow.getSystemBarInsetsFrame(),

                            unaccountedSpace,

                            Region.Op.REVERSE_DIFFERENCE);

                }



                if (unaccountedSpace.isEmpty() && focusedWindowAdded) {

                    break;

                }

            }



            // Remove child/parent references to windows that were not added.

            for (final WindowInfo window : windows) {

                if (!addedWindows.contains(window.parentToken)) {

                    window.parentToken = null;

                }

                if (window.childTokens != null) {

                    final int childTokenCount = window.childTokens.size();

                    for (int j = childTokenCount - 1; j >= 0; j--) {

                        if (!addedWindows.contains(window.childTokens.get(j))) {

                            window.childTokens.remove(j);

                        }

                    }

                    // Leave the child token list if empty.

                }

            }



            return windows;

        }



        private static boolean windowMattersToAccessibility(AccessibilityWindow a11yWindow,

                Region regionInScreen, Region unaccountedSpace) {

            if (a11yWindow.ignoreRecentsAnimationForAccessibility()) {

                return false;

            }



            if (a11yWindow.isFocused()) {

                return true;

            }



            // Ignore non-touchable windows, except the split-screen divider, which is

            // occasionally non-touchable but still useful for identifying split-screen

            // mode and the PIP menu.

            if (!a11yWindow.isTouchable()

                    && a11yWindow.getType() != TYPE_DOCK_DIVIDER && !a11yWindow.isPIPMenu()) {

                return false;

            }



            // If the window is completely covered by other windows - ignore.

            if (unaccountedSpace.quickReject(regionInScreen)) {

                return false;

            }



            // Add windows of certain types not covered by modal windows.

            if (isReportedWindowType(a11yWindow.getType())) {

                return true;

            }



            return false;

        }



        private static boolean isReportedWindowType(int windowType) {

            return (windowType != WindowManager.LayoutParams.TYPE_WALLPAPER

                    && windowType != WindowManager.LayoutParams.TYPE_BOOT_PROGRESS

                    && windowType != WindowManager.LayoutParams.TYPE_DISPLAY_OVERLAY

                    && windowType != WindowManager.LayoutParams.TYPE_DRAG

                    && windowType != WindowManager.LayoutParams.TYPE_INPUT_CONSUMER

                    && windowType != WindowManager.LayoutParams.TYPE_POINTER

                    && windowType != TYPE_MAGNIFICATION_OVERLAY

                    && windowType != WindowManager.LayoutParams.TYPE_APPLICATION_MEDIA_OVERLAY

                    && windowType != WindowManager.LayoutParams.TYPE_SECURE_SYSTEM_OVERLAY

                    && windowType != WindowManager.LayoutParams.TYPE_PRIVATE_PRESENTATION);

        }



        // Some windows should be excluded from unaccounted space computation, though they still

        // should be reported

        private static boolean windowMattersToUnaccountedSpaceComputation(

                AccessibilityWindow a11yWindow) {

            // Do not account space of trusted non-touchable windows, except the split-screen

            // divider.

            // If it's not trusted, touch events are not sent to the windows behind it.

            if (!a11yWindow.isTouchable()

                    && (a11yWindow.getType() != TYPE_DOCK_DIVIDER)

                    && a11yWindow.isTrustedOverlay()) {

                return false;

            }



            if (a11yWindow.getType() == WindowManager.LayoutParams.TYPE_ACCESSIBILITY_OVERLAY) {

                return false;

            }

            return true;

        }



        private void updateWindowsByWindowAttributesLocked(List<WindowInfo> windows) {

            for (int i = windows.size() - 1; i >= 0; i--) {

                final WindowInfo windowInfo = windows.get(i);

services/core/Android.bp

+1 −0
Original line number Diff line number Diff line
@@ -207,6 +207,7 @@ java_library_static { 
        "notification_flags_lib",

        "biometrics_flags_lib",

        "am_flags_lib",

        "com_android_server_accessibility_flags_lib",

        "com_android_systemui_shared_flags_lib",

        "com_android_wm_shell_flags_lib",

        "com.android.server.utils_aconfig-java",

services/core/java/com/android/server/wm/AccessibilityController.java

+19 −7
Original line number Diff line number Diff line
@@ -1539,8 +1539,6 @@ final class AccessibilityController { 


        private final Set<IBinder> mTempBinderSet = new ArraySet<>();



        private final Point mTempPoint = new Point();



        private final Region mTempRegion = new Region();



        private final Region mTempRegion2 = new Region();
@@ -1610,8 +1608,9 @@ final class AccessibilityController { 
                Slog.i(LOG_TAG, "computeChangedWindows()");

            }



            final List<WindowInfo> windows;

            List<WindowInfo> windows = null;

            final List<AccessibilityWindow> visibleWindows = new ArrayList<>();

            final Point screenSize = new Point();

            final int topFocusedDisplayId;

            IBinder topFocusedWindowToken = null;
@@ -1639,19 +1638,27 @@ final class AccessibilityController { 
                    return;

                }

                final Display display = dc.getDisplay();

                display.getRealSize(mTempPoint);

                display.getRealSize(screenSize);



                mA11yWindowsPopulator.populateVisibleWindowsOnScreenLocked(

                        mDisplayId, visibleWindows);



                windows = buildWindowInfoListLocked(visibleWindows, mTempPoint);

                if (!com.android.server.accessibility.Flags.computeWindowChangesOnA11y()) {

                    windows = buildWindowInfoListLocked(visibleWindows, screenSize);

                }



                // Gets the top focused display Id and window token for supporting multi-display.

                topFocusedDisplayId = mService.mRoot.getTopFocusedDisplayContent().getDisplayId();

                topFocusedWindowToken = topFocusedWindowState.mClient.asBinder();

            }



            if (com.android.server.accessibility.Flags.computeWindowChangesOnA11y()) {

                mCallback.onAccessibilityWindowsChanged(forceSend, topFocusedDisplayId,

                        topFocusedWindowToken, screenSize, visibleWindows);

            } else {

                mCallback.onWindowsForAccessibilityChanged(forceSend, topFocusedDisplayId,

                        topFocusedWindowToken, windows);

            }



            // Recycle the windows as we do not need them.

            for (final AccessibilityWindowsPopulator.AccessibilityWindow window : visibleWindows) {
@@ -1660,6 +1667,9 @@ final class AccessibilityController { 
            mInitialized = true;

        }



        // Here are old code paths, called when computeWindowChangesOnA11y flag is disabled.

        // LINT.IfChange



        /**

         * From a list of windows, decides windows to be exposed to accessibility based on touchable

         * region in the screen.
@@ -1819,6 +1829,8 @@ final class AccessibilityController { 
                    && windowType != WindowManager.LayoutParams.TYPE_PRIVATE_PRESENTATION);

        }



        // LINT.ThenChange(/services/accessibility/java/com/android/server/accessibility/AccessibilityWindowManager.java)



        private WindowState getTopFocusWindow() {

            return mService.mRoot.getTopFocusedDisplayContent().mCurrentFocus;

        }

services/core/java/com/android/server/wm/AccessibilityWindowsPopulator.java

+21 −0
Original line number Diff line number Diff line
@@ -22,6 +22,7 @@ import static com.android.internal.util.DumpUtils.dumpSparseArray; 
import static com.android.server.wm.utils.RegionUtils.forEachRect;



import android.annotation.NonNull;

import android.annotation.Nullable;

import android.graphics.Matrix;

import android.graphics.Rect;

import android.graphics.RectF;
@@ -655,6 +656,7 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { 
        private final Region mTouchableRegionInScreen = new Region();

        private final Region mTouchableRegionInWindow = new Region();

        private WindowInfo mWindowInfo;

        private Rect mSystemBarInsetFrame = null;





        /**
@@ -718,6 +720,16 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { 
                    Slog.w(TAG, "can't find spec");

                }

            }



            // Compute system bar insets frame if needed.

            if (com.android.server.accessibility.Flags.computeWindowChangesOnA11y()

                    && windowState != null && instance.isUntouchableNavigationBar()) {

                final InsetsSourceProvider provider =

                        windowState.getControllableInsetProvider();

                if (provider != null) {

                    instance.mSystemBarInsetFrame = provider.getSource().getFrame();

                }

            }

            return instance;

        }
@@ -812,6 +824,15 @@ public final class AccessibilityWindowsPopulator extends WindowInfosListener { 
            return mIsPIPMenu;

        }



        /**

         * @return the system inset frame size if the window is untouchable navigation bar.

         *  Returns null otherwise.

         */

        @Nullable

        public Rect getSystemBarInsetsFrame() {

            return mSystemBarInsetFrame;

        }



        private static void getTouchableRegionInWindow(boolean shouldMagnify, Region inRegion,

                Region outRegion, Rect frame, Matrix inverseMatrix, Matrix displayMatrix) {

            // Some modal windows, like the activity with Theme.dialog, has the full screen