Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 0aa8fb74 authored by Daniel Norman's avatar Daniel Norman
Browse files

fix(magnify_ime): Remove typing delay for magnify keyboard 

- Immediately delegates all touch events performed over the keyboard
  while not magnified, except for the first tap after triggering
  a magnification shortcut.
- Delegates all finished taps performed over the keyboard while
  magnified.

This has the effect of preventing the user from performing the triple
tap shortcut over the keyboard. The user must triple-tap outside the
keyboard region in order to activate or deactivate magnification.

This may require panning before they can deactivate, if the user has
zoomed the keyboard to take over the entire screen. As a fallback the
user can always lock the screen if they are confused or feel stuck
and magnification will reset the zoom level (existing behavior).

Fix: 413446643
Flag: com.android.server.accessibility.enable_magnification_magnify_nav_bar_and_ime
Test: atest FullScreenMagnificationControllerTest
Test: atest FullScreenMagnificationGestureHandlerTest
Change-Id: I3dafc39980c825731bde048864e6be847c45eadf
parent 41c1e2ba

services/accessibility/java/com/android/server/accessibility/magnification/FullScreenMagnificationController.java

+49 −0
Original line number Diff line number Diff line
@@ -249,6 +249,7 @@ public class FullScreenMagnificationController implements 


        private final Region mMagnificationRegion = Region.obtain();

        private final Rect mMagnificationBounds = new Rect();

        private final Region mImeRegion = Region.obtain();



        private final Rect mTempRect = new Rect();

        private final Rect mTempRect1 = new Rect();
@@ -454,6 +455,14 @@ public class FullScreenMagnificationController implements 
            mControllerCtx.getHandler().sendMessage(m);

        }



        @Override

        public void onImeRegionChanged(Region imeRegion) {

            final Message m = PooledLambda.obtainMessage(

                    DisplayMagnification::updateImeRegion, this,

                    Region.obtain(imeRegion));

            mControllerCtx.getHandler().sendMessage(m);

        }



        @Override

        public void onRectangleOnScreenRequested(int left, int top, int right, int bottom) {

            final Message m = PooledLambda.obtainMessage(
@@ -517,6 +526,15 @@ public class FullScreenMagnificationController implements 
            }

        }



        void updateImeRegion(Region imeRegion) {

            synchronized (mLock) {

                if (!mRegistered) {

                    return;

                }

                mImeRegion.set(imeRegion);

            }

        }



        void sendSpecToAnimation(MagnificationSpec spec,

                MagnificationAnimationCallback animationCallback) {

            if (DEBUG) {
@@ -632,6 +650,18 @@ public class FullScreenMagnificationController implements 
            return mMagnificationRegion.contains((int) x, (int) y);

        }



        @GuardedBy("mLock")

        boolean imeRegionContains(float x, float y) {

            if (!Flags.enableMagnificationMagnifyNavBarAndIme()) {

                return false;

            }

            // mImeRegion uses global unmagnified coordinates, so convert screen-relative

            // coordinates (x,y) to global unmagnified coordinates first.

            x = (x - mCurrentMagnificationSpec.offsetX) / mCurrentMagnificationSpec.scale;

            y = (y - mCurrentMagnificationSpec.offsetY) / mCurrentMagnificationSpec.scale;

            return mImeRegion.contains((int) x, (int) y);

        }



        @GuardedBy("mLock")

        void getMagnificationBounds(@NonNull Rect outBounds) {

            outBounds.set(mMagnificationBounds);
@@ -1364,6 +1394,25 @@ public class FullScreenMagnificationController implements 
        }

    }



    /**

     * Returns whether the keyboard (IME) region contains the specified screen-relative coordinates.

     *

     * @param displayId The logical display id.

     * @param x the screen-relative X coordinate to check

     * @param y the screen-relative Y coordinate to check

     * @return {@code true} if the coordinate is contained within the

     *         keyboard region, otherwise {@code false}

     */

    public boolean imeRegionContains(int displayId, float x, float y) {

        synchronized (mLock) {

            final DisplayMagnification display = mDisplays.get(displayId);

            if (display == null) {

                return false;

            }

            return display.imeRegionContains(x, y);

        }

    }



    /**

     * Gets the full screen magnification data needed by

     * {@link #FullScreenMagnificationPointerMotionEventFilter}.

services/accessibility/java/com/android/server/accessibility/magnification/FullScreenMagnificationGestureHandler.java

+33 −8
Original line number Diff line number Diff line
@@ -924,6 +924,13 @@ public class FullScreenMagnificationGestureHandler extends MagnificationGestureH 


                        transitionToDelegatingStateAndClear();



                    } else if (!isActivated() && !mShortcutTriggered

                            && mFullScreenMagnificationController.imeRegionContains(

                                    mDisplayId, event.getX(), event.getY())) {

                        // Delegate new taps performed over the IME while unmagnified. This removes

                        // any observable delay while typing on an unmagnified keyboard.

                        transitionToDelegatingStateAndClear();



                    } else if (isMultiTapTriggered(2 /* taps */)) {



                        // 3tap and hold
@@ -963,7 +970,7 @@ public class FullScreenMagnificationGestureHandler extends MagnificationGestureH 
                    } else {

                        transitionToDelegatingStateAndClear();

                    }

                    // LINT.ThenChange(:action_pointer_down_with_multi_finger)

                    // LINT.ThenChange(FullScreenMagnificationGestureHandler.java:action_pointer_down_with_multi_finger)

                }

                break;

                case ACTION_POINTER_UP: {
@@ -973,7 +980,7 @@ public class FullScreenMagnificationGestureHandler extends MagnificationGestureH 
                    }

                    // LINT.IfChange(action_pointer_up)

                    transitionToDelegatingStateAndClear();

                    // LINT.ThenChange(:action_pointer_up_with_multi_finger)

                    // LINT.ThenChange(FullScreenMagnificationGestureHandler.java:action_pointer_up_with_multi_finger)

                }

                break;

                case ACTION_MOVE: {
@@ -1023,7 +1030,7 @@ public class FullScreenMagnificationGestureHandler extends MagnificationGestureH 
                        //Second pointer is swiping, so transit to PanningScalingState

                        transitToPanningScalingStateAndClear();

                    }

                    // LINT.ThenChange(:action_move_with_multi_finger)

                    // LINT.ThenChange(FullScreenMagnificationGestureHandler.java:action_move_with_multi_finger)

                }

                break;

                case ACTION_UP: {
@@ -1041,6 +1048,15 @@ public class FullScreenMagnificationGestureHandler extends MagnificationGestureH 
                            mDisplayId, event.getX(), event.getY())) {

                        transitionToDelegatingStateAndClear();



                    } else if (isActivated() && !mShortcutTriggered

                            && mFullScreenMagnificationController.imeRegionContains(

                                    mDisplayId, event.getX(), event.getY())) {

                        // Delegate completed taps performed over the IME while magnified. Benefits:

                        // - Removes any observable delay while typing on a magnified keyboard.

                        // - Ensures that quick taps (e.g. "www") do not accidentally trigger the

                        //   triple-tap shortcut and deactivate magnification.

                        transitionToDelegatingStateAndClear();



                    } else if (isMultiTapTriggered(3 /* taps */)) {

                        onTripleTap(/* up */ event);
@@ -1053,7 +1069,7 @@ public class FullScreenMagnificationGestureHandler extends MagnificationGestureH 
                        transitionToDelegatingStateAndClear();



                    }

                    // LINT.ThenChange(:action_up_with_multi_finger)

                    // LINT.ThenChange(FullScreenMagnificationGestureHandler.java:action_up_with_multi_finger)

                }

                break;

            }
@@ -1390,7 +1406,7 @@ public class FullScreenMagnificationGestureHandler extends MagnificationGestureH 
                } else {

                    transitionToDelegatingStateAndClear();

                }

                // LINT.ThenChange(:action_pointer_down)

                // LINT.ThenChange(FullScreenMagnificationGestureHandler.java:action_pointer_down)

            }



            private void onMove(MotionEvent event) {
@@ -1447,7 +1463,7 @@ public class FullScreenMagnificationGestureHandler extends MagnificationGestureH 
                            MESSAGE_TRANSITION_TO_PANNINGSCALING_STATE,

                            ViewConfiguration.getTapTimeout());

                }

                // LINT.ThenChange(:action_move)

                // LINT.ThenChange(FullScreenMagnificationGestureHandler.java:action_move)

            }



            private void onPointerUp() {
@@ -1458,7 +1474,7 @@ public class FullScreenMagnificationGestureHandler extends MagnificationGestureH 
                if (!mIsTwoFingerCountReached) {

                    transitionToDelegatingStateAndClear();

                }

                // LINT.ThenChange(:action_pointer_up)

                // LINT.ThenChange(FullScreenMagnificationGestureHandler.java:action_pointer_up)

            }



            private void onUp(MotionEvent event) {
@@ -1467,6 +1483,15 @@ public class FullScreenMagnificationGestureHandler extends MagnificationGestureH 
                        mDisplayId, event.getX(), event.getY())) {

                    transitionToDelegatingStateAndClear();



                } else if (isActivated() && !mShortcutTriggered

                            && mFullScreenMagnificationController.imeRegionContains(

                                    mDisplayId, event.getX(), event.getY())) {

                    // Delegate completed taps performed over the IME while magnified. Benefits:

                    // - Removes any observable delay while typing on a magnified keyboard.

                    // - Ensures that quick taps (e.g. "www") do not accidentally trigger the

                    //   triple-tap shortcut and deactivate magnification.

                    transitionToDelegatingStateAndClear();



                } else if (isMultiFingerMultiTapTriggered(

                        TWO_FINGER_GESTURE_MAX_TAPS, event)) {

                    // Placing multiple fingers before a single finger, because achieving a
@@ -1489,7 +1514,7 @@ public class FullScreenMagnificationGestureHandler extends MagnificationGestureH 
                                && mCompletedTapCount == 0) {

                    transitionToDelegatingStateAndClear();

                }

                // LINT.ThenChange(:action_up)

                // LINT.ThenChange(FullScreenMagnificationGestureHandler.java:action_up)

            }



            private boolean isMultiFingerMultiTapTriggered(int targetTapCount, MotionEvent event) {

services/core/java/com/android/server/wm/AccessibilityController.java

+28 −1
Original line number Diff line number Diff line
@@ -22,6 +22,7 @@ import static android.os.Build.IS_USER; 
import static android.view.WindowManager.LayoutParams.PRIVATE_FLAG_EXCLUDE_FROM_SCREEN_MAGNIFICATION;

import static android.view.WindowManager.LayoutParams.PRIVATE_FLAG_IS_ROUNDED_CORNERS_OVERLAY;

import static android.view.WindowManager.LayoutParams.TYPE_ACCESSIBILITY_MAGNIFICATION_OVERLAY;

import static android.view.WindowManager.LayoutParams.TYPE_INPUT_METHOD;

import static android.view.WindowManager.LayoutParams.TYPE_MAGNIFICATION_OVERLAY;

import static android.view.WindowManager.TRANSIT_FLAG_IS_RECENTS;
@@ -564,6 +565,8 @@ final class AccessibilityController { 
        private boolean mIsFullscreenMagnificationActivated = false;

        private final Region mMagnificationRegion = new Region();

        private final Region mOldMagnificationRegion = new Region();

        private final Region mImeRegion = new Region();

        private final Region mOldImeRegion = new Region();



        private final MagnificationSpec mMagnificationSpec = new MagnificationSpec();
@@ -813,7 +816,8 @@ final class AccessibilityController { 
            final int screenWidth = mScreenSize.x;

            final int screenHeight = mScreenSize.y;



            mMagnificationRegion.set(0, 0, 0, 0);

            mMagnificationRegion.setEmpty();

            mImeRegion.setEmpty();

            final Region availableBounds = mTempRegion1;

            availableBounds.set(0, 0, screenWidth, screenHeight);
@@ -866,6 +870,13 @@ final class AccessibilityController { 
                applyMatrixToRegion(matrix, touchableRegion);

                windowBounds.set(touchableRegion);



                if (windowType == TYPE_INPUT_METHOD) {

                    // Track the bounds of IME windows separately. This region is unrelated to

                    // mMagnificationRegion (these regions may overlap if the user chooses to

                    // magnify their keyboard) so this does not affect other calculations.

                    mImeRegion.op(windowBounds, Region.Op.UNION);

                }



                // Only update new regions

                Region portionOfWindowAlreadyAccountedFor = mTempRegion3;

                portionOfWindowAlreadyAccountedFor.set(mMagnificationRegion);
@@ -923,6 +934,14 @@ final class AccessibilityController { 
                                MyHandler.MESSAGE_NOTIFY_MAGNIFICATION_REGION_CHANGED, args)

                        .sendToTarget();

            }

            if (com.android.server.accessibility.Flags.enableMagnificationMagnifyNavBarAndIme()

                    && !mOldImeRegion.equals(mImeRegion)) {

                mOldImeRegion.set(mImeRegion);

                final SomeArgs args = SomeArgs.obtain();

                args.arg1 = Region.obtain(mImeRegion);

                mHandler.obtainMessage(MyHandler.MESSAGE_NOTIFY_IME_REGION_CHANGED, args)

                        .sendToTarget();

            }

        }



        private Region getLetterboxBounds(WindowState windowState) {
@@ -1000,6 +1019,7 @@ final class AccessibilityController { 
            public static final int MESSAGE_NOTIFY_USER_CONTEXT_CHANGED = 3;

            public static final int MESSAGE_NOTIFY_DISPLAY_SIZE_CHANGED = 4;

            public static final int MESSAGE_NOTIFY_IME_WINDOW_VISIBILITY_CHANGED = 5;

            public static final int MESSAGE_NOTIFY_IME_REGION_CHANGED = 6;



            MyHandler(Looper looper) {

                super(looper);
@@ -1027,6 +1047,13 @@ final class AccessibilityController { 
                        final boolean shown = message.arg1 == 1;

                        mCallbacks.onImeWindowVisibilityChanged(shown);

                    } break;



                    case MESSAGE_NOTIFY_IME_REGION_CHANGED: {

                        final SomeArgs args = (SomeArgs) message.obj;

                        final Region imeRegion = (Region) args.arg1;

                        mCallbacks.onImeRegionChanged(imeRegion);

                        imeRegion.recycle();

                    } break;

                }

            }

        }

services/core/java/com/android/server/wm/WindowManagerInternal.java

+7 −0
Original line number Diff line number Diff line
@@ -208,6 +208,13 @@ public abstract class WindowManagerInternal { 
         */

        void onMagnificationRegionChanged(Region magnificationRegion);



        /**

         * Called when the region used by TYPE_INPUT_METHOD windows changes.

         *

         * @param imeRegion the current region taken by TYPE_INPUT_METHOD window(s).

         */

        void onImeRegionChanged(Region imeRegion);



        /**

         * Called when an application requests a rectangle on the screen to allow

         * the client to apply the appropriate pan and scale.

services/tests/servicestests/src/com/android/server/accessibility/magnification/FullScreenMagnificationControllerTest.java

+73 −0
Original line number Diff line number Diff line
@@ -54,9 +54,11 @@ import android.graphics.Region; 
import android.hardware.display.DisplayManagerInternal;

import android.os.Looper;

import android.os.UserHandle;

import android.platform.test.annotations.EnableFlags;

import android.platform.test.annotations.RequiresFlagsEnabled;

import android.platform.test.flag.junit.CheckFlagsRule;

import android.platform.test.flag.junit.DeviceFlagsValueProvider;

import android.platform.test.flag.junit.SetFlagsRule;

import android.provider.Settings;

import android.test.mock.MockContentResolver;

import android.view.DisplayInfo;
@@ -80,6 +82,8 @@ import com.android.server.input.InputManagerInternal; 
import com.android.server.wm.WindowManagerInternal;

import com.android.server.wm.WindowManagerInternal.MagnificationCallbacks;



import com.google.common.truth.Truth;



import org.hamcrest.CoreMatchers;

import org.hamcrest.Description;

import org.hamcrest.TypeSafeMatcher;
@@ -110,6 +114,15 @@ public class FullScreenMagnificationControllerTest { 
    static final Region INITIAL_MAGNIFICATION_REGION = new Region(INITIAL_MAGNIFICATION_BOUNDS);

    static final Region OTHER_REGION_COMPAT = new Region(OTHER_MAGNIFICATION_BOUNDS_COMPAT);

    static final Region OTHER_REGION = new Region(OTHER_MAGNIFICATION_BOUNDS);



    // IME tests define bounds where the IME takes up the bottom half of the screen.

    static final Rect SCREEN_BOUNDS = new Rect(0, 0, 1000, 2000);

    static final Rect IME_BOUNDS = new Rect(

            SCREEN_BOUNDS.left, SCREEN_BOUNDS.bottom / 2,

            SCREEN_BOUNDS.right, SCREEN_BOUNDS.bottom);

    static final PointF POINT_OUTSIDE_IME_BOUNDS = new PointF(

            SCREEN_BOUNDS.right / 2, SCREEN_BOUNDS.bottom / 4);



    static final int SERVICE_ID_1 = 1;

    static final int SERVICE_ID_2 = 2;

    static final int DISPLAY_0 = 0;
@@ -121,6 +134,8 @@ public class FullScreenMagnificationControllerTest { 
    @Rule

    public final CheckFlagsRule mCheckFlagsRule = DeviceFlagsValueProvider.createCheckFlagsRule();



    @Rule public final SetFlagsRule mSetFlagsRule = new SetFlagsRule();



    final FullScreenMagnificationController.ControllerContext mMockControllerCtx =

            mock(FullScreenMagnificationController.ControllerContext.class);

    final Context mMockContext = mock(Context.class);
@@ -607,6 +622,64 @@ public class FullScreenMagnificationControllerTest { 
        );

    }



    @Test

    @EnableFlags(Flags.FLAG_ENABLE_MAGNIFICATION_MAGNIFY_NAV_BAR_AND_IME)

    public void imeRegionContains_pointInsideImeRegion_returnsTrue() {

        for (int displayId = 0; displayId < DISPLAY_COUNT; displayId++) {

            register(displayId);

            final MagnificationCallbacks callbacks = getMagnificationCallbacks(displayId);

            final PointF point = new PointF(IME_BOUNDS.centerX(), IME_BOUNDS.centerY());



            callbacks.onImeRegionChanged(new Region(IME_BOUNDS));

            mMessageCapturingHandler.sendAllMessages();



            Truth.assertThat(mFullScreenMagnificationController.imeRegionContains(

                    displayId, point.x, point.y)).isTrue();

        }

    }



    @Test

    @EnableFlags(Flags.FLAG_ENABLE_MAGNIFICATION_MAGNIFY_NAV_BAR_AND_IME)

    public void imeRegionContains_pointOutsideImeRegion_returnsFalse() {

        for (int displayId = 0; displayId < DISPLAY_COUNT; displayId++) {

            register(displayId);

            final MagnificationCallbacks callbacks = getMagnificationCallbacks(displayId);

            final PointF point = POINT_OUTSIDE_IME_BOUNDS;



            callbacks.onImeRegionChanged(new Region(IME_BOUNDS));

            mMessageCapturingHandler.sendAllMessages();



            Truth.assertThat(mFullScreenMagnificationController.imeRegionContains(

                    displayId, point.x, point.y)).isFalse();

        }

    }



    @Test

    @EnableFlags(Flags.FLAG_ENABLE_MAGNIFICATION_MAGNIFY_NAV_BAR_AND_IME)

    public void imeRegionContains_pointInsideMagnifiedImeRegion_returnsTrue() {

        for (int displayId = 0; displayId < DISPLAY_COUNT; displayId++) {

            register(displayId);

            final MagnificationCallbacks callbacks = getMagnificationCallbacks(displayId);

            final PointF point = POINT_OUTSIDE_IME_BOUNDS;

            // Set the magnification region to the full screen.

            callbacks.onMagnificationRegionChanged(new Region(SCREEN_BOUNDS));

            mMessageCapturingHandler.sendAllMessages();

            // Zoom far into the center of the IME. This in effect makes the IME fill the entire

            // screen, so that a point on screen that was previously outside of unmagnified IME

            // bounds is now inside of magnified IME bounds.

            Truth.assertThat(mFullScreenMagnificationController

                    .setScaleAndCenter(displayId, 8, IME_BOUNDS.centerX(), IME_BOUNDS.centerY(),

                            false, false, SERVICE_ID_1)).isTrue();

            mMessageCapturingHandler.sendAllMessages();



            callbacks.onImeRegionChanged(new Region(IME_BOUNDS));

            mMessageCapturingHandler.sendAllMessages();



            Truth.assertThat(mFullScreenMagnificationController.imeRegionContains(

                    displayId, point.x, point.y)).isTrue();

        }

    }



    @Test

    public void testOffsetMagnifiedRegion_whileMagnifying_offsetsMove() {

        for (int i = 0; i < DISPLAY_COUNT; i++) {