Merge "fix(magnify_ime): Remove typing delay for magnify keyboard" into main (f10e0f46) · Commits · e / os / android_frameworks_base

services/accessibility/java/com/android/server/accessibility/magnification/FullScreenMagnificationController.java

+49 −0

Original line number	Diff line number	Diff line
		@@ -249,6 +249,7 @@ public class FullScreenMagnificationController implements

		private final Region mMagnificationRegion = Region.obtain();
		private final Rect mMagnificationBounds = new Rect();
		private final Region mImeRegion = Region.obtain();

		private final Rect mTempRect = new Rect();
		private final Rect mTempRect1 = new Rect();
		@@ -454,6 +455,14 @@ public class FullScreenMagnificationController implements
		mControllerCtx.getHandler().sendMessage(m);
		}

		@Override
		public void onImeRegionChanged(Region imeRegion) {
		final Message m = PooledLambda.obtainMessage(
		DisplayMagnification::updateImeRegion, this,
		Region.obtain(imeRegion));
		mControllerCtx.getHandler().sendMessage(m);
		}

		@Override
		public void onRectangleOnScreenRequested(int left, int top, int right, int bottom) {
		final Message m = PooledLambda.obtainMessage(
		@@ -517,6 +526,15 @@ public class FullScreenMagnificationController implements
		}
		}

		void updateImeRegion(Region imeRegion) {
		synchronized (mLock) {
		if (!mRegistered) {
		return;
		}
		mImeRegion.set(imeRegion);
		}
		}

		void sendSpecToAnimation(MagnificationSpec spec,
		MagnificationAnimationCallback animationCallback) {
		if (DEBUG) {
		@@ -632,6 +650,18 @@ public class FullScreenMagnificationController implements
		return mMagnificationRegion.contains((int) x, (int) y);
		}

		@GuardedBy("mLock")
		boolean imeRegionContains(float x, float y) {
		if (!Flags.enableMagnificationMagnifyNavBarAndIme()) {
		return false;
		}
		// mImeRegion uses global unmagnified coordinates, so convert screen-relative
		// coordinates (x,y) to global unmagnified coordinates first.
		x = (x - mCurrentMagnificationSpec.offsetX) / mCurrentMagnificationSpec.scale;
		y = (y - mCurrentMagnificationSpec.offsetY) / mCurrentMagnificationSpec.scale;
		return mImeRegion.contains((int) x, (int) y);
		}

		@GuardedBy("mLock")
		void getMagnificationBounds(@NonNull Rect outBounds) {
		outBounds.set(mMagnificationBounds);
		@@ -1364,6 +1394,25 @@ public class FullScreenMagnificationController implements
		}
		}

		/**
		* Returns whether the keyboard (IME) region contains the specified screen-relative coordinates.
		*
		* @param displayId The logical display id.
		* @param x the screen-relative X coordinate to check
		* @param y the screen-relative Y coordinate to check
		* @return {@code true} if the coordinate is contained within the
		* keyboard region, otherwise {@code false}
		*/
		public boolean imeRegionContains(int displayId, float x, float y) {
		synchronized (mLock) {
		final DisplayMagnification display = mDisplays.get(displayId);
		if (display == null) {
		return false;
		}
		return display.imeRegionContains(x, y);
		}
		}

		/**
		* Gets the full screen magnification data needed by
		* {@link #FullScreenMagnificationPointerMotionEventFilter}.

services/accessibility/java/com/android/server/accessibility/magnification/FullScreenMagnificationGestureHandler.java

+33 −8

Original line number	Diff line number	Diff line
		@@ -924,6 +924,13 @@ public class FullScreenMagnificationGestureHandler extends MagnificationGestureH

		transitionToDelegatingStateAndClear();

		} else if (!isActivated() && !mShortcutTriggered
		&& mFullScreenMagnificationController.imeRegionContains(
		mDisplayId, event.getX(), event.getY())) {
		// Delegate new taps performed over the IME while unmagnified. This removes
		// any observable delay while typing on an unmagnified keyboard.
		transitionToDelegatingStateAndClear();

		} else if (isMultiTapTriggered(2 /* taps */)) {

		// 3tap and hold
		@@ -963,7 +970,7 @@ public class FullScreenMagnificationGestureHandler extends MagnificationGestureH
		} else {
		transitionToDelegatingStateAndClear();
		}
		// LINT.ThenChange(:action_pointer_down_with_multi_finger)
		// LINT.ThenChange(FullScreenMagnificationGestureHandler.java:action_pointer_down_with_multi_finger)
		}
		break;
		case ACTION_POINTER_UP: {
		@@ -973,7 +980,7 @@ public class FullScreenMagnificationGestureHandler extends MagnificationGestureH
		}
		// LINT.IfChange(action_pointer_up)
		transitionToDelegatingStateAndClear();
		// LINT.ThenChange(:action_pointer_up_with_multi_finger)
		// LINT.ThenChange(FullScreenMagnificationGestureHandler.java:action_pointer_up_with_multi_finger)
		}
		break;
		case ACTION_MOVE: {
		@@ -1023,7 +1030,7 @@ public class FullScreenMagnificationGestureHandler extends MagnificationGestureH
		//Second pointer is swiping, so transit to PanningScalingState
		transitToPanningScalingStateAndClear();
		}
		// LINT.ThenChange(:action_move_with_multi_finger)
		// LINT.ThenChange(FullScreenMagnificationGestureHandler.java:action_move_with_multi_finger)
		}
		break;
		case ACTION_UP: {
		@@ -1041,6 +1048,15 @@ public class FullScreenMagnificationGestureHandler extends MagnificationGestureH
		mDisplayId, event.getX(), event.getY())) {
		transitionToDelegatingStateAndClear();

		} else if (isActivated() && !mShortcutTriggered
		&& mFullScreenMagnificationController.imeRegionContains(
		mDisplayId, event.getX(), event.getY())) {
		// Delegate completed taps performed over the IME while magnified. Benefits:
		// - Removes any observable delay while typing on a magnified keyboard.
		// - Ensures that quick taps (e.g. "www") do not accidentally trigger the
		// triple-tap shortcut and deactivate magnification.
		transitionToDelegatingStateAndClear();

		} else if (isMultiTapTriggered(3 /* taps */)) {
		onTripleTap(/* up */ event);

		@@ -1053,7 +1069,7 @@ public class FullScreenMagnificationGestureHandler extends MagnificationGestureH
		transitionToDelegatingStateAndClear();

		}
		// LINT.ThenChange(:action_up_with_multi_finger)
		// LINT.ThenChange(FullScreenMagnificationGestureHandler.java:action_up_with_multi_finger)
		}
		break;
		}
		@@ -1390,7 +1406,7 @@ public class FullScreenMagnificationGestureHandler extends MagnificationGestureH
		} else {
		transitionToDelegatingStateAndClear();
		}
		// LINT.ThenChange(:action_pointer_down)
		// LINT.ThenChange(FullScreenMagnificationGestureHandler.java:action_pointer_down)
		}

		private void onMove(MotionEvent event) {
		@@ -1447,7 +1463,7 @@ public class FullScreenMagnificationGestureHandler extends MagnificationGestureH
		MESSAGE_TRANSITION_TO_PANNINGSCALING_STATE,
		ViewConfiguration.getTapTimeout());
		}
		// LINT.ThenChange(:action_move)
		// LINT.ThenChange(FullScreenMagnificationGestureHandler.java:action_move)
		}

		private void onPointerUp() {
		@@ -1458,7 +1474,7 @@ public class FullScreenMagnificationGestureHandler extends MagnificationGestureH
		if (!mIsTwoFingerCountReached) {
		transitionToDelegatingStateAndClear();
		}
		// LINT.ThenChange(:action_pointer_up)
		// LINT.ThenChange(FullScreenMagnificationGestureHandler.java:action_pointer_up)
		}

		private void onUp(MotionEvent event) {
		@@ -1467,6 +1483,15 @@ public class FullScreenMagnificationGestureHandler extends MagnificationGestureH
		mDisplayId, event.getX(), event.getY())) {
		transitionToDelegatingStateAndClear();

		} else if (isActivated() && !mShortcutTriggered
		&& mFullScreenMagnificationController.imeRegionContains(
		mDisplayId, event.getX(), event.getY())) {
		// Delegate completed taps performed over the IME while magnified. Benefits:
		// - Removes any observable delay while typing on a magnified keyboard.
		// - Ensures that quick taps (e.g. "www") do not accidentally trigger the
		// triple-tap shortcut and deactivate magnification.
		transitionToDelegatingStateAndClear();

		} else if (isMultiFingerMultiTapTriggered(
		TWO_FINGER_GESTURE_MAX_TAPS, event)) {
		// Placing multiple fingers before a single finger, because achieving a
		@@ -1489,7 +1514,7 @@ public class FullScreenMagnificationGestureHandler extends MagnificationGestureH
		&& mCompletedTapCount == 0) {
		transitionToDelegatingStateAndClear();
		}
		// LINT.ThenChange(:action_up)
		// LINT.ThenChange(FullScreenMagnificationGestureHandler.java:action_up)
		}

		private boolean isMultiFingerMultiTapTriggered(int targetTapCount, MotionEvent event) {

services/core/java/com/android/server/wm/AccessibilityController.java

+28 −1

Original line number	Diff line number	Diff line
		@@ -22,6 +22,7 @@ import static android.os.Build.IS_USER;
		import static android.view.WindowManager.LayoutParams.PRIVATE_FLAG_EXCLUDE_FROM_SCREEN_MAGNIFICATION;
		import static android.view.WindowManager.LayoutParams.PRIVATE_FLAG_IS_ROUNDED_CORNERS_OVERLAY;
		import static android.view.WindowManager.LayoutParams.TYPE_ACCESSIBILITY_MAGNIFICATION_OVERLAY;
		import static android.view.WindowManager.LayoutParams.TYPE_INPUT_METHOD;
		import static android.view.WindowManager.LayoutParams.TYPE_MAGNIFICATION_OVERLAY;
		import static android.view.WindowManager.TRANSIT_FLAG_IS_RECENTS;

		@@ -564,6 +565,8 @@ final class AccessibilityController {
		private boolean mIsFullscreenMagnificationActivated = false;
		private final Region mMagnificationRegion = new Region();
		private final Region mOldMagnificationRegion = new Region();
		private final Region mImeRegion = new Region();
		private final Region mOldImeRegion = new Region();

		private final MagnificationSpec mMagnificationSpec = new MagnificationSpec();

		@@ -813,7 +816,8 @@ final class AccessibilityController {
		final int screenWidth = mScreenSize.x;
		final int screenHeight = mScreenSize.y;

		mMagnificationRegion.set(0, 0, 0, 0);
		mMagnificationRegion.setEmpty();
		mImeRegion.setEmpty();
		final Region availableBounds = mTempRegion1;
		availableBounds.set(0, 0, screenWidth, screenHeight);

		@@ -866,6 +870,13 @@ final class AccessibilityController {
		applyMatrixToRegion(matrix, touchableRegion);
		windowBounds.set(touchableRegion);

		if (windowType == TYPE_INPUT_METHOD) {
		// Track the bounds of IME windows separately. This region is unrelated to
		// mMagnificationRegion (these regions may overlap if the user chooses to
		// magnify their keyboard) so this does not affect other calculations.
		mImeRegion.op(windowBounds, Region.Op.UNION);
		}

		// Only update new regions
		Region portionOfWindowAlreadyAccountedFor = mTempRegion3;
		portionOfWindowAlreadyAccountedFor.set(mMagnificationRegion);
		@@ -923,6 +934,14 @@ final class AccessibilityController {
		MyHandler.MESSAGE_NOTIFY_MAGNIFICATION_REGION_CHANGED, args)
		.sendToTarget();
		}
		if (com.android.server.accessibility.Flags.enableMagnificationMagnifyNavBarAndIme()
		&& !mOldImeRegion.equals(mImeRegion)) {
		mOldImeRegion.set(mImeRegion);
		final SomeArgs args = SomeArgs.obtain();
		args.arg1 = Region.obtain(mImeRegion);
		mHandler.obtainMessage(MyHandler.MESSAGE_NOTIFY_IME_REGION_CHANGED, args)
		.sendToTarget();
		}
		}

		private Region getLetterboxBounds(WindowState windowState) {
		@@ -1000,6 +1019,7 @@ final class AccessibilityController {
		public static final int MESSAGE_NOTIFY_USER_CONTEXT_CHANGED = 3;
		public static final int MESSAGE_NOTIFY_DISPLAY_SIZE_CHANGED = 4;
		public static final int MESSAGE_NOTIFY_IME_WINDOW_VISIBILITY_CHANGED = 5;
		public static final int MESSAGE_NOTIFY_IME_REGION_CHANGED = 6;

		MyHandler(Looper looper) {
		super(looper);
		@@ -1027,6 +1047,13 @@ final class AccessibilityController {
		final boolean shown = message.arg1 == 1;
		mCallbacks.onImeWindowVisibilityChanged(shown);
		} break;

		case MESSAGE_NOTIFY_IME_REGION_CHANGED: {
		final SomeArgs args = (SomeArgs) message.obj;
		final Region imeRegion = (Region) args.arg1;
		mCallbacks.onImeRegionChanged(imeRegion);
		imeRegion.recycle();
		} break;
		}
		}
		}

services/core/java/com/android/server/wm/WindowManagerInternal.java

+7 −0

Original line number	Diff line number	Diff line
		@@ -208,6 +208,13 @@ public abstract class WindowManagerInternal {
		*/
		void onMagnificationRegionChanged(Region magnificationRegion);

		/**
		* Called when the region used by TYPE_INPUT_METHOD windows changes.
		*
		* @param imeRegion the current region taken by TYPE_INPUT_METHOD window(s).
		*/
		void onImeRegionChanged(Region imeRegion);

		/**
		* Called when an application requests a rectangle on the screen to allow
		* the client to apply the appropriate pan and scale.

services/tests/servicestests/src/com/android/server/accessibility/magnification/FullScreenMagnificationControllerTest.java

+73 −0

Original line number	Diff line number	Diff line
		@@ -54,9 +54,11 @@ import android.graphics.Region;
		import android.hardware.display.DisplayManagerInternal;
		import android.os.Looper;
		import android.os.UserHandle;
		import android.platform.test.annotations.EnableFlags;
		import android.platform.test.annotations.RequiresFlagsEnabled;
		import android.platform.test.flag.junit.CheckFlagsRule;
		import android.platform.test.flag.junit.DeviceFlagsValueProvider;
		import android.platform.test.flag.junit.SetFlagsRule;
		import android.provider.Settings;
		import android.test.mock.MockContentResolver;
		import android.view.DisplayInfo;
		@@ -80,6 +82,8 @@ import com.android.server.input.InputManagerInternal;
		import com.android.server.wm.WindowManagerInternal;
		import com.android.server.wm.WindowManagerInternal.MagnificationCallbacks;

		import com.google.common.truth.Truth;

		import org.hamcrest.CoreMatchers;
		import org.hamcrest.Description;
		import org.hamcrest.TypeSafeMatcher;
		@@ -110,6 +114,15 @@ public class FullScreenMagnificationControllerTest {
		static final Region INITIAL_MAGNIFICATION_REGION = new Region(INITIAL_MAGNIFICATION_BOUNDS);
		static final Region OTHER_REGION_COMPAT = new Region(OTHER_MAGNIFICATION_BOUNDS_COMPAT);
		static final Region OTHER_REGION = new Region(OTHER_MAGNIFICATION_BOUNDS);

		// IME tests define bounds where the IME takes up the bottom half of the screen.
		static final Rect SCREEN_BOUNDS = new Rect(0, 0, 1000, 2000);
		static final Rect IME_BOUNDS = new Rect(
		SCREEN_BOUNDS.left, SCREEN_BOUNDS.bottom / 2,
		SCREEN_BOUNDS.right, SCREEN_BOUNDS.bottom);
		static final PointF POINT_OUTSIDE_IME_BOUNDS = new PointF(
		SCREEN_BOUNDS.right / 2, SCREEN_BOUNDS.bottom / 4);

		static final int SERVICE_ID_1 = 1;
		static final int SERVICE_ID_2 = 2;
		static final int DISPLAY_0 = 0;
		@@ -121,6 +134,8 @@ public class FullScreenMagnificationControllerTest {
		@Rule
		public final CheckFlagsRule mCheckFlagsRule = DeviceFlagsValueProvider.createCheckFlagsRule();

		@Rule public final SetFlagsRule mSetFlagsRule = new SetFlagsRule();

		final FullScreenMagnificationController.ControllerContext mMockControllerCtx =
		mock(FullScreenMagnificationController.ControllerContext.class);
		final Context mMockContext = mock(Context.class);
		@@ -607,6 +622,64 @@ public class FullScreenMagnificationControllerTest {
		);
		}

		@Test
		@EnableFlags(Flags.FLAG_ENABLE_MAGNIFICATION_MAGNIFY_NAV_BAR_AND_IME)
		public void imeRegionContains_pointInsideImeRegion_returnsTrue() {
		for (int displayId = 0; displayId < DISPLAY_COUNT; displayId++) {
		register(displayId);
		final MagnificationCallbacks callbacks = getMagnificationCallbacks(displayId);
		final PointF point = new PointF(IME_BOUNDS.centerX(), IME_BOUNDS.centerY());

		callbacks.onImeRegionChanged(new Region(IME_BOUNDS));
		mMessageCapturingHandler.sendAllMessages();

		Truth.assertThat(mFullScreenMagnificationController.imeRegionContains(
		displayId, point.x, point.y)).isTrue();
		}
		}

		@Test
		@EnableFlags(Flags.FLAG_ENABLE_MAGNIFICATION_MAGNIFY_NAV_BAR_AND_IME)
		public void imeRegionContains_pointOutsideImeRegion_returnsFalse() {
		for (int displayId = 0; displayId < DISPLAY_COUNT; displayId++) {
		register(displayId);
		final MagnificationCallbacks callbacks = getMagnificationCallbacks(displayId);
		final PointF point = POINT_OUTSIDE_IME_BOUNDS;

		callbacks.onImeRegionChanged(new Region(IME_BOUNDS));
		mMessageCapturingHandler.sendAllMessages();

		Truth.assertThat(mFullScreenMagnificationController.imeRegionContains(
		displayId, point.x, point.y)).isFalse();
		}
		}

		@Test
		@EnableFlags(Flags.FLAG_ENABLE_MAGNIFICATION_MAGNIFY_NAV_BAR_AND_IME)
		public void imeRegionContains_pointInsideMagnifiedImeRegion_returnsTrue() {
		for (int displayId = 0; displayId < DISPLAY_COUNT; displayId++) {
		register(displayId);
		final MagnificationCallbacks callbacks = getMagnificationCallbacks(displayId);
		final PointF point = POINT_OUTSIDE_IME_BOUNDS;
		// Set the magnification region to the full screen.
		callbacks.onMagnificationRegionChanged(new Region(SCREEN_BOUNDS));
		mMessageCapturingHandler.sendAllMessages();
		// Zoom far into the center of the IME. This in effect makes the IME fill the entire
		// screen, so that a point on screen that was previously outside of unmagnified IME
		// bounds is now inside of magnified IME bounds.
		Truth.assertThat(mFullScreenMagnificationController
		.setScaleAndCenter(displayId, 8, IME_BOUNDS.centerX(), IME_BOUNDS.centerY(),
		false, false, SERVICE_ID_1)).isTrue();
		mMessageCapturingHandler.sendAllMessages();

		callbacks.onImeRegionChanged(new Region(IME_BOUNDS));
		mMessageCapturingHandler.sendAllMessages();

		Truth.assertThat(mFullScreenMagnificationController.imeRegionContains(
		displayId, point.x, point.y)).isTrue();
		}
		}

		@Test
		public void testOffsetMagnifiedRegion_whileMagnifying_offsetsMove() {
		for (int i = 0; i < DISPLAY_COUNT; i++) {