Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit ae77af78 authored by Prabir Pradhan's avatar Prabir Pradhan
Browse files

Support multi-window handwriting without focus requirement on down 

Previously, the handwriting window was only initialized over the focused
window, and re-initialized only when the IME was bound. This meant that
in multi-window modes where the input method client changes without the
IME being re-bound, handwriting would only work on one of the apps.

Here, we initialize the handwriting window over the entire display so
that handwriting can be started over any app. To ensure that handwriting
only starts if the stylus stroke starts within the bounds of the app, we
do a hit test with the start of the stroke before starting handwriting.

Bug: 230381729
Test: manual: test handwriting in split screen
Change-Id: I9296d35e29387ea54afbdeabe2baa26bf6ccbc71
parent 9e4a0c1a

services/core/java/com/android/server/inputmethod/HandwritingEventReceiverSurface.java

+3 −5
Original line number Diff line number Diff line
@@ -64,7 +64,9 @@ final class HandwritingEventReceiverSurface { 
                        | InputConfig.INTERCEPTS_STYLUS

                        | InputConfig.TRUSTED_OVERLAY;



        // The touchable region of this input surface is not initially configured.

        // Configure the surface to receive stylus events across the entire display.

        mWindowHandle.replaceTouchableRegionWithCrop(null /* use this surface's bounds */);



        final SurfaceControl.Transaction t = new SurfaceControl.Transaction();

        t.setInputWindowInfo(mInputSurface, mWindowHandle);

        t.setLayer(mInputSurface, HANDWRITING_SURFACE_LAYER);
@@ -81,10 +83,6 @@ final class HandwritingEventReceiverSurface { 
        mWindowHandle.ownerUid = imeUid;

        mWindowHandle.inputConfig &= ~InputConfig.SPY;



        // Update the touchable region so that the IME can intercept stylus events

        // across the entire display.

        mWindowHandle.replaceTouchableRegionWithCrop(null /* use this surface's bounds */);



        new SurfaceControl.Transaction()

                .setInputWindowInfo(mInputSurface, mWindowHandle)

                .apply();

services/core/java/com/android/server/inputmethod/HandwritingModeController.java

+17 −10
Original line number Diff line number Diff line
@@ -91,7 +91,7 @@ final class HandwritingModeController { 
     * InputEventReceiver that batches events according to the current thread's Choreographer.

     */

    @UiThread

    void initializeHandwritingSpy(int displayId, IBinder focusedWindowToken) {

    void initializeHandwritingSpy(int displayId) {

        // When resetting, reuse resources if we are reinitializing on the same display.

        reset(displayId == mCurrentDisplayId);

        mCurrentDisplayId = displayId;
@@ -115,12 +115,6 @@ final class HandwritingModeController { 
        mHandwritingSurface = new HandwritingEventReceiverSurface(

                name, displayId, surface, channel);



        // Configure the handwriting window to receive events over the focused window's bounds.

        mWindowManagerInternal.replaceInputSurfaceTouchableRegionWithWindowCrop(

                mHandwritingSurface.getSurface(),

                mHandwritingSurface.getInputWindowHandle(),

                focusedWindowToken);



        // Use a dup of the input channel so that event processing can be paused by disposing the

        // event receiver without causing a fd hangup.

        mHandwritingEventReceiver = new BatchedInputEventReceiver.SimpleBatchedInputEventReceiver(
@@ -149,7 +143,8 @@ final class HandwritingModeController { 
     */

    @UiThread

    @Nullable

    HandwritingSession startHandwritingSession(int requestId, int imePid, int imeUid) {

    HandwritingSession startHandwritingSession(

            int requestId, int imePid, int imeUid, IBinder focusedWindowToken) {

        if (mHandwritingSurface == null) {

            Slog.e(TAG, "Cannot start handwriting session: Handwriting was not initialized.");

            return null;
@@ -158,12 +153,20 @@ final class HandwritingModeController { 
            Slog.e(TAG, "Cannot start handwriting session: Invalid request id: " + requestId);

            return null;

        }

        if (!mRecordingGesture) {

        if (!mRecordingGesture || mHandwritingBuffer.isEmpty()) {

            Slog.e(TAG, "Cannot start handwriting session: No stylus gesture is being recorded.");

            return null;

        }

        Objects.requireNonNull(mHandwritingEventReceiver,

                "Handwriting session was already transferred to IME.");

        final MotionEvent downEvent = mHandwritingBuffer.get(0);

        assert (downEvent.getActionMasked() == MotionEvent.ACTION_DOWN);

        if (!mWindowManagerInternal.isPointInsideWindow(

                focusedWindowToken, mCurrentDisplayId, downEvent.getRawX(), downEvent.getRawY())) {

            Slog.e(TAG, "Cannot start handwriting session: "

                    + "Stylus gesture did not start inside the focused window.");

            return null;

        }

        if (DEBUG) Slog.d(TAG, "Starting handwriting session in display: " + mCurrentDisplayId);



        mInputManagerInternal.pilferPointers(mHandwritingSurface.getInputChannel().getToken());
@@ -226,13 +229,17 @@ final class HandwritingModeController { 
        }



        if (!(ev instanceof MotionEvent)) {

            Slog.e("Stylus", "Received non-motion event in stylus monitor.");

            Slog.wtf(TAG, "Received non-motion event in stylus monitor.");

            return false;

        }

        final MotionEvent event = (MotionEvent) ev;

        if (!isStylusEvent(event)) {

            return false;

        }

        if (event.getDisplayId() != mCurrentDisplayId) {

            Slog.wtf(TAG, "Received stylus event associated with the incorrect display.");

            return false;

        }



        onStylusEvent(event);

        return true;

services/core/java/com/android/server/inputmethod/InputMethodManagerService.java

+5 −5
Original line number Diff line number Diff line
@@ -5099,9 +5099,8 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub 
            case MSG_RESET_HANDWRITING: {

                synchronized (ImfLock.class) {

                    if (mBindingController.supportsStylusHandwriting()

                            && getCurMethodLocked() != null && mCurFocusedWindow != null) {

                        mHwController.initializeHandwritingSpy(

                                mCurTokenDisplayId, mCurFocusedWindow);

                            && getCurMethodLocked() != null) {

                        mHwController.initializeHandwritingSpy(mCurTokenDisplayId);

                    } else {

                        mHwController.reset();

                    }
@@ -5111,14 +5110,15 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub 
            case MSG_START_HANDWRITING:

                synchronized (ImfLock.class) {

                    IInputMethodInvoker curMethod = getCurMethodLocked();

                    if (curMethod == null) {

                    if (curMethod == null || mCurFocusedWindow == null) {

                        return true;

                    }

                    final HandwritingModeController.HandwritingSession session =

                            mHwController.startHandwritingSession(

                                    msg.arg1 /*requestId*/,

                                    msg.arg2 /*pid*/,

                                    mBindingController.getCurMethodUid());

                                    mBindingController.getCurMethodUid(),

                                    mCurFocusedWindow);

                    if (session == null) {

                        Slog.e(TAG,

                                "Failed to start handwriting session for requestId: " + msg.arg1);

services/core/java/com/android/server/wm/WindowManagerInternal.java

+6 −19
Original line number Diff line number Diff line
@@ -33,7 +33,6 @@ import android.view.IInputFilter; 
import android.view.IRemoteAnimationFinishedCallback;

import android.view.IWindow;

import android.view.InputChannel;

import android.view.InputWindowHandle;

import android.view.MagnificationSpec;

import android.view.RemoteAnimationTarget;

import android.view.SurfaceControl;
@@ -841,24 +840,12 @@ public abstract class WindowManagerInternal { 
    public abstract SurfaceControl getHandwritingSurfaceForDisplay(int displayId);



    /**

     * Replaces the touchable region of the provided input surface with the crop of the window with

     * the provided token. This method will associate the inputSurface with a copy of

     * the given inputWindowHandle, where the copy is configured using

     * {@link InputWindowHandle#replaceTouchableRegionWithCrop(SurfaceControl)} with the surface

     * of the provided windowToken.

     * Returns {@code true} if the given point is within the window bounds of the given window.

     *

     * This is a no-op if windowToken is not valid or the window is not found.

     *

     * This does not change any other properties of the inputSurface.

     *

     * This method exists to avoid leaking the window's SurfaceControl outside WindowManagerService.

     *

     * @param inputSurface The surface for which the touchable region should be set.

     * @param inputWindowHandle The {@link InputWindowHandle} for the input surface.

     * @param windowToken The window whose bounds should be used as the touchable region for the

     *                    inputSurface.

     * @param windowToken the window whose bounds should be used for the hit test.

     * @param displayX the x coordinate of the test point in the display's coordinate space.

     * @param displayY the y coordinate of the test point in the display's coordinate space.

     */

    public abstract void replaceInputSurfaceTouchableRegionWithWindowCrop(

            @NonNull SurfaceControl inputSurface, @NonNull InputWindowHandle inputWindowHandle,

            @NonNull IBinder windowToken);

    public abstract boolean isPointInsideWindow(

            @NonNull IBinder windowToken, int displayId, float displayX, float displayY);

}

services/core/java/com/android/server/wm/WindowManagerService.java

+6 −14
Original line number Diff line number Diff line
@@ -8150,23 +8150,15 @@ public class WindowManagerService extends IWindowManager.Stub 
        }



        @Override

        public void replaceInputSurfaceTouchableRegionWithWindowCrop(

                @NonNull SurfaceControl inputSurface,

                @NonNull InputWindowHandle inputWindowHandle,

                @NonNull IBinder windowToken) {

        public boolean isPointInsideWindow(@NonNull IBinder windowToken, int displayId,

                float displayX, float displayY) {

            synchronized (mGlobalLock) {

                final WindowState w = mWindowMap.get(windowToken);

                if (w == null) {

                    return;

                if (w == null || w.getDisplayId() != displayId) {

                    return false;

                }

                // Make a copy of the InputWindowHandle to avoid leaking the window's

                // SurfaceControl.

                final InputWindowHandle localHandle = new InputWindowHandle(inputWindowHandle);

                localHandle.replaceTouchableRegionWithCrop(w.getSurfaceControl());

                final SurfaceControl.Transaction t = mTransactionFactory.get();

                t.setInputWindowInfo(inputSurface, localHandle);

                t.apply();

                t.close();



                return w.getBounds().contains((int) displayX, (int) displayY);

            }

        }

    }