Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit ae77af78 authored by Prabir Pradhan's avatar Prabir Pradhan
Browse files

Support multi-window handwriting without focus requirement on down 

Previously, the handwriting window was only initialized over the focused
window, and re-initialized only when the IME was bound. This meant that
in multi-window modes where the input method client changes without the
IME being re-bound, handwriting would only work on one of the apps.

Here, we initialize the handwriting window over the entire display so
that handwriting can be started over any app. To ensure that handwriting
only starts if the stylus stroke starts within the bounds of the app, we
do a hit test with the start of the stroke before starting handwriting.

Bug: 230381729
Test: manual: test handwriting in split screen
Change-Id: I9296d35e29387ea54afbdeabe2baa26bf6ccbc71
parent 9e4a0c1a

services/core/java/com/android/server/inputmethod/HandwritingEventReceiverSurface.java

+3 −5
Original line number Original line Diff line number Diff line
@@ -64,7 +64,9 @@ final class HandwritingEventReceiverSurface { 
                        | InputConfig.INTERCEPTS_STYLUS

                        | InputConfig.INTERCEPTS_STYLUS

                        | InputConfig.TRUSTED_OVERLAY;

                        | InputConfig.TRUSTED_OVERLAY;





        // The touchable region of this input surface is not initially configured.

        // Configure the surface to receive stylus events across the entire display.

        mWindowHandle.replaceTouchableRegionWithCrop(null /* use this surface's bounds */);



        final SurfaceControl.Transaction t = new SurfaceControl.Transaction();

        final SurfaceControl.Transaction t = new SurfaceControl.Transaction();

        t.setInputWindowInfo(mInputSurface, mWindowHandle);

        t.setInputWindowInfo(mInputSurface, mWindowHandle);

        t.setLayer(mInputSurface, HANDWRITING_SURFACE_LAYER);

        t.setLayer(mInputSurface, HANDWRITING_SURFACE_LAYER);
@@ -81,10 +83,6 @@ final class HandwritingEventReceiverSurface { 
        mWindowHandle.ownerUid = imeUid;

        mWindowHandle.ownerUid = imeUid;

        mWindowHandle.inputConfig &= ~InputConfig.SPY;

        mWindowHandle.inputConfig &= ~InputConfig.SPY;





        // Update the touchable region so that the IME can intercept stylus events

        // across the entire display.

        mWindowHandle.replaceTouchableRegionWithCrop(null /* use this surface's bounds */);



        new SurfaceControl.Transaction()

        new SurfaceControl.Transaction()

                .setInputWindowInfo(mInputSurface, mWindowHandle)

                .setInputWindowInfo(mInputSurface, mWindowHandle)

                .apply();

                .apply();

services/core/java/com/android/server/inputmethod/HandwritingModeController.java

+17 −10
Original line number Original line Diff line number Diff line
@@ -91,7 +91,7 @@ final class HandwritingModeController { 
     * InputEventReceiver that batches events according to the current thread's Choreographer.

     * InputEventReceiver that batches events according to the current thread's Choreographer.

     */

     */

    @UiThread

    @UiThread

    void initializeHandwritingSpy(int displayId, IBinder focusedWindowToken) {

    void initializeHandwritingSpy(int displayId) {

        // When resetting, reuse resources if we are reinitializing on the same display.

        // When resetting, reuse resources if we are reinitializing on the same display.

        reset(displayId == mCurrentDisplayId);

        reset(displayId == mCurrentDisplayId);

        mCurrentDisplayId = displayId;

        mCurrentDisplayId = displayId;
@@ -115,12 +115,6 @@ final class HandwritingModeController { 
        mHandwritingSurface = new HandwritingEventReceiverSurface(

        mHandwritingSurface = new HandwritingEventReceiverSurface(

                name, displayId, surface, channel);

                name, displayId, surface, channel);





        // Configure the handwriting window to receive events over the focused window's bounds.

        mWindowManagerInternal.replaceInputSurfaceTouchableRegionWithWindowCrop(

                mHandwritingSurface.getSurface(),

                mHandwritingSurface.getInputWindowHandle(),

                focusedWindowToken);



        // Use a dup of the input channel so that event processing can be paused by disposing the

        // Use a dup of the input channel so that event processing can be paused by disposing the

        // event receiver without causing a fd hangup.

        // event receiver without causing a fd hangup.

        mHandwritingEventReceiver = new BatchedInputEventReceiver.SimpleBatchedInputEventReceiver(

        mHandwritingEventReceiver = new BatchedInputEventReceiver.SimpleBatchedInputEventReceiver(
@@ -149,7 +143,8 @@ final class HandwritingModeController { 
     */

     */

    @UiThread

    @UiThread

    @Nullable

    @Nullable

    HandwritingSession startHandwritingSession(int requestId, int imePid, int imeUid) {

    HandwritingSession startHandwritingSession(

            int requestId, int imePid, int imeUid, IBinder focusedWindowToken) {

        if (mHandwritingSurface == null) {

        if (mHandwritingSurface == null) {

            Slog.e(TAG, "Cannot start handwriting session: Handwriting was not initialized.");

            Slog.e(TAG, "Cannot start handwriting session: Handwriting was not initialized.");

            return null;

            return null;
@@ -158,12 +153,20 @@ final class HandwritingModeController { 
            Slog.e(TAG, "Cannot start handwriting session: Invalid request id: " + requestId);

            Slog.e(TAG, "Cannot start handwriting session: Invalid request id: " + requestId);

            return null;

            return null;

        }

        }

        if (!mRecordingGesture) {

        if (!mRecordingGesture || mHandwritingBuffer.isEmpty()) {

            Slog.e(TAG, "Cannot start handwriting session: No stylus gesture is being recorded.");

            Slog.e(TAG, "Cannot start handwriting session: No stylus gesture is being recorded.");

            return null;

            return null;

        }

        }

        Objects.requireNonNull(mHandwritingEventReceiver,

        Objects.requireNonNull(mHandwritingEventReceiver,

                "Handwriting session was already transferred to IME.");

                "Handwriting session was already transferred to IME.");

        final MotionEvent downEvent = mHandwritingBuffer.get(0);

        assert (downEvent.getActionMasked() == MotionEvent.ACTION_DOWN);

        if (!mWindowManagerInternal.isPointInsideWindow(

                focusedWindowToken, mCurrentDisplayId, downEvent.getRawX(), downEvent.getRawY())) {

            Slog.e(TAG, "Cannot start handwriting session: "

                    + "Stylus gesture did not start inside the focused window.");

            return null;

        }

        if (DEBUG) Slog.d(TAG, "Starting handwriting session in display: " + mCurrentDisplayId);

        if (DEBUG) Slog.d(TAG, "Starting handwriting session in display: " + mCurrentDisplayId);





        mInputManagerInternal.pilferPointers(mHandwritingSurface.getInputChannel().getToken());

        mInputManagerInternal.pilferPointers(mHandwritingSurface.getInputChannel().getToken());
@@ -226,13 +229,17 @@ final class HandwritingModeController { 
        }

        }





        if (!(ev instanceof MotionEvent)) {

        if (!(ev instanceof MotionEvent)) {

            Slog.e("Stylus", "Received non-motion event in stylus monitor.");

            Slog.wtf(TAG, "Received non-motion event in stylus monitor.");

            return false;

            return false;

        }

        }

        final MotionEvent event = (MotionEvent) ev;

        final MotionEvent event = (MotionEvent) ev;

        if (!isStylusEvent(event)) {

        if (!isStylusEvent(event)) {

            return false;

            return false;

        }

        }

        if (event.getDisplayId() != mCurrentDisplayId) {

            Slog.wtf(TAG, "Received stylus event associated with the incorrect display.");

            return false;

        }





        onStylusEvent(event);

        onStylusEvent(event);

        return true;

        return true;

services/core/java/com/android/server/inputmethod/InputMethodManagerService.java

+5 −5
Original line number Original line Diff line number Diff line
@@ -5099,9 +5099,8 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub 
            case MSG_RESET_HANDWRITING: {

            case MSG_RESET_HANDWRITING: {

                synchronized (ImfLock.class) {

                synchronized (ImfLock.class) {

                    if (mBindingController.supportsStylusHandwriting()

                    if (mBindingController.supportsStylusHandwriting()

                            && getCurMethodLocked() != null && mCurFocusedWindow != null) {

                            && getCurMethodLocked() != null) {

                        mHwController.initializeHandwritingSpy(

                        mHwController.initializeHandwritingSpy(mCurTokenDisplayId);

                                mCurTokenDisplayId, mCurFocusedWindow);

                    } else {

                    } else {

                        mHwController.reset();

                        mHwController.reset();

                    }

                    }
@@ -5111,14 +5110,15 @@ public final class InputMethodManagerService extends IInputMethodManager.Stub 
            case MSG_START_HANDWRITING:

            case MSG_START_HANDWRITING:

                synchronized (ImfLock.class) {

                synchronized (ImfLock.class) {

                    IInputMethodInvoker curMethod = getCurMethodLocked();

                    IInputMethodInvoker curMethod = getCurMethodLocked();

                    if (curMethod == null) {

                    if (curMethod == null || mCurFocusedWindow == null) {

                        return true;

                        return true;

                    }

                    }

                    final HandwritingModeController.HandwritingSession session =

                    final HandwritingModeController.HandwritingSession session =

                            mHwController.startHandwritingSession(

                            mHwController.startHandwritingSession(

                                    msg.arg1 /*requestId*/,

                                    msg.arg1 /*requestId*/,

                                    msg.arg2 /*pid*/,

                                    msg.arg2 /*pid*/,

                                    mBindingController.getCurMethodUid());

                                    mBindingController.getCurMethodUid(),

                                    mCurFocusedWindow);

                    if (session == null) {

                    if (session == null) {

                        Slog.e(TAG,

                        Slog.e(TAG,

                                "Failed to start handwriting session for requestId: " + msg.arg1);

                                "Failed to start handwriting session for requestId: " + msg.arg1);

services/core/java/com/android/server/wm/WindowManagerInternal.java

+6 −19
Original line number Original line Diff line number Diff line
@@ -33,7 +33,6 @@ import android.view.IInputFilter; 
import android.view.IRemoteAnimationFinishedCallback;

import android.view.IRemoteAnimationFinishedCallback;

import android.view.IWindow;

import android.view.IWindow;

import android.view.InputChannel;

import android.view.InputChannel;

import android.view.InputWindowHandle;

import android.view.MagnificationSpec;

import android.view.MagnificationSpec;

import android.view.RemoteAnimationTarget;

import android.view.RemoteAnimationTarget;

import android.view.SurfaceControl;

import android.view.SurfaceControl;
@@ -841,24 +840,12 @@ public abstract class WindowManagerInternal { 
    public abstract SurfaceControl getHandwritingSurfaceForDisplay(int displayId);

    public abstract SurfaceControl getHandwritingSurfaceForDisplay(int displayId);





    /**

    /**

     * Replaces the touchable region of the provided input surface with the crop of the window with

     * Returns {@code true} if the given point is within the window bounds of the given window.

     * the provided token. This method will associate the inputSurface with a copy of

     * the given inputWindowHandle, where the copy is configured using

     * {@link InputWindowHandle#replaceTouchableRegionWithCrop(SurfaceControl)} with the surface

     * of the provided windowToken.

     *

     *

     * This is a no-op if windowToken is not valid or the window is not found.

     * @param windowToken the window whose bounds should be used for the hit test.

     *

     * @param displayX the x coordinate of the test point in the display's coordinate space.

     * This does not change any other properties of the inputSurface.

     * @param displayY the y coordinate of the test point in the display's coordinate space.

     *

     * This method exists to avoid leaking the window's SurfaceControl outside WindowManagerService.

     *

     * @param inputSurface The surface for which the touchable region should be set.

     * @param inputWindowHandle The {@link InputWindowHandle} for the input surface.

     * @param windowToken The window whose bounds should be used as the touchable region for the

     *                    inputSurface.

     */

     */

    public abstract void replaceInputSurfaceTouchableRegionWithWindowCrop(

    public abstract boolean isPointInsideWindow(

            @NonNull SurfaceControl inputSurface, @NonNull InputWindowHandle inputWindowHandle,

            @NonNull IBinder windowToken, int displayId, float displayX, float displayY);

            @NonNull IBinder windowToken);

}
 
}

services/core/java/com/android/server/wm/WindowManagerService.java

+6 −14
Original line number Original line Diff line number Diff line
@@ -8150,23 +8150,15 @@ public class WindowManagerService extends IWindowManager.Stub 
        }

        }





        @Override

        @Override

        public void replaceInputSurfaceTouchableRegionWithWindowCrop(

        public boolean isPointInsideWindow(@NonNull IBinder windowToken, int displayId,

                @NonNull SurfaceControl inputSurface,

                float displayX, float displayY) {

                @NonNull InputWindowHandle inputWindowHandle,

                @NonNull IBinder windowToken) {

            synchronized (mGlobalLock) {

            synchronized (mGlobalLock) {

                final WindowState w = mWindowMap.get(windowToken);

                final WindowState w = mWindowMap.get(windowToken);

                if (w == null) {

                if (w == null || w.getDisplayId() != displayId) {

                    return;

                    return false;

                }

                }

                // Make a copy of the InputWindowHandle to avoid leaking the window's



                // SurfaceControl.

                return w.getBounds().contains((int) displayX, (int) displayY);

                final InputWindowHandle localHandle = new InputWindowHandle(inputWindowHandle);

                localHandle.replaceTouchableRegionWithCrop(w.getSurfaceControl());

                final SurfaceControl.Transaction t = mTransactionFactory.get();

                t.setInputWindowInfo(inputSurface, localHandle);

                t.apply();

                t.close();

            }

            }

        }

        }

    }

    }