Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e35da24d authored by Taran Singh's avatar Taran Singh
Browse files

Fix handwriting support on fake editors using delegation

We originally added support for Scribe on fake editors in bug 228100684.
However, it doesn't address special cases like when real editor is a
different application package.

This CL introduces IMF APIs to suport handwriting delegation across different
view trees (with or within the same package).

Bug: 266834695
Bug: 228100684
Test: atest StylusHandwritingTest

Change-Id: I3a5d5dc574c5f78697f2a43e11729febf21cc4e2
parent dd76f4b3
Loading
Loading
Loading
Loading
+4 −0
Original line number Diff line number Diff line
@@ -55616,6 +55616,8 @@ package android.view.inputmethod {
  }
  public final class InputMethodManager {
    method public boolean acceptStylusHandwritingDelegation(@NonNull android.view.View);
    method public boolean acceptStylusHandwritingDelegation(@NonNull android.view.View, @NonNull String);
    method public void dispatchKeyEventFromInputMethod(@Nullable android.view.View, @NonNull android.view.KeyEvent);
    method public void displayCompletions(android.view.View, android.view.inputmethod.CompletionInfo[]);
    method @Nullable public android.view.inputmethod.InputMethodInfo getCurrentInputMethodInfo();
@@ -55637,6 +55639,8 @@ package android.view.inputmethod {
    method public boolean isInputMethodSuppressingSpellChecker();
    method public boolean isStylusHandwritingAvailable();
    method @Deprecated public boolean isWatchingCursor(android.view.View);
    method public void prepareStylusHandwritingDelegation(@NonNull android.view.View);
    method public void prepareStylusHandwritingDelegation(@NonNull android.view.View, @NonNull String);
    method public void restartInput(android.view.View);
    method public void sendAppPrivateCommand(android.view.View, String, android.os.Bundle);
    method @Deprecated public void setAdditionalInputMethodSubtypes(@NonNull String, @NonNull android.view.inputmethod.InputMethodSubtype[]);
+34 −0
Original line number Diff line number Diff line
@@ -504,6 +504,40 @@ final class IInputMethodManagerGlobalInvoker {
        }
    }

    @AnyThread
    static void prepareStylusHandwritingDelegation(
            @NonNull IInputMethodClient client,
            @NonNull String delegatePackageName,
            @NonNull String delegatorPackageName) {
        final IInputMethodManager service = getService();
        if (service == null) {
            return;
        }
        try {
            service.prepareStylusHandwritingDelegation(
                    client, delegatePackageName, delegatorPackageName);
        } catch (RemoteException e) {
            throw e.rethrowFromSystemServer();
        }
    }

    @AnyThread
    static boolean acceptStylusHandwritingDelegation(
            @NonNull IInputMethodClient client,
            @NonNull String delegatePackageName,
            @NonNull String delegatorPackageName) {
        final IInputMethodManager service = getService();
        if (service == null) {
            return false;
        }
        try {
            return service.acceptStylusHandwritingDelegation(
                    client, delegatePackageName, delegatorPackageName);
        } catch (RemoteException e) {
            throw e.rethrowFromSystemServer();
        }
    }

    @AnyThread
    @RequiresPermission(value = Manifest.permission.INTERACT_ACROSS_USERS_FULL, conditional = true)
    static boolean isStylusHandwritingAvailableAsUser(@UserIdInt int userId) {
+153 −18
Original line number Diff line number Diff line
@@ -99,6 +99,7 @@ import android.view.WindowManager;
import android.view.WindowManager.LayoutParams.SoftInputModeFlags;
import android.view.autofill.AutofillId;
import android.view.autofill.AutofillManager;
import android.widget.Editor;
import android.window.ImeOnBackInvokedDispatcher;
import android.window.WindowOnBackInvokedDispatcher;

@@ -1552,11 +1553,7 @@ public final class InputMethodManager {
        if (fallbackContext == null) {
            return false;
        }
        if (Settings.Global.getInt(fallbackContext.getContentResolver(),
                Settings.Global.STYLUS_HANDWRITING_ENABLED, 0) == 0) {
            if (DEBUG) {
                Log.d(TAG, "Stylus handwriting is not enabled in settings.");
            }
        if (!isStylusHandwritingEnabled(fallbackContext)) {
            return false;
        }
        return IInputMethodManagerGlobalInvoker.isStylusHandwritingAvailableAsUser(userId);
@@ -2233,35 +2230,173 @@ public final class InputMethodManager {
     * @see #isStylusHandwritingAvailable()
     */
    public void startStylusHandwriting(@NonNull View view) {
        startStylusHandwritingInternal(view, null /* delegatorPackageName */);
    }

    private boolean startStylusHandwritingInternal(
            @NonNull View view, @Nullable String delegatorPackageName) {
        Objects.requireNonNull(view);

        // Re-dispatch if there is a context mismatch.
        final InputMethodManager fallbackImm = getFallbackInputMethodManagerIfNecessary(view);
        if (fallbackImm != null) {
            fallbackImm.startStylusHandwriting(view);
            fallbackImm.startStylusHandwritingInternal(view, delegatorPackageName);
        }
        Objects.requireNonNull(view);

        if (Settings.Global.getInt(view.getContext().getContentResolver(),
                Settings.Global.STYLUS_HANDWRITING_ENABLED, 0) == 0) {
            Log.d(TAG, "Ignoring startStylusHandwriting(view) as stylus handwriting is disabled");
            return;
        boolean useDelegation = !TextUtils.isEmpty(delegatorPackageName);
        if (!isStylusHandwritingEnabled(view.getContext())) {
            Log.w(TAG, "Stylus handwriting pref is disabled. "
                    + "Ignoring calls to start stylus handwriting.");
            return false;
        }

        checkFocus();
        synchronized (mH) {
            if (!hasServedByInputMethodLocked(view)) {
                Log.w(TAG,
                        "Ignoring startStylusHandwriting() as view=" + view + " is not served.");
                return;
                        "Ignoring startStylusHandwriting as view=" + view + " is not served.");
                return false;
            }
            if (view.getViewRootImpl() != mCurRootView) {
                Log.w(TAG, "Ignoring startStylusHandwriting: View's window does not have focus.");
                Log.w(TAG,
                        "Ignoring startStylusHandwriting: View's window does not have focus.");
                return false;
            }
            if (useDelegation) {
                return IInputMethodManagerGlobalInvoker.acceptStylusHandwritingDelegation(
                        mClient, view.getContext().getOpPackageName(), delegatorPackageName);
            } else {
                IInputMethodManagerGlobalInvoker.startStylusHandwriting(mClient);
            }
            return false;
        }
    }

    private boolean isStylusHandwritingEnabled(@NonNull Context context) {
        if (Settings.Global.getInt(context.getContentResolver(),
                Settings.Global.STYLUS_HANDWRITING_ENABLED, 0) == 0) {
            Log.d(TAG, "Stylus handwriting pref is disabled.");
            return false;
        }
        return true;
    }

    /**
     * Prepares delegation of starting stylus handwriting session to a different editor in same
     * or different window than the view on which initial handwriting stroke was detected.
     *
     * Delegation can be used to start stylus handwriting session before the {@link Editor} view or
     * its {@link InputConnection} is started. Calling this method starts buffering of stylus
     * motion events until {@link #acceptStylusHandwritingDelegation(View)} is called, at which
     * point the handwriting session can be started and the buffered stylus motion events will be
     * delivered to the IME.
     * e.g. Delegation can be used when initial handwriting stroke is
     * on a pseudo {@link Editor} like widget (with no {@link InputConnection}) but actual
     * {@link Editor} is on a different window.
     *
     * <p> Note: If an actual {@link Editor} capable of {@link InputConnection} is being scribbled
     * upon using stylus, use {@link #startStylusHandwriting(View)} instead.</p>
     *
     * @param delegatorView the view that receives initial stylus stroke and delegates it to the
     *  actual editor. Its window must {@link View#hasWindowFocus have focus}.
     * @see #prepareStylusHandwritingDelegation(View, String)
     * @see #acceptStylusHandwritingDelegation(View)
     * @see #startStylusHandwriting(View)
     */
    public void prepareStylusHandwritingDelegation(@NonNull View delegatorView) {
        prepareStylusHandwritingDelegation(
                delegatorView, delegatorView.getContext().getOpPackageName());
    }

    /**
     * Prepares delegation of starting stylus handwriting session to a different editor in same or a
     * different window in a different package than the view on which initial handwriting stroke
     * was detected.
     *
     * Delegation can be used to start stylus handwriting session before the {@link Editor} view or
     * its {@link InputConnection} is started. Calling this method starts buffering of stylus
     * motion events until {@link #acceptStylusHandwritingDelegation(View, String)} is called, at
     * which point the handwriting session can be started and the buffered stylus motion events will
     * be delivered to the IME.
     * e.g. Delegation can be used when initial handwriting stroke is
     * on a pseudo {@link Editor} like widget (with no {@link InputConnection}) but actual
     * {@link Editor} is on a different window in the given package.
     *
     * <p>Note: If delegator and delegate are in same package use
     * {@link #prepareStylusHandwritingDelegation(View)} instead.</p>
     *
     * @param delegatorView  the view that receives initial stylus stroke and delegates it to the
     * actual editor. Its window must {@link View#hasWindowFocus have focus}.
     * @param delegatePackageName package name that contains actual {@link Editor} which should
     *  start stylus handwriting session by calling {@link #acceptStylusHandwritingDelegation}.
     * @see #prepareStylusHandwritingDelegation(View)
     * @see #acceptStylusHandwritingDelegation(View, String)
     */
    public void prepareStylusHandwritingDelegation(
            @NonNull View delegatorView, @NonNull String delegatePackageName) {
        Objects.requireNonNull(delegatorView);
        Objects.requireNonNull(delegatePackageName);

        // Re-dispatch if there is a context mismatch.
        final InputMethodManager fallbackImm =
                getFallbackInputMethodManagerIfNecessary(delegatorView);
        if (fallbackImm != null) {
            fallbackImm.prepareStylusHandwritingDelegation(delegatorView, delegatePackageName);
        }

        if (!isStylusHandwritingEnabled(delegatorView.getContext())) {
            Log.w(TAG, "Stylus handwriting pref is disabled. "
                    + "Ignoring prepareStylusHandwritingDelegation().");
            return;
        }
        IInputMethodManagerGlobalInvoker.prepareStylusHandwritingDelegation(
                mClient,
                delegatePackageName,
                delegatorView.getContext().getOpPackageName());
    }

            IInputMethodManagerGlobalInvoker.startStylusHandwriting(mClient);
            // TODO(b/210039666): do we need any extra work for supporting non-native
            //   UI toolkits?
    /**
     * Accepts and starts a stylus handwriting session on the delegate view, if handwriting
     * initiation delegation was previously requested using
     * {@link #prepareStylusHandwritingDelegation(View)} from the delegator.
     *
     * <p>Note: If delegator and delegate are in different application packages, use
     * {@link #acceptStylusHandwritingDelegation(View, String)} instead.</p>
     *
     * @param delegateView delegate view capable of receiving input via {@link InputConnection}
     *  on which {@link #startStylusHandwriting(View)} will be called.
     * @return {@code true} if view belongs to same application package as used in
     *  {@link #prepareStylusHandwritingDelegation(View)} and handwriting session can start.
     * @see #acceptStylusHandwritingDelegation(View, String)
     * @see #prepareStylusHandwritingDelegation(View)
     */
    public boolean acceptStylusHandwritingDelegation(@NonNull View delegateView) {
        return startStylusHandwritingInternal(
                delegateView, delegateView.getContext().getOpPackageName());
    }

    /**
     * Accepts and starts a stylus handwriting session on the delegate view, if handwriting
     * initiation delegation was previously requested using
     * {@link #prepareStylusHandwritingDelegation(View, String)} from te delegator and the view
     * belongs to a specified delegate package.
     *
     * <p>Note: If delegator and delegate are in same application package use
     * {@link #acceptStylusHandwritingDelegation(View)} instead.</p>
     *
     * @param delegateView delegate view capable of receiving input via {@link InputConnection}
     *  on which {@link #startStylusHandwriting(View)} will be called.
     * @param delegatorPackageName package name of the delegator that handled initial stylus stroke.
     * @return {@code true} if view belongs to allowed delegate package declared in
     *  {@link #prepareStylusHandwritingDelegation(View, String)} and handwriting session can start.
     * @see #prepareStylusHandwritingDelegation(View, String)
     * @see #acceptStylusHandwritingDelegation(View)
     */
    public boolean acceptStylusHandwritingDelegation(
            @NonNull View delegateView, @NonNull String delegatorPackageName) {
        Objects.requireNonNull(delegatorPackageName);

        return startStylusHandwritingInternal(delegateView, delegatorPackageName);
    }

    /**
+9 −0
Original line number Diff line number Diff line
@@ -148,6 +148,15 @@ interface IInputMethodManager {
    /** Start Stylus handwriting session **/
    void startStylusHandwriting(in IInputMethodClient client);

    /** Prepares delegation of starting stylus handwriting session to a different editor **/
    void prepareStylusHandwritingDelegation(in IInputMethodClient client,
                in String delegatePackageName,
                in String delegatorPackageName);

    /** Accepts and starts a stylus handwriting session for the delegate view **/
    boolean acceptStylusHandwritingDelegation(in IInputMethodClient client,
                in String delegatePackageName, in String delegatorPackageName);

    /** Returns {@code true} if currently selected IME supports Stylus handwriting. */
    @JavaPassthrough(annotation="@android.annotation.RequiresPermission(value = "
            + "android.Manifest.permission.INTERACT_ACROSS_USERS_FULL, conditional = true)")
+50 −1
Original line number Diff line number Diff line
@@ -20,12 +20,14 @@ import static android.view.InputDevice.SOURCE_STYLUS;

import android.Manifest;
import android.annotation.AnyThread;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.RequiresPermission;
import android.annotation.UiThread;
import android.hardware.input.InputManager;
import android.os.IBinder;
import android.os.Looper;
import android.text.TextUtils;
import android.util.Slog;
import android.view.BatchedInputEventReceiver;
import android.view.Choreographer;
@@ -35,6 +37,8 @@ import android.view.InputEvent;
import android.view.InputEventReceiver;
import android.view.MotionEvent;
import android.view.SurfaceControl;
import android.view.View;
import android.view.inputmethod.InputMethodManager;

import com.android.server.LocalServices;
import com.android.server.input.InputManagerInternal;
@@ -52,6 +56,9 @@ final class HandwritingModeController {
    // TODO(b/210039666): flip the flag.
    static final boolean DEBUG = true;
    private static final int EVENT_BUFFER_SIZE = 100;
    // A longer event buffer used for handwriting delegation
    // TODO(b/210039666): make this device touch sampling rate dependent.
    private static final int LONG_EVENT_BUFFER = EVENT_BUFFER_SIZE * 20;

    // This must be the looper for the UiThread.
    private final Looper mLooper;
@@ -63,6 +70,9 @@ final class HandwritingModeController {
    private Runnable mInkWindowInitRunnable;
    private boolean mRecordingGesture;
    private int mCurrentDisplayId;
    // when set, package names are used for handwriting delegation.
    private @Nullable String mDelegatePackageName;
    private @Nullable String mDelegatorPackageName;

    private HandwritingEventReceiverSurface mHandwritingSurface;

@@ -137,6 +147,41 @@ final class HandwritingModeController {
        return mRecordingGesture;
    }

    boolean hasOngoingStylusHandwritingSession() {
        return mHandwritingSurface != null && mHandwritingSurface.isIntercepting();
    }

    /**
     * Prepare delegation of stylus handwriting to a different editor
     * @see InputMethodManager#prepareStylusHandwritingDelegation(View, String)
     */
    void prepareStylusHandwritingDelegation(
            @NonNull String delegatePackageName, @NonNull String delegatorPackageName) {
        mDelegatePackageName = delegatePackageName;
        mDelegatorPackageName = delegatorPackageName;
        ((ArrayList) mHandwritingBuffer).ensureCapacity(LONG_EVENT_BUFFER);
        // TODO(b/210039666): cancel delegation after a timeout or next input method client binding.
    }

    @Nullable String getDelegatePackageName() {
        return mDelegatePackageName;
    }

    @Nullable String getDelegatorPackageName() {
        return mDelegatorPackageName;
    }

    /**
     * Clear any pending handwriting delegation info.
     */
    void clearPendingHandwritingDelegation() {
        if (DEBUG) {
            Slog.d(TAG, "clearPendingHandwritingDelegation");
        }
        mDelegatorPackageName = null;
        mDelegatePackageName = null;
    }

    /**
     * Starts a {@link HandwritingSession} to transfer to the IME.
     *
@@ -223,6 +268,7 @@ final class HandwritingModeController {
            }
        }

        clearPendingHandwritingDelegation();
        mRecordingGesture = false;
    }

@@ -259,7 +305,10 @@ final class HandwritingModeController {
            mInkWindowInitRunnable = null;
        }

        if (action == MotionEvent.ACTION_UP || action == MotionEvent.ACTION_CANCEL) {
        // If handwriting delegation is ongoing, don't clear the buffer so that multiple strokes
        // can be buffered across windows.
        if (TextUtils.isEmpty(mDelegatePackageName)
                && (action == MotionEvent.ACTION_UP || action == MotionEvent.ACTION_CANCEL)) {
            mRecordingGesture = false;
            mHandwritingBuffer.clear();
            return;
Loading