Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 35a2c048 authored by Justin Ghan's avatar Justin Ghan
Browse files

Connectionless handwriting APIs for InputMethodManager 

Adds APIs to start a connectionless handwriting session, and a callback
API to receive the result.

Bug: 300979854
Bug: 293898187
Test: atest StylusHandwritingTest
API-Coverage-Bug: 324492938
Change-Id: Ib33a6331460fe190a0cf2209efe6e40ce1c6c1db
parent ae197760

core/api/current.txt

+11 −0
Original line number Diff line number Diff line
@@ -55695,6 +55695,14 @@ package android.view.inputmethod { 
    field @NonNull public static final android.os.Parcelable.Creator<android.view.inputmethod.CompletionInfo> CREATOR;

  }













  @FlaggedApi("android.view.inputmethod.connectionless_handwriting") public interface ConnectionlessHandwritingCallback {













    method public void onError(int);













    method public void onResult(@NonNull CharSequence);













    field public static final int CONNECTIONLESS_HANDWRITING_ERROR_NO_TEXT_RECOGNIZED = 0; // 0x0













    field public static final int CONNECTIONLESS_HANDWRITING_ERROR_OTHER = 2; // 0x2













    field public static final int CONNECTIONLESS_HANDWRITING_ERROR_UNSUPPORTED = 1; // 0x1













  }



























  public final class CorrectionInfo implements android.os.Parcelable {















    ctor public CorrectionInfo(int, CharSequence, CharSequence);















    method public int describeContents();










@@ -56157,6 +56165,9 @@ package android.view.inputmethod {













    method public boolean showSoftInput(android.view.View, int, android.os.ResultReceiver);















    method @Deprecated public void showSoftInputFromInputMethod(android.os.IBinder, int);















    method @Deprecated public void showStatusIcon(android.os.IBinder, String, @DrawableRes int);













    method @FlaggedApi("android.view.inputmethod.connectionless_handwriting") public void startConnectionlessStylusHandwriting(@NonNull android.view.View, @Nullable android.view.inputmethod.CursorAnchorInfo, @NonNull java.util.concurrent.Executor, @NonNull android.view.inputmethod.ConnectionlessHandwritingCallback);













    method @FlaggedApi("android.view.inputmethod.connectionless_handwriting") public void startConnectionlessStylusHandwritingForDelegation(@NonNull android.view.View, @Nullable android.view.inputmethod.CursorAnchorInfo, @NonNull java.util.concurrent.Executor, @NonNull android.view.inputmethod.ConnectionlessHandwritingCallback);













    method @FlaggedApi("android.view.inputmethod.connectionless_handwriting") public void startConnectionlessStylusHandwritingForDelegation(@NonNull android.view.View, @Nullable android.view.inputmethod.CursorAnchorInfo, @NonNull String, @NonNull java.util.concurrent.Executor, @NonNull android.view.inputmethod.ConnectionlessHandwritingCallback);















    method public void startStylusHandwriting(@NonNull android.view.View);















    method @Deprecated public boolean switchToLastInputMethod(android.os.IBinder);















    method @Deprecated public boolean switchToNextInputMethod(android.os.IBinder, boolean);

























core/java/android/view/inputmethod/ConnectionlessHandwritingCallback.java
0 → 100644



+77
−0




























Original line number
Diff line number
Diff line













/*













 * Copyright (C) 2024 The Android Open Source Project













 *













 * Licensed under the Apache License, Version 2.0 (the "License");













 * you may not use this file except in compliance with the License.













 * You may obtain a copy of the License at













 *













 *      http://www.apache.org/licenses/LICENSE-2.0













 *













 * Unless required by applicable law or agreed to in writing, software













 * distributed under the License is distributed on an "AS IS" BASIS,













 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.













 * See the License for the specific language governing permissions and













 * limitations under the License.













 */



























package android.view.inputmethod;



























import android.annotation.FlaggedApi;













import android.annotation.IntDef;













import android.annotation.NonNull;













import android.view.View;



























import java.lang.annotation.Retention;













import java.lang.annotation.RetentionPolicy;













import java.util.concurrent.Executor;



























/**













 * Interface to receive the result of starting a connectionless stylus handwriting session using













 * one of {@link InputMethodManager#startConnectionlessStylusHandwriting(View, CursorAnchorInfo,













 * Executor,ConnectionlessHandwritingCallback)}, {@link













 * InputMethodManager#startConnectionlessStylusHandwritingForDelegation(View, CursorAnchorInfo,













 * Executor, ConnectionlessHandwritingCallback)}, or {@link













 * InputMethodManager#startConnectionlessStylusHandwritingForDelegation(View, CursorAnchorInfo,













 * String, Executor, ConnectionlessHandwritingCallback)}.













 */













@FlaggedApi(Flags.FLAG_CONNECTIONLESS_HANDWRITING)













public interface ConnectionlessHandwritingCallback {



























    /** @hide */













    @IntDef(prefix = {"CONNECTIONLESS_HANDWRITING_ERROR_"}, value = {













            CONNECTIONLESS_HANDWRITING_ERROR_NO_TEXT_RECOGNIZED,













            CONNECTIONLESS_HANDWRITING_ERROR_UNSUPPORTED,













            CONNECTIONLESS_HANDWRITING_ERROR_OTHER













    })













    @Retention(RetentionPolicy.SOURCE)













    @interface ConnectionlessHandwritingError {













    }



























    /**













     * Error code indicating that the connectionless handwriting session started and completed













     * but no text was recognized.













     */













    int CONNECTIONLESS_HANDWRITING_ERROR_NO_TEXT_RECOGNIZED = 0;



























    /**













     * Error code indicating that the connectionless handwriting session was not started as the













     * current IME does not support it.













     */













    int CONNECTIONLESS_HANDWRITING_ERROR_UNSUPPORTED = 1;



























    /**













     * Error code for any other reason that the connectionless handwriting session did not complete













     * successfully. Either the session could not start, or the session started but did not complete













     * successfully.













     */













    int CONNECTIONLESS_HANDWRITING_ERROR_OTHER = 2;



























    /**













     * Callback when the connectionless handwriting session completed successfully and













     * recognized text.













     */













    void onResult(@NonNull CharSequence text);



























    /** Callback when the connectionless handwriting session did not complete successfully. */













    void onError(@ConnectionlessHandwritingError int errorCode);













}
























core/java/android/view/inputmethod/IInputMethodManagerGlobalInvoker.java



+22
−0




























Original line number
Diff line number
Diff line








@@ -34,6 +34,7 @@ import android.view.WindowManager;













import android.window.ImeOnBackInvokedDispatcher;































import com.android.internal.inputmethod.DirectBootAwareness;













import com.android.internal.inputmethod.IConnectionlessHandwritingCallback;















import com.android.internal.inputmethod.IImeTracker;















import com.android.internal.inputmethod.IInputMethodClient;















import com.android.internal.inputmethod.IRemoteAccessibilityInputConnection;










@@ -491,6 +492,27 @@ final class IInputMethodManagerGlobalInvoker {













        }















    }





























    @AnyThread













    static boolean startConnectionlessStylusHandwriting(













            @NonNull IInputMethodClient client,













            @UserIdInt int userId,













            @Nullable CursorAnchorInfo cursorAnchorInfo,













            @Nullable String delegatePackageName,













            @Nullable String delegatorPackageName,













            @NonNull IConnectionlessHandwritingCallback callback) {













        final IInputMethodManager service = getService();













        if (service == null) {













            return false;













        }













        try {













            service.startConnectionlessStylusHandwriting(client, userId, cursorAnchorInfo,













                    delegatePackageName, delegatorPackageName, callback);













        } catch (RemoteException e) {













            throw e.rethrowFromSystemServer();













        }













        return true;













    }





























    @AnyThread















    static void prepareStylusHandwritingDelegation(















            @NonNull IInputMethodClient client,

































core/java/android/view/inputmethod/InputMethodManager.java



+218
−9




























Original line number
Diff line number
Diff line








@@ -38,6 +38,7 @@ import static android.view.inputmethod.InputMethodManagerProto.SERVED_VIEW;













import static com.android.internal.inputmethod.StartInputReason.BOUND_TO_IMMS;































import android.Manifest;













import android.annotation.CallbackExecutor;















import android.annotation.DisplayContext;















import android.annotation.DrawableRes;















import android.annotation.DurationMillisLong;










@@ -108,6 +109,7 @@ import android.window.WindowOnBackInvokedDispatcher;





























import com.android.internal.annotations.GuardedBy;















import com.android.internal.inputmethod.DirectBootAwareness;













import com.android.internal.inputmethod.IConnectionlessHandwritingCallback;















import com.android.internal.inputmethod.IInputMethodClient;















import com.android.internal.inputmethod.IInputMethodSession;















import com.android.internal.inputmethod.IRemoteAccessibilityInputConnection;









@@ -134,6 +136,7 @@ import java.util.List;













import java.util.Map;















import java.util.Objects;















import java.util.concurrent.CountDownLatch;













import java.util.concurrent.Executor;















import java.util.concurrent.TimeUnit;















import java.util.concurrent.atomic.AtomicBoolean;















import java.util.function.Consumer;










@@ -2474,6 +2477,127 @@ public final class InputMethodManager {













        }















    }





























    /**













     * Starts a connectionless stylus handwriting session. A connectionless session differs from a













     * regular stylus handwriting session in that the IME does not use an input connection to













     * communicate with a text editor. Instead, the IME directly returns recognised handwritten text













     * via a callback.













     *













     * <p>The {code cursorAnchorInfo} may be used by the IME to improve the handwriting recognition













     * accuracy and user experience of the handwriting session. Usually connectionless handwriting













     * is used for a view which appears like a text editor but does not really support text editing.













     * For best results, the {code cursorAnchorInfo} should be populated as it would be for a real













     * text editor (for example, the insertion marker location can be set to where the user would













     * expect it to be, even if there is no visible cursor).













     *













     * @param view the view receiving stylus events













     * @param cursorAnchorInfo positional information about the view receiving stylus events













     * @param callbackExecutor the executor to run the callback on













     * @param callback the callback to receive the result













     */













    @FlaggedApi(Flags.FLAG_CONNECTIONLESS_HANDWRITING)













    public void startConnectionlessStylusHandwriting(@NonNull View view,













            @Nullable CursorAnchorInfo cursorAnchorInfo,













            @NonNull @CallbackExecutor Executor callbackExecutor,













            @NonNull ConnectionlessHandwritingCallback callback) {













        startConnectionlessStylusHandwritingInternal(













                view, cursorAnchorInfo, null, null, callbackExecutor, callback);













    }



























    /**













     * Starts a connectionless stylus handwriting session (see {@link













     * #startConnectionlessStylusHandwriting}) and additionally enables the recognised handwritten













     * text to be later committed to a text editor using {@link













     * #acceptStylusHandwritingDelegation(View)}.













     *













     * <p>After a connectionless session started using this method completes successfully, a text













     * editor view, called the delegate view, may call {@link













     * #acceptStylusHandwritingDelegation(View)} which will request the IME to commit the recognised













     * handwritten text from the connectionless session to the delegate view.













     *













     * <p>The delegate view must belong to the same package as the delegator view for the delegation













     * to succeed. If the delegate view belongs to a different package, use {@link













     * #startConnectionlessStylusHandwritingForDelegation(View, CursorAnchorInfo, String, Executor,













     * ConnectionlessHandwritingCallback)} instead.













     *













     * @param delegatorView the view receiving stylus events













     * @param cursorAnchorInfo positional information about the view receiving stylus events













     * @param callbackExecutor the executor to run the callback on













     * @param callback the callback to receive the result













     */













    @FlaggedApi(Flags.FLAG_CONNECTIONLESS_HANDWRITING)













    public void startConnectionlessStylusHandwritingForDelegation(@NonNull View delegatorView,













            @Nullable CursorAnchorInfo cursorAnchorInfo,













            @NonNull @CallbackExecutor Executor callbackExecutor,













            @NonNull ConnectionlessHandwritingCallback callback) {













        String delegatorPackageName = delegatorView.getContext().getOpPackageName();













        startConnectionlessStylusHandwritingInternal(delegatorView, cursorAnchorInfo,













                delegatorPackageName, delegatorPackageName, callbackExecutor, callback);













    }



























    /**













     * Starts a connectionless stylus handwriting session (see {@link













     * #startConnectionlessStylusHandwriting}) and additionally enables the recognised handwritten













     * text to be later committed to a text editor using {@link













     * #acceptStylusHandwritingDelegation(View, String)}.













     *













     * <p>After a connectionless session started using this method completes successfully, a text













     * editor view, called the delegate view, may call {@link













     * #acceptStylusHandwritingDelegation(View, String)} which will request the IME to commit the













     * recognised handwritten text from the connectionless session to the delegate view.













     *













     * <p>The delegate view must belong to {@code delegatePackageName} for the delegation to













     * succeed.













     *













     * @param delegatorView the view receiving stylus events













     * @param cursorAnchorInfo positional information about the view receiving stylus events













     * @param delegatePackageName name of the package containing the delegate view which will accept













     *     the delegation













     * @param callbackExecutor the executor to run the callback on













     * @param callback the callback to receive the result













     */













    @FlaggedApi(Flags.FLAG_CONNECTIONLESS_HANDWRITING)













    public void startConnectionlessStylusHandwritingForDelegation(@NonNull View delegatorView,













            @Nullable CursorAnchorInfo cursorAnchorInfo,













            @NonNull String delegatePackageName,













            @NonNull @CallbackExecutor Executor callbackExecutor,













            @NonNull ConnectionlessHandwritingCallback callback) {













        Objects.requireNonNull(delegatePackageName);













        String delegatorPackageName = delegatorView.getContext().getOpPackageName();













        startConnectionlessStylusHandwritingInternal(delegatorView, cursorAnchorInfo,













                delegatorPackageName, delegatePackageName, callbackExecutor, callback);













    }



























    private void startConnectionlessStylusHandwritingInternal(@NonNull View view,













            @Nullable CursorAnchorInfo cursorAnchorInfo,













            @Nullable String delegatorPackageName,













            @Nullable String delegatePackageName,













            @NonNull @CallbackExecutor Executor callbackExecutor,













            @NonNull ConnectionlessHandwritingCallback callback) {













        Objects.requireNonNull(view);













        Objects.requireNonNull(callbackExecutor);













        Objects.requireNonNull(callback);













        // Re-dispatch if there is a context mismatch.













        final InputMethodManager fallbackImm = getFallbackInputMethodManagerIfNecessary(view);













        if (fallbackImm != null) {













            fallbackImm.startConnectionlessStylusHandwritingInternal(view, cursorAnchorInfo,













                    delegatorPackageName, delegatePackageName, callbackExecutor, callback);













        }



























        checkFocus();













        synchronized (mH) {













            if (view.getViewRootImpl() != mCurRootView) {













                Log.w(TAG, "Ignoring startConnectionlessStylusHandwriting: "













                        + "View's window does not have focus.");













                return;













            }













            IInputMethodManagerGlobalInvoker.startConnectionlessStylusHandwriting(













                    mClient, UserHandle.myUserId(), cursorAnchorInfo,













                    delegatePackageName, delegatorPackageName,













                    new ConnectionlessHandwritingCallbackProxy(callbackExecutor, callback));













        }













    }





























    /**















     * Prepares delegation of starting stylus handwriting session to a different editor in same















     * or different window than the view on which initial handwriting stroke was detected.










@@ -2553,12 +2677,18 @@ public final class InputMethodManager {













     * {@link #acceptStylusHandwritingDelegation(View, String)} instead.</p>















     *















     * @param delegateView delegate view capable of receiving input via {@link InputConnection}













     *  on which {@link #startStylusHandwriting(View)} will be called.















     * @return {@code true} if view belongs to same application package as used in













     *  {@link #prepareStylusHandwritingDelegation(View)} and handwriting session can start.













     * @see #acceptStylusHandwritingDelegation(View, String)













     *  {@link #prepareStylusHandwritingDelegation(View)} and delegation is accepted















     * @see #prepareStylusHandwritingDelegation(View)













     * @see #acceptStylusHandwritingDelegation(View, String)















     */













    // TODO(b/300979854): Once connectionless APIs are finalised, update documentation to add:













    // <p>Otherwise, if the delegator view previously started delegation using {@link













    // #startConnectionlessStylusHandwritingForDelegation(View, ResultReceiver, CursorAnchorInfo)},













    // requests the IME to commit the recognised handwritten text from the connectionless session to













    // the delegate view.













    // @see #startConnectionlessStylusHandwritingForDelegation(View, ResultReceiver,













    //     CursorAnchorInfo)















    public boolean acceptStylusHandwritingDelegation(@NonNull View delegateView) {















        return startStylusHandwritingInternal(















                delegateView, delegateView.getContext().getOpPackageName(),









@@ -2575,13 +2705,19 @@ public final class InputMethodManager {













     * {@link #acceptStylusHandwritingDelegation(View)} instead.</p>















     *















     * @param delegateView delegate view capable of receiving input via {@link InputConnection}













     *  on which {@link #startStylusHandwriting(View)} will be called.















     * @param delegatorPackageName package name of the delegator that handled initial stylus stroke.













     * @return {@code true} if view belongs to allowed delegate package declared in













     *  {@link #prepareStylusHandwritingDelegation(View, String)} and handwriting session can start.













     * @return {@code true} if view belongs to allowed delegate package declared in {@link













     *     #prepareStylusHandwritingDelegation(View, String)} and delegation is accepted















     * @see #prepareStylusHandwritingDelegation(View, String)















     * @see #acceptStylusHandwritingDelegation(View)















     */













    // TODO(b/300979854): Once connectionless APIs are finalised, update documentation to add:













    // <p>Otherwise, if the delegator view previously started delegation using {@link













    // #startConnectionlessStylusHandwritingForDelegation(View, ResultReceiver, CursorAnchorInfo,













    // String)}, requests the IME to commit the recognised handwritten text from the connectionless













    // session to the delegate view.













    // @see #startConnectionlessStylusHandwritingForDelegation(View, ResultReceiver,













    //     CursorAnchorInfo, String)















    public boolean acceptStylusHandwritingDelegation(















            @NonNull View delegateView, @NonNull String delegatorPackageName) {















        Objects.requireNonNull(delegatorPackageName);









@@ -2598,15 +2734,21 @@ public final class InputMethodManager {













     * <p>Note: If delegator and delegate are in the same application package, use {@link















     * #acceptStylusHandwritingDelegation(View)} instead.















     *













     * @param delegateView delegate view capable of receiving input via {@link InputConnection} on













     *     which {@link #startStylusHandwriting(View)} will be called.













     * @param delegateView delegate view capable of receiving input via {@link InputConnection}















     * @param delegatorPackageName package name of the delegator that handled initial stylus stroke.















     * @param flags {@link #HANDWRITING_DELEGATE_FLAG_HOME_DELEGATOR_ALLOWED} or {@code 0}















     * @return {@code true} if view belongs to allowed delegate package declared in {@link













     *     #prepareStylusHandwritingDelegation(View, String)} and handwriting session can start.













     *     #prepareStylusHandwritingDelegation(View, String)} and delegation is accepted















     * @see #prepareStylusHandwritingDelegation(View, String)















     * @see #acceptStylusHandwritingDelegation(View)















     */













    // TODO(b/300979854): Once connectionless APIs are finalised, update documentation to add:













    // <p>Otherwise, if the delegator view previously started delegation using {@link













    // #startConnectionlessStylusHandwritingForDelegation(View, ResultReceiver, CursorAnchorInfo,













    // String)}, requests the IME to commit the recognised handwritten text from the connectionless













    // session to the delegate view.













    // @see #startConnectionlessStylusHandwritingForDelegation(View, ResultReceiver,













    //     CursorAnchorInfo, String)















    @FlaggedApi(FLAG_HOME_SCREEN_HANDWRITING_DELEGATOR)















    public boolean acceptStylusHandwritingDelegation(















            @NonNull View delegateView, @NonNull String delegatorPackageName,










@@ -4357,6 +4499,73 @@ public final class InputMethodManager {













        public void onFinishedInputEvent(Object token, boolean handled);















    }





























    private static class ConnectionlessHandwritingCallbackProxy













            extends IConnectionlessHandwritingCallback.Stub {













        private final Object mLock = new Object();



























        @Nullable













        @GuardedBy("mLock")













        private Executor mExecutor;



























        @Nullable













        @GuardedBy("mLock")













        private ConnectionlessHandwritingCallback mCallback;



























        ConnectionlessHandwritingCallbackProxy(













                @NonNull Executor executor, @NonNull ConnectionlessHandwritingCallback callback) {













            mExecutor = executor;













            mCallback = callback;













        }



























        @Override













        public void onResult(CharSequence text) {













            Executor executor;













            ConnectionlessHandwritingCallback callback;













            synchronized (mLock) {













                if (mExecutor == null || mCallback == null) {













                    return;













                }













                executor = mExecutor;













                callback = mCallback;













                mExecutor = null;













                mCallback = null;













            }













            final long identity = Binder.clearCallingIdentity();













            try {













                if (TextUtils.isEmpty(text)) {













                    executor.execute(() -> callback.onError(













                            ConnectionlessHandwritingCallback













                                    .CONNECTIONLESS_HANDWRITING_ERROR_NO_TEXT_RECOGNIZED));













                } else {













                    executor.execute(() -> callback.onResult(text));













                }













            } finally {













                Binder.restoreCallingIdentity(identity);













            }













        }



























        @Override













        public void onError(int errorCode) {













            Executor executor;













            ConnectionlessHandwritingCallback callback;













            synchronized (mLock) {













                if (mExecutor == null || mCallback == null) {













                    return;













                }













                executor = mExecutor;













                callback = mCallback;













                mExecutor = null;













                mCallback = null;













            }













            final long identity = Binder.clearCallingIdentity();













            try {













                executor.execute(() -> callback.onError(errorCode));













            } finally {













                Binder.restoreCallingIdentity(identity);













            }













        }













    }





























    private final class ImeInputEventSender extends InputEventSender {















        public ImeInputEventSender(InputChannel inputChannel, Looper looper) {















            super(inputChannel, looper);

































core/java/com/android/internal/inputmethod/IConnectionlessHandwritingCallback.aidl
0 → 100644



+23
−0




























Original line number
Diff line number
Diff line













/*













 * Copyright (C) 2024 The Android Open Source Project













 *













 * Licensed under the Apache License, Version 2.0 (the "License");













 * you may not use this file except in compliance with the License.













 * You may obtain a copy of the License at













 *













 *      http://www.apache.org/licenses/LICENSE-2.0













 *













 * Unless required by applicable law or agreed to in writing, software













 * distributed under the License is distributed on an "AS IS" BASIS,













 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.













 * See the License for the specific language governing permissions and













 * limitations under the License.













 */



























package com.android.internal.inputmethod;



























/** Binder interface to receive a result from a connectionless stylus handwriting session. */













oneway interface IConnectionlessHandwritingCallback {













    void onResult(in CharSequence text);













    void onError(int errorCode);













}