Merge "Connectionless handwriting APIs for InputMethodManager" into main (1aec8253) · Commits · e / os / android_frameworks_base

core/api/current.txt

+11 −0

Original line number	Diff line number	Diff line
		@@ -55702,6 +55702,14 @@ package android.view.inputmethod {
		field @NonNull public static final android.os.Parcelable.Creator<android.view.inputmethod.CompletionInfo> CREATOR;
		}

		@FlaggedApi("android.view.inputmethod.connectionless_handwriting") public interface ConnectionlessHandwritingCallback {
		method public void onError(int);
		method public void onResult(@NonNull CharSequence);
		field public static final int CONNECTIONLESS_HANDWRITING_ERROR_NO_TEXT_RECOGNIZED = 0; // 0x0
		field public static final int CONNECTIONLESS_HANDWRITING_ERROR_OTHER = 2; // 0x2
		field public static final int CONNECTIONLESS_HANDWRITING_ERROR_UNSUPPORTED = 1; // 0x1
		}

		public final class CorrectionInfo implements android.os.Parcelable {
		ctor public CorrectionInfo(int, CharSequence, CharSequence);
		method public int describeContents();
		@@ -56164,6 +56172,9 @@ package android.view.inputmethod {
		method public boolean showSoftInput(android.view.View, int, android.os.ResultReceiver);
		method @Deprecated public void showSoftInputFromInputMethod(android.os.IBinder, int);
		method @Deprecated public void showStatusIcon(android.os.IBinder, String, @DrawableRes int);
		method @FlaggedApi("android.view.inputmethod.connectionless_handwriting") public void startConnectionlessStylusHandwriting(@NonNull android.view.View, @Nullable android.view.inputmethod.CursorAnchorInfo, @NonNull java.util.concurrent.Executor, @NonNull android.view.inputmethod.ConnectionlessHandwritingCallback);
		method @FlaggedApi("android.view.inputmethod.connectionless_handwriting") public void startConnectionlessStylusHandwritingForDelegation(@NonNull android.view.View, @Nullable android.view.inputmethod.CursorAnchorInfo, @NonNull java.util.concurrent.Executor, @NonNull android.view.inputmethod.ConnectionlessHandwritingCallback);
		method @FlaggedApi("android.view.inputmethod.connectionless_handwriting") public void startConnectionlessStylusHandwritingForDelegation(@NonNull android.view.View, @Nullable android.view.inputmethod.CursorAnchorInfo, @NonNull String, @NonNull java.util.concurrent.Executor, @NonNull android.view.inputmethod.ConnectionlessHandwritingCallback);
		method public void startStylusHandwriting(@NonNull android.view.View);
		method @Deprecated public boolean switchToLastInputMethod(android.os.IBinder);
		method @Deprecated public boolean switchToNextInputMethod(android.os.IBinder, boolean);

core/java/android/view/inputmethod/ConnectionlessHandwritingCallback.java

0 → 100644

+77 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2024 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package android.view.inputmethod;

		import android.annotation.FlaggedApi;
		import android.annotation.IntDef;
		import android.annotation.NonNull;
		import android.view.View;

		import java.lang.annotation.Retention;
		import java.lang.annotation.RetentionPolicy;
		import java.util.concurrent.Executor;

		/**
		* Interface to receive the result of starting a connectionless stylus handwriting session using
		* one of {@link InputMethodManager#startConnectionlessStylusHandwriting(View, CursorAnchorInfo,
		* Executor,ConnectionlessHandwritingCallback)}, {@link
		* InputMethodManager#startConnectionlessStylusHandwritingForDelegation(View, CursorAnchorInfo,
		* Executor, ConnectionlessHandwritingCallback)}, or {@link
		* InputMethodManager#startConnectionlessStylusHandwritingForDelegation(View, CursorAnchorInfo,
		* String, Executor, ConnectionlessHandwritingCallback)}.
		*/
		@FlaggedApi(Flags.FLAG_CONNECTIONLESS_HANDWRITING)
		public interface ConnectionlessHandwritingCallback {

		/** @hide */
		@IntDef(prefix = {"CONNECTIONLESS_HANDWRITING_ERROR_"}, value = {
		CONNECTIONLESS_HANDWRITING_ERROR_NO_TEXT_RECOGNIZED,
		CONNECTIONLESS_HANDWRITING_ERROR_UNSUPPORTED,
		CONNECTIONLESS_HANDWRITING_ERROR_OTHER
		})
		@Retention(RetentionPolicy.SOURCE)
		@interface ConnectionlessHandwritingError {
		}

		/**
		* Error code indicating that the connectionless handwriting session started and completed
		* but no text was recognized.
		*/
		int CONNECTIONLESS_HANDWRITING_ERROR_NO_TEXT_RECOGNIZED = 0;

		/**
		* Error code indicating that the connectionless handwriting session was not started as the
		* current IME does not support it.
		*/
		int CONNECTIONLESS_HANDWRITING_ERROR_UNSUPPORTED = 1;

		/**
		* Error code for any other reason that the connectionless handwriting session did not complete
		* successfully. Either the session could not start, or the session started but did not complete
		* successfully.
		*/
		int CONNECTIONLESS_HANDWRITING_ERROR_OTHER = 2;

		/**
		* Callback when the connectionless handwriting session completed successfully and
		* recognized text.
		*/
		void onResult(@NonNull CharSequence text);

		/** Callback when the connectionless handwriting session did not complete successfully. */
		void onError(@ConnectionlessHandwritingError int errorCode);
		}

core/java/android/view/inputmethod/IInputMethodManagerGlobalInvoker.java

+22 −0

Original line number	Diff line number	Diff line
		@@ -34,6 +34,7 @@ import android.view.WindowManager;
		import android.window.ImeOnBackInvokedDispatcher;

		import com.android.internal.inputmethod.DirectBootAwareness;
		import com.android.internal.inputmethod.IConnectionlessHandwritingCallback;
		import com.android.internal.inputmethod.IImeTracker;
		import com.android.internal.inputmethod.IInputMethodClient;
		import com.android.internal.inputmethod.IRemoteAccessibilityInputConnection;
		@@ -491,6 +492,27 @@ final class IInputMethodManagerGlobalInvoker {
		}
		}

		@AnyThread
		static boolean startConnectionlessStylusHandwriting(
		@NonNull IInputMethodClient client,
		@UserIdInt int userId,
		@Nullable CursorAnchorInfo cursorAnchorInfo,
		@Nullable String delegatePackageName,
		@Nullable String delegatorPackageName,
		@NonNull IConnectionlessHandwritingCallback callback) {
		final IInputMethodManager service = getService();
		if (service == null) {
		return false;
		}
		try {
		service.startConnectionlessStylusHandwriting(client, userId, cursorAnchorInfo,
		delegatePackageName, delegatorPackageName, callback);
		} catch (RemoteException e) {
		throw e.rethrowFromSystemServer();
		}
		return true;
		}

		@AnyThread
		static void prepareStylusHandwritingDelegation(
		@NonNull IInputMethodClient client,

core/java/android/view/inputmethod/InputMethodManager.java

+218 −9

Original line number	Diff line number	Diff line
		@@ -38,6 +38,7 @@ import static android.view.inputmethod.InputMethodManagerProto.SERVED_VIEW;
		import static com.android.internal.inputmethod.StartInputReason.BOUND_TO_IMMS;

		import android.Manifest;
		import android.annotation.CallbackExecutor;
		import android.annotation.DisplayContext;
		import android.annotation.DrawableRes;
		import android.annotation.DurationMillisLong;
		@@ -108,6 +109,7 @@ import android.window.WindowOnBackInvokedDispatcher;

		import com.android.internal.annotations.GuardedBy;
		import com.android.internal.inputmethod.DirectBootAwareness;
		import com.android.internal.inputmethod.IConnectionlessHandwritingCallback;
		import com.android.internal.inputmethod.IInputMethodClient;
		import com.android.internal.inputmethod.IInputMethodSession;
		import com.android.internal.inputmethod.IRemoteAccessibilityInputConnection;
		@@ -134,6 +136,7 @@ import java.util.List;
		import java.util.Map;
		import java.util.Objects;
		import java.util.concurrent.CountDownLatch;
		import java.util.concurrent.Executor;
		import java.util.concurrent.TimeUnit;
		import java.util.concurrent.atomic.AtomicBoolean;
		import java.util.function.Consumer;
		@@ -2474,6 +2477,127 @@ public final class InputMethodManager {
		}
		}

		/**
		* Starts a connectionless stylus handwriting session. A connectionless session differs from a
		* regular stylus handwriting session in that the IME does not use an input connection to
		* communicate with a text editor. Instead, the IME directly returns recognised handwritten text
		* via a callback.
		*
		* <p>The {code cursorAnchorInfo} may be used by the IME to improve the handwriting recognition
		* accuracy and user experience of the handwriting session. Usually connectionless handwriting
		* is used for a view which appears like a text editor but does not really support text editing.
		* For best results, the {code cursorAnchorInfo} should be populated as it would be for a real
		* text editor (for example, the insertion marker location can be set to where the user would
		* expect it to be, even if there is no visible cursor).
		*
		* @param view the view receiving stylus events
		* @param cursorAnchorInfo positional information about the view receiving stylus events
		* @param callbackExecutor the executor to run the callback on
		* @param callback the callback to receive the result
		*/
		@FlaggedApi(Flags.FLAG_CONNECTIONLESS_HANDWRITING)
		public void startConnectionlessStylusHandwriting(@NonNull View view,
		@Nullable CursorAnchorInfo cursorAnchorInfo,
		@NonNull @CallbackExecutor Executor callbackExecutor,
		@NonNull ConnectionlessHandwritingCallback callback) {
		startConnectionlessStylusHandwritingInternal(
		view, cursorAnchorInfo, null, null, callbackExecutor, callback);
		}

		/**
		* Starts a connectionless stylus handwriting session (see {@link
		* #startConnectionlessStylusHandwriting}) and additionally enables the recognised handwritten
		* text to be later committed to a text editor using {@link
		* #acceptStylusHandwritingDelegation(View)}.
		*
		* <p>After a connectionless session started using this method completes successfully, a text
		* editor view, called the delegate view, may call {@link
		* #acceptStylusHandwritingDelegation(View)} which will request the IME to commit the recognised
		* handwritten text from the connectionless session to the delegate view.
		*
		* <p>The delegate view must belong to the same package as the delegator view for the delegation
		* to succeed. If the delegate view belongs to a different package, use {@link
		* #startConnectionlessStylusHandwritingForDelegation(View, CursorAnchorInfo, String, Executor,
		* ConnectionlessHandwritingCallback)} instead.
		*
		* @param delegatorView the view receiving stylus events
		* @param cursorAnchorInfo positional information about the view receiving stylus events
		* @param callbackExecutor the executor to run the callback on
		* @param callback the callback to receive the result
		*/
		@FlaggedApi(Flags.FLAG_CONNECTIONLESS_HANDWRITING)
		public void startConnectionlessStylusHandwritingForDelegation(@NonNull View delegatorView,
		@Nullable CursorAnchorInfo cursorAnchorInfo,
		@NonNull @CallbackExecutor Executor callbackExecutor,
		@NonNull ConnectionlessHandwritingCallback callback) {
		String delegatorPackageName = delegatorView.getContext().getOpPackageName();
		startConnectionlessStylusHandwritingInternal(delegatorView, cursorAnchorInfo,
		delegatorPackageName, delegatorPackageName, callbackExecutor, callback);
		}

		/**
		* Starts a connectionless stylus handwriting session (see {@link
		* #startConnectionlessStylusHandwriting}) and additionally enables the recognised handwritten
		* text to be later committed to a text editor using {@link
		* #acceptStylusHandwritingDelegation(View, String)}.
		*
		* <p>After a connectionless session started using this method completes successfully, a text
		* editor view, called the delegate view, may call {@link
		* #acceptStylusHandwritingDelegation(View, String)} which will request the IME to commit the
		* recognised handwritten text from the connectionless session to the delegate view.
		*
		* <p>The delegate view must belong to {@code delegatePackageName} for the delegation to
		* succeed.
		*
		* @param delegatorView the view receiving stylus events
		* @param cursorAnchorInfo positional information about the view receiving stylus events
		* @param delegatePackageName name of the package containing the delegate view which will accept
		* the delegation
		* @param callbackExecutor the executor to run the callback on
		* @param callback the callback to receive the result
		*/
		@FlaggedApi(Flags.FLAG_CONNECTIONLESS_HANDWRITING)
		public void startConnectionlessStylusHandwritingForDelegation(@NonNull View delegatorView,
		@Nullable CursorAnchorInfo cursorAnchorInfo,
		@NonNull String delegatePackageName,
		@NonNull @CallbackExecutor Executor callbackExecutor,
		@NonNull ConnectionlessHandwritingCallback callback) {
		Objects.requireNonNull(delegatePackageName);
		String delegatorPackageName = delegatorView.getContext().getOpPackageName();
		startConnectionlessStylusHandwritingInternal(delegatorView, cursorAnchorInfo,
		delegatorPackageName, delegatePackageName, callbackExecutor, callback);
		}

		private void startConnectionlessStylusHandwritingInternal(@NonNull View view,
		@Nullable CursorAnchorInfo cursorAnchorInfo,
		@Nullable String delegatorPackageName,
		@Nullable String delegatePackageName,
		@NonNull @CallbackExecutor Executor callbackExecutor,
		@NonNull ConnectionlessHandwritingCallback callback) {
		Objects.requireNonNull(view);
		Objects.requireNonNull(callbackExecutor);
		Objects.requireNonNull(callback);
		// Re-dispatch if there is a context mismatch.
		final InputMethodManager fallbackImm = getFallbackInputMethodManagerIfNecessary(view);
		if (fallbackImm != null) {
		fallbackImm.startConnectionlessStylusHandwritingInternal(view, cursorAnchorInfo,
		delegatorPackageName, delegatePackageName, callbackExecutor, callback);
		}

		checkFocus();
		synchronized (mH) {
		if (view.getViewRootImpl() != mCurRootView) {
		Log.w(TAG, "Ignoring startConnectionlessStylusHandwriting: "
		+ "View's window does not have focus.");
		return;
		}
		IInputMethodManagerGlobalInvoker.startConnectionlessStylusHandwriting(
		mClient, UserHandle.myUserId(), cursorAnchorInfo,
		delegatePackageName, delegatorPackageName,
		new ConnectionlessHandwritingCallbackProxy(callbackExecutor, callback));
		}
		}

		/**
		* Prepares delegation of starting stylus handwriting session to a different editor in same
		* or different window than the view on which initial handwriting stroke was detected.
		@@ -2553,12 +2677,18 @@ public final class InputMethodManager {
		* {@link #acceptStylusHandwritingDelegation(View, String)} instead.</p>
		*
		* @param delegateView delegate view capable of receiving input via {@link InputConnection}
		* on which {@link #startStylusHandwriting(View)} will be called.
		* @return {@code true} if view belongs to same application package as used in
		* {@link #prepareStylusHandwritingDelegation(View)} and handwriting session can start.
		* @see #acceptStylusHandwritingDelegation(View, String)
		* {@link #prepareStylusHandwritingDelegation(View)} and delegation is accepted
		* @see #prepareStylusHandwritingDelegation(View)
		* @see #acceptStylusHandwritingDelegation(View, String)
		*/
		// TODO(b/300979854): Once connectionless APIs are finalised, update documentation to add:
		// <p>Otherwise, if the delegator view previously started delegation using {@link
		// #startConnectionlessStylusHandwritingForDelegation(View, ResultReceiver, CursorAnchorInfo)},
		// requests the IME to commit the recognised handwritten text from the connectionless session to
		// the delegate view.
		// @see #startConnectionlessStylusHandwritingForDelegation(View, ResultReceiver,
		// CursorAnchorInfo)
		public boolean acceptStylusHandwritingDelegation(@NonNull View delegateView) {
		return startStylusHandwritingInternal(
		delegateView, delegateView.getContext().getOpPackageName(),
		@@ -2575,13 +2705,19 @@ public final class InputMethodManager {
		* {@link #acceptStylusHandwritingDelegation(View)} instead.</p>
		*
		* @param delegateView delegate view capable of receiving input via {@link InputConnection}
		* on which {@link #startStylusHandwriting(View)} will be called.
		* @param delegatorPackageName package name of the delegator that handled initial stylus stroke.
		* @return {@code true} if view belongs to allowed delegate package declared in
		* {@link #prepareStylusHandwritingDelegation(View, String)} and handwriting session can start.
		* @return {@code true} if view belongs to allowed delegate package declared in {@link
		* #prepareStylusHandwritingDelegation(View, String)} and delegation is accepted
		* @see #prepareStylusHandwritingDelegation(View, String)
		* @see #acceptStylusHandwritingDelegation(View)
		*/
		// TODO(b/300979854): Once connectionless APIs are finalised, update documentation to add:
		// <p>Otherwise, if the delegator view previously started delegation using {@link
		// #startConnectionlessStylusHandwritingForDelegation(View, ResultReceiver, CursorAnchorInfo,
		// String)}, requests the IME to commit the recognised handwritten text from the connectionless
		// session to the delegate view.
		// @see #startConnectionlessStylusHandwritingForDelegation(View, ResultReceiver,
		// CursorAnchorInfo, String)
		public boolean acceptStylusHandwritingDelegation(
		@NonNull View delegateView, @NonNull String delegatorPackageName) {
		Objects.requireNonNull(delegatorPackageName);
		@@ -2598,15 +2734,21 @@ public final class InputMethodManager {
		* <p>Note: If delegator and delegate are in the same application package, use {@link
		* #acceptStylusHandwritingDelegation(View)} instead.
		*
		* @param delegateView delegate view capable of receiving input via {@link InputConnection} on
		* which {@link #startStylusHandwriting(View)} will be called.
		* @param delegateView delegate view capable of receiving input via {@link InputConnection}
		* @param delegatorPackageName package name of the delegator that handled initial stylus stroke.
		* @param flags {@link #HANDWRITING_DELEGATE_FLAG_HOME_DELEGATOR_ALLOWED} or {@code 0}
		* @return {@code true} if view belongs to allowed delegate package declared in {@link
		* #prepareStylusHandwritingDelegation(View, String)} and handwriting session can start.
		* #prepareStylusHandwritingDelegation(View, String)} and delegation is accepted
		* @see #prepareStylusHandwritingDelegation(View, String)
		* @see #acceptStylusHandwritingDelegation(View)
		*/
		// TODO(b/300979854): Once connectionless APIs are finalised, update documentation to add:
		// <p>Otherwise, if the delegator view previously started delegation using {@link
		// #startConnectionlessStylusHandwritingForDelegation(View, ResultReceiver, CursorAnchorInfo,
		// String)}, requests the IME to commit the recognised handwritten text from the connectionless
		// session to the delegate view.
		// @see #startConnectionlessStylusHandwritingForDelegation(View, ResultReceiver,
		// CursorAnchorInfo, String)
		@FlaggedApi(FLAG_HOME_SCREEN_HANDWRITING_DELEGATOR)
		public boolean acceptStylusHandwritingDelegation(
		@NonNull View delegateView, @NonNull String delegatorPackageName,
		@@ -4357,6 +4499,73 @@ public final class InputMethodManager {
		public void onFinishedInputEvent(Object token, boolean handled);
		}

		private static class ConnectionlessHandwritingCallbackProxy
		extends IConnectionlessHandwritingCallback.Stub {
		private final Object mLock = new Object();

		@Nullable
		@GuardedBy("mLock")
		private Executor mExecutor;

		@Nullable
		@GuardedBy("mLock")
		private ConnectionlessHandwritingCallback mCallback;

		ConnectionlessHandwritingCallbackProxy(
		@NonNull Executor executor, @NonNull ConnectionlessHandwritingCallback callback) {
		mExecutor = executor;
		mCallback = callback;
		}

		@Override
		public void onResult(CharSequence text) {
		Executor executor;
		ConnectionlessHandwritingCallback callback;
		synchronized (mLock) {
		if (mExecutor == null \|\| mCallback == null) {
		return;
		}
		executor = mExecutor;
		callback = mCallback;
		mExecutor = null;
		mCallback = null;
		}
		final long identity = Binder.clearCallingIdentity();
		try {
		if (TextUtils.isEmpty(text)) {
		executor.execute(() -> callback.onError(
		ConnectionlessHandwritingCallback
		.CONNECTIONLESS_HANDWRITING_ERROR_NO_TEXT_RECOGNIZED));
		} else {
		executor.execute(() -> callback.onResult(text));
		}
		} finally {
		Binder.restoreCallingIdentity(identity);
		}
		}

		@Override
		public void onError(int errorCode) {
		Executor executor;
		ConnectionlessHandwritingCallback callback;
		synchronized (mLock) {
		if (mExecutor == null \|\| mCallback == null) {
		return;
		}
		executor = mExecutor;
		callback = mCallback;
		mExecutor = null;
		mCallback = null;
		}
		final long identity = Binder.clearCallingIdentity();
		try {
		executor.execute(() -> callback.onError(errorCode));
		} finally {
		Binder.restoreCallingIdentity(identity);
		}
		}
		}

		private final class ImeInputEventSender extends InputEventSender {
		public ImeInputEventSender(InputChannel inputChannel, Looper looper) {
		super(inputChannel, looper);

core/java/com/android/internal/inputmethod/IConnectionlessHandwritingCallback.aidl

0 → 100644

+23 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2024 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package com.android.internal.inputmethod;

		/** Binder interface to receive a result from a connectionless stylus handwriting session. */
		oneway interface IConnectionlessHandwritingCallback {
		void onResult(in CharSequence text);
		void onError(int errorCode);
		}