Merge "Connectionless handwriting APIs for InputMethodService" into main (076ded06) · Commits · e / os / android_frameworks_base

core/api/current.txt

+2 −0

Original line number	Diff line number	Diff line
		@@ -20665,6 +20665,7 @@ package android.inputmethodservice {
		@UiContext public class InputMethodService extends android.inputmethodservice.AbstractInputMethodService {
		ctor public InputMethodService();
		method @Deprecated public boolean enableHardwareAcceleration();
		method @FlaggedApi("android.view.inputmethod.connectionless_handwriting") public final void finishConnectionlessStylusHandwriting(@Nullable CharSequence);
		method public final void finishStylusHandwriting();
		method public int getBackDisposition();
		method public int getCandidatesHiddenVisibility();
		@@ -20718,6 +20719,7 @@ package android.inputmethodservice {
		method public void onPrepareStylusHandwriting();
		method public boolean onShowInputRequested(int, boolean);
		method public void onStartCandidatesView(android.view.inputmethod.EditorInfo, boolean);
		method @FlaggedApi("android.view.inputmethod.connectionless_handwriting") public boolean onStartConnectionlessStylusHandwriting(int, @Nullable android.view.inputmethod.CursorAnchorInfo);
		method public void onStartInput(android.view.inputmethod.EditorInfo, boolean);
		method public void onStartInputView(android.view.inputmethod.EditorInfo, boolean);
		method public boolean onStartStylusHandwriting();

core/java/android/inputmethodservice/IInputMethodWrapper.java

+41 −4

Original line number	Diff line number	Diff line
		@@ -32,6 +32,7 @@ import android.os.ResultReceiver;
		import android.util.Log;
		import android.view.InputChannel;
		import android.view.MotionEvent;
		import android.view.inputmethod.CursorAnchorInfo;
		import android.view.inputmethod.ImeTracker;
		import android.view.inputmethod.InputBinding;
		import android.view.inputmethod.InputConnection;
		@@ -40,6 +41,7 @@ import android.view.inputmethod.InputMethodSession;
		import android.view.inputmethod.InputMethodSubtype;

		import com.android.internal.inputmethod.CancellationGroup;
		import com.android.internal.inputmethod.IConnectionlessHandwritingCallback;
		import com.android.internal.inputmethod.IInlineSuggestionsRequestCallback;
		import com.android.internal.inputmethod.IInputMethod;
		import com.android.internal.inputmethod.IInputMethodSession;
		@@ -85,6 +87,8 @@ class IInputMethodWrapper extends IInputMethod.Stub
		private static final int DO_UPDATE_TOOL_TYPE = 140;
		private static final int DO_REMOVE_STYLUS_HANDWRITING_WINDOW = 150;
		private static final int DO_SET_STYLUS_WINDOW_IDLE_TIMEOUT = 160;
		private static final int DO_COMMIT_HANDWRITING_DELEGATION_TEXT_IF_AVAILABLE = 170;
		private static final int DO_DISCARD_HANDWRITING_DELEGATION_TEXT = 180;

		final WeakReference<InputMethodServiceInternal> mTarget;
		final Context mContext;
		@@ -265,9 +269,13 @@ class IInputMethodWrapper extends IInputMethod.Stub
		return;
		}
		case DO_CAN_START_STYLUS_HANDWRITING: {
		final SomeArgs args = (SomeArgs) msg.obj;
		if (isValid(inputMethod, target, "DO_CAN_START_STYLUS_HANDWRITING")) {
		inputMethod.canStartStylusHandwriting(msg.arg1);
		inputMethod.canStartStylusHandwriting(msg.arg1,
		(IConnectionlessHandwritingCallback) args.arg1,
		(CursorAnchorInfo) args.arg2, msg.arg2 != 0);
		}
		args.recycle();
		return;
		}
		case DO_UPDATE_TOOL_TYPE: {
		@@ -309,6 +317,19 @@ class IInputMethodWrapper extends IInputMethod.Stub
		}
		return;
		}
		case DO_COMMIT_HANDWRITING_DELEGATION_TEXT_IF_AVAILABLE: {
		if (isValid(inputMethod, target,
		"DO_COMMIT_HANDWRITING_DELEGATION_TEXT_IF_AVAILABLE")) {
		inputMethod.commitHandwritingDelegationTextIfAvailable();
		}
		return;
		}
		case DO_DISCARD_HANDWRITING_DELEGATION_TEXT: {
		if (isValid(inputMethod, target, "DO_DISCARD_HANDWRITING_DELEGATION_TEXT")) {
		inputMethod.discardHandwritingDelegationText();
		}
		return;
		}
		}
		Log.w(TAG, "Unhandled message code: " + msg.what);
		}
		@@ -457,10 +478,13 @@ class IInputMethodWrapper extends IInputMethod.Stub

		@BinderThread
		@Override
		public void canStartStylusHandwriting(int requestId)
		public void canStartStylusHandwriting(int requestId,
		IConnectionlessHandwritingCallback connectionlessCallback,
		CursorAnchorInfo cursorAnchorInfo, boolean isConnectionlessForDelegation)
		throws RemoteException {
		mCaller.executeOrSendMessage(
		mCaller.obtainMessageI(DO_CAN_START_STYLUS_HANDWRITING, requestId));
		mCaller.executeOrSendMessage(mCaller.obtainMessageIIOO(DO_CAN_START_STYLUS_HANDWRITING,
		requestId, isConnectionlessForDelegation ? 1 : 0, connectionlessCallback,
		cursorAnchorInfo));
		}

		@BinderThread
		@@ -481,6 +505,19 @@ class IInputMethodWrapper extends IInputMethod.Stub
		stylusEvents));
		}

		@BinderThread
		@Override
		public void commitHandwritingDelegationTextIfAvailable() {
		mCaller.executeOrSendMessage(
		mCaller.obtainMessage(DO_COMMIT_HANDWRITING_DELEGATION_TEXT_IF_AVAILABLE));
		}

		@BinderThread
		@Override
		public void discardHandwritingDelegationText() {
		mCaller.executeOrSendMessage(mCaller.obtainMessage(DO_DISCARD_HANDWRITING_DELEGATION_TEXT));
		}

		@BinderThread
		@Override
		public void initInkWindow() {

core/java/android/inputmethodservice/InputMethodService.java

+148 −6

Original line number	Diff line number	Diff line
		@@ -51,6 +51,10 @@ import static android.view.ViewGroup.LayoutParams.MATCH_PARENT;
		import static android.view.ViewGroup.LayoutParams.WRAP_CONTENT;
		import static android.view.WindowInsets.Type.navigationBars;
		import static android.view.WindowInsets.Type.statusBars;
		import static android.view.inputmethod.ConnectionlessHandwritingCallback.CONNECTIONLESS_HANDWRITING_ERROR_NO_TEXT_RECOGNIZED;
		import static android.view.inputmethod.ConnectionlessHandwritingCallback.CONNECTIONLESS_HANDWRITING_ERROR_OTHER;
		import static android.view.inputmethod.ConnectionlessHandwritingCallback.CONNECTIONLESS_HANDWRITING_ERROR_UNSUPPORTED;
		import static android.view.inputmethod.Flags.FLAG_CONNECTIONLESS_HANDWRITING;

		import static java.lang.annotation.RetentionPolicy.SOURCE;

		@@ -58,6 +62,7 @@ import android.annotation.AnyThread;
		import android.annotation.CallSuper;
		import android.annotation.DrawableRes;
		import android.annotation.DurationMillisLong;
		import android.annotation.FlaggedApi;
		import android.annotation.IntDef;
		import android.annotation.MainThread;
		import android.annotation.NonNull;
		@@ -89,6 +94,7 @@ import android.os.Handler;
		import android.os.IBinder;
		import android.os.Looper;
		import android.os.Process;
		import android.os.RemoteException;
		import android.os.ResultReceiver;
		import android.os.SystemClock;
		import android.os.SystemProperties;
		@@ -97,6 +103,7 @@ import android.provider.Settings;
		import android.text.InputType;
		import android.text.Layout;
		import android.text.Spannable;
		import android.text.TextUtils;
		import android.text.method.MovementMethod;
		import android.util.Log;
		import android.util.PrintWriterPrinter;
		@@ -123,6 +130,7 @@ import android.view.WindowInsets.Type;
		import android.view.WindowManager;
		import android.view.animation.AnimationUtils;
		import android.view.inputmethod.CompletionInfo;
		import android.view.inputmethod.ConnectionlessHandwritingCallback;
		import android.view.inputmethod.CursorAnchorInfo;
		import android.view.inputmethod.EditorInfo;
		import android.view.inputmethod.ExtractedText;
		@@ -151,6 +159,7 @@ import android.window.WindowMetricsHelper;

		import com.android.internal.annotations.GuardedBy;
		import com.android.internal.annotations.VisibleForTesting;
		import com.android.internal.inputmethod.IConnectionlessHandwritingCallback;
		import com.android.internal.inputmethod.IInlineSuggestionsRequestCallback;
		import com.android.internal.inputmethod.IInputContentUriToken;
		import com.android.internal.inputmethod.IInputMethod;
		@@ -174,6 +183,7 @@ import java.util.ArrayList;
		import java.util.List;
		import java.util.Objects;
		import java.util.OptionalInt;
		import java.util.concurrent.Executor;

		/**
		* InputMethodService provides a standard implementation of an InputMethod,
		@@ -665,6 +675,12 @@ public class InputMethodService extends AbstractInputMethodService {
		/** Stylus handwriting Ink window. */
		private InkWindow mInkWindow;

		private IConnectionlessHandwritingCallback mConnectionlessHandwritingCallback;
		private boolean mIsConnectionlessHandwritingForDelegation;
		// Holds the recognized text from a connectionless handwriting session which can later be
		// committed by commitHandwritingDelegationTextIfAvailable().
		private CharSequence mHandwritingDelegationText;

		/**
		* An opaque {@link Binder} token of window requesting {@link InputMethodImpl#showSoftInput}
		* The original app window token is passed from client app window.
		@@ -1017,7 +1033,10 @@ public class InputMethodService extends AbstractInputMethodService {
		* @hide
		*/
		@Override
		public void canStartStylusHandwriting(int requestId) {
		public void canStartStylusHandwriting(int requestId,
		@Nullable IConnectionlessHandwritingCallback connectionlessCallback,
		@Nullable CursorAnchorInfo cursorAnchorInfo,
		boolean isConnectionlessForDelegation) {
		if (DEBUG) Log.v(TAG, "canStartStylusHandwriting()");
		if (mHandwritingRequestId.isPresent()) {
		Log.d(TAG, "There is an ongoing Handwriting session. ignoring.");
		@@ -1034,7 +1053,24 @@ public class InputMethodService extends AbstractInputMethodService {
		}
		// reset flag as it's not relevant after onStartStylusHandwriting().
		mOnPreparedStylusHwCalled = false;
		if (onStartStylusHandwriting()) {
		if (connectionlessCallback != null) {
		if (onStartConnectionlessStylusHandwriting(
		InputType.TYPE_CLASS_TEXT, cursorAnchorInfo)) {
		mConnectionlessHandwritingCallback = connectionlessCallback;
		mIsConnectionlessHandwritingForDelegation = isConnectionlessForDelegation;
		cancelStylusWindowIdleTimeout();
		mPrivOps.onStylusHandwritingReady(requestId, Process.myPid());
		} else {
		Log.i(TAG, "IME is not ready "
		+ "or doesn't currently support connectionless handwriting");
		try {
		connectionlessCallback.onError(
		CONNECTIONLESS_HANDWRITING_ERROR_UNSUPPORTED);
		} catch (RemoteException e) {
		Log.e(TAG, "Couldn't send connectionless handwriting error result", e);
		}
		}
		} else if (onStartStylusHandwriting()) {
		cancelStylusWindowIdleTimeout();
		mPrivOps.onStylusHandwritingReady(requestId, Process.myPid());
		} else {
		@@ -1088,6 +1124,24 @@ public class InputMethodService extends AbstractInputMethodService {
		scheduleHandwritingSessionTimeout();
		}

		/**
		* {@inheritDoc}
		* @hide
		*/
		@Override
		public void commitHandwritingDelegationTextIfAvailable() {
		InputMethodService.this.commitHandwritingDelegationTextIfAvailable();
		}

		/**
		* {@inheritDoc}
		* @hide
		*/
		@Override
		public void discardHandwritingDelegationText() {
		InputMethodService.this.discardHandwritingDelegationText();
		}

		/**
		* {@inheritDoc}
		* @hide
		@@ -2580,6 +2634,32 @@ public class InputMethodService extends AbstractInputMethodService {
		return false;
		}

		/**
		* Called when an app requests to start a connectionless stylus handwriting session using one of
		* {@link InputMethodManager#startConnectionlessStylusHandwriting(View, CursorAnchorInfo,
		* Executor, ConnectionlessHandwritingCallback)}, {@link
		* InputMethodManager#startConnectionlessStylusHandwritingForDelegation(View, CursorAnchorInfo,
		* Executor, ConnectionlessHandwritingCallback)}, or {@link
		* InputMethodManager#startConnectionlessStylusHandwritingForDelegation(View, CursorAnchorInfo,
		* String, Executor, ConnectionlessHandwritingCallback)}.
		*
		* <p>A connectionless stylus handwriting session differs from a regular session in that an
		* input connection is not used to communicate with a text editor. Instead, the recognised text
		* is delivered when the IME finishes the connectionless session using {@link
		* #finishConnectionlessStylusHandwriting(CharSequence)}.
		*
		* <p>If the IME can start the connectionless handwriting session, it should return {@code
		* true}, ensure its inking views are attached to the {@link #getStylusHandwritingWindow()}, and
		* handle stylus input received from {@link #onStylusHandwritingMotionEvent(MotionEvent)} on the
		* {@link #getStylusHandwritingWindow()}.
		*/
		@FlaggedApi(FLAG_CONNECTIONLESS_HANDWRITING)
		public boolean onStartConnectionlessStylusHandwriting(
		int inputType, @Nullable CursorAnchorInfo cursorAnchorInfo) {
		// Intentionally empty
		return false;
		}

		/**
		* Called after {@link #onStartStylusHandwriting()} returns {@code true} for every Stylus
		* {@link MotionEvent}.
		@@ -2647,16 +2727,19 @@ public class InputMethodService extends AbstractInputMethodService {
		/**
		* Finish the current stylus handwriting session.
		*
		* This dismisses the {@link #getStylusHandwritingWindow ink window} and stops intercepting
		* <p>This dismisses the {@link #getStylusHandwritingWindow ink window} and stops intercepting
		* stylus {@code MotionEvent}s.
		*
		* Note for IME developers: Call this method at any time to finish current handwriting session.
		* Generally, this should be invoked after a short timeout, giving the user enough time
		* <p>Note for IME developers: Call this method at any time to finish the current handwriting
		* session. Generally, this should be invoked after a short timeout, giving the user enough time
		* to start the next stylus stroke, if any. By default, system will time-out after few seconds.
		* To override default timeout, use {@link #setStylusHandwritingSessionTimeout(Duration)}.
		*
		* Handwriting session will be finished by framework on next {@link #onFinishInput()}.
		* <p>Handwriting session will be finished by framework on next {@link #onFinishInput()}.
		*/
		// TODO(b/300979854): Once connectionless APIs are finalised, update documentation to add:
		// <p>Connectionless handwriting sessions should be finished using {@link
		// #finishConnectionlessStylusHandwriting(CharSequence)}.
		public final void finishStylusHandwriting() {
		if (DEBUG) Log.v(TAG, "finishStylusHandwriting()");
		if (mInkWindow == null) {
		@@ -2677,11 +2760,70 @@ public class InputMethodService extends AbstractInputMethodService {
		mHandwritingEventReceiver = null;
		mInkWindow.hide(false /* remove */);

		if (mConnectionlessHandwritingCallback != null) {
		Log.i(TAG, "Connectionless handwriting session did not complete successfully");
		try {
		mConnectionlessHandwritingCallback.onError(CONNECTIONLESS_HANDWRITING_ERROR_OTHER);
		} catch (RemoteException e) {
		Log.e(TAG, "Couldn't send connectionless handwriting error result", e);
		}
		mConnectionlessHandwritingCallback = null;
		}
		mIsConnectionlessHandwritingForDelegation = false;

		mPrivOps.resetStylusHandwriting(requestId);
		mOnPreparedStylusHwCalled = false;
		onFinishStylusHandwriting();
		}

		/**
		* Finishes the current connectionless stylus handwriting session and delivers the result.
		*
		* <p>This dismisses the {@link #getStylusHandwritingWindow ink window} and stops intercepting
		* stylus {@code MotionEvent}s.
		*
		* <p>Note for IME developers: Call this method at any time to finish the current handwriting
		* session. Generally, this should be invoked after a short timeout, giving the user enough time
		* to start the next stylus stroke, if any. By default, system will time-out after few seconds.
		* To override default timeout, use {@link #setStylusHandwritingSessionTimeout(Duration)}.
		*/
		@FlaggedApi(FLAG_CONNECTIONLESS_HANDWRITING)
		public final void finishConnectionlessStylusHandwriting(@Nullable CharSequence text) {
		if (DEBUG) Log.v(TAG, "finishConnectionlessStylusHandwriting()");
		if (mConnectionlessHandwritingCallback != null) {
		try {
		if (!TextUtils.isEmpty(text)) {
		mConnectionlessHandwritingCallback.onResult(text);
		if (mIsConnectionlessHandwritingForDelegation) {
		mHandwritingDelegationText = text;
		}
		} else {
		mConnectionlessHandwritingCallback.onError(
		CONNECTIONLESS_HANDWRITING_ERROR_NO_TEXT_RECOGNIZED);
		}
		} catch (RemoteException e) {
		Log.e(TAG, "Couldn't send connectionless handwriting result", e);
		}
		mConnectionlessHandwritingCallback = null;
		}
		finishStylusHandwriting();
		}

		private void commitHandwritingDelegationTextIfAvailable() {
		if (!TextUtils.isEmpty(mHandwritingDelegationText)) {
		InputConnection ic = getCurrentInputConnection();
		if (ic != null) {
		// Place cursor after inserted text.
		ic.commitText(mHandwritingDelegationText, /* newCursorPosition= */ 1);
		}
		}
		mHandwritingDelegationText = null;
		}

		private void discardHandwritingDelegationText() {
		mHandwritingDelegationText = null;
		}

		/**
		* Remove Stylus handwriting window.
		* Typically, this is called when {@link InkWindow} should no longer be holding a surface in

core/java/android/view/inputmethod/InputMethod.java

+33 −2

Original line number	Diff line number	Diff line
		@@ -32,6 +32,7 @@ import android.view.InputChannel;
		import android.view.MotionEvent;
		import android.view.View;

		import com.android.internal.inputmethod.IConnectionlessHandwritingCallback;
		import com.android.internal.inputmethod.IInlineSuggestionsRequestCallback;
		import com.android.internal.inputmethod.IInputMethod;
		import com.android.internal.inputmethod.InlineSuggestionsRequestInfo;
		@@ -387,10 +388,20 @@ public interface InputMethod {
		/**
		* Checks if IME is ready to start stylus handwriting session.
		* If yes, {@link #startStylusHandwriting(int, InputChannel, List)} is called.
		* @param requestId
		*
		* @param requestId identifier for the session start request
		* @param connectionlessCallback the callback to receive the session result if starting a
		* connectionless handwriting session, or null if starting a regular session
		* @param cursorAnchorInfo optional positional information about the view receiving stylus
		* events for a connectionless handwriting session
		* @param isConnectionlessForDelegation whether the connectionless handwriting session is for
		* delegation. If true, the recognised text should be saved and can later be committed by
		* {@link #commitHandwritingDelegationTextIfAvailable}.
		* @hide
		*/
		default void canStartStylusHandwriting(int requestId) {
		default void canStartStylusHandwriting(int requestId,
		@Nullable IConnectionlessHandwritingCallback connectionlessCallback,
		@Nullable CursorAnchorInfo cursorAnchorInfo, boolean isConnectionlessForDelegation) {
		// intentionally empty
		}

		@@ -412,6 +423,26 @@ public interface InputMethod {
		// intentionally empty
		}

		/**
		* Commits recognised text that was previously saved from a connectionless handwriting session
		* for delegation.
		*
		* @hide
		*/
		default void commitHandwritingDelegationTextIfAvailable() {
		// intentionally empty
		}

		/**
		* Discards recognised text that was previously saved from a connectionless handwriting session
		* for delegation.
		*
		* @hide
		*/
		default void discardHandwritingDelegationText() {
		// intentionally empty
		}

		/**
		* Initialize Ink window early-on.
		* @hide

core/java/com/android/internal/inputmethod/IInputMethod.aidl

+9 −1

Original line number	Diff line number	Diff line
		@@ -20,11 +20,13 @@ import android.os.IBinder;
		import android.os.ResultReceiver;
		import android.view.InputChannel;
		import android.view.MotionEvent;
		import android.view.inputmethod.CursorAnchorInfo;
		import android.view.inputmethod.EditorInfo;
		import android.view.inputmethod.ImeTracker;
		import android.view.inputmethod.InputBinding;
		import android.view.inputmethod.InputMethodSubtype;
		import android.window.ImeOnBackInvokedDispatcher;
		import com.android.internal.inputmethod.IConnectionlessHandwritingCallback;
		import com.android.internal.inputmethod.IInlineSuggestionsRequestCallback;
		import com.android.internal.inputmethod.IInputMethodPrivilegedOperations;
		import com.android.internal.inputmethod.IInputMethodSession;
		@@ -79,11 +81,17 @@ oneway interface IInputMethod {

		void changeInputMethodSubtype(in InputMethodSubtype subtype);

		void canStartStylusHandwriting(int requestId);
		void canStartStylusHandwriting(int requestId,
		in IConnectionlessHandwritingCallback connectionlessCallback,
		in CursorAnchorInfo cursorAnchorInfo, boolean isConnectionlessForDelegation);

		void startStylusHandwriting(int requestId, in InputChannel channel,
		in List<MotionEvent> events);

		void commitHandwritingDelegationTextIfAvailable();

		void discardHandwritingDelegationText();

		void initInkWindow();

		void finishStylusHandwriting();