Merge "Notify autofill with the IME start/finish input view events" into rvc-dev am: c9611acb62 (867f2e23) · Commits · e / os / android_frameworks_base

core/java/android/inputmethodservice/InlineSuggestionSession.java

+49 −12

Original line number	Diff line number	Diff line
		@@ -28,6 +28,7 @@ import android.os.Looper;
		import android.os.RemoteException;
		import android.util.Log;
		import android.view.autofill.AutofillId;
		import android.view.inputmethod.EditorInfo;
		import android.view.inputmethod.InlineSuggestionsRequest;
		import android.view.inputmethod.InlineSuggestionsResponse;

		@@ -45,10 +46,27 @@ import java.util.function.Supplier;
		* Each session corresponds to one {@link InlineSuggestionsRequest} and one {@link
		* IInlineSuggestionsResponseCallback}, but there may be multiple invocations of the response
		* callback for the same field or different fields in the same component.
		*
		* <p>
		* The data flow from IMS point of view is:
		* Before calling {@link InputMethodService#onStartInputView(EditorInfo, boolean)}, call the {@link
		* #notifyOnStartInputView(AutofillId)}
		* ->
		* [async] {@link IInlineSuggestionsRequestCallback#onInputMethodStartInputView(AutofillId)}
		* --- process boundary ---
		* ->
		* {@link com.android.server.inputmethod.InputMethodManagerService
		* .InlineSuggestionsRequestCallbackDecorator#onInputMethodStartInputView(AutofillId)}
		* ->
		* {@link com.android.server.autofill.InlineSuggestionSession
		* .InlineSuggestionsRequestCallbackImpl#onInputMethodStartInputView(AutofillId)}
		*
		* <p>
		* The data flow for {@link #notifyOnFinishInputView(AutofillId)} is similar.
		*/
		class InlineSuggestionSession {

		private static final String TAG = InlineSuggestionSession.class.getSimpleName();
		private static final String TAG = "ImsInlineSuggestionSession";

		private final Handler mHandler = new Handler(Looper.getMainLooper(), null, true);

		@@ -77,7 +95,8 @@ class InlineSuggestionSession {
		@NonNull Supplier<AutofillId> clientAutofillIdSupplier,
		@NonNull Supplier<InlineSuggestionsRequest> requestSupplier,
		@NonNull Supplier<IBinder> hostInputTokenSupplier,
		@NonNull Consumer<InlineSuggestionsResponse> responseConsumer) {
		@NonNull Consumer<InlineSuggestionsResponse> responseConsumer,
		boolean inputViewStarted) {
		mComponentName = componentName;
		mCallback = callback;
		mResponseCallback = new InlineSuggestionsResponseCallbackImpl(this);
		@@ -87,7 +106,25 @@ class InlineSuggestionSession {
		mHostInputTokenSupplier = hostInputTokenSupplier;
		mResponseConsumer = responseConsumer;

		makeInlineSuggestionsRequest();
		makeInlineSuggestionsRequest(inputViewStarted);
		}

		void notifyOnStartInputView(AutofillId imeFieldId) {
		if (DEBUG) Log.d(TAG, "notifyOnStartInputView");
		try {
		mCallback.onInputMethodStartInputView(imeFieldId);
		} catch (RemoteException e) {
		Log.w(TAG, "onInputMethodStartInputView() remote exception:" + e);
		}
		}

		void notifyOnFinishInputView(AutofillId imeFieldId) {
		if (DEBUG) Log.d(TAG, "notifyOnFinishInputView");
		try {
		mCallback.onInputMethodFinishInputView(imeFieldId);
		} catch (RemoteException e) {
		Log.w(TAG, "onInputMethodFinishInputView() remote exception:" + e);
		}
		}

		/**
		@@ -103,7 +140,7 @@ class InlineSuggestionSession {
		* Autofill Session through
		* {@link IInlineSuggestionsRequestCallback#onInlineSuggestionsRequest}.
		*/
		private void makeInlineSuggestionsRequest() {
		private void makeInlineSuggestionsRequest(boolean inputViewStarted) {
		try {
		final InlineSuggestionsRequest request = mRequestSupplier.get();
		if (request == null) {
		@@ -113,7 +150,8 @@ class InlineSuggestionSession {
		mCallback.onInlineSuggestionsUnsupported();
		} else {
		request.setHostInputToken(mHostInputTokenSupplier.get());
		mCallback.onInlineSuggestionsRequest(request, mResponseCallback);
		mCallback.onInlineSuggestionsRequest(request, mResponseCallback,
		mClientAutofillIdSupplier.get(), inputViewStarted);
		}
		} catch (RemoteException e) {
		Log.w(TAG, "makeInlinedSuggestionsRequest() remote exception:" + e);
		@@ -128,16 +166,15 @@ class InlineSuggestionSession {
		}
		return;
		}
		// TODO(b/149522488): Verify fieldId against {@code mClientAutofillIdSupplier.get()} using
		// {@link AutofillId#equalsIgnoreSession(AutofillId)}. Right now, this seems to be
		// falsely alarmed quite often, depending whether autofill suggestions arrive earlier
		// than the IMS EditorInfo updates or not.
		if (!mComponentName.getPackageName().equals(mClientPackageNameSupplier.get())) {

		if (!mComponentName.getPackageName().equals(mClientPackageNameSupplier.get())
		\|\| !fieldId.equalsIgnoreSession(mClientAutofillIdSupplier.get())) {
		if (DEBUG) {
		Log.d(TAG,
		"handleOnInlineSuggestionsResponse() called on the wrong package "
		"handleOnInlineSuggestionsResponse() called on the wrong package/field "
		+ "name: " + mComponentName.getPackageName() + " v.s. "
		+ mClientPackageNameSupplier.get());
		+ mClientPackageNameSupplier.get() + ", " + fieldId + " v.s. "
		+ mClientAutofillIdSupplier.get());
		}
		return;
		}

core/java/android/inputmethodservice/InputMethodService.java

+33 −6

Original line number	Diff line number	Diff line
		@@ -444,6 +444,16 @@ public class InputMethodService extends AbstractInputMethodService {
		final Insets mTmpInsets = new Insets();
		final int[] mTmpLocation = new int[2];

		/**
		* We use a separate {@code mInlineLock} to make sure {@code mInlineSuggestionSession} is
		* only accessed synchronously. Although when the lock is introduced, all the calls are from
		* the main thread so the lock is not really necessarily (but for the same reason it also
		* doesn't hurt), it's still being added as a safety guard to make sure in the future we
		* don't add more code causing race condition when updating the {@code
		* mInlineSuggestionSession}.
		*/
		private final Object mInlineLock = new Object();
		@GuardedBy("mInlineLock")
		@Nullable
		private InlineSuggestionSession mInlineSuggestionSession;

		@@ -822,13 +832,15 @@ public class InputMethodService extends AbstractInputMethodService {
		return;
		}

		synchronized (mInlineLock) {
		if (mInlineSuggestionSession != null) {
		mInlineSuggestionSession.invalidateSession();
		}
		mInlineSuggestionSession = new InlineSuggestionSession(requestInfo.getComponentName(),
		callback, this::getEditorInfoPackageName, this::getEditorInfoAutofillId,
		() -> onCreateInlineSuggestionsRequest(requestInfo.getUiExtras()),
		this::getHostInputToken, this::onInlineSuggestionsResponse);
		this::getHostInputToken, this::onInlineSuggestionsResponse, mInputViewStarted);
		}
		}

		@Nullable
		@@ -2193,6 +2205,11 @@ public class InputMethodService extends AbstractInputMethodService {
		if (!mInputViewStarted) {
		if (DEBUG) Log.v(TAG, "CALL: onStartInputView");
		mInputViewStarted = true;
		synchronized (mInlineLock) {
		if (mInlineSuggestionSession != null) {
		mInlineSuggestionSession.notifyOnStartInputView(getEditorInfoAutofillId());
		}
		}
		onStartInputView(mInputEditorInfo, false);
		}
		} else if (!mCandidatesViewStarted) {
		@@ -2233,6 +2250,11 @@ public class InputMethodService extends AbstractInputMethodService {
		private void finishViews(boolean finishingInput) {
		if (mInputViewStarted) {
		if (DEBUG) Log.v(TAG, "CALL: onFinishInputView");
		synchronized (mInlineLock) {
		if (mInlineSuggestionSession != null) {
		mInlineSuggestionSession.notifyOnFinishInputView(getEditorInfoAutofillId());
		}
		}
		onFinishInputView(finishingInput);
		} else if (mCandidatesViewStarted) {
		if (DEBUG) Log.v(TAG, "CALL: onFinishCandidatesView");
		@@ -2345,6 +2367,11 @@ public class InputMethodService extends AbstractInputMethodService {
		if (mShowInputRequested) {
		if (DEBUG) Log.v(TAG, "CALL: onStartInputView");
		mInputViewStarted = true;
		synchronized (mInlineLock) {
		if (mInlineSuggestionSession != null) {
		mInlineSuggestionSession.notifyOnStartInputView(getEditorInfoAutofillId());
		}
		}
		onStartInputView(mInputEditorInfo, restarting);
		startExtractingText(true);
		} else if (mCandidatesVisibility == View.VISIBLE) {

core/java/android/view/inputmethod/EditorInfo.java

+3 −4

Original line number	Diff line number	Diff line
		@@ -431,8 +431,7 @@ public class EditorInfo implements InputType, Parcelable {
		* <p> Marked as hide since it's only used by framework.</p>
		* @hide
		*/
		@NonNull
		public AutofillId autofillId = new AutofillId(View.NO_ID);
		public AutofillId autofillId;

		/**
		* Identifier for the editor's field. This is optional, and may be
		@@ -832,7 +831,7 @@ public class EditorInfo implements InputType, Parcelable {
		TextUtils.writeToParcel(hintText, dest, flags);
		TextUtils.writeToParcel(label, dest, flags);
		dest.writeString(packageName);
		autofillId.writeToParcel(dest, flags);
		dest.writeParcelable(autofillId, flags);
		dest.writeInt(fieldId);
		dest.writeString(fieldName);
		dest.writeBundle(extras);
		@@ -864,7 +863,7 @@ public class EditorInfo implements InputType, Parcelable {
		res.hintText = TextUtils.CHAR_SEQUENCE_CREATOR.createFromParcel(source);
		res.label = TextUtils.CHAR_SEQUENCE_CREATOR.createFromParcel(source);
		res.packageName = source.readString();
		res.autofillId = AutofillId.CREATOR.createFromParcel(source);
		res.autofillId = source.readParcelable(AutofillId.class.getClassLoader());
		res.fieldId = source.readInt();
		res.fieldName = source.readString();
		res.extras = source.readBundle();

core/java/com/android/internal/view/IInlineSuggestionsRequestCallback.aidl

+5 −1

Original line number	Diff line number	Diff line
		@@ -16,6 +16,7 @@

		package com.android.internal.view;

		import android.view.autofill.AutofillId;
		import android.view.inputmethod.InlineSuggestionsRequest;

		import com.android.internal.view.IInlineSuggestionsResponseCallback;
		@@ -27,5 +28,8 @@ import com.android.internal.view.IInlineSuggestionsResponseCallback;
		oneway interface IInlineSuggestionsRequestCallback {
		void onInlineSuggestionsUnsupported();
		void onInlineSuggestionsRequest(in InlineSuggestionsRequest request,
		in IInlineSuggestionsResponseCallback callback);
		in IInlineSuggestionsResponseCallback callback, in AutofillId imeFieldId,
		boolean inputViewStarted);
		void onInputMethodStartInputView(in AutofillId imeFieldId);
		void onInputMethodFinishInputView(in AutofillId imeFieldId);
		}

services/autofill/java/com/android/server/autofill/InlineSuggestionSession.java

+140 −17

Original line number	Diff line number	Diff line
		@@ -18,6 +18,7 @@ package com.android.server.autofill;

		import static com.android.server.autofill.Helper.sDebug;

		import android.annotation.BinderThread;
		import android.annotation.NonNull;
		import android.annotation.Nullable;
		import android.content.ComponentName;
		@@ -53,11 +54,21 @@ import java.util.concurrent.TimeoutException;
		* suggestions for different input fields.
		*
		* <p>
		* This class is the sole place in Autofill responsible for directly communicating with the IME. It
		* receives the IME input view start/finish events, with the associated IME field Id. It uses the
		* information to decide when to send the {@link InlineSuggestionsResponse} to IME. As a result,
		* some of the response will be cached locally and only be sent when the IME is ready to show them.
		*
		* <p>
		* See {@link android.inputmethodservice.InlineSuggestionSession} comments for InputMethodService
		* side flow.
		*
		* <p>
		* This class is thread safe.
		*/
		final class InlineSuggestionSession {

		private static final String TAG = "InlineSuggestionSession";
		private static final String TAG = "AfInlineSuggestionSession";
		private static final int INLINE_REQUEST_TIMEOUT_MS = 1000;

		@NonNull
		@@ -67,33 +78,83 @@ final class InlineSuggestionSession {
		private final ComponentName mComponentName;
		@NonNull
		private final Object mLock;
		@NonNull
		private final ImeStatusListener mImeStatusListener;

		/**
		* To avoid the race condition, one should not access {@code mPendingImeResponse} without
		* holding the {@code mLock}. For consuming the existing value, tt's recommended to use
		* {@link #getPendingImeResponse()} to get a copy of the reference to avoid blocking call.
		*/
		@GuardedBy("mLock")
		@Nullable
		private CompletableFuture<ImeResponse> mPendingImeResponse;

		@GuardedBy("mLock")
		@Nullable
		private AutofillResponse mPendingAutofillResponse;

		@GuardedBy("mLock")
		private boolean mIsLastResponseNonEmpty = false;

		@Nullable
		@GuardedBy("mLock")
		private AutofillId mImeFieldId = null;

		@GuardedBy("mLock")
		private boolean mImeInputViewStarted = false;

		InlineSuggestionSession(InputMethodManagerInternal inputMethodManagerInternal,
		int userId, ComponentName componentName) {
		mInputMethodManagerInternal = inputMethodManagerInternal;
		mUserId = userId;
		mComponentName = componentName;
		mLock = new Object();
		mImeStatusListener = new ImeStatusListener() {
		@Override
		public void onInputMethodStartInputView(AutofillId imeFieldId) {
		synchronized (mLock) {
		mImeFieldId = imeFieldId;
		mImeInputViewStarted = true;
		AutofillResponse pendingAutofillResponse = mPendingAutofillResponse;
		if (pendingAutofillResponse != null
		&& pendingAutofillResponse.mAutofillId.equalsIgnoreSession(
		mImeFieldId)) {
		mPendingAutofillResponse = null;
		onInlineSuggestionsResponseLocked(pendingAutofillResponse.mAutofillId,
		pendingAutofillResponse.mResponse);
		}
		}
		}

		@Override
		public void onInputMethodFinishInputView(AutofillId imeFieldId) {
		synchronized (mLock) {
		mImeFieldId = imeFieldId;
		mImeInputViewStarted = false;
		}
		}
		};
		}

		public void onCreateInlineSuggestionsRequest(@NonNull AutofillId autofillId) {
		if (sDebug) Log.d(TAG, "onCreateInlineSuggestionsRequest called for " + autofillId);

		synchronized (mLock) {
		cancelCurrentRequest();
		// Clean up all the state about the previous request.
		hideInlineSuggestionsUi(autofillId);
		mImeFieldId = null;
		mImeInputViewStarted = false;
		if (mPendingImeResponse != null && !mPendingImeResponse.isDone()) {
		mPendingImeResponse.complete(null);
		}
		mPendingImeResponse = new CompletableFuture<>();
		// TODO(b/146454892): pipe the uiExtras from the ExtServices.
		mInputMethodManagerInternal.onCreateInlineSuggestionsRequest(
		mUserId,
		new InlineSuggestionsRequestInfo(mComponentName, autofillId, new Bundle()),
		new InlineSuggestionsRequestCallbackImpl(mPendingImeResponse));
		new InlineSuggestionsRequestCallbackImpl(mPendingImeResponse,
		mImeStatusListener));
		}
		}

		@@ -116,10 +177,8 @@ final class InlineSuggestionSession {
		}

		public boolean hideInlineSuggestionsUi(@NonNull AutofillId autofillId) {
		if (sDebug) Log.d(TAG, "Called hideInlineSuggestionsUi for " + autofillId);
		synchronized (mLock) {
		if (mIsLastResponseNonEmpty) {
		if (sDebug) Log.d(TAG, "Send empty suggestion to IME");
		return onInlineSuggestionsResponseLocked(autofillId,
		new InlineSuggestionsResponse(Collections.EMPTY_LIST));
		}
		@@ -138,14 +197,32 @@ final class InlineSuggestionSession {
		@NonNull InlineSuggestionsResponse inlineSuggestionsResponse) {
		final CompletableFuture<ImeResponse> completedImsResponse = getPendingImeResponse();
		if (completedImsResponse == null \|\| !completedImsResponse.isDone()) {
		if (sDebug) Log.d(TAG, "onInlineSuggestionsResponseLocked without IMS request");
		return false;
		}
		// There is no need to wait on the CompletableFuture since it should have been completed
		// when {@link #waitAndGetInlineSuggestionsRequest()} was called.
		ImeResponse imeResponse = completedImsResponse.getNow(null);
		if (imeResponse == null) {
		if (sDebug) Log.d(TAG, "onInlineSuggestionsResponseLocked with pending IMS response");
		return false;
		}

		if (!mImeInputViewStarted \|\| !autofillId.equalsIgnoreSession(mImeFieldId)) {
		if (sDebug) {
		Log.d(TAG,
		"onInlineSuggestionsResponseLocked not sent because input view is not "
		+ "started for " + autofillId);
		}
		mPendingAutofillResponse = new AutofillResponse(autofillId, inlineSuggestionsResponse);
		// TODO(b/149442582): Although we are not sending the response to IME right away, we
		// still return true to indicate that the response may be sent eventually, such that
		// the dropdown UI will not be shown. This may not be the desired behavior in the
		// auto-focus case where IME isn't shown after switching back to an activity. We may
		// revisit this.
		return true;
		}

		try {
		imeResponse.mCallback.onInlineSuggestionsResponse(autofillId,
		inlineSuggestionsResponse);
		@@ -161,13 +238,6 @@ final class InlineSuggestionSession {
		}
		}

		private void cancelCurrentRequest() {
		CompletableFuture<ImeResponse> pendingImeResponse = getPendingImeResponse();
		if (pendingImeResponse != null && !pendingImeResponse.isDone()) {
		pendingImeResponse.complete(null);
		}
		}

		@Nullable
		@GuardedBy("mLock")
		private CompletableFuture<ImeResponse> getPendingImeResponse() {
		@@ -180,31 +250,84 @@ final class InlineSuggestionSession {
		extends IInlineSuggestionsRequestCallback.Stub {

		private final CompletableFuture<ImeResponse> mResponse;
		private final ImeStatusListener mImeStatusListener;

		private InlineSuggestionsRequestCallbackImpl(CompletableFuture<ImeResponse> response) {
		private InlineSuggestionsRequestCallbackImpl(CompletableFuture<ImeResponse> response,
		ImeStatusListener imeStatusListener) {
		mResponse = response;
		mImeStatusListener = imeStatusListener;
		}

		@BinderThread
		@Override
		public void onInlineSuggestionsUnsupported() throws RemoteException {
		if (sDebug) Log.d(TAG, "onInlineSuggestionsUnsupported() called.");
		mResponse.complete(null);
		}

		@BinderThread
		@Override
		public void onInlineSuggestionsRequest(InlineSuggestionsRequest request,
		IInlineSuggestionsResponseCallback callback) {
		if (sDebug) Log.d(TAG, "onInlineSuggestionsRequest() received: " + request);
		IInlineSuggestionsResponseCallback callback, AutofillId imeFieldId,
		boolean inputViewStarted) {
		if (sDebug) {
		Log.d(TAG,
		"onInlineSuggestionsRequest() received: " + request + ", inputViewStarted="
		+ inputViewStarted + ", imeFieldId=" + imeFieldId);
		}
		if (inputViewStarted) {
		mImeStatusListener.onInputMethodStartInputView(imeFieldId);
		} else {
		mImeStatusListener.onInputMethodFinishInputView(imeFieldId);
		}
		if (request != null && callback != null) {
		mResponse.complete(new ImeResponse(request, callback));
		} else {
		mResponse.complete(null);
		}
		}

		@BinderThread
		@Override
		public void onInputMethodStartInputView(AutofillId imeFieldId) {
		if (sDebug) Log.d(TAG, "onInputMethodStartInputView() received on " + imeFieldId);
		mImeStatusListener.onInputMethodStartInputView(imeFieldId);
		}

		@BinderThread
		@Override
		public void onInputMethodFinishInputView(AutofillId imeFieldId) {
		if (sDebug) Log.d(TAG, "onInputMethodFinishInputView() received on " + imeFieldId);
		mImeStatusListener.onInputMethodFinishInputView(imeFieldId);
		}
		}

		private interface ImeStatusListener {
		void onInputMethodStartInputView(AutofillId imeFieldId);

		void onInputMethodFinishInputView(AutofillId imeFieldId);
		}

		/**
		* A data class wrapping Autofill responses for the inline suggestion request.
		*/
		private static class AutofillResponse {
		@NonNull
		final AutofillId mAutofillId;

		@NonNull
		final InlineSuggestionsResponse mResponse;

		AutofillResponse(@NonNull AutofillId autofillId,
		@NonNull InlineSuggestionsResponse response) {
		mAutofillId = autofillId;
		mResponse = response;
		}

		}

		/**
		* A data class wrapping IME responses for the inline suggestion request.
		* A data class wrapping IME responses for the create inline suggestions request.
		*/
		private static class ImeResponse {
		@NonNull