Send more IME events to autofill manager service. (97ec1c4d) · Commits · e / os / android_frameworks_base

core/java/android/inputmethodservice/InlineSuggestionSession.java

+110 −101

Original line number	Diff line number	Diff line
		@@ -20,129 +20,148 @@ import static android.inputmethodservice.InputMethodService.DEBUG;

		import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage;

		import android.annotation.BinderThread;
		import android.annotation.MainThread;
		import android.annotation.NonNull;
		import android.content.ComponentName;
		import android.annotation.Nullable;
		import android.os.Bundle;
		import android.os.Handler;
		import android.os.IBinder;
		import android.os.Looper;
		import android.os.RemoteException;
		import android.util.Log;
		import android.view.autofill.AutofillId;
		import android.view.inputmethod.EditorInfo;
		import android.view.inputmethod.InlineSuggestionsRequest;
		import android.view.inputmethod.InlineSuggestionsResponse;

		import com.android.internal.view.IInlineSuggestionsRequestCallback;
		import com.android.internal.view.IInlineSuggestionsResponseCallback;
		import com.android.internal.view.InlineSuggestionsRequestInfo;

		import java.lang.ref.WeakReference;
		import java.util.function.Consumer;
		import java.util.function.Function;
		import java.util.function.Supplier;

		/**
		* Maintains an active inline suggestion session with the autofill manager service.
		* Maintains an inline suggestion session with the autofill manager service.
		*
		* <p> Each session correspond to one request from the Autofill manager service to create an
		* {@link InlineSuggestionsRequest}. It's responsible for calling back to the Autofill manager
		* service with {@link InlineSuggestionsRequest} and receiving {@link InlineSuggestionsResponse}
		* from it.
		* <p>
		* Each session corresponds to one {@link InlineSuggestionsRequest} and one {@link
		* IInlineSuggestionsResponseCallback}, but there may be multiple invocations of the response
		* callback for the same field or different fields in the same component.
		* TODO(b/151123764): currently the session may receive responses for different views on the same
		* screen, but we will fix it so each session corresponds to one view.
		*
		* <p>
		* The data flow from IMS point of view is:
		* Before calling {@link InputMethodService#onStartInputView(EditorInfo, boolean)}, call the {@link
		* #notifyOnStartInputView(AutofillId)}
		* ->
		* [async] {@link IInlineSuggestionsRequestCallback#onInputMethodStartInputView(AutofillId)}
		* --- process boundary ---
		* ->
		* {@link com.android.server.inputmethod.InputMethodManagerService
		* .InlineSuggestionsRequestCallbackDecorator#onInputMethodStartInputView(AutofillId)}
		* ->
		* {@link com.android.server.autofill.InlineSuggestionSession
		* .InlineSuggestionsRequestCallbackImpl#onInputMethodStartInputView(AutofillId)}
		*
		* <p>
		* The data flow for {@link #notifyOnFinishInputView(AutofillId)} is similar.
		* <p> All the methods are expected to be called from the main thread, to ensure thread safety.
		*/
		class InlineSuggestionSession {

		private static final String TAG = "ImsInlineSuggestionSession";

		private final Handler mHandler = new Handler(Looper.getMainLooper(), null, true);

		@NonNull
		private final ComponentName mComponentName;
		private final Handler mMainThreadHandler;
		@NonNull
		private final IInlineSuggestionsRequestCallback mCallback;
		private final InlineSuggestionSessionController mInlineSuggestionSessionController;
		@NonNull
		private final InlineSuggestionsResponseCallbackImpl mResponseCallback;
		private final InlineSuggestionsRequestInfo mRequestInfo;
		@NonNull
		private final Supplier<String> mClientPackageNameSupplier;
		@NonNull
		private final Supplier<AutofillId> mClientAutofillIdSupplier;
		private final IInlineSuggestionsRequestCallback mCallback;
		@NonNull
		private final Supplier<InlineSuggestionsRequest> mRequestSupplier;
		private final Function<Bundle, InlineSuggestionsRequest> mRequestSupplier;
		@NonNull
		private final Supplier<IBinder> mHostInputTokenSupplier;
		@NonNull
		private final Consumer<InlineSuggestionsResponse> mResponseConsumer;

		private volatile boolean mInvalidated = false;

		InlineSuggestionSession(@NonNull ComponentName componentName,
		/**
		* Indicates whether {@link #makeInlineSuggestionRequestUncheck()} has been called or not,
		* because it should only be called at most once.
		*/
		@Nullable
		private boolean mCallbackInvoked = false;
		@Nullable
		private InlineSuggestionsResponseCallbackImpl mResponseCallback;

		InlineSuggestionSession(@NonNull InlineSuggestionsRequestInfo requestInfo,
		@NonNull IInlineSuggestionsRequestCallback callback,
		@NonNull Supplier<String> clientPackageNameSupplier,
		@NonNull Supplier<AutofillId> clientAutofillIdSupplier,
		@NonNull Supplier<InlineSuggestionsRequest> requestSupplier,
		@NonNull Function<Bundle, InlineSuggestionsRequest> requestSupplier,
		@NonNull Supplier<IBinder> hostInputTokenSupplier,
		@NonNull Consumer<InlineSuggestionsResponse> responseConsumer,
		boolean inputViewStarted) {
		mComponentName = componentName;
		@NonNull InlineSuggestionSessionController inlineSuggestionSessionController,
		@NonNull Handler mainThreadHandler) {
		mRequestInfo = requestInfo;
		mCallback = callback;
		mResponseCallback = new InlineSuggestionsResponseCallbackImpl(this);
		mClientPackageNameSupplier = clientPackageNameSupplier;
		mClientAutofillIdSupplier = clientAutofillIdSupplier;
		mRequestSupplier = requestSupplier;
		mHostInputTokenSupplier = hostInputTokenSupplier;
		mResponseConsumer = responseConsumer;

		makeInlineSuggestionsRequest(inputViewStarted);
		mInlineSuggestionSessionController = inlineSuggestionSessionController;
		mMainThreadHandler = mainThreadHandler;
		}

		void notifyOnStartInputView(AutofillId imeFieldId) {
		if (DEBUG) Log.d(TAG, "notifyOnStartInputView");
		try {
		mCallback.onInputMethodStartInputView(imeFieldId);
		} catch (RemoteException e) {
		Log.w(TAG, "onInputMethodStartInputView() remote exception:" + e);
		@MainThread
		InlineSuggestionsRequestInfo getRequestInfo() {
		return mRequestInfo;
		}

		@MainThread
		IInlineSuggestionsRequestCallback getRequestCallback() {
		return mCallback;
		}

		void notifyOnFinishInputView(AutofillId imeFieldId) {
		if (DEBUG) Log.d(TAG, "notifyOnFinishInputView");
		try {
		mCallback.onInputMethodFinishInputView(imeFieldId);
		} catch (RemoteException e) {
		Log.w(TAG, "onInputMethodFinishInputView() remote exception:" + e);
		/**
		* Returns true if the session should send Ime status updates to Autofill.
		*
		* <p> The session only starts to send Ime status updates to Autofill after the sending back
		* an {@link InlineSuggestionsRequest}.
		*/
		@MainThread
		boolean shouldSendImeStatus() {
		return mResponseCallback != null;
		}

		/**
		* Returns true if {@link #makeInlineSuggestionRequestUncheck()} is called. It doesn't not
		* necessarily mean an {@link InlineSuggestionsRequest} was sent, because it may call {@link
		* IInlineSuggestionsRequestCallback#onInlineSuggestionsUnsupported()}.
		*
		* <p> The callback should be invoked at most once for each session.
		*/
		@MainThread
		boolean isCallbackInvoked() {
		return mCallbackInvoked;
		}

		/**
		* This needs to be called before creating a new session, such that the later response callbacks
		* will be discarded.
		* Invalidates the current session so it doesn't process any further {@link
		* InlineSuggestionsResponse} from Autofill.
		*
		* <p> This method should be called when the session is de-referenced from the {@link
		* InlineSuggestionSessionController}.
		*/
		void invalidateSession() {
		mInvalidated = true;
		@MainThread
		void invalidate() {
		if (mResponseCallback != null) {
		mResponseCallback.invalidate();
		mResponseCallback = null;
		}
		}

		/**
		* Sends an {@link InlineSuggestionsRequest} obtained from {@cocde supplier} to the current
		* Autofill Session through
		* {@link IInlineSuggestionsRequestCallback#onInlineSuggestionsRequest}.
		* Gets the {@link InlineSuggestionsRequest} from IME and send it back to the Autofill if it's
		* not null.
		*
		* <p>Calling this method implies that the input is started on the view corresponding to the
		* session.
		*/
		private void makeInlineSuggestionsRequest(boolean inputViewStarted) {
		@MainThread
		void makeInlineSuggestionRequestUncheck() {
		if (mCallbackInvoked) {
		return;
		}
		try {
		final InlineSuggestionsRequest request = mRequestSupplier.get();
		final InlineSuggestionsRequest request = mRequestSupplier.apply(
		mRequestInfo.getUiExtras());
		if (request == null) {
		if (DEBUG) {
		Log.d(TAG, "onCreateInlineSuggestionsRequest() returned null request");
		@@ -150,37 +169,19 @@ class InlineSuggestionSession {
		mCallback.onInlineSuggestionsUnsupported();
		} else {
		request.setHostInputToken(mHostInputTokenSupplier.get());
		mCallback.onInlineSuggestionsRequest(request, mResponseCallback,
		mClientAutofillIdSupplier.get(), inputViewStarted);
		mResponseCallback = new InlineSuggestionsResponseCallbackImpl(this);
		mCallback.onInlineSuggestionsRequest(request, mResponseCallback);
		}
		} catch (RemoteException e) {
		Log.w(TAG, "makeInlinedSuggestionsRequest() remote exception:" + e);
		}
		mCallbackInvoked = true;
		}

		private void handleOnInlineSuggestionsResponse(@NonNull AutofillId fieldId,
		@MainThread
		void handleOnInlineSuggestionsResponse(@NonNull AutofillId fieldId,
		@NonNull InlineSuggestionsResponse response) {
		if (mInvalidated) {
		if (DEBUG) {
		Log.d(TAG, "handleOnInlineSuggestionsResponse() called on invalid session");
		}
		return;
		}
		// The IME doesn't have information about the virtual view id for the child views in the
		// web view, so we are only comparing the parent view id here. This means that for cases
		// where there are two input fields in the web view, they will have the same view id
		// (although different virtual child id), and we will not be able to distinguish them.
		final AutofillId imeClientFieldId = mClientAutofillIdSupplier.get();
		if (!mComponentName.getPackageName().equals(mClientPackageNameSupplier.get())
		\|\| imeClientFieldId == null
		\|\| fieldId.getViewId() != imeClientFieldId.getViewId()) {
		if (DEBUG) {
		Log.d(TAG,
		"handleOnInlineSuggestionsResponse() called on the wrong package/field "
		+ "name: " + mComponentName.getPackageName() + " v.s. "
		+ mClientPackageNameSupplier.get() + ", " + fieldId + " v.s. "
		+ mClientAutofillIdSupplier.get());
		}
		if (!mInlineSuggestionSessionController.match(fieldId)) {
		return;
		}
		if (DEBUG) {
		@@ -192,23 +193,31 @@ class InlineSuggestionSession {
		/**
		* Internal implementation of {@link IInlineSuggestionsResponseCallback}.
		*/
		static final class InlineSuggestionsResponseCallbackImpl
		extends IInlineSuggestionsResponseCallback.Stub {
		private final WeakReference<InlineSuggestionSession> mInlineSuggestionSession;
		private static final class InlineSuggestionsResponseCallbackImpl extends
		IInlineSuggestionsResponseCallback.Stub {
		private final WeakReference<InlineSuggestionSession> mSession;
		private volatile boolean mInvalid = false;

		private InlineSuggestionsResponseCallbackImpl(InlineSuggestionSession session) {
		mSession = new WeakReference<>(session);
		}

		private InlineSuggestionsResponseCallbackImpl(
		InlineSuggestionSession inlineSuggestionSession) {
		mInlineSuggestionSession = new WeakReference<>(inlineSuggestionSession);
		void invalidate() {
		mInvalid = true;
		}

		@BinderThread
		@Override
		public void onInlineSuggestionsResponse(AutofillId fieldId,
		InlineSuggestionsResponse response) {
		final InlineSuggestionSession session = mInlineSuggestionSession.get();
		if (mInvalid) {
		return;
		}
		final InlineSuggestionSession session = mSession.get();
		if (session != null) {
		session.mHandler.sendMessage(obtainMessage(
		InlineSuggestionSession::handleOnInlineSuggestionsResponse, session,
		fieldId, response));
		session.mMainThreadHandler.sendMessage(
		obtainMessage(InlineSuggestionSession::handleOnInlineSuggestionsResponse,
		session, fieldId, response));
		}
		}
		}

core/java/android/inputmethodservice/InlineSuggestionSessionController.java

0 → 100644

+261 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2020 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package android.inputmethodservice;

		import static android.inputmethodservice.InputMethodService.DEBUG;

		import android.annotation.MainThread;
		import android.annotation.NonNull;
		import android.annotation.Nullable;
		import android.os.Bundle;
		import android.os.Handler;
		import android.os.IBinder;
		import android.os.Looper;
		import android.os.RemoteException;
		import android.util.Log;
		import android.view.autofill.AutofillId;
		import android.view.inputmethod.EditorInfo;
		import android.view.inputmethod.InlineSuggestionsRequest;
		import android.view.inputmethod.InlineSuggestionsResponse;

		import com.android.internal.view.IInlineSuggestionsRequestCallback;
		import com.android.internal.view.InlineSuggestionsRequestInfo;

		import java.util.function.Consumer;
		import java.util.function.Function;
		import java.util.function.Supplier;

		/**
		* Manages the interaction with the autofill manager service for the inline suggestion sessions.
		*
		* <p>
		* The class maintains the inline suggestion session with the autofill service. There is at most one
		* active inline suggestion session at any given time.
		*
		* <p>
		* The class receives the IME status change events (input start/finish, input view start/finish, and
		* show input requested result), and send them through IPC to the {@link
		* com.android.server.inputmethod.InputMethodManagerService}, which sends them to {@link
		* com.android.server.autofill.InlineSuggestionSession} in the Autofill manager service. If there is
		* no open inline suggestion session, no event will be send to autofill manager service.
		*
		* <p>
		* All the methods are expected to be called from the main thread, to ensure thread safety.
		*/
		class InlineSuggestionSessionController {
		private static final String TAG = "InlineSuggestionSessionController";

		private final Handler mMainThreadHandler = new Handler(Looper.getMainLooper(), null, true);

		@NonNull
		private final Function<Bundle, InlineSuggestionsRequest> mRequestSupplier;
		@NonNull
		private final Supplier<IBinder> mHostInputTokenSupplier;
		@NonNull
		private final Consumer<InlineSuggestionsResponse> mResponseConsumer;

		/* The following variables track the IME status */
		@Nullable
		private String mImeClientPackageName;
		@Nullable
		private AutofillId mImeClientFieldId;
		private boolean mImeInputStarted;
		private boolean mImeInputViewStarted;

		@Nullable
		private InlineSuggestionSession mSession;

		InlineSuggestionSessionController(
		@NonNull Function<Bundle, InlineSuggestionsRequest> requestSupplier,
		@NonNull Supplier<IBinder> hostInputTokenSupplier,
		@NonNull Consumer<InlineSuggestionsResponse> responseConsumer) {
		mRequestSupplier = requestSupplier;
		mHostInputTokenSupplier = hostInputTokenSupplier;
		mResponseConsumer = responseConsumer;
		}

		/**
		* Called upon IME receiving a create inline suggestion request. Must be called in the main
		* thread to ensure thread safety.
		*/
		@MainThread
		void onMakeInlineSuggestionsRequest(@NonNull InlineSuggestionsRequestInfo requestInfo,
		@NonNull IInlineSuggestionsRequestCallback callback) {
		if (DEBUG) Log.d(TAG, "onMakeInlineSuggestionsRequest: " + requestInfo);
		// Creates a new session for the new create request from Autofill.
		if (mSession != null) {
		mSession.invalidate();
		}
		mSession = new InlineSuggestionSession(requestInfo, callback, mRequestSupplier,
		mHostInputTokenSupplier, mResponseConsumer, this, mMainThreadHandler);

		// If the input is started on the same view, then initiate the callback to the Autofill.
		// Otherwise wait for the input to start.
		if (mImeInputStarted && match(mSession.getRequestInfo())) {
		mSession.makeInlineSuggestionRequestUncheck();
		// ... then update the Autofill whether the input view is started.
		if (mImeInputViewStarted) {
		try {
		mSession.getRequestCallback().onInputMethodStartInputView();
		} catch (RemoteException e) {
		Log.w(TAG, "onInputMethodStartInputView() remote exception:" + e);
		}
		}
		}
		}

		/**
		* Called from IME main thread before calling {@link InputMethodService#onStartInput(EditorInfo,
		* boolean)}. This method should be quick as it makes a unblocking IPC.
		*/
		@MainThread
		void notifyOnStartInput(@Nullable String imeClientPackageName,
		@Nullable AutofillId imeFieldId) {
		if (DEBUG) Log.d(TAG, "notifyOnStartInput: " + imeClientPackageName + ", " + imeFieldId);
		if (imeClientPackageName == null \|\| imeFieldId == null) {
		return;
		}
		mImeInputStarted = true;
		mImeClientPackageName = imeClientPackageName;
		mImeClientFieldId = imeFieldId;

		if (mSession != null) {
		// Initiates the callback to Autofill if there is a pending matching session.
		// Otherwise updates the session with the Ime status.
		if (!mSession.isCallbackInvoked() && match(mSession.getRequestInfo())) {
		mSession.makeInlineSuggestionRequestUncheck();
		} else if (mSession.shouldSendImeStatus()) {
		try {
		mSession.getRequestCallback().onInputMethodStartInput(mImeClientFieldId);
		} catch (RemoteException e) {
		Log.w(TAG, "onInputMethodStartInput() remote exception:" + e);
		}
		}
		}
		}

		/**
		* Called from IME main thread after getting results from
		* {@link InputMethodService#dispatchOnShowInputRequested(int,
		* boolean)}. This method should be quick as it makes a unblocking IPC.
		*/
		@MainThread
		void notifyOnShowInputRequested(boolean requestResult) {
		if (DEBUG) Log.d(TAG, "notifyShowInputRequested");
		if (mSession != null && mSession.shouldSendImeStatus()) {
		try {
		mSession.getRequestCallback().onInputMethodShowInputRequested(requestResult);
		} catch (RemoteException e) {
		Log.w(TAG, "onInputMethodShowInputRequested() remote exception:" + e);
		}
		}
		}

		/**
		* Called from IME main thread before calling
		* {@link InputMethodService#onStartInputView(EditorInfo,
		* boolean)} . This method should be quick as it makes a unblocking IPC.
		*/
		@MainThread
		void notifyOnStartInputView() {
		if (DEBUG) Log.d(TAG, "notifyOnStartInputView");
		mImeInputViewStarted = true;
		if (mSession != null && mSession.shouldSendImeStatus()) {
		try {
		mSession.getRequestCallback().onInputMethodStartInputView();
		} catch (RemoteException e) {
		Log.w(TAG, "onInputMethodStartInputView() remote exception:" + e);
		}
		}
		}

		/**
		* Called from IME main thread before calling
		* {@link InputMethodService#onFinishInputView(boolean)}.
		* This method should be quick as it makes a unblocking IPC.
		*/
		@MainThread
		void notifyOnFinishInputView() {
		if (DEBUG) Log.d(TAG, "notifyOnFinishInputView");
		mImeInputViewStarted = false;
		if (mSession != null && mSession.shouldSendImeStatus()) {
		try {
		mSession.getRequestCallback().onInputMethodFinishInputView();
		} catch (RemoteException e) {
		Log.w(TAG, "onInputMethodFinishInputView() remote exception:" + e);
		}
		}
		}

		/**
		* Called from IME main thread before calling {@link InputMethodService#onFinishInput()}. This
		* method should be quick as it makes a unblocking IPC.
		*/
		@MainThread
		void notifyOnFinishInput() {
		if (DEBUG) Log.d(TAG, "notifyOnFinishInput");
		mImeClientPackageName = null;
		mImeClientFieldId = null;
		mImeInputViewStarted = false;
		mImeInputStarted = false;
		if (mSession != null && mSession.shouldSendImeStatus()) {
		try {
		mSession.getRequestCallback().onInputMethodFinishInput();
		} catch (RemoteException e) {
		Log.w(TAG, "onInputMethodFinishInput() remote exception:" + e);
		}
		}
		}

		/**
		* Returns true if the current Ime focused field matches the session {@code requestInfo}.
		*/
		@MainThread
		boolean match(@Nullable InlineSuggestionsRequestInfo requestInfo) {
		return match(requestInfo, mImeClientPackageName, mImeClientFieldId);
		}

		/**
		* Returns true if the current Ime focused field matches the {@code autofillId}.
		*/
		@MainThread
		boolean match(@Nullable AutofillId autofillId) {
		return match(autofillId, mImeClientFieldId);
		}

		private static boolean match(
		@Nullable InlineSuggestionsRequestInfo inlineSuggestionsRequestInfo,
		@Nullable String imeClientPackageName, @Nullable AutofillId imeClientFieldId) {
		if (inlineSuggestionsRequestInfo == null \|\| imeClientPackageName == null
		\|\| imeClientFieldId == null) {
		return false;
		}
		return inlineSuggestionsRequestInfo.getComponentName().getPackageName().equals(
		imeClientPackageName) && match(inlineSuggestionsRequestInfo.getAutofillId(),
		imeClientFieldId);
		}

		private static boolean match(@Nullable AutofillId autofillId,
		@Nullable AutofillId imeClientFieldId) {
		// The IME doesn't have information about the virtual view id for the child views in the
		// web view, so we are only comparing the parent view id here. This means that for cases
		// where there are two input fields in the web view, they will have the same view id
		// (although different virtual child id), and we will not be able to distinguish them.
		return autofillId != null && imeClientFieldId != null
		&& autofillId.getViewId() == imeClientFieldId.getViewId();
		}
		}

core/java/android/inputmethodservice/InputMethodService.java

+14 −71

File changed.

Preview size limit exceeded, changes collapsed.

core/java/com/android/internal/view/IInlineSuggestionsRequestCallback.aidl

+30 −4

File changed.

Preview size limit exceeded, changes collapsed.

services/autofill/java/com/android/server/autofill/InlineSuggestionSession.java

+57 −25

File changed.

Preview size limit exceeded, changes collapsed.