Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 97ec1c4d authored by Feng Cao's avatar Feng Cao
Browse files

Send more IME events to autofill manager service.

* In IME side, wait for the input start before calling back to Autofill,
  rather than returning inline unsupported immediately.
* Also adds an InlineSuggestionManager to simplify code in the
  InputMethodService

Test: atest CtsAutoFillServiceTestCases
Test: atest CtsInputMethodTestCases
Bug: 151123764

Change-Id: I199925d77aa508f259e98a8929120aeb96015b57
parent 64d3ed60
Loading
Loading
Loading
Loading
+110 −101
Original line number Diff line number Diff line
@@ -20,129 +20,148 @@ import static android.inputmethodservice.InputMethodService.DEBUG;

import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage;

import android.annotation.BinderThread;
import android.annotation.MainThread;
import android.annotation.NonNull;
import android.content.ComponentName;
import android.annotation.Nullable;
import android.os.Bundle;
import android.os.Handler;
import android.os.IBinder;
import android.os.Looper;
import android.os.RemoteException;
import android.util.Log;
import android.view.autofill.AutofillId;
import android.view.inputmethod.EditorInfo;
import android.view.inputmethod.InlineSuggestionsRequest;
import android.view.inputmethod.InlineSuggestionsResponse;

import com.android.internal.view.IInlineSuggestionsRequestCallback;
import com.android.internal.view.IInlineSuggestionsResponseCallback;
import com.android.internal.view.InlineSuggestionsRequestInfo;

import java.lang.ref.WeakReference;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Supplier;

/**
 * Maintains an active inline suggestion session with the autofill manager service.
 * Maintains an inline suggestion session with the autofill manager service.
 *
 * <p> Each session correspond to one request from the Autofill manager service to create an
 * {@link InlineSuggestionsRequest}. It's responsible for calling back to the Autofill manager
 * service with {@link InlineSuggestionsRequest} and receiving {@link InlineSuggestionsResponse}
 * from it.
 * <p>
 * Each session corresponds to one {@link InlineSuggestionsRequest} and one {@link
 * IInlineSuggestionsResponseCallback}, but there may be multiple invocations of the response
 * callback for the same field or different fields in the same component.
 * TODO(b/151123764): currently the session may receive responses for different views on the same
 * screen, but we will fix it so each session corresponds to one view.
 *
 * <p>
 * The data flow from IMS point of view is:
 * Before calling {@link InputMethodService#onStartInputView(EditorInfo, boolean)}, call the {@link
 * #notifyOnStartInputView(AutofillId)}
 * ->
 * [async] {@link IInlineSuggestionsRequestCallback#onInputMethodStartInputView(AutofillId)}
 * --- process boundary ---
 * ->
 * {@link com.android.server.inputmethod.InputMethodManagerService
 * .InlineSuggestionsRequestCallbackDecorator#onInputMethodStartInputView(AutofillId)}
 * ->
 * {@link com.android.server.autofill.InlineSuggestionSession
 * .InlineSuggestionsRequestCallbackImpl#onInputMethodStartInputView(AutofillId)}
 *
 * <p>
 * The data flow for {@link #notifyOnFinishInputView(AutofillId)} is similar.
 * <p> All the methods are expected to be called from the main thread, to ensure thread safety.
 */
class InlineSuggestionSession {

    private static final String TAG = "ImsInlineSuggestionSession";

    private final Handler mHandler = new Handler(Looper.getMainLooper(), null, true);

    @NonNull
    private final ComponentName mComponentName;
    private final Handler mMainThreadHandler;
    @NonNull
    private final IInlineSuggestionsRequestCallback mCallback;
    private final InlineSuggestionSessionController mInlineSuggestionSessionController;
    @NonNull
    private final InlineSuggestionsResponseCallbackImpl mResponseCallback;
    private final InlineSuggestionsRequestInfo mRequestInfo;
    @NonNull
    private final Supplier<String> mClientPackageNameSupplier;
    @NonNull
    private final Supplier<AutofillId> mClientAutofillIdSupplier;
    private final IInlineSuggestionsRequestCallback mCallback;
    @NonNull
    private final Supplier<InlineSuggestionsRequest> mRequestSupplier;
    private final Function<Bundle, InlineSuggestionsRequest> mRequestSupplier;
    @NonNull
    private final Supplier<IBinder> mHostInputTokenSupplier;
    @NonNull
    private final Consumer<InlineSuggestionsResponse> mResponseConsumer;

    private volatile boolean mInvalidated = false;

    InlineSuggestionSession(@NonNull ComponentName componentName,
    /**
     * Indicates whether {@link #makeInlineSuggestionRequestUncheck()} has been called or not,
     * because it should only be called at most once.
     */
    @Nullable
    private boolean mCallbackInvoked = false;
    @Nullable
    private InlineSuggestionsResponseCallbackImpl mResponseCallback;

    InlineSuggestionSession(@NonNull InlineSuggestionsRequestInfo requestInfo,
            @NonNull IInlineSuggestionsRequestCallback callback,
            @NonNull Supplier<String> clientPackageNameSupplier,
            @NonNull Supplier<AutofillId> clientAutofillIdSupplier,
            @NonNull Supplier<InlineSuggestionsRequest> requestSupplier,
            @NonNull Function<Bundle, InlineSuggestionsRequest> requestSupplier,
            @NonNull Supplier<IBinder> hostInputTokenSupplier,
            @NonNull Consumer<InlineSuggestionsResponse> responseConsumer,
            boolean inputViewStarted) {
        mComponentName = componentName;
            @NonNull InlineSuggestionSessionController inlineSuggestionSessionController,
            @NonNull Handler mainThreadHandler) {
        mRequestInfo = requestInfo;
        mCallback = callback;
        mResponseCallback = new InlineSuggestionsResponseCallbackImpl(this);
        mClientPackageNameSupplier = clientPackageNameSupplier;
        mClientAutofillIdSupplier = clientAutofillIdSupplier;
        mRequestSupplier = requestSupplier;
        mHostInputTokenSupplier = hostInputTokenSupplier;
        mResponseConsumer = responseConsumer;

        makeInlineSuggestionsRequest(inputViewStarted);
        mInlineSuggestionSessionController = inlineSuggestionSessionController;
        mMainThreadHandler = mainThreadHandler;
    }

    void notifyOnStartInputView(AutofillId imeFieldId) {
        if (DEBUG) Log.d(TAG, "notifyOnStartInputView");
        try {
            mCallback.onInputMethodStartInputView(imeFieldId);
        } catch (RemoteException e) {
            Log.w(TAG, "onInputMethodStartInputView() remote exception:" + e);
    @MainThread
    InlineSuggestionsRequestInfo getRequestInfo() {
        return mRequestInfo;
    }

    @MainThread
    IInlineSuggestionsRequestCallback getRequestCallback() {
        return mCallback;
    }

    void notifyOnFinishInputView(AutofillId imeFieldId) {
        if (DEBUG) Log.d(TAG, "notifyOnFinishInputView");
        try {
            mCallback.onInputMethodFinishInputView(imeFieldId);
        } catch (RemoteException e) {
            Log.w(TAG, "onInputMethodFinishInputView() remote exception:" + e);
    /**
     * Returns true if the session should send Ime status updates to Autofill.
     *
     * <p> The session only starts to send Ime status updates to Autofill after the sending back
     * an {@link InlineSuggestionsRequest}.
     */
    @MainThread
    boolean shouldSendImeStatus() {
        return mResponseCallback != null;
    }

    /**
     * Returns true if {@link #makeInlineSuggestionRequestUncheck()} is called. It doesn't not
     * necessarily mean an {@link InlineSuggestionsRequest} was sent, because it may call {@link
     * IInlineSuggestionsRequestCallback#onInlineSuggestionsUnsupported()}.
     *
     * <p> The callback should be invoked at most once for each session.
     */
    @MainThread
    boolean isCallbackInvoked() {
        return mCallbackInvoked;
    }

    /**
     * This needs to be called before creating a new session, such that the later response callbacks
     * will be discarded.
     * Invalidates the current session so it doesn't process any further {@link
     * InlineSuggestionsResponse} from Autofill.
     *
     * <p> This method should be called when the session is de-referenced from the {@link
     * InlineSuggestionSessionController}.
     */
    void invalidateSession() {
        mInvalidated = true;
    @MainThread
    void invalidate() {
        if (mResponseCallback != null) {
            mResponseCallback.invalidate();
            mResponseCallback = null;
        }
    }

    /**
     * Sends an {@link InlineSuggestionsRequest} obtained from {@cocde supplier} to the current
     * Autofill Session through
     * {@link IInlineSuggestionsRequestCallback#onInlineSuggestionsRequest}.
     * Gets the {@link InlineSuggestionsRequest} from IME and send it back to the Autofill if it's
     * not null.
     *
     * <p>Calling this method implies that the input is started on the view corresponding to the
     * session.
     */
    private void makeInlineSuggestionsRequest(boolean inputViewStarted) {
    @MainThread
    void makeInlineSuggestionRequestUncheck() {
        if (mCallbackInvoked) {
            return;
        }
        try {
            final InlineSuggestionsRequest request = mRequestSupplier.get();
            final InlineSuggestionsRequest request = mRequestSupplier.apply(
                    mRequestInfo.getUiExtras());
            if (request == null) {
                if (DEBUG) {
                    Log.d(TAG, "onCreateInlineSuggestionsRequest() returned null request");
@@ -150,37 +169,19 @@ class InlineSuggestionSession {
                mCallback.onInlineSuggestionsUnsupported();
            } else {
                request.setHostInputToken(mHostInputTokenSupplier.get());
                mCallback.onInlineSuggestionsRequest(request, mResponseCallback,
                        mClientAutofillIdSupplier.get(), inputViewStarted);
                mResponseCallback = new InlineSuggestionsResponseCallbackImpl(this);
                mCallback.onInlineSuggestionsRequest(request, mResponseCallback);
            }
        } catch (RemoteException e) {
            Log.w(TAG, "makeInlinedSuggestionsRequest() remote exception:" + e);
        }
        mCallbackInvoked = true;
    }

    private void handleOnInlineSuggestionsResponse(@NonNull AutofillId fieldId,
    @MainThread
    void handleOnInlineSuggestionsResponse(@NonNull AutofillId fieldId,
            @NonNull InlineSuggestionsResponse response) {
        if (mInvalidated) {
            if (DEBUG) {
                Log.d(TAG, "handleOnInlineSuggestionsResponse() called on invalid session");
            }
            return;
        }
        // The IME doesn't have information about the virtual view id for the child views in the
        // web view, so we are only comparing the parent view id here. This means that for cases
        // where there are two input fields in the web view, they will have the same view id
        // (although different virtual child id), and we will not be able to distinguish them.
        final AutofillId imeClientFieldId = mClientAutofillIdSupplier.get();
        if (!mComponentName.getPackageName().equals(mClientPackageNameSupplier.get())
                || imeClientFieldId == null
                || fieldId.getViewId() != imeClientFieldId.getViewId()) {
            if (DEBUG) {
                Log.d(TAG,
                        "handleOnInlineSuggestionsResponse() called on the wrong package/field "
                                + "name: " + mComponentName.getPackageName() + " v.s. "
                                + mClientPackageNameSupplier.get() + ", " + fieldId + " v.s. "
                                + mClientAutofillIdSupplier.get());
            }
        if (!mInlineSuggestionSessionController.match(fieldId)) {
            return;
        }
        if (DEBUG) {
@@ -192,23 +193,31 @@ class InlineSuggestionSession {
    /**
     * Internal implementation of {@link IInlineSuggestionsResponseCallback}.
     */
    static final class InlineSuggestionsResponseCallbackImpl
            extends IInlineSuggestionsResponseCallback.Stub {
        private final WeakReference<InlineSuggestionSession> mInlineSuggestionSession;
    private static final class InlineSuggestionsResponseCallbackImpl extends
            IInlineSuggestionsResponseCallback.Stub {
        private final WeakReference<InlineSuggestionSession> mSession;
        private volatile boolean mInvalid = false;

        private InlineSuggestionsResponseCallbackImpl(InlineSuggestionSession session) {
            mSession = new WeakReference<>(session);
        }

        private InlineSuggestionsResponseCallbackImpl(
                InlineSuggestionSession inlineSuggestionSession) {
            mInlineSuggestionSession = new WeakReference<>(inlineSuggestionSession);
        void invalidate() {
            mInvalid = true;
        }

        @BinderThread
        @Override
        public void onInlineSuggestionsResponse(AutofillId fieldId,
                InlineSuggestionsResponse response) {
            final InlineSuggestionSession session = mInlineSuggestionSession.get();
            if (mInvalid) {
                return;
            }
            final InlineSuggestionSession session = mSession.get();
            if (session != null) {
                session.mHandler.sendMessage(obtainMessage(
                        InlineSuggestionSession::handleOnInlineSuggestionsResponse, session,
                        fieldId, response));
                session.mMainThreadHandler.sendMessage(
                        obtainMessage(InlineSuggestionSession::handleOnInlineSuggestionsResponse,
                                session, fieldId, response));
            }
        }
    }
+261 −0
Original line number Diff line number Diff line
/*
 * Copyright (C) 2020 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package android.inputmethodservice;

import static android.inputmethodservice.InputMethodService.DEBUG;

import android.annotation.MainThread;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.os.Bundle;
import android.os.Handler;
import android.os.IBinder;
import android.os.Looper;
import android.os.RemoteException;
import android.util.Log;
import android.view.autofill.AutofillId;
import android.view.inputmethod.EditorInfo;
import android.view.inputmethod.InlineSuggestionsRequest;
import android.view.inputmethod.InlineSuggestionsResponse;

import com.android.internal.view.IInlineSuggestionsRequestCallback;
import com.android.internal.view.InlineSuggestionsRequestInfo;

import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Supplier;

/**
 * Manages the interaction with the autofill manager service for the inline suggestion sessions.
 *
 * <p>
 * The class maintains the inline suggestion session with the autofill service. There is at most one
 * active inline suggestion session at any given time.
 *
 * <p>
 * The class receives the IME status change events (input start/finish, input view start/finish, and
 * show input requested result), and send them through IPC to the {@link
 * com.android.server.inputmethod.InputMethodManagerService}, which sends them to {@link
 * com.android.server.autofill.InlineSuggestionSession} in the Autofill manager service. If there is
 * no open inline suggestion session, no event will be send to autofill manager service.
 *
 * <p>
 * All the methods are expected to be called from the main thread, to ensure thread safety.
 */
class InlineSuggestionSessionController {
    private static final String TAG = "InlineSuggestionSessionController";

    private final Handler mMainThreadHandler = new Handler(Looper.getMainLooper(), null, true);

    @NonNull
    private final Function<Bundle, InlineSuggestionsRequest> mRequestSupplier;
    @NonNull
    private final Supplier<IBinder> mHostInputTokenSupplier;
    @NonNull
    private final Consumer<InlineSuggestionsResponse> mResponseConsumer;

    /* The following variables track the IME status */
    @Nullable
    private String mImeClientPackageName;
    @Nullable
    private AutofillId mImeClientFieldId;
    private boolean mImeInputStarted;
    private boolean mImeInputViewStarted;

    @Nullable
    private InlineSuggestionSession mSession;

    InlineSuggestionSessionController(
            @NonNull Function<Bundle, InlineSuggestionsRequest> requestSupplier,
            @NonNull Supplier<IBinder> hostInputTokenSupplier,
            @NonNull Consumer<InlineSuggestionsResponse> responseConsumer) {
        mRequestSupplier = requestSupplier;
        mHostInputTokenSupplier = hostInputTokenSupplier;
        mResponseConsumer = responseConsumer;
    }

    /**
     * Called upon IME receiving a create inline suggestion request. Must be called in the main
     * thread to ensure thread safety.
     */
    @MainThread
    void onMakeInlineSuggestionsRequest(@NonNull InlineSuggestionsRequestInfo requestInfo,
            @NonNull IInlineSuggestionsRequestCallback callback) {
        if (DEBUG) Log.d(TAG, "onMakeInlineSuggestionsRequest: " + requestInfo);
        // Creates a new session for the new create request from Autofill.
        if (mSession != null) {
            mSession.invalidate();
        }
        mSession = new InlineSuggestionSession(requestInfo, callback, mRequestSupplier,
                mHostInputTokenSupplier, mResponseConsumer, this, mMainThreadHandler);

        // If the input is started on the same view, then initiate the callback to the Autofill.
        // Otherwise wait for the input to start.
        if (mImeInputStarted && match(mSession.getRequestInfo())) {
            mSession.makeInlineSuggestionRequestUncheck();
            // ... then update the Autofill whether the input view is started.
            if (mImeInputViewStarted) {
                try {
                    mSession.getRequestCallback().onInputMethodStartInputView();
                } catch (RemoteException e) {
                    Log.w(TAG, "onInputMethodStartInputView() remote exception:" + e);
                }
            }
        }
    }

    /**
     * Called from IME main thread before calling {@link InputMethodService#onStartInput(EditorInfo,
     * boolean)}. This method should be quick as it makes a unblocking IPC.
     */
    @MainThread
    void notifyOnStartInput(@Nullable String imeClientPackageName,
            @Nullable AutofillId imeFieldId) {
        if (DEBUG) Log.d(TAG, "notifyOnStartInput: " + imeClientPackageName + ", " + imeFieldId);
        if (imeClientPackageName == null || imeFieldId == null) {
            return;
        }
        mImeInputStarted = true;
        mImeClientPackageName = imeClientPackageName;
        mImeClientFieldId = imeFieldId;

        if (mSession != null) {
            // Initiates the callback to Autofill if there is a pending matching session.
            // Otherwise updates the session with the Ime status.
            if (!mSession.isCallbackInvoked() && match(mSession.getRequestInfo())) {
                mSession.makeInlineSuggestionRequestUncheck();
            } else if (mSession.shouldSendImeStatus()) {
                try {
                    mSession.getRequestCallback().onInputMethodStartInput(mImeClientFieldId);
                } catch (RemoteException e) {
                    Log.w(TAG, "onInputMethodStartInput() remote exception:" + e);
                }
            }
        }
    }

    /**
     * Called from IME main thread after getting results from
     * {@link InputMethodService#dispatchOnShowInputRequested(int,
     * boolean)}. This method should be quick as it makes a unblocking IPC.
     */
    @MainThread
    void notifyOnShowInputRequested(boolean requestResult) {
        if (DEBUG) Log.d(TAG, "notifyShowInputRequested");
        if (mSession != null && mSession.shouldSendImeStatus()) {
            try {
                mSession.getRequestCallback().onInputMethodShowInputRequested(requestResult);
            } catch (RemoteException e) {
                Log.w(TAG, "onInputMethodShowInputRequested() remote exception:" + e);
            }
        }
    }

    /**
     * Called from IME main thread before calling
     * {@link InputMethodService#onStartInputView(EditorInfo,
     * boolean)} . This method should be quick as it makes a unblocking IPC.
     */
    @MainThread
    void notifyOnStartInputView() {
        if (DEBUG) Log.d(TAG, "notifyOnStartInputView");
        mImeInputViewStarted = true;
        if (mSession != null && mSession.shouldSendImeStatus()) {
            try {
                mSession.getRequestCallback().onInputMethodStartInputView();
            } catch (RemoteException e) {
                Log.w(TAG, "onInputMethodStartInputView() remote exception:" + e);
            }
        }
    }

    /**
     * Called from IME main thread before calling
     * {@link InputMethodService#onFinishInputView(boolean)}.
     * This method should be quick as it makes a unblocking IPC.
     */
    @MainThread
    void notifyOnFinishInputView() {
        if (DEBUG) Log.d(TAG, "notifyOnFinishInputView");
        mImeInputViewStarted = false;
        if (mSession != null && mSession.shouldSendImeStatus()) {
            try {
                mSession.getRequestCallback().onInputMethodFinishInputView();
            } catch (RemoteException e) {
                Log.w(TAG, "onInputMethodFinishInputView() remote exception:" + e);
            }
        }
    }

    /**
     * Called from IME main thread before calling {@link InputMethodService#onFinishInput()}. This
     * method should be quick as it makes a unblocking IPC.
     */
    @MainThread
    void notifyOnFinishInput() {
        if (DEBUG) Log.d(TAG, "notifyOnFinishInput");
        mImeClientPackageName = null;
        mImeClientFieldId = null;
        mImeInputViewStarted = false;
        mImeInputStarted = false;
        if (mSession != null && mSession.shouldSendImeStatus()) {
            try {
                mSession.getRequestCallback().onInputMethodFinishInput();
            } catch (RemoteException e) {
                Log.w(TAG, "onInputMethodFinishInput() remote exception:" + e);
            }
        }
    }

    /**
     * Returns true if the current Ime focused field matches the session {@code requestInfo}.
     */
    @MainThread
    boolean match(@Nullable InlineSuggestionsRequestInfo requestInfo) {
        return match(requestInfo, mImeClientPackageName, mImeClientFieldId);
    }

    /**
     * Returns true if the current Ime focused field matches the {@code autofillId}.
     */
    @MainThread
    boolean match(@Nullable AutofillId autofillId) {
        return match(autofillId, mImeClientFieldId);
    }

    private static boolean match(
            @Nullable InlineSuggestionsRequestInfo inlineSuggestionsRequestInfo,
            @Nullable String imeClientPackageName, @Nullable AutofillId imeClientFieldId) {
        if (inlineSuggestionsRequestInfo == null || imeClientPackageName == null
                || imeClientFieldId == null) {
            return false;
        }
        return inlineSuggestionsRequestInfo.getComponentName().getPackageName().equals(
                imeClientPackageName) && match(inlineSuggestionsRequestInfo.getAutofillId(),
                imeClientFieldId);
    }

    private static boolean match(@Nullable AutofillId autofillId,
            @Nullable AutofillId imeClientFieldId) {
        // The IME doesn't have information about the virtual view id for the child views in the
        // web view, so we are only comparing the parent view id here. This means that for cases
        // where there are two input fields in the web view, they will have the same view id
        // (although different virtual child id), and we will not be able to distinguish them.
        return autofillId != null && imeClientFieldId != null
                && autofillId.getViewId() == imeClientFieldId.getViewId();
    }
}
+14 −71

File changed.

Preview size limit exceeded, changes collapsed.

+30 −4

File changed.

Preview size limit exceeded, changes collapsed.

+57 −25

File changed.

Preview size limit exceeded, changes collapsed.

Loading