Merge "Show augmented autofill suggestion inline iff both augmented autofill... (79ca0c84) · Commits · e / os / android_frameworks_base

services/autofill/java/com/android/server/autofill/InlineSuggestionSession.java

0 → 100644

+168 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2020 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package com.android.server.autofill;

		import static com.android.server.autofill.Helper.sDebug;

		import android.annotation.NonNull;
		import android.annotation.Nullable;
		import android.content.ComponentName;
		import android.os.RemoteException;
		import android.util.Log;
		import android.view.autofill.AutofillId;
		import android.view.inputmethod.InlineSuggestionsRequest;

		import com.android.internal.annotations.GuardedBy;
		import com.android.internal.view.IInlineSuggestionsRequestCallback;
		import com.android.internal.view.IInlineSuggestionsResponseCallback;
		import com.android.server.inputmethod.InputMethodManagerInternal;

		import java.util.concurrent.CancellationException;
		import java.util.concurrent.CompletableFuture;
		import java.util.concurrent.ExecutionException;
		import java.util.concurrent.TimeUnit;
		import java.util.concurrent.TimeoutException;

		/**
		* Maintains an inline suggestion autofill session.
		*
		* <p> This class is thread safe.
		*/
		final class InlineSuggestionSession {

		private static final String TAG = "InlineSuggestionSession";
		private static final int INLINE_REQUEST_TIMEOUT_MS = 1000;

		@NonNull
		private final InputMethodManagerInternal mInputMethodManagerInternal;
		private final int mUserId;
		@NonNull
		private final ComponentName mComponentName;
		@NonNull
		private final Object mLock;

		@GuardedBy("mLock")
		@Nullable
		private CompletableFuture<ImeResponse> mPendingImeResponse;

		InlineSuggestionSession(InputMethodManagerInternal inputMethodManagerInternal,
		int userId, ComponentName componentName) {
		mInputMethodManagerInternal = inputMethodManagerInternal;
		mUserId = userId;
		mComponentName = componentName;
		mLock = new Object();
		}

		public void createRequest(@NonNull AutofillId currentViewId) {
		synchronized (mLock) {
		cancelCurrentRequest();
		mPendingImeResponse = new CompletableFuture<>();
		mInputMethodManagerInternal.onCreateInlineSuggestionsRequest(
		mUserId, mComponentName, currentViewId,
		new InlineSuggestionsRequestCallbackImpl(mPendingImeResponse));
		}
		}

		@Nullable
		public ImeResponse waitAndGetImeResponse() {
		CompletableFuture<ImeResponse> pendingImeResponse = getPendingImeResponse();
		if (pendingImeResponse == null \|\| pendingImeResponse.isCancelled()) {
		return null;
		}
		try {
		return pendingImeResponse.get(INLINE_REQUEST_TIMEOUT_MS, TimeUnit.MILLISECONDS);
		} catch (TimeoutException e) {
		Log.w(TAG, "Exception getting inline suggestions request in time: " + e);
		} catch (CancellationException e) {
		Log.w(TAG, "Inline suggestions request cancelled");
		} catch (InterruptedException \| ExecutionException e) {
		throw new RuntimeException(e);
		}
		return null;
		}

		private void cancelCurrentRequest() {
		CompletableFuture<ImeResponse> pendingImeResponse = getPendingImeResponse();
		if (pendingImeResponse != null) {
		pendingImeResponse.cancel(true);
		}
		}

		@Nullable
		@GuardedBy("mLock")
		private CompletableFuture<ImeResponse> getPendingImeResponse() {
		synchronized (mLock) {
		return mPendingImeResponse;
		}
		}

		private static final class InlineSuggestionsRequestCallbackImpl
		extends IInlineSuggestionsRequestCallback.Stub {

		private final CompletableFuture<ImeResponse> mResponse;

		private InlineSuggestionsRequestCallbackImpl(CompletableFuture<ImeResponse> response) {
		mResponse = response;
		}

		@Override
		public void onInlineSuggestionsUnsupported() throws RemoteException {
		if (sDebug) {
		Log.d(TAG, "onInlineSuggestionsUnsupported() called.");
		}
		mResponse.cancel(true);
		}

		@Override
		public void onInlineSuggestionsRequest(InlineSuggestionsRequest request,
		IInlineSuggestionsResponseCallback callback) throws RemoteException {
		if (sDebug) {
		Log.d(TAG, "onInlineSuggestionsRequest() received: " + request);
		}
		if (request != null && callback != null) {
		mResponse.complete(new ImeResponse(request, callback));
		} else {
		mResponse.cancel(true);
		}
		}
		}

		/**
		* A data class wrapping IME responses for the inline suggestion request.
		*/
		public static class ImeResponse {
		@NonNull
		private final InlineSuggestionsRequest mRequest;

		@NonNull
		private final IInlineSuggestionsResponseCallback mCallback;

		ImeResponse(@NonNull InlineSuggestionsRequest request,
		@NonNull IInlineSuggestionsResponseCallback callback) {
		mRequest = request;
		mCallback = callback;
		}

		public InlineSuggestionsRequest getRequest() {
		return mRequest;
		}

		public IInlineSuggestionsResponseCallback getCallback() {
		return mCallback;
		}
		}
		}

services/autofill/java/com/android/server/autofill/Session.java

+27 −95

Original line number	Diff line number	Diff line
		@@ -100,7 +100,6 @@ import com.android.internal.annotations.GuardedBy;
		import com.android.internal.logging.MetricsLogger;
		import com.android.internal.logging.nano.MetricsProto.MetricsEvent;
		import com.android.internal.util.ArrayUtils;
		import com.android.internal.view.IInlineSuggestionsRequestCallback;
		import com.android.internal.view.IInlineSuggestionsResponseCallback;
		import com.android.server.autofill.ui.AutoFillUI;
		import com.android.server.autofill.ui.InlineSuggestionFactory;
		@@ -114,11 +113,6 @@ import java.util.Collection;
		import java.util.Collections;
		import java.util.List;
		import java.util.Objects;
		import java.util.concurrent.CancellationException;
		import java.util.concurrent.CompletableFuture;
		import java.util.concurrent.ExecutionException;
		import java.util.concurrent.TimeUnit;
		import java.util.concurrent.TimeoutException;
		import java.util.concurrent.atomic.AtomicInteger;

		/**
		@@ -159,9 +153,6 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState
		/** uid the session is for */
		public final int uid;

		/** user id the session is for */
		public final int userId;

		/** ID of the task associated with this session's activity */
		public final int taskId;

		@@ -310,12 +301,8 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState
		@GuardedBy("mLock")
		private boolean mForAugmentedAutofillOnly;

		@NonNull
		private final InputMethodManagerInternal mInputMethodManagerInternal;

		@Nullable
		@GuardedBy("mLock")
		private InlineSuggestionsRequestCallbackImpl mInlineSuggestionsRequestCallback;
		private final InlineSuggestionSession mInlineSuggestionSession;

		/**
		* Receiver of assist data from the app's {@link Activity}.
		@@ -416,12 +403,11 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState
		final ArrayList<FillContext> contexts =
		mergePreviousSessionLocked(/* forSave= */ false);

		final InlineSuggestionsRequest suggestionsRequest =
		mInlineSuggestionsRequestCallback != null
		? mInlineSuggestionsRequestCallback.getRequest() : null;
		final InlineSuggestionSession.ImeResponse imeResponse =
		mInlineSuggestionSession.waitAndGetImeResponse();

		request = new FillRequest(requestId, contexts, mClientState, flags,
		suggestionsRequest);
		imeResponse != null ? imeResponse.getRequest() : null);
		}

		if (mActivityToken != null) {
		@@ -619,75 +605,12 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState
		private void maybeRequestInlineSuggestionsRequestThenFillLocked(@NonNull ViewState viewState,
		int newState, int flags) {
		if (isInlineSuggestionsEnabled()) {
		mInlineSuggestionsRequestCallback = new InlineSuggestionsRequestCallbackImpl();
		mInputMethodManagerInternal.onCreateInlineSuggestionsRequest(userId,
		mComponentName, mCurrentViewId, mInlineSuggestionsRequestCallback);
		mInlineSuggestionSession.createRequest(mCurrentViewId);
		}

		requestNewFillResponseLocked(viewState, newState, flags);
		}

		private static final class InlineSuggestionsRequestCallbackImpl
		extends IInlineSuggestionsRequestCallback.Stub {
		private static final int INLINE_REQUEST_TIMEOUT_MS = 1000;

		private final CompletableFuture<InlineSuggestionsRequest> mRequest;
		private final CompletableFuture<IInlineSuggestionsResponseCallback> mResponseCallback;

		private InlineSuggestionsRequestCallbackImpl() {
		mRequest = new CompletableFuture<>();
		mResponseCallback = new CompletableFuture<>();
		}

		@Override
		public void onInlineSuggestionsUnsupported() throws RemoteException {
		if (sDebug) {
		Log.d(TAG, "inline suggestions request unsupported, "
		+ "falling back to regular autofill");
		}
		mRequest.cancel(true);
		mResponseCallback.cancel(true);
		}

		@Override
		public void onInlineSuggestionsRequest(InlineSuggestionsRequest request,
		IInlineSuggestionsResponseCallback callback) throws RemoteException {
		if (sDebug) {
		Log.d(TAG, "onInlineSuggestionsRequest() received: " + request);
		}
		mRequest.complete(request);
		mResponseCallback.complete(callback);
		}

		@Nullable
		private InlineSuggestionsRequest getRequest() {
		try {
		return mRequest.get(INLINE_REQUEST_TIMEOUT_MS, TimeUnit.MILLISECONDS);
		} catch (TimeoutException e) {
		Log.w(TAG, "Exception getting inline suggestions request in time: " + e);
		} catch (CancellationException e) {
		Log.w(TAG, "Inline suggestions request cancelled");
		} catch (InterruptedException \| ExecutionException e) {
		throw new RuntimeException(e);
		}
		return null;
		}

		@Nullable
		private IInlineSuggestionsResponseCallback getResponseCallback() {
		try {
		return mResponseCallback.get(INLINE_REQUEST_TIMEOUT_MS, TimeUnit.MILLISECONDS);
		} catch (TimeoutException e) {
		Log.w(TAG, "Exception getting inline suggestions callback in time: " + e);
		} catch (CancellationException e) {
		Log.w(TAG, "Inline suggestions callback cancelled");
		} catch (InterruptedException \| ExecutionException e) {
		throw new RuntimeException(e);
		}
		return null;
		}
		}

		/**
		* Reads a new structure and then request a new fill response from the fill service.
		*/
		@@ -767,7 +690,6 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState
		mFlags = flags;
		this.taskId = taskId;
		this.uid = uid;
		this.userId = userId;
		mStartTime = SystemClock.elapsedRealtime();
		mService = service;
		mLock = lock;
		@@ -786,7 +708,8 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState
		mForAugmentedAutofillOnly = forAugmentedAutofillOnly;
		setClientLocked(client);

		mInputMethodManagerInternal = inputMethodManagerInternal;
		mInlineSuggestionSession = new InlineSuggestionSession(inputMethodManagerInternal, userId,
		componentName);

		mMetricsLogger.write(newLogMaker(MetricsEvent.AUTOFILL_SESSION_STARTED)
		.addTaggedData(MetricsEvent.FIELD_AUTOFILL_FLAGS, flags));
		@@ -2679,7 +2602,7 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState

		if (response.supportsInlineSuggestions()) {
		synchronized (mLock) {
		if (requestShowInlineSuggestions(response, mInlineSuggestionsRequestCallback)) {
		if (requestShowInlineSuggestionsLocked(response)) {
		//TODO(b/137800469): Fix it to log showed only when IME asks for inflation,
		// rather than here where framework sends back the response.
		mService.logDatasetShown(id, mClientState);
		@@ -2722,21 +2645,21 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState
		/**
		* Returns whether we made a request to show inline suggestions.
		*/
		private boolean requestShowInlineSuggestions(@NonNull FillResponse response,
		@Nullable InlineSuggestionsRequestCallbackImpl callback) {
		private boolean requestShowInlineSuggestionsLocked(@NonNull FillResponse response) {
		final List<Dataset> datasets = response.getDatasets();
		if (datasets == null) {
		Log.w(TAG, "response returned null datasets");
		return false;
		}

		if (callback == null \|\| callback.getRequest() == null
		\|\| callback.getResponseCallback() == null) {
		final InlineSuggestionSession.ImeResponse imeResponse =
		mInlineSuggestionSession.waitAndGetImeResponse();
		if (imeResponse == null) {
		Log.w(TAG, "Session input method callback is not set yet");
		return false;
		}

		final InlineSuggestionsRequest request = callback.getRequest();
		final InlineSuggestionsRequest request = imeResponse.getRequest();
		InlineSuggestionsResponse inlineSuggestionsResponse =
		InlineSuggestionFactory.createInlineSuggestionsResponse(request,
		response.getRequestId(),
		@@ -2747,7 +2670,7 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState
		}
		});
		try {
		callback.getResponseCallback().onInlineSuggestionsResponse(inlineSuggestionsResponse);
		imeResponse.getCallback().onInlineSuggestionsResponse(inlineSuggestionsResponse);
		} catch (RemoteException e) {
		Log.w(TAG, "onFillReady() remote error calling onInlineSuggestionsResponse()");
		return false;
		@@ -3029,12 +2952,21 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState

		final AutofillId focusedId = AutofillId.withoutSession(mCurrentViewId);

		// There are 3 cases when augmented autofill should ask IME for a new request:
		// 1. standard autofill provider is None
		// 2. standard autofill provider doesn't support inline (and returns null response)
		// 3. standard autofill provider supports inline, but isn't called because the field
		// doesn't want autofill
		if (mForAugmentedAutofillOnly \|\| !isInlineSuggestionsEnabled()) {
		if (sDebug) Slog.d(TAG, "Create inline request for augmented autofill");
		mInlineSuggestionSession.createRequest(mCurrentViewId);
		}
		InlineSuggestionSession.ImeResponse imeResponse =
		mInlineSuggestionSession.waitAndGetImeResponse();
		final InlineSuggestionsRequest inlineSuggestionsRequest =
		mInlineSuggestionsRequestCallback != null
		? mInlineSuggestionsRequestCallback.getRequest() : null;
		imeResponse != null ? imeResponse.getRequest() : null;
		final IInlineSuggestionsResponseCallback inlineSuggestionsResponseCallback =
		mInlineSuggestionsRequestCallback != null
		? mInlineSuggestionsRequestCallback.getResponseCallback() : null;
		imeResponse != null ? imeResponse.getCallback() : null;
		remoteService.onRequestAutofillLocked(id, mClient, taskId, mComponentName, focusedId,
		currentValue, inlineSuggestionsRequest, inlineSuggestionsResponseCallback, () -> {
		synchronized (mLock) {