Merge "Fix on click listener for the inline suggestion, and some refactoring." (e796b879) · Commits · e / os / android_frameworks_base

services/autofill/java/com/android/server/autofill/InlineSuggestionFactory.java

+83 −37

Original line number	Diff line number	Diff line
		@@ -20,23 +20,24 @@ import static com.android.server.autofill.Helper.sDebug;

		import android.annotation.NonNull;
		import android.annotation.Nullable;
		import android.app.slice.Slice;
		import android.content.Context;
		import android.os.Handler;
		import android.os.RemoteException;
		import android.service.autofill.Dataset;
		import android.service.autofill.InlinePresentation;
		import android.util.Slog;
		import android.view.SurfaceControl;
		import android.view.View;
		import android.view.autofill.AutofillId;
		import android.view.autofill.IAutoFillManagerClient;
		import android.view.inline.InlinePresentationSpec;
		import android.view.inputmethod.InlineSuggestion;
		import android.view.inputmethod.InlineSuggestionInfo;
		import android.view.inputmethod.InlineSuggestionsRequest;
		import android.view.inputmethod.InlineSuggestionsResponse;

		import com.android.internal.view.inline.IInlineContentCallback;
		import com.android.internal.view.inline.IInlineContentProvider;
		import com.android.server.UiThread;
		import com.android.server.autofill.ui.AutoFillUI;
		import com.android.server.autofill.ui.InlineSuggestionUi;

		import java.util.ArrayList;
		@@ -56,23 +57,59 @@ public final class InlineSuggestionFactory {
		int sessionId,
		@NonNull Dataset[] datasets,
		@NonNull AutofillId autofillId,
		@NonNull InlineSuggestionsRequest request,
		@NonNull Handler uiHandler,
		@NonNull Context context,
		@NonNull IAutoFillManagerClient client) {
		if (sDebug) Slog.d(TAG, "createInlineSuggestionsResponse called");
		if (sDebug) Slog.d(TAG, "createAugmentedInlineSuggestionsResponse called");

		final ArrayList<InlineSuggestion> inlineSuggestions = new ArrayList<>();
		final InlineSuggestionUi inlineSuggestionUi = new InlineSuggestionUi(context);
		for (Dataset dataset : datasets) {
		// TODO(b/146453195): use the spec in the dataset.
		InlinePresentationSpec spec = request.getPresentationSpecs().get(0);
		if (spec == null) {
		Slog.w(TAG, "InlinePresentationSpec is not provided in the response data set");
		continue;
		final int fieldIndex = dataset.getFieldIds().indexOf(autofillId);
		if (fieldIndex < 0) {
		Slog.w(TAG, "AutofillId=" + autofillId + " not found in dataset");
		return null;
		}
		final InlinePresentation inlinePresentation = dataset.getFieldInlinePresentation(
		fieldIndex);
		if (inlinePresentation == null) {
		Slog.w(TAG, "InlinePresentation not found in dataset");
		return null;
		}
		InlineSuggestion inlineSuggestion = createAugmentedInlineSuggestion(sessionId, dataset,
		autofillId, spec, uiHandler, inlineSuggestionUi, client);
		inlinePresentation, inlineSuggestionUi, client);
		inlineSuggestions.add(inlineSuggestion);
		}
		return new InlineSuggestionsResponse(inlineSuggestions);
		}

		/**
		* Creates an {@link InlineSuggestionsResponse} with the {@code datasets} provided by the
		* autofill service.
		*/
		public static InlineSuggestionsResponse createInlineSuggestionsResponse(int requestId,
		@NonNull Dataset[] datasets,
		@NonNull AutofillId autofillId,
		@NonNull Context context,
		@NonNull AutoFillUI.AutoFillUiCallback client) {
		if (sDebug) Slog.d(TAG, "createInlineSuggestionsResponse called");

		final ArrayList<InlineSuggestion> inlineSuggestions = new ArrayList<>();
		final InlineSuggestionUi inlineSuggestionUi = new InlineSuggestionUi(context);
		for (Dataset dataset : datasets) {
		final int fieldIndex = dataset.getFieldIds().indexOf(autofillId);
		if (fieldIndex < 0) {
		Slog.w(TAG, "AutofillId=" + autofillId + " not found in dataset");
		return null;
		}
		final InlinePresentation inlinePresentation = dataset.getFieldInlinePresentation(
		fieldIndex);
		if (inlinePresentation == null) {
		Slog.w(TAG, "InlinePresentation not found in dataset");
		return null;
		}
		InlineSuggestion inlineSuggestion = createInlineSuggestion(requestId, dataset,
		fieldIndex,
		inlinePresentation, inlineSuggestionUi, client);
		inlineSuggestions.add(inlineSuggestion);
		}
		return new InlineSuggestionsResponse(inlineSuggestions);
		@@ -80,33 +117,55 @@ public final class InlineSuggestionFactory {

		private static InlineSuggestion createAugmentedInlineSuggestion(int sessionId,
		@NonNull Dataset dataset,
		@NonNull AutofillId autofillId,
		@NonNull InlinePresentationSpec spec,
		@NonNull Handler uiHandler,
		@NonNull InlinePresentation inlinePresentation,
		@NonNull InlineSuggestionUi inlineSuggestionUi,
		@NonNull IAutoFillManagerClient client) {
		// TODO(b/146453195): fill in the autofill hint properly.
		final InlineSuggestionInfo inlineSuggestionInfo = new InlineSuggestionInfo(
		spec, InlineSuggestionInfo.SOURCE_PLATFORM, new String[]{""});
		final View.OnClickListener onClickListener = createOnClickListener(sessionId, dataset,
		client);
		inlinePresentation.getInlinePresentationSpec(),
		InlineSuggestionInfo.SOURCE_PLATFORM, new String[]{""});
		final View.OnClickListener onClickListener = v -> {
		try {
		client.autofill(sessionId, dataset.getFieldIds(), dataset.getFieldValues());
		} catch (RemoteException e) {
		Slog.w(TAG, "Encounter exception autofilling the values");
		}
		};
		final InlineSuggestion inlineSuggestion = new InlineSuggestion(inlineSuggestionInfo,
		createInlineContentProvider(autofillId, dataset, uiHandler, inlineSuggestionUi,
		createInlineContentProvider(inlinePresentation.getSlice(), inlineSuggestionUi,
		onClickListener));
		return inlineSuggestion;
		}

		private static IInlineContentProvider.Stub createInlineContentProvider(
		@NonNull AutofillId autofillId, @NonNull Dataset dataset, @NonNull Handler uiHandler,
		private static InlineSuggestion createInlineSuggestion(int requestId,
		@NonNull Dataset dataset,
		int fieldIndex,
		@NonNull InlinePresentation inlinePresentation,
		@NonNull InlineSuggestionUi inlineSuggestionUi,
		@NonNull AutoFillUI.AutoFillUiCallback client) {
		// TODO(b/146453195): fill in the autofill hint properly.
		final InlineSuggestionInfo inlineSuggestionInfo = new InlineSuggestionInfo(
		inlinePresentation.getInlinePresentationSpec(),
		InlineSuggestionInfo.SOURCE_AUTOFILL, new String[]{""});
		final View.OnClickListener onClickListener = v -> {
		client.fill(requestId, fieldIndex, dataset);
		};
		final InlineSuggestion inlineSuggestion = new InlineSuggestion(inlineSuggestionInfo,
		createInlineContentProvider(inlinePresentation.getSlice(), inlineSuggestionUi,
		onClickListener));
		return inlineSuggestion;
		}

		private static IInlineContentProvider.Stub createInlineContentProvider(
		@NonNull Slice slice, @NonNull InlineSuggestionUi inlineSuggestionUi,
		@Nullable View.OnClickListener onClickListener) {
		return new IInlineContentProvider.Stub() {
		@Override
		public void provideContent(int width, int height,
		IInlineContentCallback callback) {
		uiHandler.post(() -> {
		SurfaceControl sc = inlineSuggestionUi.inflate(dataset, autofillId,
		width, height, onClickListener);
		UiThread.getHandler().post(() -> {
		SurfaceControl sc = inlineSuggestionUi.inflate(slice, width, height,
		onClickListener);
		try {
		callback.onContent(sc);
		} catch (RemoteException e) {
		@@ -117,19 +176,6 @@ public final class InlineSuggestionFactory {
		};
		}

		private static View.OnClickListener createOnClickListener(int sessionId,
		@NonNull Dataset dataset,
		@NonNull IAutoFillManagerClient client) {
		return v -> {
		if (sDebug) Slog.d(TAG, "Inline suggestion clicked");
		try {
		client.autofill(sessionId, dataset.getFieldIds(), dataset.getFieldValues());
		} catch (RemoteException e) {
		Slog.w(TAG, "Encounter exception autofilling the values");
		}
		};
		}

		private InlineSuggestionFactory() {
		}
		}

services/autofill/java/com/android/server/autofill/RemoteAugmentedAutofillService.java

+4 −9

Original line number	Diff line number	Diff line
		@@ -161,8 +161,7 @@ final class RemoteAugmentedAutofillService
		@Override
		public void onSuccess(@Nullable Dataset[] inlineSuggestionsData) {
		maybeHandleInlineSuggestions(sessionId, inlineSuggestionsData,
		focusedId, inlineSuggestionsRequest,
		inlineSuggestionsCallback, client);
		focusedId, inlineSuggestionsCallback, client);
		requestAutofill.complete(null);
		}

		@@ -226,19 +225,15 @@ final class RemoteAugmentedAutofillService

		private void maybeHandleInlineSuggestions(int sessionId,
		@Nullable Dataset[] inlineSuggestionsData, @NonNull AutofillId focusedId,
		@Nullable InlineSuggestionsRequest inlineSuggestionsRequest,
		@Nullable IInlineSuggestionsResponseCallback inlineSuggestionsCallback,
		@NonNull IAutoFillManagerClient client) {
		if (inlineSuggestionsRequest == null
		\|\| ArrayUtils.isEmpty(inlineSuggestionsData)
		\|\| inlineSuggestionsCallback == null) {
		if (ArrayUtils.isEmpty(inlineSuggestionsData) \|\| inlineSuggestionsCallback == null) {
		return;
		}
		try {
		inlineSuggestionsCallback.onInlineSuggestionsResponse(
		InlineSuggestionFactory.createAugmentedInlineSuggestionsResponse(
		sessionId, inlineSuggestionsData, focusedId, inlineSuggestionsRequest,
		getJobHandler(), mContext, client));
		InlineSuggestionFactory.createAugmentedInlineSuggestionsResponse(sessionId,
		inlineSuggestionsData, focusedId, mContext, client));
		} catch (RemoteException e) {
		Slog.w(TAG, "Exception sending inline suggestions response back to IME.");
		}

services/autofill/java/com/android/server/autofill/Session.java

+6 −34

Original line number	Diff line number	Diff line
		@@ -71,7 +71,6 @@ import android.service.autofill.FieldClassificationUserData;
		import android.service.autofill.FillContext;
		import android.service.autofill.FillRequest;
		import android.service.autofill.FillResponse;
		import android.service.autofill.InlinePresentation;
		import android.service.autofill.InternalSanitizer;
		import android.service.autofill.InternalValidator;
		import android.service.autofill.SaveInfo;
		@@ -93,9 +92,6 @@ import android.view.autofill.AutofillManager.SmartSuggestionMode;
		import android.view.autofill.AutofillValue;
		import android.view.autofill.IAutoFillManagerClient;
		import android.view.autofill.IAutofillWindowPresenter;
		import android.view.inline.InlinePresentationSpec;
		import android.view.inputmethod.InlineSuggestion;
		import android.view.inputmethod.InlineSuggestionInfo;
		import android.view.inputmethod.InlineSuggestionsRequest;
		import android.view.inputmethod.InlineSuggestionsResponse;

		@@ -106,8 +102,6 @@ import com.android.internal.logging.nano.MetricsProto.MetricsEvent;
		import com.android.internal.util.ArrayUtils;
		import com.android.internal.view.IInlineSuggestionsRequestCallback;
		import com.android.internal.view.IInlineSuggestionsResponseCallback;
		import com.android.internal.view.inline.IInlineContentCallback;
		import com.android.internal.view.inline.IInlineContentProvider;
		import com.android.server.autofill.ui.AutoFillUI;
		import com.android.server.autofill.ui.PendingUi;
		import com.android.server.inputmethod.InputMethodManagerInternal;
		@@ -148,6 +142,7 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState
		private final Handler mHandler;
		private final Object mLock;
		private final AutoFillUI mUi;
		private final Context mContext;

		private final MetricsLogger mMetricsLogger = new MetricsLogger();

		@@ -773,6 +768,7 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState
		mLock = lock;
		mUi = ui;
		mHandler = handler;
		mContext = context;
		mRemoteFillService = serviceComponentName == null ? null
		: new RemoteFillService(context, serviceComponentName, userId, this,
		bindInstantServiceAllowed);
		@@ -2733,35 +2729,11 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState
		return false;
		}

		final int size = datasets.size();
		final ArrayList<InlineSuggestion> inlineSuggestions = new ArrayList<>();

		for (int index = 0; index < size; index++) {
		final Dataset dataset = datasets.get(index);
		//TODO(b/146453536): Use the proper presentation/spec for currently focused view.
		final InlinePresentation inlinePresentation = dataset.getFieldInlinePresentation(0);
		if (inlinePresentation == null) {
		if (sDebug) Log.d(TAG, "Missing InlinePresentation on dataset=" + dataset);
		continue;
		}
		final InlinePresentationSpec spec = inlinePresentation.getInlinePresentationSpec();
		final InlineSuggestionInfo inlineSuggestionInfo = new InlineSuggestionInfo(
		spec, InlineSuggestionInfo.SOURCE_AUTOFILL, new String[] { "" });

		inlineSuggestions.add(new InlineSuggestion(inlineSuggestionInfo,
		new IInlineContentProvider.Stub() {
		@Override
		public void provideContent(int width, int height,
		IInlineContentCallback callback) throws RemoteException {
		getUiForShowing().getSuggestionSurfaceForShowing(dataset, response,
		mCurrentViewId, width, height, callback);
		}
		}));
		}

		InlineSuggestionsResponse inlineSuggestionsResponse =
		InlineSuggestionFactory.createInlineSuggestionsResponse(response.getRequestId(),
		datasets.toArray(new Dataset[]{}), mCurrentViewId, mContext, this);
		try {
		inlineContentCallback.onInlineSuggestionsResponse(
		new InlineSuggestionsResponse(inlineSuggestions));
		inlineContentCallback.onInlineSuggestionsResponse(inlineSuggestionsResponse);
		} catch (RemoteException e) {
		Log.w(TAG, "onFillReady() remote error calling onInlineSuggestionsResponse()");
		return false;

services/autofill/java/com/android/server/autofill/ui/AutoFillUI.java

+0 −32

Original line number	Diff line number	Diff line
		@@ -37,7 +37,6 @@ import android.service.autofill.ValueFinder;
		import android.text.TextUtils;
		import android.util.Slog;
		import android.view.KeyEvent;
		import android.view.SurfaceControl;
		import android.view.autofill.AutofillId;
		import android.view.autofill.AutofillManager;
		import android.view.autofill.IAutofillWindowPresenter;
		@@ -45,7 +44,6 @@ import android.widget.Toast;

		import com.android.internal.logging.MetricsLogger;
		import com.android.internal.logging.nano.MetricsProto.MetricsEvent;
		import com.android.internal.view.inline.IInlineContentCallback;
		import com.android.server.LocalServices;
		import com.android.server.UiModeManagerInternal;
		import com.android.server.UiThread;
		@@ -172,36 +170,6 @@ public final class AutoFillUI {
		});
		}

		/**
		* TODO(b/137800469): Fill in javadoc.
		* TODO(b/137800469): peoperly manage lifecycle of suggestions surfaces.
		*/
		public void getSuggestionSurfaceForShowing(@NonNull Dataset dataset,
		@NonNull FillResponse response, AutofillId autofillId, int width, int height,
		IInlineContentCallback cb) {
		if (dataset == null) {
		Slog.w(TAG, "getSuggestionSurfaceForShowing() called with null dataset");
		}
		mHandler.post(() -> {
		final InlineSuggestionUi inlineSuggestionUi = new InlineSuggestionUi(mContext);
		final SurfaceControl suggestionSurface = inlineSuggestionUi.inflate(dataset,
		autofillId, width, height, v -> {
		Slog.d(TAG, "Inline suggestion clicked");
		hideFillUiUiThread(mCallback, true);
		if (mCallback != null) {
		final int datasetIndex = response.getDatasets().indexOf(dataset);
		mCallback.fill(response.getRequestId(), datasetIndex, dataset);
		}
		});

		try {
		cb.onContent(suggestionSurface);
		} catch (RemoteException e) {
		Slog.w(TAG, "RemoteException replying onContent(" + suggestionSurface + "): " + e);
		}
		});
		}

		/**
		* Shows the fill UI, removing the previous fill UI if the has changed.
		*

services/autofill/java/com/android/server/autofill/ui/InlineSuggestionUi.java

+7 −16

Original line number	Diff line number	Diff line
		@@ -28,17 +28,13 @@ import android.content.Context;
		import android.graphics.PixelFormat;
		import android.graphics.drawable.Icon;
		import android.os.IBinder;
		import android.service.autofill.Dataset;
		import android.util.Log;
		import android.util.Slog;
		import android.view.LayoutInflater;
		import android.view.SurfaceControl;
		import android.view.SurfaceControlViewHost;
		import android.view.View;
		import android.view.ViewGroup;
		import android.view.WindowManager;
		import android.view.autofill.AutofillId;
		import android.view.autofill.AutofillValue;
		import android.widget.ImageView;
		import android.widget.TextView;

		@@ -69,23 +65,18 @@ public class InlineSuggestionUi {
		*/
		@MainThread
		@Nullable
		public SurfaceControl inflate(@NonNull Dataset dataset, @NonNull AutofillId autofillId,
		int width, int height, @Nullable View.OnClickListener onClickListener) {
		public SurfaceControl inflate(@NonNull Slice slice, int width, int height,
		@Nullable View.OnClickListener onClickListener) {
		Log.d(TAG, "Inflating the inline suggestion UI");
		final int index = dataset.getFieldIds().indexOf(autofillId);
		if (index < 0) {
		Slog.w(TAG, "inflateInlineSuggestion(): AutofillId=" + autofillId
		+ " not found in dataset");
		return null;
		}
		final AutofillValue datasetValue = dataset.getFieldValues().get(index);

		//TODO(b/137800469): Pass in inputToken from IME.
		final SurfaceControlViewHost wvr = new SurfaceControlViewHost(mContext,
		mContext.getDisplay(), (IBinder) null);
		final SurfaceControl sc = wvr.getSurfacePackage().getSurfaceControl();

		final ViewGroup suggestionView =
		(ViewGroup) renderSlice(dataset.getFieldInlinePresentation(index).getSlice());
		final ViewGroup suggestionView = (ViewGroup) renderSlice(slice);
		if (onClickListener != null) {
		suggestionView.setOnClickListener(onClickListener);
		}

		WindowManager.LayoutParams lp =
		new WindowManager.LayoutParams(width, height,