Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit aaccc8f2 authored by Feng Cao's avatar Feng Cao
Browse files

Add inline actions to the response to IME. Use inline style from the IME request in the response. 

Test: manual
Bug: 147394280

Change-Id: Ic5bc56b3020476998f227ff32ce4d22c765aeee4
parent 08b29115

services/autofill/java/com/android/server/autofill/RemoteAugmentedAutofillService.java

+7 −5
Original line number Diff line number Diff line
@@ -166,8 +166,8 @@ final class RemoteAugmentedAutofillService 
                                public void onSuccess(@Nullable Dataset[] inlineSuggestionsData) {

                                    mCallbacks.resetLastResponse();

                                    maybeRequestShowInlineSuggestions(sessionId,

                                            inlineSuggestionsData, focusedId,

                                            inlineSuggestionsCallback, client,

                                            inlineSuggestionsRequest, inlineSuggestionsData,

                                            focusedId, inlineSuggestionsCallback, client,

                                            onErrorCallback);

                                    requestAutofill.complete(null);

                                }
@@ -231,10 +231,12 @@ final class RemoteAugmentedAutofillService 
    }



    private void maybeRequestShowInlineSuggestions(int sessionId,

            @Nullable Dataset[] inlineSuggestionsData, @NonNull AutofillId focusedId,

            @Nullable InlineSuggestionsRequest request, @Nullable Dataset[] inlineSuggestionsData,

            @NonNull AutofillId focusedId,

            @Nullable IInlineSuggestionsResponseCallback inlineSuggestionsCallback,

            @NonNull IAutoFillManagerClient client, @NonNull Runnable onErrorCallback) {

        if (ArrayUtils.isEmpty(inlineSuggestionsData) || inlineSuggestionsCallback == null) {

        if (ArrayUtils.isEmpty(inlineSuggestionsData) || inlineSuggestionsCallback == null

                || request == null) {

            return;

        }

        mCallbacks.setLastResponse(sessionId);
@@ -242,7 +244,7 @@ final class RemoteAugmentedAutofillService 
        try {

            inlineSuggestionsCallback.onInlineSuggestionsResponse(

                    InlineSuggestionFactory.createAugmentedInlineSuggestionsResponse(

                            inlineSuggestionsData, focusedId, mContext,

                            request, inlineSuggestionsData, focusedId, mContext,

                            dataset -> {

                                mCallbacks.logAugmentedAutofillSelected(sessionId,

                                        dataset.getId());

services/autofill/java/com/android/server/autofill/Session.java

+27 −22
Original line number Diff line number Diff line
@@ -314,6 +314,7 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState 
    private final InputMethodManagerInternal mInputMethodManagerInternal;



    @Nullable

    @GuardedBy("mLock")

    private InlineSuggestionsRequestCallbackImpl mInlineSuggestionsRequestCallback;



    /**
@@ -2674,13 +2675,15 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState 
        }



        if (response.supportsInlineSuggestions()) {

            if (requestShowInlineSuggestions(response)) {

                //TODO(b/137800469): Fix it to log showed only when IME asks for inflation, rather

                // than here where framework sends back the response.

            synchronized (mLock) {

                if (requestShowInlineSuggestions(response, mInlineSuggestionsRequestCallback)) {

                    //TODO(b/137800469): Fix it to log showed only when IME asks for inflation,

                    // rather than here where framework sends back the response.

                    mService.logDatasetShown(id, mClientState);

                    return;

                }

            }

        }



        getUiForShowing().showFillUi(filledId, response, filterText,

                mService.getServicePackageName(), mComponentName,
@@ -2716,30 +2719,32 @@ final class Session implements RemoteFillService.FillServiceCallbacks, ViewState 
    /**

     * Returns whether we made a request to show inline suggestions.

     */

    private boolean requestShowInlineSuggestions(FillResponse response) {

        final IInlineSuggestionsResponseCallback inlineContentCallback =

                mInlineSuggestionsRequestCallback != null

                        ? mInlineSuggestionsRequestCallback.getResponseCallback() : null;

        if (inlineContentCallback == null) {

            Log.w(TAG, "Session input method callback is not set yet");

            return false;

        }



    private boolean requestShowInlineSuggestions(@NonNull FillResponse response,

            @Nullable InlineSuggestionsRequestCallbackImpl callback) {

        final List<Dataset> datasets = response.getDatasets();

        if (datasets == null) {

            Log.w(TAG, "response returned null datasets");

            return false;

        }



        if (callback == null || callback.getRequest() == null

                || callback.getResponseCallback() == null) {

            Log.w(TAG, "Session input method callback is not set yet");

            return false;

        }



        final InlineSuggestionsRequest request = callback.getRequest();

        InlineSuggestionsResponse inlineSuggestionsResponse =

                InlineSuggestionFactory.createInlineSuggestionsResponse(response.getRequestId(),

                        datasets.toArray(new Dataset[]{}), mCurrentViewId, mContext, this, () -> {

                InlineSuggestionFactory.createInlineSuggestionsResponse(request,

                        response.getRequestId(),

                        datasets.toArray(new Dataset[]{}), response.getInlineActions(),

                        mCurrentViewId, mContext, this, () -> {

                            synchronized (mLock) {

                                requestHideFillUi(mCurrentViewId);

                            }

                        });

        try {

            inlineContentCallback.onInlineSuggestionsResponse(inlineSuggestionsResponse);

            callback.getResponseCallback().onInlineSuggestionsResponse(inlineSuggestionsResponse);

        } catch (RemoteException e) {

            Log.w(TAG, "onFillReady() remote error calling onInlineSuggestionsResponse()");

            return false;

services/autofill/java/com/android/server/autofill/ui/InlineSuggestionFactory.java

+74 −37
Original line number Diff line number Diff line
@@ -28,16 +28,20 @@ import android.util.Slog; 
import android.view.SurfaceControl;

import android.view.View;

import android.view.autofill.AutofillId;

import android.view.inline.InlinePresentationSpec;

import android.view.inputmethod.InlineSuggestion;

import android.view.inputmethod.InlineSuggestionInfo;

import android.view.inputmethod.InlineSuggestionsRequest;

import android.view.inputmethod.InlineSuggestionsResponse;

import android.widget.Toast;



import com.android.internal.util.function.QuadFunction;

import com.android.internal.view.inline.IInlineContentCallback;

import com.android.internal.view.inline.IInlineContentProvider;

import com.android.server.UiThread;



import java.util.ArrayList;

import java.util.List;

import java.util.function.BiFunction;



public final class InlineSuggestionFactory {

    private static final String TAG = "InlineSuggestionFactory";
@@ -57,46 +61,47 @@ public final class InlineSuggestionFactory { 
     * augmented autofill service.

     */

    public static InlineSuggestionsResponse createAugmentedInlineSuggestionsResponse(

            @NonNull InlineSuggestionsRequest request,

            @NonNull Dataset[] datasets,

            @NonNull AutofillId autofillId,

            @NonNull Context context,

            @NonNull InlineSuggestionUiCallback inlineSuggestionUiCallback,

            @NonNull Runnable onErrorCallback) {

        return createInlineSuggestionsResponseInternal(datasets, autofillId,

                context, onErrorCallback, (dataset, inlinePresentation, inlineSuggestionUi,

                        filedIndex) -> createAugmentedInlineSuggestion(dataset,

                    inlinePresentation, inlineSuggestionUi, inlineSuggestionUiCallback));

        if (sDebug) Slog.d(TAG, "createAugmentedInlineSuggestionsResponse called");

        return createInlineSuggestionsResponseInternal(/* isAugmented= */ true, request,

                datasets, /* inlineActions= */ null, autofillId, context, onErrorCallback,

                (dataset, filedIndex) -> (v -> inlineSuggestionUiCallback.autofill(dataset)));

    }



    /**

     * Creates an {@link InlineSuggestionsResponse} with the {@code datasets} provided by the

     * autofill service.

     */

    public static InlineSuggestionsResponse createInlineSuggestionsResponse(int requestId,

    public static InlineSuggestionsResponse createInlineSuggestionsResponse(

            @NonNull InlineSuggestionsRequest request, int requestId,

            @NonNull Dataset[] datasets,

            @Nullable List<InlinePresentation> inlineActions,

            @NonNull AutofillId autofillId,

            @NonNull Context context,

            @NonNull AutoFillUI.AutoFillUiCallback client,

            @NonNull Runnable onErrorCallback) {

        return createInlineSuggestionsResponseInternal(datasets, autofillId,

                context, onErrorCallback, (dataset, inlinePresentation, inlineSuggestionUi,

                        filedIndex) -> createInlineSuggestion(requestId, dataset, filedIndex,

                        inlinePresentation, inlineSuggestionUi, client));

        if (sDebug) Slog.d(TAG, "createInlineSuggestionsResponse called");

        return createInlineSuggestionsResponseInternal(/* isAugmented= */ false, request, datasets,

                inlineActions, autofillId, context, onErrorCallback,

                (dataset, filedIndex) -> (v -> client.fill(requestId, filedIndex, dataset)));

    }



    private static InlineSuggestionsResponse createInlineSuggestionsResponseInternal(

            @NonNull Dataset[] datasets,

            @NonNull AutofillId autofillId,

            @NonNull Context context,

            boolean isAugmented, @NonNull InlineSuggestionsRequest request,

            @NonNull Dataset[] datasets, @Nullable List<InlinePresentation> inlineActions,

            @NonNull AutofillId autofillId, @NonNull Context context,

            @NonNull Runnable onErrorCallback,

            @NonNull QuadFunction<Dataset, InlinePresentation, InlineSuggestionUi,

                    Integer, InlineSuggestion> suggestionFactory) {

        if (sDebug) Slog.d(TAG, "createAugmentedInlineSuggestionsResponse called");



            @NonNull BiFunction<Dataset, Integer, View.OnClickListener> onClickListenerFactory) {

        final ArrayList<InlineSuggestion> inlineSuggestions = new ArrayList<>();

        final InlineSuggestionUi inlineSuggestionUi = new InlineSuggestionUi(context,

                onErrorCallback);

        for (Dataset dataset : datasets) {

        for (int i = 0; i < datasets.length; i++) {

            final Dataset dataset = datasets[i];

            final int fieldIndex = dataset.getFieldIds().indexOf(autofillId);

            if (fieldIndex < 0) {

                Slog.w(TAG, "AutofillId=" + autofillId + " not found in dataset");
@@ -108,50 +113,82 @@ public final class InlineSuggestionFactory { 
                Slog.w(TAG, "InlinePresentation not found in dataset");

                return null;

            }

            InlineSuggestion inlineSuggestion = suggestionFactory.apply(dataset,

                    inlinePresentation, inlineSuggestionUi, fieldIndex);

            InlineSuggestion inlineSuggestion = createInlineSuggestion(isAugmented, dataset,

                    fieldIndex, mergedInlinePresentation(request, i, inlinePresentation),

                    inlineSuggestionUi, onClickListenerFactory);

            inlineSuggestions.add(inlineSuggestion);

        }

        if (inlineActions != null) {

            for (InlinePresentation inlinePresentation : inlineActions) {

                final InlineSuggestion inlineAction = createInlineAction(isAugmented, context,

                        mergedInlinePresentation(request, 0, inlinePresentation),

                        inlineSuggestionUi);

                inlineSuggestions.add(inlineAction);

            }

        }

        return new InlineSuggestionsResponse(inlineSuggestions);

    }



    private static InlineSuggestion createAugmentedInlineSuggestion(@NonNull Dataset dataset,

    private static InlineSuggestion createInlineAction(boolean isAugmented,

            @NonNull Context context,

            @NonNull InlinePresentation inlinePresentation,

            @NonNull InlineSuggestionUi inlineSuggestionUi,

            @NonNull InlineSuggestionUiCallback inlineSuggestionUiCallback) {

            @NonNull InlineSuggestionUi inlineSuggestionUi) {

        // TODO(b/146453195): fill in the autofill hint properly.

        final InlineSuggestionInfo inlineSuggestionInfo = new InlineSuggestionInfo(

                inlinePresentation.getInlinePresentationSpec(),

                InlineSuggestionInfo.SOURCE_PLATFORM, new String[]{""},

                InlineSuggestionInfo.TYPE_SUGGESTION);

        final View.OnClickListener onClickListener = v ->

            inlineSuggestionUiCallback.autofill(dataset);

        final InlineSuggestion inlineSuggestion = new InlineSuggestion(inlineSuggestionInfo,

                isAugmented ? InlineSuggestionInfo.SOURCE_PLATFORM

                        : InlineSuggestionInfo.SOURCE_AUTOFILL, new String[]{""},

                InlineSuggestionInfo.TYPE_ACTION);

        final View.OnClickListener onClickListener = v -> {

            // TODO(b/148567875): Launch the intent provided through the slice. This

            //  should be part of the UI renderer therefore will be moved to the support

            //  library.

            Toast.makeText(context, "icon clicked", Toast.LENGTH_SHORT).show();

        };

        return new InlineSuggestion(inlineSuggestionInfo,

                createInlineContentProvider(inlinePresentation, inlineSuggestionUi,

                        onClickListener));

        return inlineSuggestion;

    }



    private static InlineSuggestion createInlineSuggestion(int requestId,

    private static InlineSuggestion createInlineSuggestion(boolean isAugmented,

            @NonNull Dataset dataset,

            int fieldIndex,

            @NonNull InlinePresentation inlinePresentation,

            int fieldIndex, @NonNull InlinePresentation inlinePresentation,

            @NonNull InlineSuggestionUi inlineSuggestionUi,

            @NonNull AutoFillUI.AutoFillUiCallback client) {

            @NonNull BiFunction<Dataset, Integer, View.OnClickListener> onClickListenerFactory) {

        // TODO(b/146453195): fill in the autofill hint properly.

        final InlineSuggestionInfo inlineSuggestionInfo = new InlineSuggestionInfo(

                inlinePresentation.getInlinePresentationSpec(),

                InlineSuggestionInfo.SOURCE_AUTOFILL, new String[]{""},

                isAugmented ? InlineSuggestionInfo.SOURCE_PLATFORM

                        : InlineSuggestionInfo.SOURCE_AUTOFILL, new String[]{""},

                InlineSuggestionInfo.TYPE_SUGGESTION);

        final View.OnClickListener onClickListener = v -> {

            client.fill(requestId, fieldIndex, dataset);

        };

        final View.OnClickListener onClickListener = onClickListenerFactory.apply(dataset,

                fieldIndex);

        final InlineSuggestion inlineSuggestion = new InlineSuggestion(inlineSuggestionInfo,

                createInlineContentProvider(inlinePresentation, inlineSuggestionUi,

                        onClickListener));

        return inlineSuggestion;

    }



    /**

     * Returns an {@link InlinePresentation} with the style spec from the request/host, and

     * everything else from the provided {@code inlinePresentation}.

     */

    private static InlinePresentation mergedInlinePresentation(

            @NonNull InlineSuggestionsRequest request,

            int index, @NonNull InlinePresentation inlinePresentation) {

        final List<InlinePresentationSpec> specs = request.getPresentationSpecs();

        if (specs.isEmpty()) {

            return inlinePresentation;

        }

        InlinePresentationSpec specFromHost = specs.get(Math.min(specs.size() - 1, index));

        InlinePresentationSpec mergedInlinePresentation = new InlinePresentationSpec.Builder(

                inlinePresentation.getInlinePresentationSpec().getMinSize(),

                inlinePresentation.getInlinePresentationSpec().getMaxSize()).setStyle(

                specFromHost.getStyle()).build();

        return new InlinePresentation(inlinePresentation.getSlice(), mergedInlinePresentation,

                inlinePresentation.isPinned());

    }



    private static IInlineContentProvider.Stub createInlineContentProvider(

            @NonNull InlinePresentation inlinePresentation,

            @NonNull InlineSuggestionUi inlineSuggestionUi,