Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 17d9d2f2 authored by Svet Ganov's avatar Svet Ganov
Browse files

Scrollable inline keyboard suggestions 

Inlined keyboard suggestions are provided by the OS while
hosted by the keyboard. The clicks are detected by the OS
trusted code to release the data. Not all suggestions may
fit on the keyboard suggestion strip and the user may need
to scroll. To support this we are delegating the touch
event stream to the keyboard window as soon as the OS
detects that the user is not clicking, i.e. after a pointer
went down and moved more than the touch slop.

Bug:146535667

Test: atest CtsAutofillTestCases
      atest inputflinger_tests
      atest com.android.server.wm.DragDropControllerTests
      atest android.server.wm.DragDropTest
      atest android.server.wm.CrossAppDragAndDropTests

      added dedicated tests to inputflinger_tests

Change-Id: Iefb732f70a7b593b31ab2f323b0c277e3819f7b7
parent bbc78d5d

core/java/android/service/autofill/augmented/AugmentedAutofillService.java

+41 −29
Original line number Original line Diff line number Diff line
@@ -31,7 +31,6 @@ import android.content.ComponentName; 
import android.content.Intent;

import android.content.Intent;

import android.graphics.Rect;

import android.graphics.Rect;

import android.os.Build;

import android.os.Build;

import android.os.Bundle;

import android.os.CancellationSignal;

import android.os.CancellationSignal;

import android.os.Handler;

import android.os.Handler;

import android.os.IBinder;

import android.os.IBinder;
@@ -93,7 +92,7 @@ public abstract class AugmentedAutofillService extends Service { 
    // Used for metrics / debug only

    // Used for metrics / debug only

    private ComponentName mServiceComponentName;

    private ComponentName mServiceComponentName;





    private final IAugmentedAutofillService mInterface = new IAugmentedAutofillService.Stub() {

    private final class AugmentedAutofillServiceImpl extends IAugmentedAutofillService.Stub {





        @Override

        @Override

        public void onConnected(boolean debug, boolean verbose) {

        public void onConnected(boolean debug, boolean verbose) {
@@ -137,7 +136,7 @@ public abstract class AugmentedAutofillService extends Service { 
    public final IBinder onBind(Intent intent) {

    public final IBinder onBind(Intent intent) {

        mServiceComponentName = intent.getComponent();

        mServiceComponentName = intent.getComponent();

        if (SERVICE_INTERFACE.equals(intent.getAction())) {

        if (SERVICE_INTERFACE.equals(intent.getAction())) {

            return mInterface.asBinder();

            return new AugmentedAutofillServiceImpl();

        }

        }

        Log.w(TAG, "Tried to bind to wrong intent (should be " + SERVICE_INTERFACE + ": " + intent);

        Log.w(TAG, "Tried to bind to wrong intent (should be " + SERVICE_INTERFACE + ": " + intent);

        return null;

        return null;
@@ -352,11 +351,13 @@ public abstract class AugmentedAutofillService extends Service { 
        static final int REPORT_EVENT_NO_RESPONSE = 1;

        static final int REPORT_EVENT_NO_RESPONSE = 1;

        static final int REPORT_EVENT_UI_SHOWN = 2;

        static final int REPORT_EVENT_UI_SHOWN = 2;

        static final int REPORT_EVENT_UI_DESTROYED = 3;

        static final int REPORT_EVENT_UI_DESTROYED = 3;

        static final int REPORT_EVENT_INLINE_RESPONSE = 4;





        @IntDef(prefix = { "REPORT_EVENT_" }, value = {

        @IntDef(prefix = { "REPORT_EVENT_" }, value = {

                REPORT_EVENT_NO_RESPONSE,

                REPORT_EVENT_NO_RESPONSE,

                REPORT_EVENT_UI_SHOWN,

                REPORT_EVENT_UI_SHOWN,

                REPORT_EVENT_UI_DESTROYED

                REPORT_EVENT_UI_DESTROYED,

                REPORT_EVENT_INLINE_RESPONSE

        })

        })

        @Retention(RetentionPolicy.SOURCE)

        @Retention(RetentionPolicy.SOURCE)

        @interface ReportEvent{}

        @interface ReportEvent{}
@@ -365,8 +366,8 @@ public abstract class AugmentedAutofillService extends Service { 
        private final Object mLock = new Object();

        private final Object mLock = new Object();

        private final IAugmentedAutofillManagerClient mClient;

        private final IAugmentedAutofillManagerClient mClient;

        private final int mSessionId;

        private final int mSessionId;

        public final int taskId;

        public final int mTaskId;

        public final ComponentName componentName;

        public final ComponentName mComponentName;

        // Used for metrics / debug only

        // Used for metrics / debug only

        private String mServicePackageName;

        private String mServicePackageName;

        @GuardedBy("mLock")

        @GuardedBy("mLock")
@@ -406,8 +407,8 @@ public abstract class AugmentedAutofillService extends Service { 
            mSessionId = sessionId;

            mSessionId = sessionId;

            mClient = IAugmentedAutofillManagerClient.Stub.asInterface(client);

            mClient = IAugmentedAutofillManagerClient.Stub.asInterface(client);

            mCallback = callback;

            mCallback = callback;

            this.taskId = taskId;

            mTaskId = taskId;

            this.componentName = componentName;

            mComponentName = componentName;

            mServicePackageName = serviceComponentName.getPackageName();

            mServicePackageName = serviceComponentName.getPackageName();

            mFocusedId = focusedId;

            mFocusedId = focusedId;

            mFocusedValue = focusedValue;

            mFocusedValue = focusedValue;
@@ -514,22 +515,24 @@ public abstract class AugmentedAutofillService extends Service { 
            }

            }

        }

        }





        public void onInlineSuggestionsDataReady(@NonNull List<Dataset> inlineSuggestionsData,

        void reportResult(@Nullable List<Dataset> inlineSuggestionsData) {

                @Nullable Bundle clientState) {

            try {

            try {

                mCallback.onSuccess(inlineSuggestionsData.toArray(new Dataset[]{}), clientState);

                final Dataset[] inlineSuggestions = (inlineSuggestionsData != null)

                        ? inlineSuggestionsData.toArray(new Dataset[inlineSuggestionsData.size()])

                        : null;

                mCallback.onSuccess(inlineSuggestions);

            } catch (RemoteException e) {

            } catch (RemoteException e) {

                Log.e(TAG, "Error calling back with the inline suggestions data: " + e);

                Log.e(TAG, "Error calling back with the inline suggestions data: " + e);

            }

            }

        }

        }





        // Used (mostly) for metrics.

        void logEvent(@ReportEvent int event) {

        public void report(@ReportEvent int event) {

            if (sVerbose) Log.v(TAG, "returnAndLogResult(): " + event);

            if (sVerbose) Log.v(TAG, "report(): " + event);

            long duration = -1;

            long duration = -1;

            int type = MetricsEvent.TYPE_UNKNOWN;

            int type = MetricsEvent.TYPE_UNKNOWN;



            switch (event) {

            switch (event) {

                case REPORT_EVENT_NO_RESPONSE:

                case REPORT_EVENT_NO_RESPONSE: {

                    type = MetricsEvent.TYPE_SUCCESS;

                    type = MetricsEvent.TYPE_SUCCESS;

                    if (mFirstOnSuccessTime == 0) {

                    if (mFirstOnSuccessTime == 0) {

                        mFirstOnSuccessTime = SystemClock.elapsedRealtime();

                        mFirstOnSuccessTime = SystemClock.elapsedRealtime();
@@ -538,40 +541,49 @@ public abstract class AugmentedAutofillService extends Service { 
                            Log.d(TAG, "Service responded nothing in " + formatDuration(duration));

                            Log.d(TAG, "Service responded nothing in " + formatDuration(duration));

                        }

                        }

                    }

                    }

                    try {

                } break;

                        mCallback.onSuccess(/* inlineSuggestionsData= */null, /* clientState=*/



                                null);

                case REPORT_EVENT_INLINE_RESPONSE: {

                    } catch (RemoteException e) {

                    // TODO: Define a constant and log this event

                        Log.e(TAG, "Error reporting success: " + e);

                    // type = MetricsEvent.TYPE_SUCCESS_INLINE;

                    if (mFirstOnSuccessTime == 0) {

                        mFirstOnSuccessTime = SystemClock.elapsedRealtime();

                        duration = mFirstOnSuccessTime - mFirstRequestTime;

                        if (sDebug) {

                            Log.d(TAG, "Service responded nothing in " + formatDuration(duration));

                        }

                    }

                    }

                    break;

                } break;

                case REPORT_EVENT_UI_SHOWN:



                case REPORT_EVENT_UI_SHOWN: {

                    type = MetricsEvent.TYPE_OPEN;

                    type = MetricsEvent.TYPE_OPEN;

                    if (mUiFirstShownTime == 0) {

                    if (mUiFirstShownTime == 0) {

                        mUiFirstShownTime = SystemClock.elapsedRealtime();

                        mUiFirstShownTime = SystemClock.elapsedRealtime();

                        duration = mUiFirstShownTime - mFirstRequestTime;

                        duration = mUiFirstShownTime - mFirstRequestTime;

                        if (sDebug) Log.d(TAG, "UI shown in " + formatDuration(duration));

                        if (sDebug) Log.d(TAG, "UI shown in " + formatDuration(duration));

                    }

                    }

                    break;

                } break;

                case REPORT_EVENT_UI_DESTROYED:



                case REPORT_EVENT_UI_DESTROYED: {

                    type = MetricsEvent.TYPE_CLOSE;

                    type = MetricsEvent.TYPE_CLOSE;

                    if (mUiFirstDestroyedTime == 0) {

                    if (mUiFirstDestroyedTime == 0) {

                        mUiFirstDestroyedTime = SystemClock.elapsedRealtime();

                        mUiFirstDestroyedTime = SystemClock.elapsedRealtime();

                        duration = mUiFirstDestroyedTime - mFirstRequestTime;

                        duration = mUiFirstDestroyedTime - mFirstRequestTime;

                        if (sDebug) Log.d(TAG, "UI destroyed in " + formatDuration(duration));

                        if (sDebug) Log.d(TAG, "UI destroyed in " + formatDuration(duration));

                    }

                    }

                    break;

                } break;



                default:

                default:

                    Log.w(TAG, "invalid event reported: " + event);

                    Log.w(TAG, "invalid event reported: " + event);

            }

            }

            logResponse(type, mServicePackageName, componentName, mSessionId, duration);

            logResponse(type, mServicePackageName, mComponentName, mSessionId, duration);

        }

        }





        public void dump(@NonNull String prefix, @NonNull PrintWriter pw) {

        public void dump(@NonNull String prefix, @NonNull PrintWriter pw) {

            pw.print(prefix); pw.print("sessionId: "); pw.println(mSessionId);

            pw.print(prefix); pw.print("sessionId: "); pw.println(mSessionId);

            pw.print(prefix); pw.print("taskId: "); pw.println(taskId);

            pw.print(prefix); pw.print("taskId: "); pw.println(mTaskId);

            pw.print(prefix); pw.print("component: ");

            pw.print(prefix); pw.print("component: ");

            pw.println(componentName.flattenToShortString());

            pw.println(mComponentName.flattenToShortString());

            pw.print(prefix); pw.print("focusedId: "); pw.println(mFocusedId);

            pw.print(prefix); pw.print("focusedId: "); pw.println(mFocusedId);

            if (mFocusedValue != null) {

            if (mFocusedValue != null) {

                pw.print(prefix); pw.print("focusedValue: "); pw.println(mFocusedValue);

                pw.print(prefix); pw.print("focusedValue: "); pw.println(mFocusedValue);

core/java/android/service/autofill/augmented/FillCallback.java

+4 −2
Original line number Original line Diff line number Diff line
@@ -54,13 +54,15 @@ public final class FillCallback { 
        if (sDebug) Log.d(TAG, "onSuccess(): " + response);

        if (sDebug) Log.d(TAG, "onSuccess(): " + response);





        if (response == null) {

        if (response == null) {

            mProxy.report(AutofillProxy.REPORT_EVENT_NO_RESPONSE);

            mProxy.logEvent(AutofillProxy.REPORT_EVENT_NO_RESPONSE);

            mProxy.reportResult(null /*inlineSuggestions*/);

            return;

            return;

        }

        }





        List<Dataset> inlineSuggestions = response.getInlineSuggestions();

        List<Dataset> inlineSuggestions = response.getInlineSuggestions();

        if (inlineSuggestions != null && !inlineSuggestions.isEmpty()) {

        if (inlineSuggestions != null && !inlineSuggestions.isEmpty()) {

            mProxy.onInlineSuggestionsDataReady(inlineSuggestions, response.getClientState());

            mProxy.logEvent(AutofillProxy.REPORT_EVENT_INLINE_RESPONSE);

            mProxy.reportResult(inlineSuggestions);

            return;

            return;

        }

        }

core/java/android/service/autofill/augmented/FillController.java

+5 −4
Original line number Original line Diff line number Diff line
@@ -62,12 +62,13 @@ public final class FillController { 




        try {

        try {

            mProxy.autofill(values);

            mProxy.autofill(values);

        } catch (RemoteException e) {

            e.rethrowAsRuntimeException();

        }



        final FillWindow fillWindow = mProxy.getFillWindow();

        final FillWindow fillWindow = mProxy.getFillWindow();

        if (fillWindow != null) {

        if (fillWindow != null) {

            fillWindow.destroy();

            fillWindow.destroy();

        }

        }

        } catch (RemoteException e) {

            e.rethrowAsRuntimeException();

        }

    }

    }

}
 
}

core/java/android/service/autofill/augmented/FillRequest.java

+2 −2
Original line number Original line Diff line number Diff line
@@ -53,7 +53,7 @@ public final class FillRequest { 
     * Gets the task of the activity associated with this request.

     * Gets the task of the activity associated with this request.

     */

     */

    public int getTaskId() {

    public int getTaskId() {

        return mProxy.taskId;

        return mProxy.mTaskId;

    }

    }





    /**

    /**
@@ -61,7 +61,7 @@ public final class FillRequest { 
     */

     */

    @NonNull

    @NonNull

    public ComponentName getActivityComponent() {

    public ComponentName getActivityComponent() {

        return mProxy.componentName;

        return mProxy.mComponentName;

    }

    }





    /**

    /**

core/java/android/service/autofill/augmented/FillWindow.java

+27 −14
Original line number Original line Diff line number Diff line
@@ -21,6 +21,7 @@ import static android.service.autofill.augmented.AugmentedAutofillService.sVerbo 
import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage;

import static com.android.internal.util.function.pooled.PooledLambda.obtainMessage;





import android.annotation.NonNull;

import android.annotation.NonNull;

import android.annotation.Nullable;

import android.annotation.SystemApi;

import android.annotation.SystemApi;

import android.annotation.TestApi;

import android.annotation.TestApi;

import android.graphics.Rect;

import android.graphics.Rect;
@@ -41,6 +42,7 @@ import com.android.internal.util.Preconditions; 
import dalvik.system.CloseGuard;

import dalvik.system.CloseGuard;





import java.io.PrintWriter;

import java.io.PrintWriter;

import java.lang.ref.WeakReference;





/**

/**

 * Handle to a window used to display the augmented autofill UI.

 * Handle to a window used to display the augmented autofill UI.
@@ -70,23 +72,22 @@ public final class FillWindow implements AutoCloseable { 
    private final CloseGuard mCloseGuard = CloseGuard.get();

    private final CloseGuard mCloseGuard = CloseGuard.get();





    private final @NonNull Handler mUiThreadHandler = new Handler(Looper.getMainLooper());

    private final @NonNull Handler mUiThreadHandler = new Handler(Looper.getMainLooper());

    private final @NonNull FillWindowPresenter mFillWindowPresenter = new FillWindowPresenter();





    @GuardedBy("mLock")

    @GuardedBy("mLock")

    private WindowManager mWm;

    private @NonNull WindowManager mWm;

    @GuardedBy("mLock")

    @GuardedBy("mLock")

    private View mFillView;

    private View mFillView;

    @GuardedBy("mLock")

    @GuardedBy("mLock")

    private boolean mShowing;

    private boolean mShowing;

    @GuardedBy("mLock")

    @GuardedBy("mLock")

    private Rect mBounds;

    private @Nullable Rect mBounds;





    @GuardedBy("mLock")

    @GuardedBy("mLock")

    private boolean mUpdateCalled;

    private boolean mUpdateCalled;

    @GuardedBy("mLock")

    @GuardedBy("mLock")

    private boolean mDestroyed;

    private boolean mDestroyed;





    private AutofillProxy mProxy;

    private @NonNull AutofillProxy mProxy;





    /**

    /**

     * Updates the content of the window.

     * Updates the content of the window.
@@ -172,11 +173,11 @@ public final class FillWindow implements AutoCloseable { 
                try {

                try {

                    mProxy.requestShowFillUi(mBounds.right - mBounds.left,

                    mProxy.requestShowFillUi(mBounds.right - mBounds.left,

                            mBounds.bottom - mBounds.top,

                            mBounds.bottom - mBounds.top,

                            /*anchorBounds=*/ null, mFillWindowPresenter);

                            /*anchorBounds=*/ null, new FillWindowPresenter(this));

                } catch (RemoteException e) {

                } catch (RemoteException e) {

                    Log.w(TAG, "Error requesting to show fill window", e);

                    Log.w(TAG, "Error requesting to show fill window", e);

                }

                }

                mProxy.report(AutofillProxy.REPORT_EVENT_UI_SHOWN);

                mProxy.logEvent(AutofillProxy.REPORT_EVENT_UI_SHOWN);

            }

            }

        }

        }

    }

    }
@@ -244,7 +245,7 @@ public final class FillWindow implements AutoCloseable { 
            if (mUpdateCalled) {

            if (mUpdateCalled) {

                mFillView.setOnClickListener(null);

                mFillView.setOnClickListener(null);

                hide();

                hide();

                mProxy.report(AutofillProxy.REPORT_EVENT_UI_DESTROYED);

                mProxy.logEvent(AutofillProxy.REPORT_EVENT_UI_DESTROYED);

            }

            }

            mDestroyed = true;

            mDestroyed = true;

            mCloseGuard.close();

            mCloseGuard.close();
@@ -254,9 +255,7 @@ public final class FillWindow implements AutoCloseable { 
    @Override

    @Override

    protected void finalize() throws Throwable {

    protected void finalize() throws Throwable {

        try {

        try {

            if (mCloseGuard != null) {

            mCloseGuard.warnIfOpen();

            mCloseGuard.warnIfOpen();

            }

            destroy();

            destroy();

        } finally {

        } finally {

            super.finalize();

            super.finalize();
@@ -289,22 +288,36 @@ public final class FillWindow implements AutoCloseable { 




    /** @hide */

    /** @hide */

    @Override

    @Override

    public void close() throws Exception {

    public void close() {

        destroy();

        destroy();

    }

    }





    private final class FillWindowPresenter extends IAutofillWindowPresenter.Stub {

    private static final class FillWindowPresenter extends IAutofillWindowPresenter.Stub {

        private final @NonNull WeakReference<FillWindow> mFillWindowReference;



        FillWindowPresenter(@NonNull FillWindow fillWindow) {

            mFillWindowReference = new WeakReference<>(fillWindow);

        }



        @Override

        @Override

        public void show(WindowManager.LayoutParams p, Rect transitionEpicenter,

        public void show(WindowManager.LayoutParams p, Rect transitionEpicenter,

                boolean fitsSystemWindows, int layoutDirection) {

                boolean fitsSystemWindows, int layoutDirection) {

            if (sDebug) Log.d(TAG, "FillWindowPresenter.show()");

            if (sDebug) Log.d(TAG, "FillWindowPresenter.show()");

            mUiThreadHandler.sendMessage(obtainMessage(FillWindow::handleShow, FillWindow.this, p));

            final FillWindow fillWindow = mFillWindowReference.get();

            if (fillWindow != null) {

                fillWindow.mUiThreadHandler.sendMessage(

                        obtainMessage(FillWindow::handleShow, fillWindow, p));

            }

        }

        }





        @Override

        @Override

        public void hide(Rect transitionEpicenter) {

        public void hide(Rect transitionEpicenter) {

            if (sDebug) Log.d(TAG, "FillWindowPresenter.hide()");

            if (sDebug) Log.d(TAG, "FillWindowPresenter.hide()");

            mUiThreadHandler.sendMessage(obtainMessage(FillWindow::handleHide, FillWindow.this));

            final FillWindow fillWindow = mFillWindowReference.get();

            if (fillWindow != null) {

                fillWindow.mUiThreadHandler.sendMessage(

                        obtainMessage(FillWindow::handleHide, fillWindow));

            }

        }

        }

    }

    }

}
 
}