Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit fb17e5ae authored by yingleiw's avatar yingleiw
Browse files

Allow a11y services to enter text via the path IMEs use 

InputMethodService is the primary and a11y is the secondary.
InputMethodService is not
affected by a11y status. When the session from input method is
established, app can start input (pass input context to input method).
When an a11y session comes back, it will be passed to the app. When
InputMethodManagerService binds to/start input with
InputMethodService, it does the same to a11y services which
requested IME functionalities.

It is possible that input method can edit text before ally sessios are
established. So the EditorInfo passed to a11y could be stale.
So when an a11y session is passed to client, client will send
a notification (input method doesn't have this extra notification) for
the current selection. I think since the time for a11y session establish
shouldn't be long, and we get the current state later, it should be fine
for a11y services.

When input method is disconnected from app (client) (even for input
method switching), we cleared a11y and sessions too. When
input method request sessions, we must rerequest sessions for a11y. This
is mainly because when we unbindCurrentClientLocked(SWITCH_IME), we set
active to false for the current client. Suppose we don't want to
change the current structure of input method, an inactive client
probably should clear accessibility sessions too.

When we switch to a client which already has a session with input
method, there might be some a11y sessions with this client, and
some a11y services might be disabled or enabled while the client was
switched out. We pass unchanged a11y
sessions to client, and request sessions for newly enabled a11y services.
When an a11y service is disabled, it removes its session
from all clients in InputMethodManagerService.

Test: type word through modified "switchToInputMethod". Tested session
notification through logs. Tested client switching, input method
switching, a11y service enabled/disable, multiple a11y services,
a11y service enabled before device reboot.
Also tested work profile.

Bug: 187453053

Change-Id: Ia651a811093a939d00c081be1961e24ed3ad0356
parent 370b27ad

core/api/current.txt

+24 −0
Original line number Original line Diff line number Diff line
@@ -3059,6 +3059,7 @@ package android.accessibilityservice { 
    method @NonNull public final android.accessibilityservice.AccessibilityButtonController getAccessibilityButtonController();

    method @NonNull public final android.accessibilityservice.AccessibilityButtonController getAccessibilityButtonController();

    method @NonNull public final android.accessibilityservice.AccessibilityButtonController getAccessibilityButtonController(int);

    method @NonNull public final android.accessibilityservice.AccessibilityButtonController getAccessibilityButtonController(int);

    method @NonNull @RequiresPermission(android.Manifest.permission.USE_FINGERPRINT) public final android.accessibilityservice.FingerprintGestureController getFingerprintGestureController();

    method @NonNull @RequiresPermission(android.Manifest.permission.USE_FINGERPRINT) public final android.accessibilityservice.FingerprintGestureController getFingerprintGestureController();

    method @Nullable public final android.accessibilityservice.InputMethod getInputMethod();

    method @NonNull public final android.accessibilityservice.AccessibilityService.MagnificationController getMagnificationController();

    method @NonNull public final android.accessibilityservice.AccessibilityService.MagnificationController getMagnificationController();

    method public android.view.accessibility.AccessibilityNodeInfo getRootInActiveWindow();

    method public android.view.accessibility.AccessibilityNodeInfo getRootInActiveWindow();

    method public final android.accessibilityservice.AccessibilityServiceInfo getServiceInfo();

    method public final android.accessibilityservice.AccessibilityServiceInfo getServiceInfo();
@@ -3071,6 +3072,7 @@ package android.accessibilityservice { 
    method public boolean isNodeInCache(@NonNull android.view.accessibility.AccessibilityNodeInfo);

    method public boolean isNodeInCache(@NonNull android.view.accessibility.AccessibilityNodeInfo);

    method public abstract void onAccessibilityEvent(android.view.accessibility.AccessibilityEvent);

    method public abstract void onAccessibilityEvent(android.view.accessibility.AccessibilityEvent);

    method public final android.os.IBinder onBind(android.content.Intent);

    method public final android.os.IBinder onBind(android.content.Intent);

    method @NonNull public android.accessibilityservice.InputMethod onCreateInputMethod();

    method @Deprecated protected boolean onGesture(int);

    method @Deprecated protected boolean onGesture(int);

    method public boolean onGesture(@NonNull android.accessibilityservice.AccessibilityGestureEvent);

    method public boolean onGesture(@NonNull android.accessibilityservice.AccessibilityGestureEvent);

    method public abstract void onInterrupt();

    method public abstract void onInterrupt();
@@ -3317,6 +3319,28 @@ package android.accessibilityservice { 
    method public boolean willContinue();

    method public boolean willContinue();

  }

  }




















  public class InputMethod {














    ctor protected InputMethod(@NonNull android.accessibilityservice.AccessibilityService);














    method @Nullable public final android.accessibilityservice.InputMethod.AccessibilityInputConnection getCurrentInputConnection();














    method @Nullable public final android.view.inputmethod.EditorInfo getCurrentInputEditorInfo();














    method public final boolean getCurrentInputStarted();














    method public void onFinishInput();














    method public void onStartInput(@NonNull android.view.inputmethod.EditorInfo, boolean);














    method public void onUpdateSelection(int, int, int, int, int, int);














  }



























  public final class InputMethod.AccessibilityInputConnection {














    method public void clearMetaKeyStates(int);














    method public void commitText(@NonNull CharSequence, int, @Nullable android.view.inputmethod.TextAttribute);














    method public void deleteSurroundingText(int, int);














    method public int getCursorCapsMode(int);














    method @Nullable public android.view.inputmethod.SurroundingText getSurroundingText(@IntRange(from=0) int, @IntRange(from=0) int, int);














    method public void performContextMenuAction(int);














    method public void performEditorAction(int);














    method public void sendKeyEvent(@NonNull android.view.KeyEvent);














    method public void setSelection(int, int);














  }
























  public final class MagnificationConfig implements android.os.Parcelable {







  public final class MagnificationConfig implements android.os.Parcelable {











    method public int describeContents();







    method public int describeContents();











    method public float getCenterX();







    method public float getCenterX();

























core/java/android/accessibilityservice/AccessibilityService.java



+229
−0




























Original line number
Original line
Diff line number
Diff line








@@ -40,6 +40,8 @@ import android.graphics.ParcelableColorSpace;









import android.graphics.Region;







import android.graphics.Region;











import android.hardware.HardwareBuffer;







import android.hardware.HardwareBuffer;











import android.hardware.display.DisplayManager;







import android.hardware.display.DisplayManager;














import android.inputmethodservice.IInputMethodSessionWrapper;














import android.inputmethodservice.RemoteInputConnection;











import android.os.Build;







import android.os.Build;











import android.os.Bundle;







import android.os.Bundle;











import android.os.Handler;







import android.os.Handler;









@@ -65,14 +67,23 @@ import android.view.accessibility.AccessibilityInteractionClient;









import android.view.accessibility.AccessibilityNodeInfo;







import android.view.accessibility.AccessibilityNodeInfo;











import android.view.accessibility.AccessibilityNodeInfo.AccessibilityAction;







import android.view.accessibility.AccessibilityNodeInfo.AccessibilityAction;











import android.view.accessibility.AccessibilityWindowInfo;







import android.view.accessibility.AccessibilityWindowInfo;














import android.view.inputmethod.EditorInfo;














import android.view.inputmethod.InputBinding;














import android.view.inputmethod.InputConnection;














import android.view.inputmethod.InputMethodSession;


































import com.android.internal.inputmethod.CancellationGroup;











import com.android.internal.os.HandlerCaller;







import com.android.internal.os.HandlerCaller;











import com.android.internal.os.SomeArgs;







import com.android.internal.os.SomeArgs;











import com.android.internal.util.Preconditions;







import com.android.internal.util.Preconditions;











import com.android.internal.util.function.pooled.PooledLambda;







import com.android.internal.util.function.pooled.PooledLambda;














import com.android.internal.view.IInputContext;














import com.android.internal.view.IInputMethodSession;














import com.android.internal.view.IInputSessionWithIdCallback;































import java.lang.annotation.Retention;







import java.lang.annotation.Retention;











import java.lang.annotation.RetentionPolicy;







import java.lang.annotation.RetentionPolicy;














import java.lang.ref.WeakReference;











import java.util.Collections;







import java.util.Collections;











import java.util.List;







import java.util.List;











import java.util.concurrent.Executor;







import java.util.concurrent.Executor;










@@ -627,6 +638,21 @@ public abstract class AccessibilityService extends Service {









        void onAccessibilityButtonAvailabilityChanged(boolean available);







        void onAccessibilityButtonAvailabilityChanged(boolean available);











        /** This is called when the system action list is changed. */







        /** This is called when the system action list is changed. */











        void onSystemActionsChanged();







        void onSystemActionsChanged();














        /** This is called when an app requests ime sessions or when the service is enabled. */














        void createImeSession(IInputSessionWithIdCallback callback);














        /**














         * This is called when InputMethodManagerService requests to set the session enabled or














         * disabled














         */














        void setImeSessionEnabled(InputMethodSession session, boolean enabled);














        /** This is called when an app binds input or when the service is enabled. */














        void bindInput(InputBinding binding);














        /** This is called when an app unbinds input or when the service is disabled. */














        void unbindInput();














        /** This is called when an app starts input or when the service is enabled. */














        void startInput(@Nullable InputConnection inputConnection,














                @NonNull EditorInfo editorInfo, boolean restarting,














                @NonNull IBinder startInputToken);











    }







    }































    /**







    /**










@@ -763,6 +789,8 @@ public abstract class AccessibilityService extends Service {









            new SparseArray<>(0);







            new SparseArray<>(0);































    private SoftKeyboardController mSoftKeyboardController;







    private SoftKeyboardController mSoftKeyboardController;














    private InputMethod mInputMethod;














    private boolean mInputMethodInitialized = false;











    private final SparseArray<AccessibilityButtonController> mAccessibilityButtonControllers =







    private final SparseArray<AccessibilityButtonController> mAccessibilityButtonControllers =











            new SparseArray<>(0);







            new SparseArray<>(0);






























@@ -797,6 +825,11 @@ public abstract class AccessibilityService extends Service {









            for (int i = 0; i < mMagnificationControllers.size(); i++) {







            for (int i = 0; i < mMagnificationControllers.size(); i++) {











                mMagnificationControllers.valueAt(i).onServiceConnectedLocked();







                mMagnificationControllers.valueAt(i).onServiceConnectedLocked();











            }







            }














            // TODO(b/187453053): If service requested ime capabilities














            if (!mInputMethodInitialized) {














                mInputMethod = onCreateInputMethod();














                mInputMethodInitialized = true;














            }











        }







        }











        if (mSoftKeyboardController != null) {







        if (mSoftKeyboardController != null) {











            mSoftKeyboardController.onServiceConnected();







            mSoftKeyboardController.onServiceConnected();










@@ -1849,6 +1882,32 @@ public abstract class AccessibilityService extends Service {









        }







        }











    }







    }


































    /**














     * The default implementation returns our default {@link InputMethod}. Subclasses can override














     * it to provide their own customized version. Accessibility services need to set the














     * {@link AccessibilityServiceInfo#FLAG_REQUEST_IME_APIS} flag to use input method APIs.














     *














     * @return the InputMethod.














     */














    @NonNull














    public InputMethod onCreateInputMethod() {














        return new InputMethod(this);














    }





























    /**














     * Returns the InputMethod instance after the system calls {@link #onCreateInputMethod()},














     * which may be used to input text or get editable text selection change notifications. It will














     * return null if the accessibility service doesn't set the














     * {@link AccessibilityServiceInfo#FLAG_REQUEST_IME_APIS} flag or the system doesn't call














     * {@link #onCreateInputMethod()}.














     *














     * @return the InputMethod instance














     */














    @Nullable














    public final InputMethod getInputMethod() {














        return mInputMethod;














    }


























    private void onSoftKeyboardShowModeChanged(int showMode) {







    private void onSoftKeyboardShowModeChanged(int showMode) {











        if (mSoftKeyboardController != null) {







        if (mSoftKeyboardController != null) {











            mSoftKeyboardController.dispatchSoftKeyboardShowModeChanged(showMode);







            mSoftKeyboardController.dispatchSoftKeyboardShowModeChanged(showMode);










@@ -2657,6 +2716,47 @@ public abstract class AccessibilityService extends Service {









            public void onSystemActionsChanged() {







            public void onSystemActionsChanged() {











                AccessibilityService.this.onSystemActionsChanged();







                AccessibilityService.this.onSystemActionsChanged();











            }







            }





























            @Override














            public void createImeSession(IInputSessionWithIdCallback callback) {














                if (mInputMethod != null) {














                    mInputMethod.createImeSession(callback);














                }














            }





























            @Override














            public void setImeSessionEnabled(InputMethodSession session, boolean enabled) {














                if (mInputMethod != null) {














                    mInputMethod.setImeSessionEnabled(session, enabled);














                }














            }





























            @Override














            public void bindInput(InputBinding binding) {














                if (mInputMethod != null) {














                    mInputMethod.bindInput(binding);














                }














            }





























            @Override














            public void unbindInput() {














                if (mInputMethod != null) {














                    mInputMethod.unbindInput();














                }














            }





























            @Override














            public void startInput(@Nullable InputConnection inputConnection,














                    @NonNull EditorInfo editorInfo, boolean restarting,














                    @NonNull IBinder startInputToken) {














                if (mInputMethod != null) {














                    if (restarting) {














                        mInputMethod.restartInput(inputConnection, editorInfo);














                    } else {














                        mInputMethod.startInput(inputConnection, editorInfo);














                    }














                }














            }











        });







        });











    }







    }





























@@ -2682,6 +2782,11 @@ public abstract class AccessibilityService extends Service {









        private static final int DO_ACCESSIBILITY_BUTTON_CLICKED = 12;







        private static final int DO_ACCESSIBILITY_BUTTON_CLICKED = 12;











        private static final int DO_ACCESSIBILITY_BUTTON_AVAILABILITY_CHANGED = 13;







        private static final int DO_ACCESSIBILITY_BUTTON_AVAILABILITY_CHANGED = 13;











        private static final int DO_ON_SYSTEM_ACTIONS_CHANGED = 14;







        private static final int DO_ON_SYSTEM_ACTIONS_CHANGED = 14;














        private static final int DO_CREATE_IME_SESSION = 15;














        private static final int DO_SET_IME_SESSION_ENABLED = 16;














        private static final int DO_BIND_INPUT = 17;














        private static final int DO_UNBIND_INPUT = 18;














        private static final int DO_START_INPUT = 19;































        private final HandlerCaller mCaller;







        private final HandlerCaller mCaller;





























@@ -2690,6 +2795,22 @@ public abstract class AccessibilityService extends Service {





























        private int mConnectionId = AccessibilityInteractionClient.NO_ID;







        private int mConnectionId = AccessibilityInteractionClient.NO_ID;


































        /**














         * This is not {@null} only between {@link #bindInput(InputBinding)} and














         * {@link #unbindInput()} so that {@link RemoteInputConnection} can query if














         * {@link #unbindInput()} has already been called or not, mainly to avoid unnecessary














         * blocking operations.














         *














         * <p>This field must be set and cleared only from the binder thread(s), where the system














         * guarantees that {@link #bindInput(InputBinding)},














         * {@link #startInput(IBinder, IInputContext, EditorInfo, boolean)}, and














         * {@link #unbindInput()} are called with the same order as the original calls














         * in {@link com.android.server.inputmethod.InputMethodManagerService}.














         * See {@link IBinder#FLAG_ONEWAY} for detailed semantics.</p>














         */














        @Nullable














        CancellationGroup mCancellationGroup = null;


























        public IAccessibilityServiceClientWrapper(Context context, Looper looper,







        public IAccessibilityServiceClientWrapper(Context context, Looper looper,











                Callbacks callback) {







                Callbacks callback) {











            mCallback = callback;







            mCallback = callback;










@@ -2783,6 +2904,70 @@ public abstract class AccessibilityService extends Service {









            mCaller.sendMessage(mCaller.obtainMessage(DO_ON_SYSTEM_ACTIONS_CHANGED));







            mCaller.sendMessage(mCaller.obtainMessage(DO_ON_SYSTEM_ACTIONS_CHANGED));











        }







        }


































        /** This is called when an app requests ime sessions or when the service is enabled. */














        public void createImeSession(IInputSessionWithIdCallback callback) {














            final Message message = mCaller.obtainMessageO(DO_CREATE_IME_SESSION, callback);














            mCaller.sendMessage(message);














        }





























        /**














         * This is called when InputMethodManagerService requests to set the session enabled or














         * disabled














         */














        public void setImeSessionEnabled(IInputMethodSession session, boolean enabled) {














            try {














                InputMethodSession ls = ((IInputMethodSessionWrapper)














                        session).getInternalInputMethodSession();














                if (ls == null) {














                    Log.w(LOG_TAG, "Session is already finished: " + session);














                    return;














                }














                mCaller.sendMessage(mCaller.obtainMessageIO(














                        DO_SET_IME_SESSION_ENABLED, enabled ? 1 : 0, ls));














            } catch (ClassCastException e) {














                Log.w(LOG_TAG, "Incoming session not of correct type: " + session, e);














            }














        }





























        /** This is called when an app binds input or when the service is enabled. */














        public void bindInput(InputBinding binding) {














            if (mCancellationGroup != null) {














                Log.e(LOG_TAG, "bindInput must be paired with unbindInput.");














            }














            mCancellationGroup = new CancellationGroup();














            InputConnection ic = new RemoteInputConnection(new WeakReference<>(() -> mContext),














                    IInputContext.Stub.asInterface(binding.getConnectionToken()),














                    mCancellationGroup);














            InputBinding nu = new InputBinding(ic, binding);














            final Message message = mCaller.obtainMessageO(DO_BIND_INPUT, nu);














            mCaller.sendMessage(message);














        }





























        /** This is called when an app unbinds input or when the service is disabled. */














        public void unbindInput() {














            if (mCancellationGroup != null) {














                // Signal the flag then forget it.














                mCancellationGroup.cancelAll();














                mCancellationGroup = null;














            } else {














                Log.e(LOG_TAG, "unbindInput must be paired with bindInput.");














            }














            mCaller.sendMessage(mCaller.obtainMessage(DO_UNBIND_INPUT));














        }





























        /** This is called when an app starts input or when the service is enabled. */














        public void startInput(IBinder startInputToken, IInputContext inputContext,














                EditorInfo editorInfo, boolean restarting) {














            if (mCancellationGroup == null) {














                Log.e(LOG_TAG, "startInput must be called after bindInput.");














                mCancellationGroup = new CancellationGroup();














            }














            final Message message = mCaller.obtainMessageOOOOII(DO_START_INPUT, startInputToken,














                    inputContext, editorInfo, mCancellationGroup, restarting ? 1 : 0,














                    0 /* unused */);














            mCaller.sendMessage(message);














        }


























        @Override







        @Override











        public void onMotionEvent(MotionEvent event) {







        public void onMotionEvent(MotionEvent event) {











            final Message message = PooledLambda.obtainMessage(







            final Message message = PooledLambda.obtainMessage(










@@ -2948,6 +3133,50 @@ public abstract class AccessibilityService extends Service {









                    }







                    }











                    return;







                    return;











                }







                }














                case DO_CREATE_IME_SESSION: {














                    if (mConnectionId != AccessibilityInteractionClient.NO_ID) {














                        IInputSessionWithIdCallback callback =














                                (IInputSessionWithIdCallback) message.obj;














                        mCallback.createImeSession(callback);














                    }














                    return;














                }














                case DO_SET_IME_SESSION_ENABLED: {














                    if (mConnectionId != AccessibilityInteractionClient.NO_ID) {














                        mCallback.setImeSessionEnabled((InputMethodSession) message.obj,














                                message.arg1 != 0);














                    }














                    return;














                }














                case DO_BIND_INPUT: {














                    if (mConnectionId != AccessibilityInteractionClient.NO_ID) {














                        mCallback.bindInput((InputBinding) message.obj);














                    }














                    return;














                }














                case DO_UNBIND_INPUT: {














                    if (mConnectionId != AccessibilityInteractionClient.NO_ID) {














                        mCallback.unbindInput();














                    }














                    return;














                }














                case DO_START_INPUT: {














                    if (mConnectionId != AccessibilityInteractionClient.NO_ID) {














                        final SomeArgs args = (SomeArgs) message.obj;














                        final IBinder startInputToken = (IBinder) args.arg1;














                        final IInputContext inputContext = (IInputContext) args.arg2;














                        final EditorInfo info = (EditorInfo) args.arg3;














                        final CancellationGroup cancellationGroup = (CancellationGroup) args.arg4;














                        final boolean restarting = args.argi5 == 1;














                        final InputConnection ic = inputContext != null














                                ? new RemoteInputConnection(new WeakReference<>(() -> mContext),














                                inputContext, cancellationGroup) : null;














                        info.makeCompatible(mContext.getApplicationInfo().targetSdkVersion);














                        mCallback.startInput(ic, info, restarting, startInputToken);














                        args.recycle();














                    }














                    return;














                }











                default:







                default:











                    Log.w(LOG_TAG, "Unknown message type " + message.what);







                    Log.w(LOG_TAG, "Unknown message type " + message.what);











            }







            }

































core/java/android/accessibilityservice/IAccessibilityServiceClient.aidl



+16
−0




























Original line number
Original line
Diff line number
Diff line








@@ -24,6 +24,11 @@ import android.accessibilityservice.AccessibilityGestureEvent;









import android.accessibilityservice.MagnificationConfig;







import android.accessibilityservice.MagnificationConfig;











import android.view.KeyEvent;







import android.view.KeyEvent;











import android.view.MotionEvent;







import android.view.MotionEvent;














import android.view.inputmethod.EditorInfo;














import android.view.inputmethod.InputBinding;














import com.android.internal.view.IInputContext;














import com.android.internal.view.IInputMethodSession;














import com.android.internal.view.IInputSessionWithIdCallback;































/**







/**











 * Top-level interface to an accessibility service component.







 * Top-level interface to an accessibility service component.










@@ -63,4 +68,15 @@ import android.view.MotionEvent;









    void onAccessibilityButtonAvailabilityChanged(boolean available);







    void onAccessibilityButtonAvailabilityChanged(boolean available);































    void onSystemActionsChanged();







    void onSystemActionsChanged();





























    void createImeSession(IInputSessionWithIdCallback callback);





























    void setImeSessionEnabled(IInputMethodSession session, boolean enabled);





























    void bindInput(in InputBinding binding);





























    void unbindInput();





























    void startInput(in IBinder startInputToken, in IInputContext inputContext,














                in EditorInfo editorInfo, boolean restarting);











}






}
























core/java/android/accessibilityservice/InputMethod.java
0 → 100644



+637
−0


























File added.
Preview size limit exceeded, changes collapsed.
























core/java/android/app/UiAutomation.java



+28
−0




























Original line number
Original line
Diff line number
Diff line








@@ -63,9 +63,14 @@ import android.view.accessibility.AccessibilityInteractionClient;









import android.view.accessibility.AccessibilityNodeInfo;







import android.view.accessibility.AccessibilityNodeInfo;











import android.view.accessibility.AccessibilityWindowInfo;







import android.view.accessibility.AccessibilityWindowInfo;











import android.view.accessibility.IAccessibilityInteractionConnection;







import android.view.accessibility.IAccessibilityInteractionConnection;














import android.view.inputmethod.EditorInfo;














import android.view.inputmethod.InputBinding;














import android.view.inputmethod.InputConnection;














import android.view.inputmethod.InputMethodSession;































import com.android.internal.annotations.GuardedBy;







import com.android.internal.annotations.GuardedBy;











import com.android.internal.util.function.pooled.PooledLambda;







import com.android.internal.util.function.pooled.PooledLambda;














import com.android.internal.view.IInputSessionWithIdCallback;































import libcore.io.IoUtils;







import libcore.io.IoUtils;






























@@ -1565,6 +1570,29 @@ public final class UiAutomation {









                    /* do nothing */







                    /* do nothing */











                }







                }


































                @Override














                public void createImeSession(IInputSessionWithIdCallback callback) {














                    /* do nothing */














                }





























                @Override














                public void setImeSessionEnabled(InputMethodSession session, boolean enabled) {














                }





























                @Override














                public void bindInput(InputBinding binding) {














                }





























                @Override














                public void unbindInput() {














                }





























                @Override














                public void startInput(@Nullable InputConnection inputConnection,














                        @NonNull EditorInfo editorInfo, boolean restarting,














                        @NonNull IBinder startInputToken) {














                }


























                @Override







                @Override











                public boolean onGesture(AccessibilityGestureEvent gestureEvent) {







                public boolean onGesture(AccessibilityGestureEvent gestureEvent) {











                    /* do nothing */







                    /* do nothing */