Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c4fccd18 authored by Svetoslav's avatar Svetoslav
Browse files

Adding APIs for an accessibility service to intercept key events. 

Now that we have gestures which are detected by the system and
interpreted by an accessibility service, there is an inconsistent
behavior between using the gestures and the keyboard. Some devices
have both. Therefore, an accessibility service should be able to
interpret keys in addition to gestures to provide consistent user
experience. Now an accessibility service can expose shortcuts for
each gestural action.

This change adds APIs for an accessibility service to observe and
intercept at will key events before they are dispatched to the
rest of the system. The service can return true or false from its
onKeyEvent to either consume the event or to let it be delivered
to the rest of the system. However, the service will *not* be
able to inject key events or modify the observed ones.

Previous ideas of allowing the service to say it "tracks" the event
so the latter is not delivered to the system until a subsequent
event is either "handled" or "not handled" will not work. If the
service tracks a key but no other key is pressed essentially this
key is not delivered to the app and at potentially much later point
this stashed event will be delivered in maybe a completely different
context.The correct way of implementing shortcuts is a combination
of modifier keys plus some other key/key sequence. Key events already
contain information about which modifier keys are down as well as
the service can track them as well.

bug:8088812

Change-Id: I81ba9a7de9f19ca6662661f27fdc852323e38c00
parent dbf500aa

api/current.txt

+1 −0
Original line number Diff line number Diff line
@@ -2074,6 +2074,7 @@ package android.accessibilityservice { 
    method public final android.os.IBinder onBind(android.content.Intent);

    method protected boolean onGesture(int);

    method public abstract void onInterrupt();

    method protected boolean onKeyEvent(android.view.KeyEvent);

    method protected void onServiceConnected();

    method public final boolean performGlobalAction(int);

    method public final void setServiceInfo(android.accessibilityservice.AccessibilityServiceInfo);

core/java/android/accessibilityservice/AccessibilityService.java

+74 −14
Original line number Diff line number Diff line
@@ -24,6 +24,7 @@ import android.os.Looper; 
import android.os.Message;

import android.os.RemoteException;

import android.util.Log;

import android.view.KeyEvent;

import android.view.accessibility.AccessibilityEvent;

import android.view.accessibility.AccessibilityInteractionClient;

import android.view.accessibility.AccessibilityNodeInfo;
@@ -348,6 +349,7 @@ public abstract class AccessibilityService extends Service { 
        public void onServiceConnected();

        public void onSetConnectionId(int connectionId);

        public boolean onGesture(int gestureId);

        public boolean onKeyEvent(KeyEvent event);

    }



    private int mConnectionId;
@@ -412,6 +414,32 @@ public abstract class AccessibilityService extends Service { 
        return false;

    }



    /**

     * Callback that allows an accessibility service to observe the key events

     * before they are passed to the rest of the system. This means that the events

     * are first delivered here before they are passed to the device policy, the

     * input method, or applications.

     * <p>

     * <strong>Note:</strong> It is important that key events are handled in such

     * a way that the event stream that would be passed to the rest of the system

     * is well-formed. For example, handling the down event but not the up event

     * and vice versa would generate an inconsistent event stream.

     * </p>

     * <p>

     * <strong>Note:</strong> The key events delivered in this method are copies

     * and modifying them will have no effect on the events that will be passed

     * to the system. This method is intended to perform purely filtering

     * functionality.

     * <p>

     *

     * @param event The event to be processed.

     * @return If true then the event will be consumed and not delivered to

     *         applications, otherwise it will be delivered as usual.

     */

    protected boolean onKeyEvent(KeyEvent event) {

        return false;

    }



    /**

     * Gets the root node in the currently active window if this service

     * can retrieve window content.
@@ -535,6 +563,11 @@ public abstract class AccessibilityService extends Service { 
            public boolean onGesture(int gestureId) {

                return AccessibilityService.this.onGesture(gestureId);

            }



            @Override

            public boolean onKeyEvent(KeyEvent event) {

                return AccessibilityService.this.onKeyEvent(event);

            }

        });

    }
@@ -554,11 +587,14 @@ public abstract class AccessibilityService extends Service { 
        private static final int DO_ON_ACCESSIBILITY_EVENT = 30;

        private static final int DO_ON_GESTURE = 40;

        private static final int DO_CLEAR_ACCESSIBILITY_NODE_INFO_CACHE = 50;

        private static final int DO_ON_KEY_EVENT = 60;



        private final HandlerCaller mCaller;



        private final Callbacks mCallback;



        private int mConnectionId;



        public IAccessibilityServiceClientWrapper(Context context, Looper looper,

                Callbacks callback) {

            mCallback = callback;
@@ -591,41 +627,65 @@ public abstract class AccessibilityService extends Service { 
            mCaller.sendMessage(message);

        }



        @Override

        public void onKeyEvent(KeyEvent event, int sequence) {

            Message message = mCaller.obtainMessageIO(DO_ON_KEY_EVENT, sequence, event);

            mCaller.sendMessage(message);

        }



        public void executeMessage(Message message) {

            switch (message.what) {

                case DO_ON_ACCESSIBILITY_EVENT :

                case DO_ON_ACCESSIBILITY_EVENT: {

                    AccessibilityEvent event = (AccessibilityEvent) message.obj;

                    if (event != null) {

                        AccessibilityInteractionClient.getInstance().onAccessibilityEvent(event);

                        mCallback.onAccessibilityEvent(event);

                        event.recycle();

                    }

                    return;

                case DO_ON_INTERRUPT :

                } return;

                case DO_ON_INTERRUPT: {

                    mCallback.onInterrupt();

                    return;

                case DO_SET_SET_CONNECTION :

                    final int connectionId = message.arg1;

                } return;

                case DO_SET_SET_CONNECTION: {

                    mConnectionId = message.arg1;

                    IAccessibilityServiceConnection connection =

                        (IAccessibilityServiceConnection) message.obj;

                    if (connection != null) {

                        AccessibilityInteractionClient.getInstance().addConnection(connectionId,

                        AccessibilityInteractionClient.getInstance().addConnection(mConnectionId,

                                connection);

                        mCallback.onSetConnectionId(connectionId);

                        mCallback.onSetConnectionId(mConnectionId);

                        mCallback.onServiceConnected();

                    } else {

                        AccessibilityInteractionClient.getInstance().removeConnection(connectionId);

                        AccessibilityInteractionClient.getInstance().removeConnection(mConnectionId);

                        AccessibilityInteractionClient.getInstance().clearCache();

                        mCallback.onSetConnectionId(AccessibilityInteractionClient.NO_ID);

                    }

                    return;

                case DO_ON_GESTURE :

                } return;

                case DO_ON_GESTURE: {

                    final int gestureId = message.arg1;

                    mCallback.onGesture(gestureId);

                    return;

                case DO_CLEAR_ACCESSIBILITY_NODE_INFO_CACHE:

                } return;

                case DO_CLEAR_ACCESSIBILITY_NODE_INFO_CACHE: {

                    AccessibilityInteractionClient.getInstance().clearCache();

                    return;

                } return;

                case DO_ON_KEY_EVENT: {

                    KeyEvent event = (KeyEvent) message.obj;

                    try {

                        IAccessibilityServiceConnection connection = AccessibilityInteractionClient

                                .getInstance().getConnection(mConnectionId);

                        if (connection != null) {

                            final boolean result = mCallback.onKeyEvent(event);

                            final int sequence = message.arg1;

                            try {

                                connection.setOnKeyEventResult(result, sequence);

                            } catch (RemoteException re) {

                                /* ignore */

                            }

                        }

                    } finally {

                        event.recycle();

                    }

                } return;

                default :

                    Log.w(LOG_TAG, "Unknown message type " + message.what);

            }

core/java/android/accessibilityservice/IAccessibilityServiceClient.aidl

+3 −0
Original line number Diff line number Diff line
@@ -18,6 +18,7 @@ package android.accessibilityservice; 


import android.accessibilityservice.IAccessibilityServiceConnection;

import android.view.accessibility.AccessibilityEvent;

import android.view.KeyEvent;



/**

 * Top-level interface to an accessibility service component.
@@ -35,4 +36,6 @@ import android.view.accessibility.AccessibilityEvent; 
    void onGesture(int gesture);



    void clearAccessibilityNodeInfoCache();



    void onKeyEvent(in KeyEvent event, int sequence);

}

core/java/android/accessibilityservice/IAccessibilityServiceConnection.aidl

+2 −115
Original line number Diff line number Diff line
@@ -31,144 +31,31 @@ interface IAccessibilityServiceConnection { 


    void setServiceInfo(in AccessibilityServiceInfo info);



    /**

     * Finds an {@link android.view.accessibility.AccessibilityNodeInfo} by accessibility id.

     *

     * @param accessibilityWindowId A unique window id. Use

     *     {@link android.view.accessibility.AccessibilityNodeInfo#ACTIVE_WINDOW_ID}

     *     to query the currently active window.

     * @param accessibilityNodeId A unique view id or virtual descendant id from

     *     where to start the search. Use

     *     {@link android.view.accessibility.AccessibilityNodeInfo#ROOT_NODE_ID}

     *     to start from the root.

     * @param interactionId The id of the interaction for matching with the callback result.

     * @param callback Callback which to receive the result.

     * @param flags Additional flags.

     * @param threadId The id of the calling thread.

     * @return Whether the call succeeded.

     */

    boolean findAccessibilityNodeInfoByAccessibilityId(int accessibilityWindowId,

        long accessibilityNodeId, int interactionId,

        IAccessibilityInteractionConnectionCallback callback, int flags, long threadId);



    /**

     * Finds {@link android.view.accessibility.AccessibilityNodeInfo}s by View text.

     * The match is case insensitive containment. The search is performed in the window

     * whose id is specified and starts from the node whose accessibility id is specified.

     *

     * @param accessibilityWindowId A unique window id. Use

     *     {@link android.view.accessibility.AccessibilityNodeInfo#ACTIVE_WINDOW_ID}

     *     to query the currently active window.

     * @param accessibilityNodeId A unique view id or virtual descendant id from

     *     where to start the search. Use

     *     {@link android.view.accessibility.AccessibilityNodeInfo#ROOT_NODE_ID}

     *     to start from the root.

     * @param text The searched text.

     * @param interactionId The id of the interaction for matching with the callback result.

     * @param callback Callback which to receive the result.

     * @param threadId The id of the calling thread.

     * @return Whether the call succeeded.

     */

    boolean findAccessibilityNodeInfosByText(int accessibilityWindowId, long accessibilityNodeId,

        String text, int interactionId, IAccessibilityInteractionConnectionCallback callback,

        long threadId);



    /**

     * Finds an {@link android.view.accessibility.AccessibilityNodeInfo} by View id. The search

     * is performed in the window whose id is specified and starts from the node whose

     * accessibility id is specified.

     *

     * @param accessibilityWindowId A unique window id. Use

     *     {@link android.view.accessibility.AccessibilityNodeInfo#ACTIVE_WINDOW_ID}

     *     to query the currently active window.

     * @param accessibilityNodeId A unique view id or virtual descendant id from

     *     where to start the search. Use

     *     {@link android.view.accessibility.AccessibilityNodeInfo#ROOT_NODE_ID}

     *     to start from the root.

     * @param viewId The fully qualified resource name of the view id to find.

     * @param interactionId The id of the interaction for matching with the callback result.

     * @param callback Callback which to receive the result.

     * @param threadId The id of the calling thread.

     * @return Whether the call succeeded.

     */

    boolean findAccessibilityNodeInfosByViewId(int accessibilityWindowId,

        long accessibilityNodeId, String viewId, int interactionId,

        IAccessibilityInteractionConnectionCallback callback, long threadId);



    /**

     * Finds the {@link android.view.accessibility.AccessibilityNodeInfo} that has the specified

     * focus type. The search is performed in the window whose id is specified and starts from

     * the node whose accessibility id is specified.

     *

     * @param accessibilityWindowId A unique window id. Use

     *     {@link android.view.accessibility.AccessibilityNodeInfo#ACTIVE_WINDOW_ID}

     *     to query the currently active window.

     * @param accessibilityNodeId A unique view id or virtual descendant id from

     *     where to start the search. Use

     *     {@link android.view.accessibility.AccessibilityNodeInfo#ROOT_NODE_ID}

     *     to start from the root.

     * @param focusType The type of focus to find.

     * @param interactionId The id of the interaction for matching with the callback result.

     * @param callback Callback which to receive the result.

     * @param threadId The id of the calling thread.

     * @return Whether the call succeeded.

     */

    boolean findFocus(int accessibilityWindowId, long accessibilityNodeId, int focusType,

        int interactionId, IAccessibilityInteractionConnectionCallback callback, long threadId);



    /**

     * Finds an {@link android.view.accessibility.AccessibilityNodeInfo} to take accessibility

     * focus in the given direction. The search is performed in the window whose id is

     * specified and starts from the node whose accessibility id is specified.

     *

     * @param accessibilityWindowId A unique window id. Use

     *     {@link android.view.accessibility.AccessibilityNodeInfo#ACTIVE_WINDOW_ID}

     *     to query the currently active window.

     * @param accessibilityNodeId A unique view id or virtual descendant id from

     *     where to start the search. Use

     *     {@link android.view.accessibility.AccessibilityNodeInfo#ROOT_NODE_ID}

     *     to start from the root.

     * @param direction The direction in which to search for focusable.

     * @param interactionId The id of the interaction for matching with the callback result.

     * @param callback Callback which to receive the result.

     * @param threadId The id of the calling thread.

     * @return Whether the call succeeded.

     */

    boolean focusSearch(int accessibilityWindowId, long accessibilityNodeId, int direction,

        int interactionId, IAccessibilityInteractionConnectionCallback callback, long threadId);



    /**

     * Performs an accessibility action on an

     * {@link android.view.accessibility.AccessibilityNodeInfo}.

     *

     * @param accessibilityWindowId A unique window id. Use

     *     {@link android.view.accessibility.AccessibilityNodeInfo#ACTIVE_WINDOW_ID}

     *     to query the currently active window.

     * @param accessibilityNodeId A unique view id or virtual descendant id from

     *     where to start the search. Use

     *     {@link android.view.accessibility.AccessibilityNodeInfo#ROOT_NODE_ID}

     *     to start from the root.

     * @param action The action to perform.

     * @param arguments Optional action arguments.

     * @param interactionId The id of the interaction for matching with the callback result.

     * @param callback Callback which to receive the result.

     * @param threadId The id of the calling thread.

     * @return Whether the action was performed.

     */

    boolean performAccessibilityAction(int accessibilityWindowId, long accessibilityNodeId,

        int action, in Bundle arguments, int interactionId,

        IAccessibilityInteractionConnectionCallback callback, long threadId);



    /**

     * @return The associated accessibility service info.

     */

    AccessibilityServiceInfo getServiceInfo();



    /**

     * Performs a global action, such as going home, going back, etc.

     *

     * @param action The action to perform.

     * @return Whether the action was performed.

     */

    boolean performGlobalAction(int action);



    oneway void setOnKeyEventResult(boolean handled, int sequence);

}

core/java/android/app/UiAutomation.java

+6 −0
Original line number Diff line number Diff line
@@ -31,6 +31,7 @@ import android.os.SystemClock; 
import android.util.Log;

import android.view.Display;

import android.view.InputEvent;

import android.view.KeyEvent;

import android.view.Surface;

import android.view.accessibility.AccessibilityEvent;

import android.view.accessibility.AccessibilityInteractionClient;
@@ -693,6 +694,11 @@ public final class UiAutomation { 
                        listener.onAccessibilityEvent(AccessibilityEvent.obtain(event));

                    }

                }



                @Override

                public boolean onKeyEvent(KeyEvent event) {

                    return false;

                }

            });

        }

    }