Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit a766c12e authored by Daniel Norman's avatar Daniel Norman
Browse files

Introduce AccessibilityService#onMotionEvent() 

This API allows accessibility services to intercept motions events, to
expand device control capabilities to more types of input devices
such as watch rotary wheels and gamepad joysticks.

Unlike KeyEvents which are optionally consumed, for MotionEvents the
service must declare the event sources it wishes to consume and then
events from these sources are *always* consumed. The source list can be
changed by the service at runtime. By always consuming the requested
events we avoid the potential "wait and see" latency concerns that
onKeyEvent deals with.

Bug: 247550565
Test: atest AccessibilityEndtoEndTest
Test: use in an a11yservice to observe joystick events
Change-Id: Iea54b00b21f5348f50d94d6e2213df24c0c8a36a
parent 83e20e86

core/api/current.txt

+3 −0
Original line number Diff line number Diff line
@@ -3115,6 +3115,7 @@ package android.accessibilityservice { 
    method public boolean onGesture(@NonNull android.accessibilityservice.AccessibilityGestureEvent);

    method public abstract void onInterrupt();

    method protected boolean onKeyEvent(android.view.KeyEvent);

    method public void onMotionEvent(@NonNull android.view.MotionEvent);

    method protected void onServiceConnected();

    method public void onSystemActionsChanged();

    method public final boolean performGlobalAction(int);
@@ -3272,6 +3273,7 @@ package android.accessibilityservice { 
    method @Deprecated public String getDescription();

    method public String getId();

    method public int getInteractiveUiTimeoutMillis();

    method public int getMotionEventSources();

    method public int getNonInteractiveUiTimeoutMillis();

    method public android.content.pm.ResolveInfo getResolveInfo();

    method public String getSettingsActivityName();
@@ -3281,6 +3283,7 @@ package android.accessibilityservice { 
    method @Nullable public CharSequence loadIntro(@NonNull android.content.pm.PackageManager);

    method public CharSequence loadSummary(android.content.pm.PackageManager);

    method public void setInteractiveUiTimeoutMillis(@IntRange(from=0) int);

    method public void setMotionEventSources(int);

    method public void setNonInteractiveUiTimeoutMillis(@IntRange(from=0) int);

    method public void writeToParcel(android.os.Parcel, int);

    field public static final int CAPABILITY_CAN_CONTROL_MAGNIFICATION = 16; // 0x10

core/java/android/accessibilityservice/AccessibilityService.java

+45 −9
Original line number Diff line number Diff line
@@ -54,6 +54,7 @@ import android.util.Log; 
import android.util.Slog;

import android.util.SparseArray;

import android.view.Display;

import android.view.InputDevice;

import android.view.KeyEvent;

import android.view.MotionEvent;

import android.view.SurfaceControl;
@@ -797,6 +798,8 @@ public abstract class AccessibilityService extends Service { 


    private FingerprintGestureController mFingerprintGestureController;



    private int mMotionEventSources;



    /**

     * Callback for {@link android.view.accessibility.AccessibilityEvent}s.

     *
@@ -820,7 +823,11 @@ public abstract class AccessibilityService extends Service { 
            for (int i = 0; i < mMagnificationControllers.size(); i++) {

                mMagnificationControllers.valueAt(i).onServiceConnectedLocked();

            }

            updateInputMethod(getServiceInfo());

            final AccessibilityServiceInfo info = getServiceInfo();

            if (info != null) {

                updateInputMethod(info);

                mMotionEventSources = info.getMotionEventSources();

            }

        }

        if (mSoftKeyboardController != null) {

            mSoftKeyboardController.onServiceConnected();
@@ -945,6 +952,25 @@ public abstract class AccessibilityService extends Service { 
        return false;

    }



    /**

     * Callback that allows an accessibility service to observe generic {@link MotionEvent}s.

     * <p>

     * Prefer {@link TouchInteractionController} to observe and control touchscreen events,

     * including touch gestures. If this or any enabled service is using

     * {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} then

     * {@link #onMotionEvent} will not receive touchscreen events.

     * </p>

     * <p>

     * <strong>Note:</strong> The service must first request to listen to events using

     * {@link AccessibilityServiceInfo#setMotionEventSources}.

     * {@link MotionEvent}s from sources in {@link AccessibilityServiceInfo#getMotionEventSources()}

     * are not sent to the rest of the system. To stop listening to events from a given source, call

     * {@link AccessibilityServiceInfo#setMotionEventSources} with that source removed.

     * </p>

     * @param event The event to be processed.

     */

    public void onMotionEvent(@NonNull MotionEvent event) { }



    /**

     * Gets the windows on the screen of the default display. This method returns only the windows

     * that a sighted user can interact with, as opposed to all windows.
@@ -2521,6 +2547,7 @@ public abstract class AccessibilityService extends Service { 
    public final void setServiceInfo(AccessibilityServiceInfo info) {

        mInfo = info;

        updateInputMethod(info);

        mMotionEventSources = info.getMotionEventSources();

        sendServiceInfo();

    }
@@ -2724,7 +2751,7 @@ public abstract class AccessibilityService extends Service { 


            @Override

            public void onMotionEvent(MotionEvent event) {

                AccessibilityService.this.onMotionEvent(event);

                AccessibilityService.this.sendMotionEventToCallback(event);

            }



            @Override
@@ -3359,16 +3386,25 @@ public abstract class AccessibilityService extends Service { 
        }

    }



    void onMotionEvent(MotionEvent event) {

    void sendMotionEventToCallback(MotionEvent event) {

        boolean sendingTouchEventToTouchInteractionController = false;

        if (event.isFromSource(InputDevice.SOURCE_TOUCHSCREEN)) {

            TouchInteractionController controller;

            synchronized (mLock) {

                int displayId = event.getDisplayId();

                controller = mTouchInteractionControllers.get(displayId);

            }

            if (controller != null) {

                sendingTouchEventToTouchInteractionController = true;

                controller.onMotionEvent(event);

            }

        }

        final int eventSourceWithoutClass = event.getSource() & ~InputDevice.SOURCE_CLASS_MASK;

        if ((mMotionEventSources & eventSourceWithoutClass) != 0

                && !sendingTouchEventToTouchInteractionController) {

            onMotionEvent(event);

        }

    }



    void onTouchStateChanged(int displayId, int state) {

        TouchInteractionController controller;

core/java/android/accessibilityservice/AccessibilityServiceInfo.java

+47 −0
Original line number Diff line number Diff line
@@ -611,6 +611,12 @@ public class AccessibilityServiceInfo implements Parcelable { 
     */

    private boolean mIsAccessibilityTool = false;



    /**

     * The bit mask of {@link android.view.InputDevice} sources that the accessibility

     * service wants to listen to for generic {@link android.view.MotionEvent}s.

     */

    private int mMotionEventSources = 0;



    /**

     * Creates a new instance.

     */
@@ -785,6 +791,7 @@ public class AccessibilityServiceInfo implements Parcelable { 
        mNonInteractiveUiTimeout = other.mNonInteractiveUiTimeout;

        mInteractiveUiTimeout = other.mInteractiveUiTimeout;

        flags = other.flags;

        mMotionEventSources = other.mMotionEventSources;

        // NOTE: Ensure that only properties that are safe to be modified by the service itself

        // are included here (regardless of hidden setters, etc.).

    }
@@ -955,6 +962,44 @@ public class AccessibilityServiceInfo implements Parcelable { 
        mCapabilities = capabilities;

    }



    /**

     * Returns the bit mask of {@link android.view.InputDevice} sources that the accessibility

     * service wants to listen to for generic {@link android.view.MotionEvent}s.

     */

    public int getMotionEventSources() {

        return mMotionEventSources;

    }



    /**

     * Sets the bit mask of {@link android.view.InputDevice} sources that the accessibility

     * service wants to listen to for generic {@link android.view.MotionEvent}s.

     *

     * <p>

     * Note: including an {@link android.view.InputDevice} source that does not send

     * {@link android.view.MotionEvent}s is effectively a no-op for that source, since you will

     * not receive any events from that source.

     * </p>

     * <p>

     * Allowed sources include:

     * <li>{@link android.view.InputDevice#SOURCE_MOUSE}</li>

     * <li>{@link android.view.InputDevice#SOURCE_STYLUS}</li>

     * <li>{@link android.view.InputDevice#SOURCE_BLUETOOTH_STYLUS}</li>

     * <li>{@link android.view.InputDevice#SOURCE_TRACKBALL}</li>

     * <li>{@link android.view.InputDevice#SOURCE_MOUSE_RELATIVE}</li>

     * <li>{@link android.view.InputDevice#SOURCE_TOUCHPAD}</li>

     * <li>{@link android.view.InputDevice#SOURCE_TOUCH_NAVIGATION}</li>

     * <li>{@link android.view.InputDevice#SOURCE_ROTARY_ENCODER}</li>

     * <li>{@link android.view.InputDevice#SOURCE_JOYSTICK}</li>

     * <li>{@link android.view.InputDevice#SOURCE_SENSOR}</li>

     * </p>

     *

     * @param motionEventSources A bit mask of {@link android.view.InputDevice} sources.

     * @see AccessibilityService#onMotionEvent

     */

    public void setMotionEventSources(int motionEventSources) {

        mMotionEventSources = motionEventSources;

    }



    /**

     * The localized summary of the accessibility service.

     * <p>
@@ -1179,6 +1224,7 @@ public class AccessibilityServiceInfo implements Parcelable { 
        parcel.writeBoolean(mIsAccessibilityTool);

        parcel.writeString(mTileServiceName);

        parcel.writeInt(mIntroResId);

        parcel.writeInt(mMotionEventSources);

    }



    private void initFromParcel(Parcel parcel) {
@@ -1203,6 +1249,7 @@ public class AccessibilityServiceInfo implements Parcelable { 
        mIsAccessibilityTool = parcel.readBoolean();

        mTileServiceName = parcel.readString();

        mIntroResId = parcel.readInt();

        mMotionEventSources = parcel.readInt();

    }



    @Override

services/accessibility/java/com/android/server/accessibility/AbstractAccessibilityServiceConnection.java

+10 −0
Original line number Diff line number Diff line
@@ -74,8 +74,10 @@ import android.util.Pair; 
import android.util.Slog;

import android.util.SparseArray;

import android.view.Display;

import android.view.InputDevice;

import android.view.KeyEvent;

import android.view.MagnificationSpec;

import android.view.MotionEvent;

import android.view.SurfaceControl;

import android.view.View;

import android.view.WindowInfo;
@@ -200,6 +202,8 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ 


    final ComponentName mComponentName;



    int mGenericMotionEventSources;



    // the events pending events to be dispatched to this service

    final SparseArray<AccessibilityEvent> mPendingEvents = new SparseArray<>();
@@ -362,6 +366,7 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ 
        }

        mNotificationTimeout = info.notificationTimeout;

        mIsDefault = (info.flags & DEFAULT) != 0;

        mGenericMotionEventSources = info.getMotionEventSources();



        if (supportsFlagForNotImportantViews(info)) {

            if ((info.flags & AccessibilityServiceInfo.FLAG_INCLUDE_NOT_IMPORTANT_VIEWS) != 0) {
@@ -1751,6 +1756,11 @@ abstract class AbstractAccessibilityServiceConnection extends IAccessibilityServ 
        return mSystemSupport.getWindowTransformationMatrixAndMagnificationSpec(resolvedWindowId);

    }



    public boolean wantsGenericMotionEvent(MotionEvent event) {

        final int eventSourceWithoutClass = event.getSource() & ~InputDevice.SOURCE_CLASS_MASK;

        return (mGenericMotionEventSources & eventSourceWithoutClass) != 0;

    }



    /**

     * Called by the invocation handler to notify the service that the

     * state of magnification has changed.

services/accessibility/java/com/android/server/accessibility/AccessibilityInputFilter.java

+59 −3
Original line number Diff line number Diff line
@@ -141,6 +141,9 @@ class AccessibilityInputFilter extends InputFilter implements EventStreamTransfo 
     */

    static final int FLAG_SEND_MOTION_EVENTS = 0x00000400;



    /** Flag for intercepting generic motion events. */

    static final int FLAG_FEATURE_INTERCEPT_GENERIC_MOTION_EVENTS = 0x00000800;



    static final int FEATURES_AFFECTING_MOTION_EVENTS =

            FLAG_FEATURE_INJECT_MOTION_EVENTS

                    | FLAG_FEATURE_AUTOCLICK
@@ -149,7 +152,8 @@ class AccessibilityInputFilter extends InputFilter implements EventStreamTransfo 
                    | FLAG_FEATURE_TRIGGERED_SCREEN_MAGNIFIER

                    | FLAG_SERVICE_HANDLES_DOUBLE_TAP

                    | FLAG_REQUEST_MULTI_FINGER_GESTURES

                    | FLAG_REQUEST_2_FINGER_PASSTHROUGH;

                    | FLAG_REQUEST_2_FINGER_PASSTHROUGH

                    | FLAG_FEATURE_INTERCEPT_GENERIC_MOTION_EVENTS;



    private final Context mContext;
@@ -182,6 +186,10 @@ class AccessibilityInputFilter extends InputFilter implements EventStreamTransfo 


    private final SparseArray<EventStreamState> mTouchScreenStreamStates = new SparseArray<>(0);



    // State tracking for generic MotionEvents is display-agnostic so we only need one.

    private GenericMotionEventStreamState mGenericMotionEventStreamState;

    private int mCombinedGenericMotionEventSources = 0;



    private EventStreamState mKeyboardStreamState;



    AccessibilityInputFilter(Context context, AccessibilityManagerService service) {
@@ -298,7 +306,13 @@ class AccessibilityInputFilter extends InputFilter implements EventStreamTransfo 
    private EventStreamState getEventStreamState(InputEvent event) {

        if (event instanceof MotionEvent) {

            final int displayId = event.getDisplayId();

            if (mGenericMotionEventStreamState == null) {

                mGenericMotionEventStreamState = new GenericMotionEventStreamState();

            }



            if (mGenericMotionEventStreamState.shouldProcessMotionEvent((MotionEvent) event)) {

                return mGenericMotionEventStreamState;

            }

            if (event.isFromSource(InputDevice.SOURCE_TOUCHSCREEN)) {

                EventStreamState touchScreenStreamState = mTouchScreenStreamStates.get(displayId);

                if (touchScreenStreamState == null) {
@@ -364,8 +378,11 @@ class AccessibilityInputFilter extends InputFilter implements EventStreamTransfo 
        mPm.userActivity(event.getEventTime(), false);

        MotionEvent transformedEvent = MotionEvent.obtain(event);

        final int displayId = event.getDisplayId();

        mEventHandler.get(isDisplayIdValid(displayId) ? displayId : Display.DEFAULT_DISPLAY)

                .onMotionEvent(transformedEvent, event, policyFlags);

        EventStreamTransformation eventStreamTransformation = mEventHandler.get(

                isDisplayIdValid(displayId) ? displayId : Display.DEFAULT_DISPLAY);

        if (eventStreamTransformation != null) {

            eventStreamTransformation.onMotionEvent(transformedEvent, event, policyFlags);

        }

        transformedEvent.recycle();

    }
@@ -492,6 +509,19 @@ class AccessibilityInputFilter extends InputFilter implements EventStreamTransfo 
            mTouchExplorer.put(displayId, explorer);

        }



        if ((mEnabledFeatures & FLAG_FEATURE_INTERCEPT_GENERIC_MOTION_EVENTS) != 0) {

            addFirstEventHandler(displayId, new BaseEventStreamTransformation() {

                @Override

                public void onMotionEvent(MotionEvent event, MotionEvent rawEvent,

                        int policyFlags) {

                    if (!anyServiceWantsGenericMotionEvent(rawEvent)

                            || !mAms.sendMotionEventToListeningServices(rawEvent)) {

                        super.onMotionEvent(event, rawEvent, policyFlags);

                    }

                }

            });

        }



        if ((mEnabledFeatures & FLAG_FEATURE_CONTROL_SCREEN_MAGNIFIER) != 0

                || ((mEnabledFeatures & FLAG_FEATURE_SCREEN_MAGNIFIER) != 0)

                || ((mEnabledFeatures & FLAG_FEATURE_TRIGGERED_SCREEN_MAGNIFIER) != 0)) {
@@ -844,6 +874,32 @@ class AccessibilityInputFilter extends InputFilter implements EventStreamTransfo 
        }

    }



    private class GenericMotionEventStreamState extends EventStreamState {

        @Override

        public boolean shouldProcessMotionEvent(MotionEvent event) {

            return anyServiceWantsGenericMotionEvent(event);

        }

        @Override

        public boolean shouldProcessScroll() {

            return true;

        }

    }



    private boolean anyServiceWantsGenericMotionEvent(MotionEvent event) {

        // Disable SOURCE_TOUCHSCREEN generic event interception if any service is performing

        // touch exploration.

        if (event.isFromSource(InputDevice.SOURCE_TOUCHSCREEN)

                && (mEnabledFeatures & FLAG_FEATURE_TOUCH_EXPLORATION) != 0) {

            return false;

        }

        final int eventSourceWithoutClass = event.getSource() & ~InputDevice.SOURCE_CLASS_MASK;

        return (mCombinedGenericMotionEventSources & eventSourceWithoutClass) != 0;

    }



    public void setCombinedGenericMotionEventSources(int sources) {

        mCombinedGenericMotionEventSources = sources;

    }



    /**

     * Keeps state of streams of events from all keyboard devices.

     */