Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 96b878aa authored by Sally Yuen's avatar Sally Yuen Committed by Android (Google) Code Review
Browse files

Merge "Implement multi-finger multi-tap gesture matchers for accessibility"

parents 04346f01 3cbd098e
Loading
Loading
Loading
Loading
+6 −0
Original line number Original line Diff line number Diff line
@@ -2873,6 +2873,12 @@ package android.accessibilityservice {
    method public final boolean performGlobalAction(int);
    method public final boolean performGlobalAction(int);
    method public final void setServiceInfo(android.accessibilityservice.AccessibilityServiceInfo);
    method public final void setServiceInfo(android.accessibilityservice.AccessibilityServiceInfo);
    method public boolean takeScreenshot(int, @NonNull java.util.concurrent.Executor, @NonNull java.util.function.Consumer<android.graphics.Bitmap>);
    method public boolean takeScreenshot(int, @NonNull java.util.concurrent.Executor, @NonNull java.util.function.Consumer<android.graphics.Bitmap>);
    field public static final int GESTURE_2_FINGER_DOUBLE_TAP = 20; // 0x14
    field public static final int GESTURE_2_FINGER_SINGLE_TAP = 19; // 0x13
    field public static final int GESTURE_2_FINGER_TRIPLE_TAP = 21; // 0x15
    field public static final int GESTURE_3_FINGER_DOUBLE_TAP = 23; // 0x17
    field public static final int GESTURE_3_FINGER_SINGLE_TAP = 22; // 0x16
    field public static final int GESTURE_3_FINGER_TRIPLE_TAP = 24; // 0x18
    field public static final int GESTURE_DOUBLE_TAP = 17; // 0x11
    field public static final int GESTURE_DOUBLE_TAP = 17; // 0x11
    field public static final int GESTURE_DOUBLE_TAP_AND_HOLD = 18; // 0x12
    field public static final int GESTURE_DOUBLE_TAP_AND_HOLD = 18; // 0x12
    field public static final int GESTURE_SWIPE_DOWN = 2; // 0x2
    field public static final int GESTURE_SWIPE_DOWN = 2; // 0x2
+43 −1
Original line number Original line Diff line number Diff line
@@ -17,6 +17,12 @@
package android.accessibilityservice;
package android.accessibilityservice;




import static android.accessibilityservice.AccessibilityService.GESTURE_2_FINGER_DOUBLE_TAP;
import static android.accessibilityservice.AccessibilityService.GESTURE_2_FINGER_SINGLE_TAP;
import static android.accessibilityservice.AccessibilityService.GESTURE_2_FINGER_TRIPLE_TAP;
import static android.accessibilityservice.AccessibilityService.GESTURE_3_FINGER_DOUBLE_TAP;
import static android.accessibilityservice.AccessibilityService.GESTURE_3_FINGER_SINGLE_TAP;
import static android.accessibilityservice.AccessibilityService.GESTURE_3_FINGER_TRIPLE_TAP;
import static android.accessibilityservice.AccessibilityService.GESTURE_DOUBLE_TAP;
import static android.accessibilityservice.AccessibilityService.GESTURE_DOUBLE_TAP;
import static android.accessibilityservice.AccessibilityService.GESTURE_DOUBLE_TAP_AND_HOLD;
import static android.accessibilityservice.AccessibilityService.GESTURE_DOUBLE_TAP_AND_HOLD;
import static android.accessibilityservice.AccessibilityService.GESTURE_SWIPE_DOWN;
import static android.accessibilityservice.AccessibilityService.GESTURE_SWIPE_DOWN;
@@ -60,6 +66,12 @@ public final class AccessibilityGestureEvent implements Parcelable {


    /** @hide */
    /** @hide */
    @IntDef(prefix = { "GESTURE_" }, value = {
    @IntDef(prefix = { "GESTURE_" }, value = {
            GESTURE_2_FINGER_SINGLE_TAP,
            GESTURE_2_FINGER_DOUBLE_TAP,
            GESTURE_2_FINGER_TRIPLE_TAP,
            GESTURE_3_FINGER_SINGLE_TAP,
            GESTURE_3_FINGER_DOUBLE_TAP,
            GESTURE_3_FINGER_TRIPLE_TAP,
            GESTURE_DOUBLE_TAP,
            GESTURE_DOUBLE_TAP,
            GESTURE_DOUBLE_TAP_AND_HOLD,
            GESTURE_DOUBLE_TAP_AND_HOLD,
            GESTURE_SWIPE_UP,
            GESTURE_SWIPE_UP,
@@ -122,13 +134,43 @@ public final class AccessibilityGestureEvent implements Parcelable {
    @Override
    @Override
    public String toString() {
    public String toString() {
        StringBuilder stringBuilder = new StringBuilder("AccessibilityGestureEvent[");
        StringBuilder stringBuilder = new StringBuilder("AccessibilityGestureEvent[");
        stringBuilder.append("gestureId: ").append(mGestureId);
        stringBuilder.append("gestureId: ").append(eventTypeToString(mGestureId));
        stringBuilder.append(", ");
        stringBuilder.append(", ");
        stringBuilder.append("displayId: ").append(mDisplayId);
        stringBuilder.append("displayId: ").append(mDisplayId);
        stringBuilder.append(']');
        stringBuilder.append(']');
        return stringBuilder.toString();
        return stringBuilder.toString();
    }
    }


    private static String eventTypeToString(int eventType) {
        switch (eventType) {
            case GESTURE_2_FINGER_SINGLE_TAP: return "GESTURE_2_FINGER_SINGLE_TAP";
            case GESTURE_2_FINGER_DOUBLE_TAP: return "GESTURE_2_FINGER_DOUBLE_TAP";
            case GESTURE_2_FINGER_TRIPLE_TAP: return "GESTURE_2_FINGER_TRIPLE_TAP";
            case GESTURE_3_FINGER_SINGLE_TAP: return "GESTURE_3_FINGER_SINGLE_TAP";
            case GESTURE_3_FINGER_DOUBLE_TAP: return "GESTURE_3_FINGER_DOUBLE_TAP";
            case GESTURE_3_FINGER_TRIPLE_TAP: return "GESTURE_3_FINGER_TRIPLE_TAP";
            case GESTURE_DOUBLE_TAP: return "GESTURE_DOUBLE_TAP";
            case GESTURE_DOUBLE_TAP_AND_HOLD: return "GESTURE_DOUBLE_TAP_AND_HOLD";
            case GESTURE_SWIPE_DOWN: return "GESTURE_SWIPE_DOWN";
            case GESTURE_SWIPE_DOWN_AND_LEFT: return "GESTURE_SWIPE_DOWN_AND_LEFT";
            case GESTURE_SWIPE_DOWN_AND_UP: return "GESTURE_SWIPE_DOWN_AND_UP";
            case GESTURE_SWIPE_DOWN_AND_RIGHT: return "GESTURE_SWIPE_DOWN_AND_RIGHT";
            case GESTURE_SWIPE_LEFT: return "GESTURE_SWIPE_LEFT";
            case GESTURE_SWIPE_LEFT_AND_UP: return "GESTURE_SWIPE_LEFT_AND_UP";
            case GESTURE_SWIPE_LEFT_AND_RIGHT: return "GESTURE_SWIPE_LEFT_AND_RIGHT";
            case GESTURE_SWIPE_LEFT_AND_DOWN: return "GESTURE_SWIPE_LEFT_AND_DOWN";
            case GESTURE_SWIPE_RIGHT: return "GESTURE_SWIPE_RIGHT";
            case GESTURE_SWIPE_RIGHT_AND_UP: return "GESTURE_SWIPE_RIGHT_AND_UP";
            case GESTURE_SWIPE_RIGHT_AND_LEFT: return "GESTURE_SWIPE_RIGHT_AND_LEFT";
            case GESTURE_SWIPE_RIGHT_AND_DOWN: return "GESTURE_SWIPE_RIGHT_AND_DOWN";
            case GESTURE_SWIPE_UP: return "GESTURE_SWIPE_UP";
            case GESTURE_SWIPE_UP_AND_LEFT: return "GESTURE_SWIPE_UP_AND_LEFT";
            case GESTURE_SWIPE_UP_AND_DOWN: return "GESTURE_SWIPE_UP_AND_DOWN";
            case GESTURE_SWIPE_UP_AND_RIGHT: return "GESTURE_SWIPE_UP_AND_RIGHT";
            default: return Integer.toHexString(eventType);
        }
    }

    /**
    /**
     * {@inheritDoc}
     * {@inheritDoc}
     */
     */
+30 −0
Original line number Original line Diff line number Diff line
@@ -318,6 +318,36 @@ public abstract class AccessibilityService extends Service {
     */
     */
    public static final int GESTURE_DOUBLE_TAP_AND_HOLD = 18;
    public static final int GESTURE_DOUBLE_TAP_AND_HOLD = 18;


    /**
     * The user has performed a two-finger single tap gesture on the touch screen.
     */
    public static final int GESTURE_2_FINGER_SINGLE_TAP = 19;

    /**
     * The user has performed a two-finger double tap gesture on the touch screen.
     */
    public static final int GESTURE_2_FINGER_DOUBLE_TAP = 20;

    /**
     * The user has performed a two-finger triple tap gesture on the touch screen.
     */
    public static final int GESTURE_2_FINGER_TRIPLE_TAP = 21;

    /**
     * The user has performed a three-finger single tap gesture on the touch screen.
     */
    public static final int GESTURE_3_FINGER_SINGLE_TAP = 22;

    /**
     * The user has performed a three-finger double tap gesture on the touch screen.
     */
    public static final int GESTURE_3_FINGER_DOUBLE_TAP = 23;

    /**
     * The user has performed a three-finger triple tap gesture on the touch screen.
     */
    public static final int GESTURE_3_FINGER_TRIPLE_TAP = 24;

    /**
    /**
     * The {@link Intent} that must be declared as handled by the service.
     * The {@link Intent} that must be declared as handled by the service.
     */
     */
+20 −0
Original line number Original line Diff line number Diff line
@@ -16,6 +16,12 @@


package com.android.server.accessibility.gestures;
package com.android.server.accessibility.gestures;


import static android.accessibilityservice.AccessibilityService.GESTURE_2_FINGER_DOUBLE_TAP;
import static android.accessibilityservice.AccessibilityService.GESTURE_2_FINGER_SINGLE_TAP;
import static android.accessibilityservice.AccessibilityService.GESTURE_2_FINGER_TRIPLE_TAP;
import static android.accessibilityservice.AccessibilityService.GESTURE_3_FINGER_DOUBLE_TAP;
import static android.accessibilityservice.AccessibilityService.GESTURE_3_FINGER_SINGLE_TAP;
import static android.accessibilityservice.AccessibilityService.GESTURE_3_FINGER_TRIPLE_TAP;
import static android.accessibilityservice.AccessibilityService.GESTURE_DOUBLE_TAP;
import static android.accessibilityservice.AccessibilityService.GESTURE_DOUBLE_TAP;
import static android.accessibilityservice.AccessibilityService.GESTURE_DOUBLE_TAP_AND_HOLD;
import static android.accessibilityservice.AccessibilityService.GESTURE_DOUBLE_TAP_AND_HOLD;
import static android.accessibilityservice.AccessibilityService.GESTURE_SWIPE_DOWN;
import static android.accessibilityservice.AccessibilityService.GESTURE_SWIPE_DOWN;
@@ -104,6 +110,20 @@ class GestureManifold implements GestureMatcher.StateChangeListener {
        mGestures.add(new Swipe(context, UP, DOWN, GESTURE_SWIPE_UP_AND_DOWN, this));
        mGestures.add(new Swipe(context, UP, DOWN, GESTURE_SWIPE_UP_AND_DOWN, this));
        mGestures.add(new Swipe(context, UP, LEFT, GESTURE_SWIPE_UP_AND_LEFT, this));
        mGestures.add(new Swipe(context, UP, LEFT, GESTURE_SWIPE_UP_AND_LEFT, this));
        mGestures.add(new Swipe(context, UP, RIGHT, GESTURE_SWIPE_UP_AND_RIGHT, this));
        mGestures.add(new Swipe(context, UP, RIGHT, GESTURE_SWIPE_UP_AND_RIGHT, this));
        // Two-finger taps.
        mMultiFingerGestures.add(
                new MultiFingerMultiTap(mContext, 2, 1, GESTURE_2_FINGER_SINGLE_TAP, this));
        mMultiFingerGestures.add(
                new MultiFingerMultiTap(mContext, 2, 2, GESTURE_2_FINGER_DOUBLE_TAP, this));
        mMultiFingerGestures.add(
                new MultiFingerMultiTap(mContext, 2, 3, GESTURE_2_FINGER_TRIPLE_TAP, this));
        // Three-finger taps.
        mMultiFingerGestures.add(
                new MultiFingerMultiTap(mContext, 3, 1, GESTURE_3_FINGER_SINGLE_TAP, this));
        mMultiFingerGestures.add(
                new MultiFingerMultiTap(mContext, 3, 2, GESTURE_3_FINGER_DOUBLE_TAP, this));
        mMultiFingerGestures.add(
                new MultiFingerMultiTap(mContext, 3, 3, GESTURE_3_FINGER_TRIPLE_TAP, this));
    }
    }


    /**
    /**
+291 −0
Original line number Original line Diff line number Diff line
/*
 * Copyright (C) 2020 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.android.server.accessibility.gestures;

import android.content.Context;
import android.graphics.PointF;
import android.os.Handler;
import android.view.MotionEvent;
import android.view.ViewConfiguration;

import com.android.internal.util.Preconditions;

import java.util.ArrayList;
import java.util.Arrays;

/**
 * This class matches multi-finger multi-tap gestures. The number of fingers and the number of taps
 * for each instance is specified in the constructor.
 */
class MultiFingerMultiTap extends GestureMatcher {

    // The target number of taps.
    final int mTargetTapCount;
    // The target number of fingers.
    final int mTargetFingerCount;
    // The acceptable distance between two taps of a finger.
    private int mDoubleTapSlop;
    // The acceptable distance the pointer can move and still count as a tap.
    private int mTouchSlop;
    // A tap counts when target number of fingers are down and up once.
    private int mCompletedTapCount;
    // A flag set to true when target number of fingers have touched down at once before.
    // Used to indicate what next finger action should be. Down when false and lift when true.
    private boolean mIsTargetFingerCountReached = false;
    // Store initial down points for slop checking and update when next down if is inside slop.
    private PointF[] mBases;
    // The points in bases that already have slop checked when onDown or onPointerDown.
    // It prevents excluded points matched multiple times by other pointers from next check.
    private ArrayList<PointF> mExcludedPointsForDownSlopChecked;

    /**
     * @throws IllegalArgumentException if <code>fingers<code/> is less than 2
     *                                  or <code>taps<code/> is not positive.
     */
    MultiFingerMultiTap(Context context, int fingers, int taps, int gestureId,
            GestureMatcher.StateChangeListener listener) {
        super(gestureId, new Handler(context.getMainLooper()), listener);
        Preconditions.checkArgument(fingers >= 2);
        Preconditions.checkArgumentPositive(taps, "Tap count must greater than 0.");
        mTargetTapCount = taps;
        mTargetFingerCount = fingers;
        mDoubleTapSlop = ViewConfiguration.get(context).getScaledDoubleTapSlop();
        mTouchSlop = ViewConfiguration.get(context).getScaledTouchSlop();

        mBases = new PointF[mTargetFingerCount];
        for (int i = 0; i < mBases.length; i++) {
            mBases[i] = new PointF();
        }
        mExcludedPointsForDownSlopChecked = new ArrayList<>(mTargetFingerCount);
        clear();
    }

    @Override
    protected void clear() {
        mCompletedTapCount = 0;
        mIsTargetFingerCountReached = false;
        for (int i = 0; i < mBases.length; i++) {
            mBases[i].set(Float.NaN, Float.NaN);
        }
        mExcludedPointsForDownSlopChecked.clear();
        super.clear();
    }

    @Override
    protected void onDown(MotionEvent event, MotionEvent rawEvent, int policyFlags) {
        // Before the matcher state transit to completed,
        // Cancel when an additional down arrived after reaching the target number of taps.
        if (mCompletedTapCount == mTargetTapCount) {
            cancelGesture(event, rawEvent, policyFlags);
            return;
        }
        cancelAfterTapTimeout(event, rawEvent, policyFlags);

        if (mCompletedTapCount == 0) {
            initBaseLocation(rawEvent);
            return;
        }
        // As fingers go up and down, their pointer ids will not be the same.
        // Therefore we require that a given finger be in slop range of any one
        // of the fingers from the previous tap.
        final PointF nearest = findNearestPoint(rawEvent, mDoubleTapSlop, true);
        if (nearest != null) {
            // Update pointer location to nearest one as a new base for next slop check.
            final int index = event.getActionIndex();
            nearest.set(event.getX(index), event.getY(index));
        } else {
            cancelGesture(event, rawEvent, policyFlags);
        }
    }

    @Override
    protected void onUp(MotionEvent event, MotionEvent rawEvent, int policyFlags) {
        cancelAfterDoubleTapTimeout(event, rawEvent, policyFlags);

        final PointF nearest = findNearestPoint(rawEvent, mTouchSlop, false);
        if ((getState() == STATE_GESTURE_STARTED || getState() == STATE_CLEAR)
                && null != nearest) {
            // Increase current tap count when the user have all fingers lifted
            // within the tap timeout since the target number of fingers are down.
            if (mIsTargetFingerCountReached) {
                mCompletedTapCount++;
                mIsTargetFingerCountReached = false;
                mExcludedPointsForDownSlopChecked.clear();
            }

            // Start gesture detection here to avoid the conflict to 2nd finger double tap
            // that never actually started gesture detection.
            if (mCompletedTapCount == 1) {
                startGesture(event, rawEvent, policyFlags);
            }
            if (mCompletedTapCount == mTargetTapCount) {
                // Done.
                completeAfterDoubleTapTimeout(event, rawEvent, policyFlags);
            }
        } else {
            // Either too many taps or nonsensical event stream.
            cancelGesture(event, rawEvent, policyFlags);
        }
    }

    @Override
    protected void onMove(MotionEvent event, MotionEvent rawEvent, int policyFlags) {
        // Outside the touch slop
        if (null == findNearestPoint(rawEvent, mTouchSlop, false)) {
            cancelGesture(event, rawEvent, policyFlags);
        }
    }

    @Override
    protected void onPointerDown(MotionEvent event, MotionEvent rawEvent, int policyFlags) {
        // Reset timeout to ease the use for some people
        // with certain impairments to get all their fingers down.
        cancelAfterTapTimeout(event, rawEvent, policyFlags);
        final int currentFingerCount = event.getPointerCount();
        // Accept down only before target number of fingers are down
        // or the finger count is not more than target.
        if ((currentFingerCount > mTargetFingerCount) || mIsTargetFingerCountReached) {
            cancelGesture(event, rawEvent, policyFlags);
            return;
        }

        final PointF nearest;
        if (mCompletedTapCount == 0) {
            nearest = initBaseLocation(rawEvent);
        } else {
            nearest = findNearestPoint(rawEvent, mDoubleTapSlop, true);
        }
        if ((getState() == STATE_GESTURE_STARTED || getState() == STATE_CLEAR)
                && nearest != null) {
            // The user have all fingers down within the tap timeout since first finger down,
            // setting the timeout for fingers to be lifted.
            if (currentFingerCount == mTargetFingerCount) {
                mIsTargetFingerCountReached = true;
            }
            // Update pointer location to nearest one as a new base for next slop check.
            final int index = event.getActionIndex();
            nearest.set(event.getX(index), event.getY(index));
        } else {
            cancelGesture(event, rawEvent, policyFlags);
        }
    }

    @Override
    protected void onPointerUp(MotionEvent event, MotionEvent rawEvent, int policyFlags) {
        // Accept up only after target number of fingers are down.
        if (!mIsTargetFingerCountReached) {
            cancelGesture(event, rawEvent, policyFlags);
            return;
        }

        if (getState() == STATE_GESTURE_STARTED || getState() == STATE_CLEAR) {
            // Needs more fingers lifted within the tap timeout
            // after reaching the target number of fingers are down.
        } else {
            cancelGesture(event, rawEvent, policyFlags);
        }
    }

    @Override
    public String getGestureName() {
        final StringBuilder builder = new StringBuilder();
        builder.append(mTargetFingerCount).append("-Finger ");
        if (mTargetTapCount == 1) {
            builder.append("Single");
        } else if (mTargetTapCount == 2) {
            builder.append("Double");
        } else if (mTargetTapCount == 3) {
            builder.append("Triple");
        } else if (mTargetTapCount > 3) {
            builder.append(mTargetTapCount);
        }
        return builder.append(" Tap").toString();
    }

    private PointF initBaseLocation(MotionEvent event) {
        final int index = event.getActionIndex();
        final int baseIndex = event.getPointerCount() - 1;
        final PointF p = mBases[baseIndex];
        if (Float.isNaN(p.x) && Float.isNaN(p.y)) {
            p.set(event.getX(index), event.getY(index));
        }
        return p;
    }

    /**
     * Find the nearest location to the given event in the bases.
     * If no one found, it could be not inside {@code slop}, filtered or empty bases.
     * When {@code filterMatched} is true, if the location of given event matches one of the points
     * in {@link #mExcludedPointsForDownSlopChecked} it would be ignored. Otherwise, the location
     * will be added to {@link #mExcludedPointsForDownSlopChecked}.
     *
     * @param event to find nearest point in bases.
     * @param slop to check to the given location of the event.
     * @param filterMatched true to exclude points already matched other pointers.
     * @return the point in bases closed to the location of the given event.
     */
    private PointF findNearestPoint(MotionEvent event, float slop, boolean filterMatched) {
        float moveDelta = Float.MAX_VALUE;
        PointF nearest = null;
        for (int i = 0; i < mBases.length; i++) {
            final PointF p = mBases[i];
            if (Float.isNaN(p.x) && Float.isNaN(p.y)) {
                continue;
            }
            if (filterMatched && mExcludedPointsForDownSlopChecked.contains(p)) {
                continue;
            }
            final int index = event.getActionIndex();
            final float dX = p.x - event.getX(index);
            final float dY = p.y - event.getY(index);
            if (dX == 0 && dY == 0) {
                if (filterMatched) {
                    mExcludedPointsForDownSlopChecked.add(p);
                }
                return p;
            }
            final float delta = (float) Math.hypot(dX, dY);
            if (moveDelta > delta) {
                moveDelta = delta;
                nearest = p;
            }
        }
        if (moveDelta < slop) {
            if (filterMatched) {
                mExcludedPointsForDownSlopChecked.add(nearest);
            }
            return nearest;
        }
        return null;
    }

    @Override
    public String toString() {
        final StringBuilder builder = new StringBuilder(super.toString());
        if (getState() != STATE_GESTURE_CANCELED) {
            builder.append(", CompletedTapCount: ");
            builder.append(mCompletedTapCount);
            builder.append(", IsTargetFingerCountReached: ");
            builder.append(mIsTargetFingerCountReached);
            builder.append(", Bases: ");
            builder.append(Arrays.toString(mBases));
            builder.append(", ExcludedPointsForDownSlopChecked: ");
            builder.append(mExcludedPointsForDownSlopChecked.toString());
        }
        return builder.toString();
    }
}
Loading