Implement multi-finger multi-tap gesture matchers for accessibility (3cbd098e) · Commits · e / os / android_frameworks_base

api/current.txt

+6 −0

Original line number	Diff line number	Diff line
		@@ -2873,6 +2873,12 @@ package android.accessibilityservice {
		method public final boolean performGlobalAction(int);
		method public final void setServiceInfo(android.accessibilityservice.AccessibilityServiceInfo);
		method public boolean takeScreenshot(int, @NonNull java.util.concurrent.Executor, @NonNull java.util.function.Consumer<android.graphics.Bitmap>);
		field public static final int GESTURE_2_FINGER_DOUBLE_TAP = 20; // 0x14
		field public static final int GESTURE_2_FINGER_SINGLE_TAP = 19; // 0x13
		field public static final int GESTURE_2_FINGER_TRIPLE_TAP = 21; // 0x15
		field public static final int GESTURE_3_FINGER_DOUBLE_TAP = 23; // 0x17
		field public static final int GESTURE_3_FINGER_SINGLE_TAP = 22; // 0x16
		field public static final int GESTURE_3_FINGER_TRIPLE_TAP = 24; // 0x18
		field public static final int GESTURE_DOUBLE_TAP = 17; // 0x11
		field public static final int GESTURE_DOUBLE_TAP_AND_HOLD = 18; // 0x12
		field public static final int GESTURE_SWIPE_DOWN = 2; // 0x2

core/java/android/accessibilityservice/AccessibilityGestureEvent.java

+43 −1

Original line number	Diff line number	Diff line
		@@ -17,6 +17,12 @@
		package android.accessibilityservice;


		import static android.accessibilityservice.AccessibilityService.GESTURE_2_FINGER_DOUBLE_TAP;
		import static android.accessibilityservice.AccessibilityService.GESTURE_2_FINGER_SINGLE_TAP;
		import static android.accessibilityservice.AccessibilityService.GESTURE_2_FINGER_TRIPLE_TAP;
		import static android.accessibilityservice.AccessibilityService.GESTURE_3_FINGER_DOUBLE_TAP;
		import static android.accessibilityservice.AccessibilityService.GESTURE_3_FINGER_SINGLE_TAP;
		import static android.accessibilityservice.AccessibilityService.GESTURE_3_FINGER_TRIPLE_TAP;
		import static android.accessibilityservice.AccessibilityService.GESTURE_DOUBLE_TAP;
		import static android.accessibilityservice.AccessibilityService.GESTURE_DOUBLE_TAP_AND_HOLD;
		import static android.accessibilityservice.AccessibilityService.GESTURE_SWIPE_DOWN;
		@@ -60,6 +66,12 @@ public final class AccessibilityGestureEvent implements Parcelable {

		/** @hide */
		@IntDef(prefix = { "GESTURE_" }, value = {
		GESTURE_2_FINGER_SINGLE_TAP,
		GESTURE_2_FINGER_DOUBLE_TAP,
		GESTURE_2_FINGER_TRIPLE_TAP,
		GESTURE_3_FINGER_SINGLE_TAP,
		GESTURE_3_FINGER_DOUBLE_TAP,
		GESTURE_3_FINGER_TRIPLE_TAP,
		GESTURE_DOUBLE_TAP,
		GESTURE_DOUBLE_TAP_AND_HOLD,
		GESTURE_SWIPE_UP,
		@@ -122,13 +134,43 @@ public final class AccessibilityGestureEvent implements Parcelable {
		@Override
		public String toString() {
		StringBuilder stringBuilder = new StringBuilder("AccessibilityGestureEvent[");
		stringBuilder.append("gestureId: ").append(mGestureId);
		stringBuilder.append("gestureId: ").append(eventTypeToString(mGestureId));
		stringBuilder.append(", ");
		stringBuilder.append("displayId: ").append(mDisplayId);
		stringBuilder.append(']');
		return stringBuilder.toString();
		}

		private static String eventTypeToString(int eventType) {
		switch (eventType) {
		case GESTURE_2_FINGER_SINGLE_TAP: return "GESTURE_2_FINGER_SINGLE_TAP";
		case GESTURE_2_FINGER_DOUBLE_TAP: return "GESTURE_2_FINGER_DOUBLE_TAP";
		case GESTURE_2_FINGER_TRIPLE_TAP: return "GESTURE_2_FINGER_TRIPLE_TAP";
		case GESTURE_3_FINGER_SINGLE_TAP: return "GESTURE_3_FINGER_SINGLE_TAP";
		case GESTURE_3_FINGER_DOUBLE_TAP: return "GESTURE_3_FINGER_DOUBLE_TAP";
		case GESTURE_3_FINGER_TRIPLE_TAP: return "GESTURE_3_FINGER_TRIPLE_TAP";
		case GESTURE_DOUBLE_TAP: return "GESTURE_DOUBLE_TAP";
		case GESTURE_DOUBLE_TAP_AND_HOLD: return "GESTURE_DOUBLE_TAP_AND_HOLD";
		case GESTURE_SWIPE_DOWN: return "GESTURE_SWIPE_DOWN";
		case GESTURE_SWIPE_DOWN_AND_LEFT: return "GESTURE_SWIPE_DOWN_AND_LEFT";
		case GESTURE_SWIPE_DOWN_AND_UP: return "GESTURE_SWIPE_DOWN_AND_UP";
		case GESTURE_SWIPE_DOWN_AND_RIGHT: return "GESTURE_SWIPE_DOWN_AND_RIGHT";
		case GESTURE_SWIPE_LEFT: return "GESTURE_SWIPE_LEFT";
		case GESTURE_SWIPE_LEFT_AND_UP: return "GESTURE_SWIPE_LEFT_AND_UP";
		case GESTURE_SWIPE_LEFT_AND_RIGHT: return "GESTURE_SWIPE_LEFT_AND_RIGHT";
		case GESTURE_SWIPE_LEFT_AND_DOWN: return "GESTURE_SWIPE_LEFT_AND_DOWN";
		case GESTURE_SWIPE_RIGHT: return "GESTURE_SWIPE_RIGHT";
		case GESTURE_SWIPE_RIGHT_AND_UP: return "GESTURE_SWIPE_RIGHT_AND_UP";
		case GESTURE_SWIPE_RIGHT_AND_LEFT: return "GESTURE_SWIPE_RIGHT_AND_LEFT";
		case GESTURE_SWIPE_RIGHT_AND_DOWN: return "GESTURE_SWIPE_RIGHT_AND_DOWN";
		case GESTURE_SWIPE_UP: return "GESTURE_SWIPE_UP";
		case GESTURE_SWIPE_UP_AND_LEFT: return "GESTURE_SWIPE_UP_AND_LEFT";
		case GESTURE_SWIPE_UP_AND_DOWN: return "GESTURE_SWIPE_UP_AND_DOWN";
		case GESTURE_SWIPE_UP_AND_RIGHT: return "GESTURE_SWIPE_UP_AND_RIGHT";
		default: return Integer.toHexString(eventType);
		}
		}

		/**
		* {@inheritDoc}
		*/

core/java/android/accessibilityservice/AccessibilityService.java

+30 −0

Original line number	Diff line number	Diff line
		@@ -318,6 +318,36 @@ public abstract class AccessibilityService extends Service {
		*/
		public static final int GESTURE_DOUBLE_TAP_AND_HOLD = 18;

		/**
		* The user has performed a two-finger single tap gesture on the touch screen.
		*/
		public static final int GESTURE_2_FINGER_SINGLE_TAP = 19;

		/**
		* The user has performed a two-finger double tap gesture on the touch screen.
		*/
		public static final int GESTURE_2_FINGER_DOUBLE_TAP = 20;

		/**
		* The user has performed a two-finger triple tap gesture on the touch screen.
		*/
		public static final int GESTURE_2_FINGER_TRIPLE_TAP = 21;

		/**
		* The user has performed a three-finger single tap gesture on the touch screen.
		*/
		public static final int GESTURE_3_FINGER_SINGLE_TAP = 22;

		/**
		* The user has performed a three-finger double tap gesture on the touch screen.
		*/
		public static final int GESTURE_3_FINGER_DOUBLE_TAP = 23;

		/**
		* The user has performed a three-finger triple tap gesture on the touch screen.
		*/
		public static final int GESTURE_3_FINGER_TRIPLE_TAP = 24;

		/**
		* The {@link Intent} that must be declared as handled by the service.
		*/

services/accessibility/java/com/android/server/accessibility/gestures/GestureManifold.java

+20 −0

Original line number	Diff line number	Diff line
		@@ -16,6 +16,12 @@

		package com.android.server.accessibility.gestures;

		import static android.accessibilityservice.AccessibilityService.GESTURE_2_FINGER_DOUBLE_TAP;
		import static android.accessibilityservice.AccessibilityService.GESTURE_2_FINGER_SINGLE_TAP;
		import static android.accessibilityservice.AccessibilityService.GESTURE_2_FINGER_TRIPLE_TAP;
		import static android.accessibilityservice.AccessibilityService.GESTURE_3_FINGER_DOUBLE_TAP;
		import static android.accessibilityservice.AccessibilityService.GESTURE_3_FINGER_SINGLE_TAP;
		import static android.accessibilityservice.AccessibilityService.GESTURE_3_FINGER_TRIPLE_TAP;
		import static android.accessibilityservice.AccessibilityService.GESTURE_DOUBLE_TAP;
		import static android.accessibilityservice.AccessibilityService.GESTURE_DOUBLE_TAP_AND_HOLD;
		import static android.accessibilityservice.AccessibilityService.GESTURE_SWIPE_DOWN;
		@@ -104,6 +110,20 @@ class GestureManifold implements GestureMatcher.StateChangeListener {
		mGestures.add(new Swipe(context, UP, DOWN, GESTURE_SWIPE_UP_AND_DOWN, this));
		mGestures.add(new Swipe(context, UP, LEFT, GESTURE_SWIPE_UP_AND_LEFT, this));
		mGestures.add(new Swipe(context, UP, RIGHT, GESTURE_SWIPE_UP_AND_RIGHT, this));
		// Two-finger taps.
		mMultiFingerGestures.add(
		new MultiFingerMultiTap(mContext, 2, 1, GESTURE_2_FINGER_SINGLE_TAP, this));
		mMultiFingerGestures.add(
		new MultiFingerMultiTap(mContext, 2, 2, GESTURE_2_FINGER_DOUBLE_TAP, this));
		mMultiFingerGestures.add(
		new MultiFingerMultiTap(mContext, 2, 3, GESTURE_2_FINGER_TRIPLE_TAP, this));
		// Three-finger taps.
		mMultiFingerGestures.add(
		new MultiFingerMultiTap(mContext, 3, 1, GESTURE_3_FINGER_SINGLE_TAP, this));
		mMultiFingerGestures.add(
		new MultiFingerMultiTap(mContext, 3, 2, GESTURE_3_FINGER_DOUBLE_TAP, this));
		mMultiFingerGestures.add(
		new MultiFingerMultiTap(mContext, 3, 3, GESTURE_3_FINGER_TRIPLE_TAP, this));
		}

		/**

services/accessibility/java/com/android/server/accessibility/gestures/MultiFingerMultiTap.java

0 → 100644

+291 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2020 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package com.android.server.accessibility.gestures;

		import android.content.Context;
		import android.graphics.PointF;
		import android.os.Handler;
		import android.view.MotionEvent;
		import android.view.ViewConfiguration;

		import com.android.internal.util.Preconditions;

		import java.util.ArrayList;
		import java.util.Arrays;

		/**
		* This class matches multi-finger multi-tap gestures. The number of fingers and the number of taps
		* for each instance is specified in the constructor.
		*/
		class MultiFingerMultiTap extends GestureMatcher {

		// The target number of taps.
		final int mTargetTapCount;
		// The target number of fingers.
		final int mTargetFingerCount;
		// The acceptable distance between two taps of a finger.
		private int mDoubleTapSlop;
		// The acceptable distance the pointer can move and still count as a tap.
		private int mTouchSlop;
		// A tap counts when target number of fingers are down and up once.
		private int mCompletedTapCount;
		// A flag set to true when target number of fingers have touched down at once before.
		// Used to indicate what next finger action should be. Down when false and lift when true.
		private boolean mIsTargetFingerCountReached = false;
		// Store initial down points for slop checking and update when next down if is inside slop.
		private PointF[] mBases;
		// The points in bases that already have slop checked when onDown or onPointerDown.
		// It prevents excluded points matched multiple times by other pointers from next check.
		private ArrayList<PointF> mExcludedPointsForDownSlopChecked;

		/**
		* @throws IllegalArgumentException if <code>fingers<code/> is less than 2
		* or <code>taps<code/> is not positive.
		*/
		MultiFingerMultiTap(Context context, int fingers, int taps, int gestureId,
		GestureMatcher.StateChangeListener listener) {
		super(gestureId, new Handler(context.getMainLooper()), listener);
		Preconditions.checkArgument(fingers >= 2);
		Preconditions.checkArgumentPositive(taps, "Tap count must greater than 0.");
		mTargetTapCount = taps;
		mTargetFingerCount = fingers;
		mDoubleTapSlop = ViewConfiguration.get(context).getScaledDoubleTapSlop();
		mTouchSlop = ViewConfiguration.get(context).getScaledTouchSlop();

		mBases = new PointF[mTargetFingerCount];
		for (int i = 0; i < mBases.length; i++) {
		mBases[i] = new PointF();
		}
		mExcludedPointsForDownSlopChecked = new ArrayList<>(mTargetFingerCount);
		clear();
		}

		@Override
		protected void clear() {
		mCompletedTapCount = 0;
		mIsTargetFingerCountReached = false;
		for (int i = 0; i < mBases.length; i++) {
		mBases[i].set(Float.NaN, Float.NaN);
		}
		mExcludedPointsForDownSlopChecked.clear();
		super.clear();
		}

		@Override
		protected void onDown(MotionEvent event, MotionEvent rawEvent, int policyFlags) {
		// Before the matcher state transit to completed,
		// Cancel when an additional down arrived after reaching the target number of taps.
		if (mCompletedTapCount == mTargetTapCount) {
		cancelGesture(event, rawEvent, policyFlags);
		return;
		}
		cancelAfterTapTimeout(event, rawEvent, policyFlags);

		if (mCompletedTapCount == 0) {
		initBaseLocation(rawEvent);
		return;
		}
		// As fingers go up and down, their pointer ids will not be the same.
		// Therefore we require that a given finger be in slop range of any one
		// of the fingers from the previous tap.
		final PointF nearest = findNearestPoint(rawEvent, mDoubleTapSlop, true);
		if (nearest != null) {
		// Update pointer location to nearest one as a new base for next slop check.
		final int index = event.getActionIndex();
		nearest.set(event.getX(index), event.getY(index));
		} else {
		cancelGesture(event, rawEvent, policyFlags);
		}
		}

		@Override
		protected void onUp(MotionEvent event, MotionEvent rawEvent, int policyFlags) {
		cancelAfterDoubleTapTimeout(event, rawEvent, policyFlags);

		final PointF nearest = findNearestPoint(rawEvent, mTouchSlop, false);
		if ((getState() == STATE_GESTURE_STARTED \|\| getState() == STATE_CLEAR)
		&& null != nearest) {
		// Increase current tap count when the user have all fingers lifted
		// within the tap timeout since the target number of fingers are down.
		if (mIsTargetFingerCountReached) {
		mCompletedTapCount++;
		mIsTargetFingerCountReached = false;
		mExcludedPointsForDownSlopChecked.clear();
		}

		// Start gesture detection here to avoid the conflict to 2nd finger double tap
		// that never actually started gesture detection.
		if (mCompletedTapCount == 1) {
		startGesture(event, rawEvent, policyFlags);
		}
		if (mCompletedTapCount == mTargetTapCount) {
		// Done.
		completeAfterDoubleTapTimeout(event, rawEvent, policyFlags);
		}
		} else {
		// Either too many taps or nonsensical event stream.
		cancelGesture(event, rawEvent, policyFlags);
		}
		}

		@Override
		protected void onMove(MotionEvent event, MotionEvent rawEvent, int policyFlags) {
		// Outside the touch slop
		if (null == findNearestPoint(rawEvent, mTouchSlop, false)) {
		cancelGesture(event, rawEvent, policyFlags);
		}
		}

		@Override
		protected void onPointerDown(MotionEvent event, MotionEvent rawEvent, int policyFlags) {
		// Reset timeout to ease the use for some people
		// with certain impairments to get all their fingers down.
		cancelAfterTapTimeout(event, rawEvent, policyFlags);
		final int currentFingerCount = event.getPointerCount();
		// Accept down only before target number of fingers are down
		// or the finger count is not more than target.
		if ((currentFingerCount > mTargetFingerCount) \|\| mIsTargetFingerCountReached) {
		cancelGesture(event, rawEvent, policyFlags);
		return;
		}

		final PointF nearest;
		if (mCompletedTapCount == 0) {
		nearest = initBaseLocation(rawEvent);
		} else {
		nearest = findNearestPoint(rawEvent, mDoubleTapSlop, true);
		}
		if ((getState() == STATE_GESTURE_STARTED \|\| getState() == STATE_CLEAR)
		&& nearest != null) {
		// The user have all fingers down within the tap timeout since first finger down,
		// setting the timeout for fingers to be lifted.
		if (currentFingerCount == mTargetFingerCount) {
		mIsTargetFingerCountReached = true;
		}
		// Update pointer location to nearest one as a new base for next slop check.
		final int index = event.getActionIndex();
		nearest.set(event.getX(index), event.getY(index));
		} else {
		cancelGesture(event, rawEvent, policyFlags);
		}
		}

		@Override
		protected void onPointerUp(MotionEvent event, MotionEvent rawEvent, int policyFlags) {
		// Accept up only after target number of fingers are down.
		if (!mIsTargetFingerCountReached) {
		cancelGesture(event, rawEvent, policyFlags);
		return;
		}

		if (getState() == STATE_GESTURE_STARTED \|\| getState() == STATE_CLEAR) {
		// Needs more fingers lifted within the tap timeout
		// after reaching the target number of fingers are down.
		} else {
		cancelGesture(event, rawEvent, policyFlags);
		}
		}

		@Override
		public String getGestureName() {
		final StringBuilder builder = new StringBuilder();
		builder.append(mTargetFingerCount).append("-Finger ");
		if (mTargetTapCount == 1) {
		builder.append("Single");
		} else if (mTargetTapCount == 2) {
		builder.append("Double");
		} else if (mTargetTapCount == 3) {
		builder.append("Triple");
		} else if (mTargetTapCount > 3) {
		builder.append(mTargetTapCount);
		}
		return builder.append(" Tap").toString();
		}

		private PointF initBaseLocation(MotionEvent event) {
		final int index = event.getActionIndex();
		final int baseIndex = event.getPointerCount() - 1;
		final PointF p = mBases[baseIndex];
		if (Float.isNaN(p.x) && Float.isNaN(p.y)) {
		p.set(event.getX(index), event.getY(index));
		}
		return p;
		}

		/**
		* Find the nearest location to the given event in the bases.
		* If no one found, it could be not inside {@code slop}, filtered or empty bases.
		* When {@code filterMatched} is true, if the location of given event matches one of the points
		* in {@link #mExcludedPointsForDownSlopChecked} it would be ignored. Otherwise, the location
		* will be added to {@link #mExcludedPointsForDownSlopChecked}.
		*
		* @param event to find nearest point in bases.
		* @param slop to check to the given location of the event.
		* @param filterMatched true to exclude points already matched other pointers.
		* @return the point in bases closed to the location of the given event.
		*/
		private PointF findNearestPoint(MotionEvent event, float slop, boolean filterMatched) {
		float moveDelta = Float.MAX_VALUE;
		PointF nearest = null;
		for (int i = 0; i < mBases.length; i++) {
		final PointF p = mBases[i];
		if (Float.isNaN(p.x) && Float.isNaN(p.y)) {
		continue;
		}
		if (filterMatched && mExcludedPointsForDownSlopChecked.contains(p)) {
		continue;
		}
		final int index = event.getActionIndex();
		final float dX = p.x - event.getX(index);
		final float dY = p.y - event.getY(index);
		if (dX == 0 && dY == 0) {
		if (filterMatched) {
		mExcludedPointsForDownSlopChecked.add(p);
		}
		return p;
		}
		final float delta = (float) Math.hypot(dX, dY);
		if (moveDelta > delta) {
		moveDelta = delta;
		nearest = p;
		}
		}
		if (moveDelta < slop) {
		if (filterMatched) {
		mExcludedPointsForDownSlopChecked.add(nearest);
		}
		return nearest;
		}
		return null;
		}

		@Override
		public String toString() {
		final StringBuilder builder = new StringBuilder(super.toString());
		if (getState() != STATE_GESTURE_CANCELED) {
		builder.append(", CompletedTapCount: ");
		builder.append(mCompletedTapCount);
		builder.append(", IsTargetFingerCountReached: ");
		builder.append(mIsTargetFingerCountReached);
		builder.append(", Bases: ");
		builder.append(Arrays.toString(mBases));
		builder.append(", ExcludedPointsForDownSlopChecked: ");
		builder.append(mExcludedPointsForDownSlopChecked.toString());
		}
		return builder.toString();
		}
		}