Merge "Implement multi-finger multi-tap gesture matchers for accessibility" (96b878aa) · Commits · e / os / android_frameworks_base

api/current.txt

+6 −0

Original line number	Original line	Diff line number	Diff line
	@@ -2873,6 +2873,12 @@ package android.accessibilityservice {
	method public final boolean performGlobalAction(int);		method public final boolean performGlobalAction(int);
	method public final void setServiceInfo(android.accessibilityservice.AccessibilityServiceInfo);		method public final void setServiceInfo(android.accessibilityservice.AccessibilityServiceInfo);
	method public boolean takeScreenshot(int, @NonNull java.util.concurrent.Executor, @NonNull java.util.function.Consumer<android.graphics.Bitmap>);		method public boolean takeScreenshot(int, @NonNull java.util.concurrent.Executor, @NonNull java.util.function.Consumer<android.graphics.Bitmap>);
			field public static final int GESTURE_2_FINGER_DOUBLE_TAP = 20; // 0x14
			field public static final int GESTURE_2_FINGER_SINGLE_TAP = 19; // 0x13
			field public static final int GESTURE_2_FINGER_TRIPLE_TAP = 21; // 0x15
			field public static final int GESTURE_3_FINGER_DOUBLE_TAP = 23; // 0x17
			field public static final int GESTURE_3_FINGER_SINGLE_TAP = 22; // 0x16
			field public static final int GESTURE_3_FINGER_TRIPLE_TAP = 24; // 0x18
	field public static final int GESTURE_DOUBLE_TAP = 17; // 0x11		field public static final int GESTURE_DOUBLE_TAP = 17; // 0x11
	field public static final int GESTURE_DOUBLE_TAP_AND_HOLD = 18; // 0x12		field public static final int GESTURE_DOUBLE_TAP_AND_HOLD = 18; // 0x12
	field public static final int GESTURE_SWIPE_DOWN = 2; // 0x2		field public static final int GESTURE_SWIPE_DOWN = 2; // 0x2

core/java/android/accessibilityservice/AccessibilityGestureEvent.java

+43 −1

Original line number	Original line	Diff line number	Diff line
	@@ -17,6 +17,12 @@
	package android.accessibilityservice;		package android.accessibilityservice;


			import static android.accessibilityservice.AccessibilityService.GESTURE_2_FINGER_DOUBLE_TAP;
			import static android.accessibilityservice.AccessibilityService.GESTURE_2_FINGER_SINGLE_TAP;
			import static android.accessibilityservice.AccessibilityService.GESTURE_2_FINGER_TRIPLE_TAP;
			import static android.accessibilityservice.AccessibilityService.GESTURE_3_FINGER_DOUBLE_TAP;
			import static android.accessibilityservice.AccessibilityService.GESTURE_3_FINGER_SINGLE_TAP;
			import static android.accessibilityservice.AccessibilityService.GESTURE_3_FINGER_TRIPLE_TAP;
	import static android.accessibilityservice.AccessibilityService.GESTURE_DOUBLE_TAP;		import static android.accessibilityservice.AccessibilityService.GESTURE_DOUBLE_TAP;
	import static android.accessibilityservice.AccessibilityService.GESTURE_DOUBLE_TAP_AND_HOLD;		import static android.accessibilityservice.AccessibilityService.GESTURE_DOUBLE_TAP_AND_HOLD;
	import static android.accessibilityservice.AccessibilityService.GESTURE_SWIPE_DOWN;		import static android.accessibilityservice.AccessibilityService.GESTURE_SWIPE_DOWN;
	@@ -60,6 +66,12 @@ public final class AccessibilityGestureEvent implements Parcelable {

	/** @hide */		/** @hide */
	@IntDef(prefix = { "GESTURE_" }, value = {		@IntDef(prefix = { "GESTURE_" }, value = {
			GESTURE_2_FINGER_SINGLE_TAP,
			GESTURE_2_FINGER_DOUBLE_TAP,
			GESTURE_2_FINGER_TRIPLE_TAP,
			GESTURE_3_FINGER_SINGLE_TAP,
			GESTURE_3_FINGER_DOUBLE_TAP,
			GESTURE_3_FINGER_TRIPLE_TAP,
	GESTURE_DOUBLE_TAP,		GESTURE_DOUBLE_TAP,
	GESTURE_DOUBLE_TAP_AND_HOLD,		GESTURE_DOUBLE_TAP_AND_HOLD,
	GESTURE_SWIPE_UP,		GESTURE_SWIPE_UP,
	@@ -122,13 +134,43 @@ public final class AccessibilityGestureEvent implements Parcelable {
	@Override		@Override
	public String toString() {		public String toString() {
	StringBuilder stringBuilder = new StringBuilder("AccessibilityGestureEvent[");		StringBuilder stringBuilder = new StringBuilder("AccessibilityGestureEvent[");
	stringBuilder.append("gestureId: ").append(mGestureId);		stringBuilder.append("gestureId: ").append(eventTypeToString(mGestureId));
	stringBuilder.append(", ");		stringBuilder.append(", ");
	stringBuilder.append("displayId: ").append(mDisplayId);		stringBuilder.append("displayId: ").append(mDisplayId);
	stringBuilder.append(']');		stringBuilder.append(']');
	return stringBuilder.toString();		return stringBuilder.toString();
	}		}

			private static String eventTypeToString(int eventType) {
			switch (eventType) {
			case GESTURE_2_FINGER_SINGLE_TAP: return "GESTURE_2_FINGER_SINGLE_TAP";
			case GESTURE_2_FINGER_DOUBLE_TAP: return "GESTURE_2_FINGER_DOUBLE_TAP";
			case GESTURE_2_FINGER_TRIPLE_TAP: return "GESTURE_2_FINGER_TRIPLE_TAP";
			case GESTURE_3_FINGER_SINGLE_TAP: return "GESTURE_3_FINGER_SINGLE_TAP";
			case GESTURE_3_FINGER_DOUBLE_TAP: return "GESTURE_3_FINGER_DOUBLE_TAP";
			case GESTURE_3_FINGER_TRIPLE_TAP: return "GESTURE_3_FINGER_TRIPLE_TAP";
			case GESTURE_DOUBLE_TAP: return "GESTURE_DOUBLE_TAP";
			case GESTURE_DOUBLE_TAP_AND_HOLD: return "GESTURE_DOUBLE_TAP_AND_HOLD";
			case GESTURE_SWIPE_DOWN: return "GESTURE_SWIPE_DOWN";
			case GESTURE_SWIPE_DOWN_AND_LEFT: return "GESTURE_SWIPE_DOWN_AND_LEFT";
			case GESTURE_SWIPE_DOWN_AND_UP: return "GESTURE_SWIPE_DOWN_AND_UP";
			case GESTURE_SWIPE_DOWN_AND_RIGHT: return "GESTURE_SWIPE_DOWN_AND_RIGHT";
			case GESTURE_SWIPE_LEFT: return "GESTURE_SWIPE_LEFT";
			case GESTURE_SWIPE_LEFT_AND_UP: return "GESTURE_SWIPE_LEFT_AND_UP";
			case GESTURE_SWIPE_LEFT_AND_RIGHT: return "GESTURE_SWIPE_LEFT_AND_RIGHT";
			case GESTURE_SWIPE_LEFT_AND_DOWN: return "GESTURE_SWIPE_LEFT_AND_DOWN";
			case GESTURE_SWIPE_RIGHT: return "GESTURE_SWIPE_RIGHT";
			case GESTURE_SWIPE_RIGHT_AND_UP: return "GESTURE_SWIPE_RIGHT_AND_UP";
			case GESTURE_SWIPE_RIGHT_AND_LEFT: return "GESTURE_SWIPE_RIGHT_AND_LEFT";
			case GESTURE_SWIPE_RIGHT_AND_DOWN: return "GESTURE_SWIPE_RIGHT_AND_DOWN";
			case GESTURE_SWIPE_UP: return "GESTURE_SWIPE_UP";
			case GESTURE_SWIPE_UP_AND_LEFT: return "GESTURE_SWIPE_UP_AND_LEFT";
			case GESTURE_SWIPE_UP_AND_DOWN: return "GESTURE_SWIPE_UP_AND_DOWN";
			case GESTURE_SWIPE_UP_AND_RIGHT: return "GESTURE_SWIPE_UP_AND_RIGHT";
			default: return Integer.toHexString(eventType);
			}
			}

	/**		/**
	* {@inheritDoc}		* {@inheritDoc}
	*/		*/

core/java/android/accessibilityservice/AccessibilityService.java

+30 −0

Original line number	Original line	Diff line number	Diff line
	@@ -318,6 +318,36 @@ public abstract class AccessibilityService extends Service {
	*/		*/
	public static final int GESTURE_DOUBLE_TAP_AND_HOLD = 18;		public static final int GESTURE_DOUBLE_TAP_AND_HOLD = 18;

			/**
			* The user has performed a two-finger single tap gesture on the touch screen.
			*/
			public static final int GESTURE_2_FINGER_SINGLE_TAP = 19;

			/**
			* The user has performed a two-finger double tap gesture on the touch screen.
			*/
			public static final int GESTURE_2_FINGER_DOUBLE_TAP = 20;

			/**
			* The user has performed a two-finger triple tap gesture on the touch screen.
			*/
			public static final int GESTURE_2_FINGER_TRIPLE_TAP = 21;

			/**
			* The user has performed a three-finger single tap gesture on the touch screen.
			*/
			public static final int GESTURE_3_FINGER_SINGLE_TAP = 22;

			/**
			* The user has performed a three-finger double tap gesture on the touch screen.
			*/
			public static final int GESTURE_3_FINGER_DOUBLE_TAP = 23;

			/**
			* The user has performed a three-finger triple tap gesture on the touch screen.
			*/
			public static final int GESTURE_3_FINGER_TRIPLE_TAP = 24;

	/**		/**
	* The {@link Intent} that must be declared as handled by the service.		* The {@link Intent} that must be declared as handled by the service.
	*/		*/

services/accessibility/java/com/android/server/accessibility/gestures/GestureManifold.java

+20 −0

Original line number	Original line	Diff line number	Diff line
	@@ -16,6 +16,12 @@

	package com.android.server.accessibility.gestures;		package com.android.server.accessibility.gestures;

			import static android.accessibilityservice.AccessibilityService.GESTURE_2_FINGER_DOUBLE_TAP;
			import static android.accessibilityservice.AccessibilityService.GESTURE_2_FINGER_SINGLE_TAP;
			import static android.accessibilityservice.AccessibilityService.GESTURE_2_FINGER_TRIPLE_TAP;
			import static android.accessibilityservice.AccessibilityService.GESTURE_3_FINGER_DOUBLE_TAP;
			import static android.accessibilityservice.AccessibilityService.GESTURE_3_FINGER_SINGLE_TAP;
			import static android.accessibilityservice.AccessibilityService.GESTURE_3_FINGER_TRIPLE_TAP;
	import static android.accessibilityservice.AccessibilityService.GESTURE_DOUBLE_TAP;		import static android.accessibilityservice.AccessibilityService.GESTURE_DOUBLE_TAP;
	import static android.accessibilityservice.AccessibilityService.GESTURE_DOUBLE_TAP_AND_HOLD;		import static android.accessibilityservice.AccessibilityService.GESTURE_DOUBLE_TAP_AND_HOLD;
	import static android.accessibilityservice.AccessibilityService.GESTURE_SWIPE_DOWN;		import static android.accessibilityservice.AccessibilityService.GESTURE_SWIPE_DOWN;
	@@ -104,6 +110,20 @@ class GestureManifold implements GestureMatcher.StateChangeListener {
	mGestures.add(new Swipe(context, UP, DOWN, GESTURE_SWIPE_UP_AND_DOWN, this));		mGestures.add(new Swipe(context, UP, DOWN, GESTURE_SWIPE_UP_AND_DOWN, this));
	mGestures.add(new Swipe(context, UP, LEFT, GESTURE_SWIPE_UP_AND_LEFT, this));		mGestures.add(new Swipe(context, UP, LEFT, GESTURE_SWIPE_UP_AND_LEFT, this));
	mGestures.add(new Swipe(context, UP, RIGHT, GESTURE_SWIPE_UP_AND_RIGHT, this));		mGestures.add(new Swipe(context, UP, RIGHT, GESTURE_SWIPE_UP_AND_RIGHT, this));
			// Two-finger taps.
			mMultiFingerGestures.add(
			new MultiFingerMultiTap(mContext, 2, 1, GESTURE_2_FINGER_SINGLE_TAP, this));
			mMultiFingerGestures.add(
			new MultiFingerMultiTap(mContext, 2, 2, GESTURE_2_FINGER_DOUBLE_TAP, this));
			mMultiFingerGestures.add(
			new MultiFingerMultiTap(mContext, 2, 3, GESTURE_2_FINGER_TRIPLE_TAP, this));
			// Three-finger taps.
			mMultiFingerGestures.add(
			new MultiFingerMultiTap(mContext, 3, 1, GESTURE_3_FINGER_SINGLE_TAP, this));
			mMultiFingerGestures.add(
			new MultiFingerMultiTap(mContext, 3, 2, GESTURE_3_FINGER_DOUBLE_TAP, this));
			mMultiFingerGestures.add(
			new MultiFingerMultiTap(mContext, 3, 3, GESTURE_3_FINGER_TRIPLE_TAP, this));
	}		}

	/**		/**

services/accessibility/java/com/android/server/accessibility/gestures/MultiFingerMultiTap.java

0 → 100644

+291 −0

Original line number	Original line	Diff line number	Diff line
			/*
			* Copyright (C) 2020 The Android Open Source Project
			*
			* Licensed under the Apache License, Version 2.0 (the "License");
			* you may not use this file except in compliance with the License.
			* You may obtain a copy of the License at
			*
			* http://www.apache.org/licenses/LICENSE-2.0
			*
			* Unless required by applicable law or agreed to in writing, software
			* distributed under the License is distributed on an "AS IS" BASIS,
			* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
			* See the License for the specific language governing permissions and
			* limitations under the License.
			*/

			package com.android.server.accessibility.gestures;

			import android.content.Context;
			import android.graphics.PointF;
			import android.os.Handler;
			import android.view.MotionEvent;
			import android.view.ViewConfiguration;

			import com.android.internal.util.Preconditions;

			import java.util.ArrayList;
			import java.util.Arrays;

			/**
			* This class matches multi-finger multi-tap gestures. The number of fingers and the number of taps
			* for each instance is specified in the constructor.
			*/
			class MultiFingerMultiTap extends GestureMatcher {

			// The target number of taps.
			final int mTargetTapCount;
			// The target number of fingers.
			final int mTargetFingerCount;
			// The acceptable distance between two taps of a finger.
			private int mDoubleTapSlop;
			// The acceptable distance the pointer can move and still count as a tap.
			private int mTouchSlop;
			// A tap counts when target number of fingers are down and up once.
			private int mCompletedTapCount;
			// A flag set to true when target number of fingers have touched down at once before.
			// Used to indicate what next finger action should be. Down when false and lift when true.
			private boolean mIsTargetFingerCountReached = false;
			// Store initial down points for slop checking and update when next down if is inside slop.
			private PointF[] mBases;
			// The points in bases that already have slop checked when onDown or onPointerDown.
			// It prevents excluded points matched multiple times by other pointers from next check.
			private ArrayList<PointF> mExcludedPointsForDownSlopChecked;

			/**
			* @throws IllegalArgumentException if <code>fingers<code/> is less than 2
			* or <code>taps<code/> is not positive.
			*/
			MultiFingerMultiTap(Context context, int fingers, int taps, int gestureId,
			GestureMatcher.StateChangeListener listener) {
			super(gestureId, new Handler(context.getMainLooper()), listener);
			Preconditions.checkArgument(fingers >= 2);
			Preconditions.checkArgumentPositive(taps, "Tap count must greater than 0.");
			mTargetTapCount = taps;
			mTargetFingerCount = fingers;
			mDoubleTapSlop = ViewConfiguration.get(context).getScaledDoubleTapSlop();
			mTouchSlop = ViewConfiguration.get(context).getScaledTouchSlop();

			mBases = new PointF[mTargetFingerCount];
			for (int i = 0; i < mBases.length; i++) {
			mBases[i] = new PointF();
			}
			mExcludedPointsForDownSlopChecked = new ArrayList<>(mTargetFingerCount);
			clear();
			}

			@Override
			protected void clear() {
			mCompletedTapCount = 0;
			mIsTargetFingerCountReached = false;
			for (int i = 0; i < mBases.length; i++) {
			mBases[i].set(Float.NaN, Float.NaN);
			}
			mExcludedPointsForDownSlopChecked.clear();
			super.clear();
			}

			@Override
			protected void onDown(MotionEvent event, MotionEvent rawEvent, int policyFlags) {
			// Before the matcher state transit to completed,
			// Cancel when an additional down arrived after reaching the target number of taps.
			if (mCompletedTapCount == mTargetTapCount) {
			cancelGesture(event, rawEvent, policyFlags);
			return;
			}
			cancelAfterTapTimeout(event, rawEvent, policyFlags);

			if (mCompletedTapCount == 0) {
			initBaseLocation(rawEvent);
			return;
			}
			// As fingers go up and down, their pointer ids will not be the same.
			// Therefore we require that a given finger be in slop range of any one
			// of the fingers from the previous tap.
			final PointF nearest = findNearestPoint(rawEvent, mDoubleTapSlop, true);
			if (nearest != null) {
			// Update pointer location to nearest one as a new base for next slop check.
			final int index = event.getActionIndex();
			nearest.set(event.getX(index), event.getY(index));
			} else {
			cancelGesture(event, rawEvent, policyFlags);
			}
			}

			@Override
			protected void onUp(MotionEvent event, MotionEvent rawEvent, int policyFlags) {
			cancelAfterDoubleTapTimeout(event, rawEvent, policyFlags);

			final PointF nearest = findNearestPoint(rawEvent, mTouchSlop, false);
			if ((getState() == STATE_GESTURE_STARTED \|\| getState() == STATE_CLEAR)
			&& null != nearest) {
			// Increase current tap count when the user have all fingers lifted
			// within the tap timeout since the target number of fingers are down.
			if (mIsTargetFingerCountReached) {
			mCompletedTapCount++;
			mIsTargetFingerCountReached = false;
			mExcludedPointsForDownSlopChecked.clear();
			}

			// Start gesture detection here to avoid the conflict to 2nd finger double tap
			// that never actually started gesture detection.
			if (mCompletedTapCount == 1) {
			startGesture(event, rawEvent, policyFlags);
			}
			if (mCompletedTapCount == mTargetTapCount) {
			// Done.
			completeAfterDoubleTapTimeout(event, rawEvent, policyFlags);
			}
			} else {
			// Either too many taps or nonsensical event stream.
			cancelGesture(event, rawEvent, policyFlags);
			}
			}

			@Override
			protected void onMove(MotionEvent event, MotionEvent rawEvent, int policyFlags) {
			// Outside the touch slop
			if (null == findNearestPoint(rawEvent, mTouchSlop, false)) {
			cancelGesture(event, rawEvent, policyFlags);
			}
			}

			@Override
			protected void onPointerDown(MotionEvent event, MotionEvent rawEvent, int policyFlags) {
			// Reset timeout to ease the use for some people
			// with certain impairments to get all their fingers down.
			cancelAfterTapTimeout(event, rawEvent, policyFlags);
			final int currentFingerCount = event.getPointerCount();
			// Accept down only before target number of fingers are down
			// or the finger count is not more than target.
			if ((currentFingerCount > mTargetFingerCount) \|\| mIsTargetFingerCountReached) {
			cancelGesture(event, rawEvent, policyFlags);
			return;
			}

			final PointF nearest;
			if (mCompletedTapCount == 0) {
			nearest = initBaseLocation(rawEvent);
			} else {
			nearest = findNearestPoint(rawEvent, mDoubleTapSlop, true);
			}
			if ((getState() == STATE_GESTURE_STARTED \|\| getState() == STATE_CLEAR)
			&& nearest != null) {
			// The user have all fingers down within the tap timeout since first finger down,
			// setting the timeout for fingers to be lifted.
			if (currentFingerCount == mTargetFingerCount) {
			mIsTargetFingerCountReached = true;
			}
			// Update pointer location to nearest one as a new base for next slop check.
			final int index = event.getActionIndex();
			nearest.set(event.getX(index), event.getY(index));
			} else {
			cancelGesture(event, rawEvent, policyFlags);
			}
			}

			@Override
			protected void onPointerUp(MotionEvent event, MotionEvent rawEvent, int policyFlags) {
			// Accept up only after target number of fingers are down.
			if (!mIsTargetFingerCountReached) {
			cancelGesture(event, rawEvent, policyFlags);
			return;
			}

			if (getState() == STATE_GESTURE_STARTED \|\| getState() == STATE_CLEAR) {
			// Needs more fingers lifted within the tap timeout
			// after reaching the target number of fingers are down.
			} else {
			cancelGesture(event, rawEvent, policyFlags);
			}
			}

			@Override
			public String getGestureName() {
			final StringBuilder builder = new StringBuilder();
			builder.append(mTargetFingerCount).append("-Finger ");
			if (mTargetTapCount == 1) {
			builder.append("Single");
			} else if (mTargetTapCount == 2) {
			builder.append("Double");
			} else if (mTargetTapCount == 3) {
			builder.append("Triple");
			} else if (mTargetTapCount > 3) {
			builder.append(mTargetTapCount);
			}
			return builder.append(" Tap").toString();
			}

			private PointF initBaseLocation(MotionEvent event) {
			final int index = event.getActionIndex();
			final int baseIndex = event.getPointerCount() - 1;
			final PointF p = mBases[baseIndex];
			if (Float.isNaN(p.x) && Float.isNaN(p.y)) {
			p.set(event.getX(index), event.getY(index));
			}
			return p;
			}

			/**
			* Find the nearest location to the given event in the bases.
			* If no one found, it could be not inside {@code slop}, filtered or empty bases.
			* When {@code filterMatched} is true, if the location of given event matches one of the points
			* in {@link #mExcludedPointsForDownSlopChecked} it would be ignored. Otherwise, the location
			* will be added to {@link #mExcludedPointsForDownSlopChecked}.
			*
			* @param event to find nearest point in bases.
			* @param slop to check to the given location of the event.
			* @param filterMatched true to exclude points already matched other pointers.
			* @return the point in bases closed to the location of the given event.
			*/
			private PointF findNearestPoint(MotionEvent event, float slop, boolean filterMatched) {
			float moveDelta = Float.MAX_VALUE;
			PointF nearest = null;
			for (int i = 0; i < mBases.length; i++) {
			final PointF p = mBases[i];
			if (Float.isNaN(p.x) && Float.isNaN(p.y)) {
			continue;
			}
			if (filterMatched && mExcludedPointsForDownSlopChecked.contains(p)) {
			continue;
			}
			final int index = event.getActionIndex();
			final float dX = p.x - event.getX(index);
			final float dY = p.y - event.getY(index);
			if (dX == 0 && dY == 0) {
			if (filterMatched) {
			mExcludedPointsForDownSlopChecked.add(p);
			}
			return p;
			}
			final float delta = (float) Math.hypot(dX, dY);
			if (moveDelta > delta) {
			moveDelta = delta;
			nearest = p;
			}
			}
			if (moveDelta < slop) {
			if (filterMatched) {
			mExcludedPointsForDownSlopChecked.add(nearest);
			}
			return nearest;
			}
			return null;
			}

			@Override
			public String toString() {
			final StringBuilder builder = new StringBuilder(super.toString());
			if (getState() != STATE_GESTURE_CANCELED) {
			builder.append(", CompletedTapCount: ");
			builder.append(mCompletedTapCount);
			builder.append(", IsTargetFingerCountReached: ");
			builder.append(mIsTargetFingerCountReached);
			builder.append(", Bases: ");
			builder.append(Arrays.toString(mBases));
			builder.append(", ExcludedPointsForDownSlopChecked: ");
			builder.append(mExcludedPointsForDownSlopChecked.toString());
			}
			return builder.toString();
			}
			}