Merge "a11y: Implement autoclick drag and move" into main (6085f394) · Commits · e / os / android_frameworks_base

services/accessibility/java/com/android/server/accessibility/autoclick/AutoclickController.java

+118 −4

Original line number	Diff line number	Diff line
		@@ -24,8 +24,9 @@ import static android.view.accessibility.AccessibilityManager.AUTOCLICK_IGNORE_M
		import static android.view.accessibility.AccessibilityManager.AUTOCLICK_REVERT_TO_LEFT_CLICK_DEFAULT;

		import static com.android.server.accessibility.autoclick.AutoclickIndicatorView.SHOW_INDICATOR_DELAY_TIME;
		import static com.android.server.accessibility.autoclick.AutoclickTypePanel.AUTOCLICK_TYPE_DOUBLE_CLICK;
		import static com.android.server.accessibility.autoclick.AutoclickScrollPanel.DIRECTION_NONE;
		import static com.android.server.accessibility.autoclick.AutoclickTypePanel.AUTOCLICK_TYPE_DOUBLE_CLICK;
		import static com.android.server.accessibility.autoclick.AutoclickTypePanel.AUTOCLICK_TYPE_DRAG;
		import static com.android.server.accessibility.autoclick.AutoclickTypePanel.AUTOCLICK_TYPE_LEFT_CLICK;
		import static com.android.server.accessibility.autoclick.AutoclickTypePanel.AUTOCLICK_TYPE_RIGHT_CLICK;
		import static com.android.server.accessibility.autoclick.AutoclickTypePanel.AUTOCLICK_TYPE_SCROLL;
		@@ -108,6 +109,12 @@ public class AutoclickController extends BaseEventStreamTransformation {
		// Default scroll direction is DIRECTION_NONE.
		private @AutoclickScrollPanel.ScrollDirection int mHoveredDirection = DIRECTION_NONE;

		// True during the duration of a dragging event.
		private boolean mDragModeIsDragging = false;
		// The MotionEvent downTime attribute associated with the originating click for a dragging
		// move.
		private long mDragModeClickDownTime;

		@VisibleForTesting
		final ClickPanelControllerInterface clickPanelController =
		new ClickPanelControllerInterface() {
		@@ -206,7 +213,16 @@ public class AutoclickController extends BaseEventStreamTransformation {
		}

		if (!isPaused()) {
		handleMouseMotion(event, policyFlags);
		scheduleClick(event, policyFlags);

		// When dragging, HOVER_MOVE events need to be manually converted to MOVE events
		// using the initiating click's down time to simulate dragging.
		if (mDragModeIsDragging
		&& event.getActionMasked() == MotionEvent.ACTION_HOVER_MOVE) {
		event.setAction(MotionEvent.ACTION_MOVE);
		event.setDownTime(mDragModeClickDownTime);
		event.setButtonState(BUTTON_PRIMARY);
		}
		}
		} else {
		cancelPendingClick();
		@@ -283,7 +299,7 @@ public class AutoclickController extends BaseEventStreamTransformation {
		}
		}

		private void handleMouseMotion(MotionEvent event, int policyFlags) {
		private void scheduleClick(MotionEvent event, int policyFlags) {
		switch (event.getActionMasked()) {
		case MotionEvent.ACTION_HOVER_MOVE: {
		if (event.getPointerCount() == 1) {
		@@ -390,6 +406,11 @@ public class AutoclickController extends BaseEventStreamTransformation {
		mAutoclickSettingsObserver.onChange(selfChange, uri);
		}

		@VisibleForTesting
		boolean isDraggingForTesting() {
		return mDragModeIsDragging;
		}

		/**
		* Observes autoclick setting values, and updates ClickScheduler delay and indicator size
		* whenever the setting value changes.
		@@ -687,8 +708,14 @@ public class AutoclickController extends BaseEventStreamTransformation {

		sendClick();
		resetInternalState();

		// If the user is currently dragging, do not reset their click type.
		boolean stillDragging = mActiveClickType == AUTOCLICK_TYPE_DRAG
		&& mDragModeIsDragging;
		if (!stillDragging) {
		resetSelectedClickTypeIfNecessary();
		}
		}

		/**
		* Updates properties that should be used for click event sequence initiated by this object,
		@@ -722,10 +749,31 @@ public class AutoclickController extends BaseEventStreamTransformation {
		if (!mActive) {
		return;
		}

		if (mDragModeIsDragging) {
		clearDraggingState();
		}
		resetInternalState();
		mHandler.removeCallbacks(this);
		}

		// Reset the drag state after a canceled click to avoid potential side effects from
		// leaving it in an inconsistent state.
		private void clearDraggingState() {
		if (mLastMotionEvent != null) {
		// A final ACTION_UP event needs to be sent to alert the system that dragging has
		// ended.
		MotionEvent upEvent = MotionEvent.obtain(mLastMotionEvent);
		upEvent.setAction(MotionEvent.ACTION_UP);
		upEvent.setDownTime(mDragModeClickDownTime);
		AutoclickController.super.onMotionEvent(upEvent, upEvent,
		mEventPolicyFlags);
		}

		resetSelectedClickTypeIfNecessary();
		mDragModeIsDragging = false;
		}

		/**
		* Updates the meta state that should be used for click sequence.
		*/
		@@ -948,6 +996,13 @@ public class AutoclickController extends BaseEventStreamTransformation {
		sendMotionEvent(actionButton, now);
		sendMotionEvent(actionButton, now + doubleTapMinimumTimeout);
		return;
		case AUTOCLICK_TYPE_DRAG:
		if (mDragModeIsDragging) {
		endDragEvent();
		} else {
		startDragEvent();
		}
		return;
		default:
		break;
		}
		@@ -1000,6 +1055,65 @@ public class AutoclickController extends BaseEventStreamTransformation {
		upEvent.recycle();
		}

		// To start a drag event, only send the DOWN and BUTTON_PRESS events.
		private void startDragEvent() {
		mDragModeClickDownTime = SystemClock.uptimeMillis();
		mDragModeIsDragging = true;

		MotionEvent downEvent =
		MotionEvent.obtain(
		/* downTime= */ mDragModeClickDownTime,
		/* eventTime= */ mDragModeClickDownTime,
		MotionEvent.ACTION_DOWN,
		/* pointerCount= */ 1,
		mTempPointerProperties,
		mTempPointerCoords,
		mMetaState,
		BUTTON_PRIMARY,
		/* xPrecision= */ 1.0f,
		/* yPrecision= */ 1.0f,
		mLastMotionEvent.getDeviceId(),
		/* edgeFlags= */ 0,
		mLastMotionEvent.getSource(),
		mLastMotionEvent.getFlags());
		MotionEvent pressEvent = MotionEvent.obtain(downEvent);
		pressEvent.setAction(MotionEvent.ACTION_BUTTON_PRESS);
		AutoclickController.super.onMotionEvent(downEvent, downEvent,
		mEventPolicyFlags);
		downEvent.recycle();
		AutoclickController.super.onMotionEvent(pressEvent, pressEvent,
		mEventPolicyFlags);
		pressEvent.recycle();
		}

		// To end a drag event, only send the BUTTON_RELEASE and UP events, making sure to
		// include the originating drag click's down time.
		private void endDragEvent() {
		mDragModeIsDragging = false;

		MotionEvent releaseEvent =
		MotionEvent.obtain(
		/* downTime= */ mDragModeClickDownTime,
		/* eventTime= */ mDragModeClickDownTime,
		MotionEvent.ACTION_BUTTON_RELEASE,
		/* pointerCount= */ 1,
		mTempPointerProperties,
		mTempPointerCoords,
		mMetaState,
		BUTTON_PRIMARY,
		/* xPrecision= */ 1.0f,
		/* yPrecision= */ 1.0f,
		mLastMotionEvent.getDeviceId(),
		/* edgeFlags= */ 0,
		mLastMotionEvent.getSource(),
		mLastMotionEvent.getFlags());
		MotionEvent upEvent = MotionEvent.obtain(releaseEvent);
		upEvent.setAction(MotionEvent.ACTION_UP);
		AutoclickController.super.onMotionEvent(releaseEvent, releaseEvent,
		mEventPolicyFlags);
		AutoclickController.super.onMotionEvent(upEvent, upEvent, mEventPolicyFlags);
		}

		@Override
		public String toString() {
		StringBuilder builder = new StringBuilder();

services/tests/servicestests/src/com/android/server/accessibility/autoclick/AutoclickControllerTest.java

+155 −1

Original line number	Diff line number	Diff line
		@@ -55,6 +55,7 @@ import org.junit.Rule;
		import org.junit.Test;
		import org.junit.runner.RunWith;
		import org.mockito.Mock;
		import org.mockito.Mockito;
		import org.mockito.junit.MockitoJUnit;
		import org.mockito.junit.MockitoRule;

		@@ -77,12 +78,33 @@ public class AutoclickControllerTest {

		private static class MotionEventCaptor extends BaseEventStreamTransformation {
		public MotionEvent downEvent;
		public MotionEvent buttonPressEvent;
		public MotionEvent buttonReleaseEvent;
		public MotionEvent upEvent;
		public MotionEvent moveEvent;
		public int eventCount = 0;
		@Override
		public void onMotionEvent(MotionEvent event, MotionEvent rawEvent, int policyFlags) {
		MotionEvent eventCopy = MotionEvent.obtain(event);
		switch (event.getAction()) {
		case MotionEvent.ACTION_DOWN:
		downEvent = event;
		downEvent = eventCopy;
		eventCount++;
		break;
		case MotionEvent.ACTION_BUTTON_PRESS:
		buttonPressEvent = eventCopy;
		eventCount++;
		break;
		case MotionEvent.ACTION_BUTTON_RELEASE:
		buttonReleaseEvent = eventCopy;
		eventCount++;
		break;
		case MotionEvent.ACTION_UP:
		upEvent = eventCopy;
		eventCount++;
		break;
		case MotionEvent.ACTION_MOVE:
		moveEvent = eventCopy;
		eventCount++;
		break;
		}
		@@ -904,7 +926,134 @@ public class AutoclickControllerTest {
		assertThat(motionEventCaptor.downEvent).isNotNull();
		assertThat(motionEventCaptor.downEvent.getButtonState()).isEqualTo(
		MotionEvent.BUTTON_PRIMARY);
		assertThat(motionEventCaptor.eventCount).isEqualTo(
		getNumEventsExpectedFromClick(/* numClicks= */ 2));
		}

		@Test
		@EnableFlags(com.android.server.accessibility.Flags.FLAG_ENABLE_AUTOCLICK_INDICATOR)
		public void sendClick_clickType_drag_simulateDragging() {
		MotionEventCaptor motionEventCaptor = new MotionEventCaptor();
		mController.setNext(motionEventCaptor);

		injectFakeMouseActionHoverMoveEvent();
		// Set delay to zero so click is scheduled to run immediately.
		mController.mClickScheduler.updateDelay(0);

		// Set click type to drag click.
		mController.clickPanelController.handleAutoclickTypeChange(
		AutoclickTypePanel.AUTOCLICK_TYPE_DRAG);

		injectFakeMouseMoveEvent(/* x= / 30, / y= */ 0, MotionEvent.ACTION_HOVER_MOVE);
		mTestableLooper.processAllMessages();

		// Verify only two motion events were sent.
		assertThat(motionEventCaptor.eventCount).isEqualTo(2);

		// Verify both events have the same down time.
		assertThat(motionEventCaptor.downEvent).isNotNull();
		assertThat(motionEventCaptor.buttonPressEvent).isNotNull();
		assertThat(motionEventCaptor.downEvent.getDownTime()).isEqualTo(
		motionEventCaptor.buttonPressEvent.getDownTime());

		// Move the mouse again to simulate dragging and verify the new mouse event is
		// transformed to a MOVE action and its down time matches the drag initiating click's
		// down time.
		injectFakeMouseMoveEvent(/* x= / 40, / y= */ 0, MotionEvent.ACTION_HOVER_MOVE);
		mTestableLooper.processAllMessages();
		assertThat(motionEventCaptor.eventCount).isEqualTo(3);
		assertThat(motionEventCaptor.moveEvent).isNotNull();
		assertThat(motionEventCaptor.moveEvent.getDownTime()).isEqualTo(
		motionEventCaptor.downEvent.getDownTime());

		// Move the mouse again further now to simulate ending the drag session.
		motionEventCaptor.moveEvent = null;
		motionEventCaptor.eventCount = 0;
		injectFakeMouseMoveEvent(/* x= / 300, / y= */ 300, MotionEvent.ACTION_HOVER_MOVE);
		mTestableLooper.processAllMessages();

		// Verify the final 3 clicks were sent: the 1 move event + 2 up type events to end the drag.
		assertThat(motionEventCaptor.eventCount).isEqualTo(3);

		// Verify each event matches the same down time as the initiating drag click.
		assertThat(motionEventCaptor.moveEvent).isNotNull();
		assertThat(motionEventCaptor.moveEvent.getDownTime()).isEqualTo(
		motionEventCaptor.downEvent.getDownTime());
		assertThat(motionEventCaptor.buttonReleaseEvent).isNotNull();
		assertThat(motionEventCaptor.buttonReleaseEvent.getDownTime()).isEqualTo(
		motionEventCaptor.downEvent.getDownTime());
		assertThat(motionEventCaptor.upEvent).isNotNull();
		assertThat(motionEventCaptor.upEvent.getDownTime()).isEqualTo(
		motionEventCaptor.downEvent.getDownTime());
		}


		@Test
		@EnableFlags(com.android.server.accessibility.Flags.FLAG_ENABLE_AUTOCLICK_INDICATOR)
		public void sendClick_clickType_drag_keyEventCancelsDrag() {
		MotionEventCaptor motionEventCaptor = new MotionEventCaptor();
		mController.setNext(motionEventCaptor);

		injectFakeMouseActionHoverMoveEvent();
		// Set delay to zero so click is scheduled to run immediately.
		mController.mClickScheduler.updateDelay(0);

		// Set click type to drag click.
		mController.clickPanelController.handleAutoclickTypeChange(
		AutoclickTypePanel.AUTOCLICK_TYPE_DRAG);

		// Initiate drag event.
		injectFakeMouseMoveEvent(/* x= / 100, / y= */ 0, MotionEvent.ACTION_HOVER_MOVE);
		mTestableLooper.processAllMessages();
		assertThat(mController.isDraggingForTesting()).isTrue();

		// Move the mouse to start the click scheduler.
		injectFakeMouseActionHoverMoveEvent();
		injectFakeMouseMoveEvent(/* x= / 200, / y= */ 0, MotionEvent.ACTION_HOVER_MOVE);
		assertThat(mController.isDraggingForTesting()).isTrue();

		// Press a key to see the drag canceled and reset.
		injectFakeKeyEvent(KeyEvent.KEYCODE_A, /* modifiers= */ 0);
		assertThat(mController.isDraggingForTesting()).isFalse();

		// Verify the ACTION_UP was sent for alerting the system that dragging has ended.
		assertThat(motionEventCaptor.upEvent).isNotNull();
		assertThat(motionEventCaptor.downEvent).isNotNull();
		assertThat(motionEventCaptor.upEvent.getDownTime()).isEqualTo(
		motionEventCaptor.downEvent.getDownTime());
		}

		@Test
		@EnableFlags(com.android.server.accessibility.Flags.FLAG_ENABLE_AUTOCLICK_INDICATOR)
		public void sendClick_clickType_drag_clickTypeDoesNotRevertAfterFirstClick() {
		MotionEventCaptor motionEventCaptor = new MotionEventCaptor();
		mController.setNext(motionEventCaptor);

		injectFakeMouseActionHoverMoveEvent();
		// Set delay to zero so click is scheduled to run immediately.
		mController.mClickScheduler.updateDelay(0);

		// Set ACCESSIBILITY_AUTOCLICK_REVERT_TO_LEFT_CLICK to true.
		Settings.Secure.putIntForUser(mTestableContext.getContentResolver(),
		Settings.Secure.ACCESSIBILITY_AUTOCLICK_REVERT_TO_LEFT_CLICK,
		AccessibilityUtils.State.ON,
		mTestableContext.getUserId());
		mController.onChangeForTesting(/* selfChange= */ true,
		Settings.Secure.getUriFor(
		Settings.Secure.ACCESSIBILITY_AUTOCLICK_REVERT_TO_LEFT_CLICK));

		// Set click type to drag click.
		AutoclickTypePanel mockAutoclickTypePanel = mock(AutoclickTypePanel.class);
		mController.mAutoclickTypePanel = mockAutoclickTypePanel;
		mController.clickPanelController.handleAutoclickTypeChange(
		AutoclickTypePanel.AUTOCLICK_TYPE_DRAG);

		// Initiate drag event.
		injectFakeMouseMoveEvent(/* x= / 100, / y= */ 0, MotionEvent.ACTION_HOVER_MOVE);
		mTestableLooper.processAllMessages();

		// Even after the click, the click type should not be reset.
		verify(mockAutoclickTypePanel, Mockito.never()).resetSelectedClickType();
		}

		/**
		@@ -965,4 +1114,9 @@ public class AutoclickControllerTest {
		Settings.Secure.getUriFor(
		Settings.Secure.ACCESSIBILITY_AUTOCLICK_IGNORE_MINOR_CURSOR_MOVEMENT));
		}

		// The 4 events represented are DOWN, BUTTON_PRESS, BUTTON_RELEASE, and UP.
		private int getNumEventsExpectedFromClick(int numClicks) {
		return numClicks * 4;
		}
		}