Merge "Scribe for View: handwriting initiation for unfocused" (e96b6383) · Commits · e / os / android_frameworks_base

core/java/android/view/HandwritingInitiator.java

+225 −17

Original line number	Diff line number	Diff line
		@@ -24,6 +24,9 @@ import android.view.inputmethod.InputMethodManager;
		import com.android.internal.annotations.VisibleForTesting;

		import java.lang.ref.WeakReference;
		import java.util.ArrayList;
		import java.util.Iterator;
		import java.util.List;

		/**
		* Initiates handwriting mode once it detects stylus movement in handwritable areas.
		@@ -58,6 +61,7 @@ public class HandwritingInitiator {
		private final long mTapTimeoutInMillis;

		private State mState = new State();
		private final HandwritingAreaTracker mHandwritingAreasTracker = new HandwritingAreaTracker();

		/**
		* Helper method to reset the internal state of this class.
		@@ -83,8 +87,8 @@ public class HandwritingInitiator {
		private final InputMethodManager mImm;

		@VisibleForTesting
		public HandwritingInitiator(ViewConfiguration viewConfiguration,
		InputMethodManager inputMethodManager) {
		public HandwritingInitiator(@NonNull ViewConfiguration viewConfiguration,
		@NonNull InputMethodManager inputMethodManager) {
		mTouchSlop = viewConfiguration.getScaledTouchSlop();
		mTapTimeoutInMillis = ViewConfiguration.getTapTimeout();
		mImm = inputMethodManager;
		@@ -98,7 +102,7 @@ public class HandwritingInitiator {
		* @param motionEvent the stylus MotionEvent.
		*/
		@VisibleForTesting
		public void onTouchEvent(MotionEvent motionEvent) {
		public void onTouchEvent(@NonNull MotionEvent motionEvent) {
		final int maskedAction = motionEvent.getActionMasked();
		switch (maskedAction) {
		case MotionEvent.ACTION_DOWN:
		@@ -151,11 +155,20 @@ public class HandwritingInitiator {
		final float y = motionEvent.getY(pointerIndex);
		if (largerThanTouchSlop(x, y, mState.mStylusDownX, mState.mStylusDownY)) {
		mState.mExceedTouchSlop = true;
		tryStartHandwriting();
		View candidateView =
		findBestCandidateView(mState.mStylusDownX, mState.mStylusDownY);
		if (candidateView != null) {
		if (candidateView == getConnectedView()) {
		startHandwriting(candidateView);
		} else {
		candidateView.requestFocus();
		}
		}
		}
		}
		}

		@Nullable
		private View getConnectedView() {
		if (mConnectedView == null) return null;
		return mConnectedView.get();
		@@ -178,15 +191,18 @@ public class HandwritingInitiator {
		clearConnectedView();
		return;
		}

		final View connectedView = getConnectedView();
		if (connectedView == view) {
		++mConnectionCount;
		} else {
		mConnectedView = new WeakReference<>(view);
		mConnectionCount = 1;
		if (mState.mShouldInitHandwriting) {
		tryStartHandwriting();
		}
		}
		}

		/**
		* Notify HandwritingInitiator that the InputConnection has closed for the given view.
		@@ -233,28 +249,90 @@ public class HandwritingInitiator {
		return;
		}

		final ViewParent viewParent = connectedView.getParent();
		// Do a final check before startHandwriting.
		if (viewParent != null && connectedView.isAttachedToWindow()) {
		final Rect editorBounds =
		new Rect(0, 0, connectedView.getWidth(), connectedView.getHeight());
		if (viewParent.getChildVisibleRect(connectedView, editorBounds, null)) {
		final int roundedInitX = Math.round(mState.mStylusDownX);
		final int roundedInitY = Math.round(mState.mStylusDownY);
		if (editorBounds.contains(roundedInitX, roundedInitY)) {
		Rect handwritingArea = getViewHandwritingArea(connectedView);
		if (handwritingArea != null) {
		if (contains(handwritingArea, mState.mStylusDownX, mState.mStylusDownY)) {
		startHandwriting(connectedView);
		}
		}
		}
		reset();
		}

		/** For test only. */
		@VisibleForTesting
		public void startHandwriting(View view) {
		public void startHandwriting(@NonNull View view) {
		mImm.startStylusHandwriting(view);
		}

		/**
		* Notify that the handwriting area for the given view might be updated.
		* @param view the view whose handwriting area might be updated.
		*/
		public void updateHandwritingAreasForView(@NonNull View view) {
		mHandwritingAreasTracker.updateHandwritingAreaForView(view);
		}

		/**
		* Given the location of the stylus event, return the best candidate view to initialize
		* handwriting mode.
		*
		* @param x the x coordinates of the stylus event, in the coordinates of the window.
		* @param y the y coordinates of the stylus event, in the coordinates of the window.
		*/
		@Nullable
		private View findBestCandidateView(float x, float y) {
		// If the connectedView is not null and do not set any handwriting area, it will check
		// whether the connectedView's boundary contains the initial stylus position. If true,
		// directly return the connectedView.
		final View connectedView = getConnectedView();
		if (connectedView != null && connectedView.isAutoHandwritingEnabled()) {
		final Rect handwritingArea = getViewHandwritingArea(connectedView);
		if (handwritingArea != null && contains(handwritingArea, x, y)) {
		return connectedView;
		}
		}

		// Check the registered handwriting areas.
		final List<HandwritableViewInfo> handwritableViewInfos =
		mHandwritingAreasTracker.computeViewInfos();
		for (HandwritableViewInfo viewInfo : handwritableViewInfos) {
		final View view = viewInfo.getView();
		if (!view.isAutoHandwritingEnabled()) continue;
		final Rect rect = viewInfo.getHandwritingArea();
		if (rect != null && contains(rect, x, y)) {
		return viewInfo.getView();
		}
		}
		return null;
		}

		/**
		* Return the handwriting area of the given view, represented in the window's coordinate.
		* If the view didn't set any handwriting area, it will return the view's boundary.
		* It will return null if the view or its handwriting area is not visible.
		*/
		@Nullable
		private static Rect getViewHandwritingArea(@NonNull View view) {
		final ViewParent viewParent = view.getParent();
		if (viewParent != null && view.isAttachedToWindow() && view.isAggregatedVisible()) {
		Rect handwritingArea = view.getHandwritingArea();
		if (handwritingArea == null) {
		handwritingArea = new Rect(0, 0, view.getWidth(), view.getHeight());
		}
		if (viewParent.getChildVisibleRect(view, handwritingArea, null)) {
		return handwritingArea;
		}
		}
		return null;
		}

		/**
		* Return true if the (x, y) is inside by the given {@link Rect}.
		*/
		private boolean contains(@NonNull Rect rect, float x, float y) {
		return x >= rect.left && x < rect.right && y >= rect.top && y < rect.bottom;
		}

		private boolean largerThanTouchSlop(float x1, float y1, float x2, float y2) {
		float dx = x1 - x2;
		float dy = y1 - y2;
		@@ -291,4 +369,134 @@ public class HandwritingInitiator {
		private float mStylusDownX = Float.NaN;
		private float mStylusDownY = Float.NaN;
		}

		/** The helper method to check if the given view is still active for handwriting. */
		private static boolean isViewActive(@Nullable View view) {
		return view != null && view.isAttachedToWindow() && view.isAggregatedVisible()
		&& view.isAutoHandwritingEnabled();
		}

		/**
		* A class used to track the handwriting areas set by the Views.
		*
		* @hide
		*/
		@VisibleForTesting
		public static class HandwritingAreaTracker {
		private final List<HandwritableViewInfo> mHandwritableViewInfos;

		public HandwritingAreaTracker() {
		mHandwritableViewInfos = new ArrayList<>();
		}

		/**
		* Notify this tracker that the handwriting area of the given view has been updated.
		* This method does three things:
		* a) iterate over the all the tracked ViewInfos and remove those already invalid ones.
		* b) mark the given view's ViewInfo to be dirty. So that next time when
		* {@link #computeViewInfos} is called, this view's handwriting area will be recomputed.
		* c) If no the given view is not in the tracked ViewInfo list, a new ViewInfo object will
		* be created and added to the list.
		*
		* @param view the view whose handwriting area is updated.
		*/
		public void updateHandwritingAreaForView(@NonNull View view) {
		Iterator<HandwritableViewInfo> iterator = mHandwritableViewInfos.iterator();
		boolean found = false;
		while (iterator.hasNext()) {
		final HandwritableViewInfo handwritableViewInfo = iterator.next();
		final View curView = handwritableViewInfo.getView();
		if (!isViewActive(curView)) {
		iterator.remove();
		}
		if (curView == view) {
		found = true;
		handwritableViewInfo.mIsDirty = true;
		}
		}
		if (!found && isViewActive(view)) {
		// The given view is not tracked. Create a new HandwritableViewInfo for it and add
		// to the list.
		mHandwritableViewInfos.add(new HandwritableViewInfo(view));
		}
		}

		/**
		* Update the handwriting areas and return a list of ViewInfos containing the view
		* reference and its handwriting area.
		*/
		@NonNull
		public List<HandwritableViewInfo> computeViewInfos() {
		mHandwritableViewInfos.removeIf(viewInfo -> !viewInfo.update());
		return mHandwritableViewInfos;
		}
		}

		/**
		* A class that reference to a View and its handwriting area(in the ViewRoot's coordinate.)
		*
		* @hide
		*/
		@VisibleForTesting
		public static class HandwritableViewInfo {
		final WeakReference<View> mViewRef;
		Rect mHandwritingArea = null;
		@VisibleForTesting
		public boolean mIsDirty = true;

		@VisibleForTesting
		public HandwritableViewInfo(@NonNull View view) {
		mViewRef = new WeakReference<>(view);
		}

		/** Return the tracked view. */
		@Nullable
		public View getView() {
		return mViewRef.get();
		}

		/**
		* Return the tracked handwriting area, represented in the ViewRoot's coordinates.
		* Notice, the caller should not modify the returned Rect.
		*/
		@Nullable
		public Rect getHandwritingArea() {
		return mHandwritingArea;
		}

		/**
		* Update the handwriting area in this ViewInfo.
		*
		* @return true if this ViewInfo is still valid. Or false if this ViewInfo has become
		* invalid due to either view is no longer visible, or the handwriting area set by the
		* view is removed. {@link HandwritingAreaTracker} no longer need to keep track of this
		* HandwritableViewInfo this method returns false.
		*/
		public boolean update() {
		final View view = getView();
		if (!isViewActive(view)) {
		return false;
		}

		if (!mIsDirty) {
		return true;
		}
		final Rect localRect = view.getHandwritingArea();
		if (localRect == null) {
		return false;
		}

		ViewParent parent = view.getParent();
		if (parent != null) {
		final Rect newRect = new Rect(localRect);
		if (parent.getChildVisibleRect(view, newRect, null /* offset */)) {
		mHandwritingArea = newRect;
		} else {
		mHandwritingArea = null;
		}
		}
		mIsDirty = false;
		return true;
		}
		}
		}

core/java/android/view/View.java

+53 −2

Original line number	Diff line number	Diff line
		@@ -4745,9 +4745,11 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
		private List<Rect> mSystemGestureExclusionRects = null;
		private List<Rect> mKeepClearRects = null;
		private boolean mPreferKeepClear = false;
		private Rect mHandwritingArea = null;

		/**
		* Used to track {@link #mSystemGestureExclusionRects} and {@link #mKeepClearRects}
		* Used to track {@link #mSystemGestureExclusionRects}, {@link #mKeepClearRects} and
		* {@link #mHandwritingArea}.
		*/
		public RenderNode.PositionUpdateListener mPositionUpdateListener;
		private Runnable mPositionChangedUpdate;
		@@ -11710,7 +11712,8 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
		private void updatePositionUpdateListener() {
		final ListenerInfo info = getListenerInfo();
		if (getSystemGestureExclusionRects().isEmpty()
		&& collectPreferKeepClearRects().isEmpty()) {
		&& collectPreferKeepClearRects().isEmpty()
		&& (info.mHandwritingArea == null \|\| !isAutoHandwritingEnabled())) {
		if (info.mPositionUpdateListener != null) {
		mRenderNode.removePositionUpdateListener(info.mPositionUpdateListener);
		info.mPositionChangedUpdate = null;
		@@ -11720,6 +11723,7 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
		info.mPositionChangedUpdate = () -> {
		updateSystemGestureExclusionRects();
		updateKeepClearRects();
		updateHandwritingArea();
		};
		info.mPositionUpdateListener = new RenderNode.PositionUpdateListener() {
		@Override
		@@ -11875,6 +11879,51 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
		return Collections.emptyList();
		}

		/**
		* Set a list of handwriting areas in this view. If there is any stylus {@link MotionEvent}
		* occurs within those areas, it will trigger stylus handwriting mode. This can be disabled by
		* disabling the auto handwriting initiation by calling
		* {@link #setAutoHandwritingEnabled(boolean)} with false.
		*
		* @attr rects a list of handwriting area in the view's local coordiniates.
		*
		* @see android.view.inputmethod.InputMethodManager#startStylusHandwriting(View)
		* @see #setAutoHandwritingEnabled(boolean)
		*
		* @hide
		*/
		public void setHandwritingArea(@Nullable Rect rect) {
		final ListenerInfo info = getListenerInfo();
		info.mHandwritingArea = rect;
		updatePositionUpdateListener();
		postUpdate(this::updateHandwritingArea);
		}

		/**
		* Return the handwriting areas set on this view, in its local coordinates.
		* Notice: the caller of this method should not modify the Rect returned.
		* @see #setHandwritingArea(Rect)
		*
		* @hide
		*/
		@Nullable
		public Rect getHandwritingArea() {
		final ListenerInfo info = mListenerInfo;
		if (info != null) {
		return info.mHandwritingArea;
		}
		return null;
		}

		void updateHandwritingArea() {
		// If autoHandwritingArea is not enabled, do nothing.
		if (!isAutoHandwritingEnabled()) return;
		final AttachInfo ai = mAttachInfo;
		if (ai != null) {
		ai.mViewRootImpl.getHandwritingInitiator().updateHandwritingAreasForView(this);
		}
		}

		/**
		* Compute the view's coordinate within the surface.
		*
		@@ -31181,6 +31230,8 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
		} else {
		mPrivateFlags4 &= ~PFLAG4_AUTO_HANDWRITING_ENABLED;
		}
		updatePositionUpdateListener();
		postUpdate(this::updateHandwritingArea);
		}

		/**

core/java/android/widget/EditText.java

+7 −0

Original line number	Diff line number	Diff line
		@@ -17,6 +17,7 @@
		package android.widget;

		import android.content.Context;
		import android.graphics.Rect;
		import android.text.Editable;
		import android.text.Selection;
		import android.text.Spannable;
		@@ -173,6 +174,12 @@ public class EditText extends TextView {
		return EditText.class.getName();
		}

		@Override
		protected void onSizeChanged(int w, int h, int oldw, int oldh) {
		super.onSizeChanged(w, h, oldw, oldh);
		setHandwritingArea(new Rect(0, 0, w, h));
		}

		/** @hide */
		@Override
		protected boolean supportsAutoSizeText() {

core/tests/coretests/src/android/view/stylus/HandwritableViewInfoTest.java

0 → 100644

+87 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2022 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package android.view.stylus;

		import static android.view.stylus.HandwritingTestUtil.createView;

		import static com.google.common.truth.Truth.assertThat;

		import static junit.framework.Assert.assertFalse;
		import static junit.framework.Assert.assertTrue;

		import android.graphics.Rect;
		import android.platform.test.annotations.Presubmit;
		import android.view.HandwritingInitiator;
		import android.view.View;

		import androidx.test.ext.junit.runners.AndroidJUnit4;
		import androidx.test.filters.SmallTest;

		import org.junit.Test;
		import org.junit.runner.RunWith;

		@Presubmit
		@SmallTest
		@RunWith(AndroidJUnit4.class)
		public class HandwritableViewInfoTest {

		@Test
		public void constructorTest() {
		final Rect rect = new Rect(1, 2, 3, 4);
		final View view = createView(rect);
		final HandwritingInitiator.HandwritableViewInfo handwritableViewInfo =
		new HandwritingInitiator.HandwritableViewInfo(view);

		assertThat(handwritableViewInfo.getView()).isEqualTo(view);
		// It's labeled dirty by default.
		assertTrue(handwritableViewInfo.mIsDirty);
		}

		@Test
		public void update() {
		final Rect rect = new Rect(1, 2, 3, 4);
		final View view = createView(rect);
		final HandwritingInitiator.HandwritableViewInfo handwritableViewInfo =
		new HandwritingInitiator.HandwritableViewInfo(view);

		assertThat(handwritableViewInfo.getView()).isEqualTo(view);

		final boolean isViewInfoValid = handwritableViewInfo.update();

		assertTrue(isViewInfoValid);
		assertThat(handwritableViewInfo.getHandwritingArea()).isEqualTo(rect);
		assertFalse(handwritableViewInfo.mIsDirty);
		}

		@Test
		public void update_viewDisableAutoHandwriting() {
		final Rect rect = new Rect(1, 2, 3, 4);
		final View view = HandwritingTestUtil.createView(rect, false /* autoHandwritingEnabled */);
		final HandwritingInitiator.HandwritableViewInfo handwritableViewInfo =
		new HandwritingInitiator.HandwritableViewInfo(view);

		assertThat(handwritableViewInfo.getView()).isEqualTo(view);

		final boolean isViewInfoValid = handwritableViewInfo.update();

		// Return false because the view disabled autoHandwriting.
		assertFalse(isViewInfoValid);
		// The view disabled the autoHandwriting, and it won't update the handwriting area.
		assertThat(handwritableViewInfo.getHandwritingArea()).isNull();
		}

		}

core/tests/coretests/src/android/view/stylus/HandwritingAreaTrackerTest.java

0 → 100644

+149 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2022 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package android.view.stylus;

		import static android.view.stylus.HandwritingTestUtil.createView;

		import static com.google.common.truth.Truth.assertThat;

		import android.app.Instrumentation;
		import android.content.Context;
		import android.graphics.Rect;
		import android.platform.test.annotations.Presubmit;
		import android.view.HandwritingInitiator;
		import android.view.View;

		import androidx.test.ext.junit.runners.AndroidJUnit4;
		import androidx.test.filters.SmallTest;
		import androidx.test.platform.app.InstrumentationRegistry;

		import org.junit.Before;
		import org.junit.Test;
		import org.junit.runner.RunWith;

		import java.util.List;


		/**
		* Tests for {@link HandwritingInitiator.HandwritingAreaTracker}
		*
		* Build/Install/Run:
		* atest FrameworksCoreTests:android.view.stylus.HandwritingAreaTrackerTest
		*/
		@Presubmit
		@SmallTest
		@RunWith(AndroidJUnit4.class)
		public class HandwritingAreaTrackerTest {
		HandwritingInitiator.HandwritingAreaTracker mHandwritingAreaTracker;
		Context mContext;

		@Before
		public void setup() {
		final Instrumentation mInstrumentation = InstrumentationRegistry.getInstrumentation();
		mContext = mInstrumentation.getTargetContext();
		mHandwritingAreaTracker = new HandwritingInitiator.HandwritingAreaTracker();
		}

		@Test
		public void updateHandwritingAreaForView_singleView() {
		Rect rect = new Rect(0, 0, 100, 100);
		View view = createView(rect);
		mHandwritingAreaTracker.updateHandwritingAreaForView(view);

		List<HandwritingInitiator.HandwritableViewInfo> viewInfos =
		mHandwritingAreaTracker.computeViewInfos();

		assertThat(viewInfos.size()).isEqualTo(1);
		assertThat(viewInfos.get(0).getHandwritingArea()).isEqualTo(rect);
		assertThat(viewInfos.get(0).getView()).isEqualTo(view);
		}

		@Test
		public void updateHandwritingAreaForView_multipleViews() {
		Rect rect1 = new Rect(0, 0, 100, 100);
		Rect rect2 = new Rect(100, 100, 200, 200);

		View view1 = createView(rect1);
		View view2 = createView(rect2);
		mHandwritingAreaTracker.updateHandwritingAreaForView(view1);
		mHandwritingAreaTracker.updateHandwritingAreaForView(view2);

		List<HandwritingInitiator.HandwritableViewInfo> viewInfos =
		mHandwritingAreaTracker.computeViewInfos();

		assertThat(viewInfos.size()).isEqualTo(2);
		assertThat(viewInfos.get(0).getView()).isEqualTo(view1);
		assertThat(viewInfos.get(0).getHandwritingArea()).isEqualTo(rect1);

		assertThat(viewInfos.get(1).getView()).isEqualTo(view2);
		assertThat(viewInfos.get(1).getHandwritingArea()).isEqualTo(rect2);
		}

		@Test
		public void updateHandwritingAreaForView_afterDisableAutoHandwriting() {
		Rect rect1 = new Rect(0, 0, 100, 100);
		Rect rect2 = new Rect(100, 100, 200, 200);

		View view1 = createView(rect1);
		View view2 = createView(rect2);
		mHandwritingAreaTracker.updateHandwritingAreaForView(view1);
		mHandwritingAreaTracker.updateHandwritingAreaForView(view2);

		// There should be 2 views tracked.
		assertThat(mHandwritingAreaTracker.computeViewInfos().size()).isEqualTo(2);

		// Disable autoHandwriting for view1 and update handwriting area.
		view1.setAutoHandwritingEnabled(false);
		mHandwritingAreaTracker.updateHandwritingAreaForView(view1);

		List<HandwritingInitiator.HandwritableViewInfo> viewInfos =
		mHandwritingAreaTracker.computeViewInfos();
		// The view1 has disabled the autoHandwriting, it's not tracked anymore.
		assertThat(viewInfos.size()).isEqualTo(1);

		// view2 is still tracked.
		assertThat(viewInfos.get(0).getView()).isEqualTo(view2);
		assertThat(viewInfos.get(0).getHandwritingArea()).isEqualTo(rect2);
		}

		@Test
		public void updateHandwritingAreaForView_removesInactiveView() {
		Rect rect1 = new Rect(0, 0, 100, 100);
		Rect rect2 = new Rect(100, 100, 200, 200);

		View view1 = createView(rect1);
		View view2 = createView(rect2);
		mHandwritingAreaTracker.updateHandwritingAreaForView(view1);
		mHandwritingAreaTracker.updateHandwritingAreaForView(view2);

		// There should be 2 viewInfos tracked.
		assertThat(mHandwritingAreaTracker.computeViewInfos().size()).isEqualTo(2);

		// Disable autoHandwriting for view1, but update handwriting area for view2.
		view1.setAutoHandwritingEnabled(false);
		mHandwritingAreaTracker.updateHandwritingAreaForView(view2);

		List<HandwritingInitiator.HandwritableViewInfo> viewInfos =
		mHandwritingAreaTracker.computeViewInfos();
		// The view1 has disabled the autoHandwriting, it's not tracked anymore.
		assertThat(viewInfos.size()).isEqualTo(1);

		// view2 is still tracked.
		assertThat(viewInfos.get(0).getView()).isEqualTo(view2);
		assertThat(viewInfos.get(0).getHandwritingArea()).isEqualTo(rect2);
		}
		}