Clicking on partially coverd views by other views or windows. (7498efdc) · Commits · e / os / android_frameworks_base

core/java/android/accessibilityservice/IAccessibilityServiceConnection.aidl

+4 −0

Original line number	Diff line number	Diff line
		@@ -54,6 +54,10 @@ interface IAccessibilityServiceConnection {
		int action, in Bundle arguments, int interactionId,
		IAccessibilityInteractionConnectionCallback callback, long threadId);

		boolean computeClickPointInScreen(int accessibilityWindowId, long accessibilityNodeId,
		int interactionId, IAccessibilityInteractionConnectionCallback callback,
		long threadId);

		AccessibilityWindowInfo getWindow(int windowId);

		List<AccessibilityWindowInfo> getWindows();

core/java/android/view/AccessibilityInteractionController.java

+130 −16

Original line number	Diff line number	Diff line
		@@ -19,7 +19,6 @@ package android.view;
		import android.graphics.Point;
		import android.graphics.Rect;
		import android.graphics.Region;
		import android.os.Build;
		import android.os.Bundle;
		import android.os.Handler;
		import android.os.Looper;
		@@ -101,7 +100,7 @@ final class AccessibilityInteractionController {
		IAccessibilityInteractionConnectionCallback callback, int flags, int interrogatingPid,
		long interrogatingTid, MagnificationSpec spec) {
		Message message = mHandler.obtainMessage();
		message.what = PrivateHandler.MSG_FIND_ACCESSIBLITY_NODE_INFO_BY_ACCESSIBILITY_ID;
		message.what = PrivateHandler.MSG_FIND_ACCESSIBILITY_NODE_INFO_BY_ACCESSIBILITY_ID;
		message.arg1 = flags;

		SomeArgs args = SomeArgs.obtain();
		@@ -176,7 +175,7 @@ final class AccessibilityInteractionController {
		IAccessibilityInteractionConnectionCallback callback, int flags, int interrogatingPid,
		long interrogatingTid, MagnificationSpec spec) {
		Message message = mHandler.obtainMessage();
		message.what = PrivateHandler.MSG_FIND_ACCESSIBLITY_NODE_INFOS_BY_VIEW_ID;
		message.what = PrivateHandler.MSG_FIND_ACCESSIBILITY_NODE_INFOS_BY_VIEW_ID;
		message.arg1 = flags;
		message.arg2 = AccessibilityNodeInfo.getAccessibilityViewId(accessibilityNodeId);

		@@ -261,7 +260,7 @@ final class AccessibilityInteractionController {
		IAccessibilityInteractionConnectionCallback callback, int flags, int interrogatingPid,
		long interrogatingTid, MagnificationSpec spec) {
		Message message = mHandler.obtainMessage();
		message.what = PrivateHandler.MSG_FIND_ACCESSIBLITY_NODE_INFO_BY_TEXT;
		message.what = PrivateHandler.MSG_FIND_ACCESSIBILITY_NODE_INFO_BY_TEXT;
		message.arg1 = flags;

		SomeArgs args = SomeArgs.obtain();
		@@ -637,6 +636,95 @@ final class AccessibilityInteractionController {
		}
		}

		public void computeClickPointInScreenClientThread(long accessibilityNodeId,
		Region interactiveRegion, int interactionId,
		IAccessibilityInteractionConnectionCallback callback, int interrogatingPid,
		long interrogatingTid, MagnificationSpec spec) {
		Message message = mHandler.obtainMessage();
		message.what = PrivateHandler.MSG_COMPUTE_CLICK_POINT_IN_SCREEN;

		SomeArgs args = SomeArgs.obtain();
		args.argi1 = AccessibilityNodeInfo.getAccessibilityViewId(accessibilityNodeId);
		args.argi2 = AccessibilityNodeInfo.getVirtualDescendantId(accessibilityNodeId);
		args.argi3 = interactionId;
		args.arg1 = callback;
		args.arg2 = spec;
		args.arg3 = interactiveRegion;

		message.obj = args;

		// If the interrogation is performed by the same thread as the main UI
		// thread in this process, set the message as a static reference so
		// after this call completes the same thread but in the interrogating
		// client can handle the message to generate the result.
		if (interrogatingPid == mMyProcessId && interrogatingTid == mMyLooperThreadId) {
		AccessibilityInteractionClient.getInstanceForThread(
		interrogatingTid).setSameThreadMessage(message);
		} else {
		mHandler.sendMessage(message);
		}
		}

		private void computeClickPointInScreenUiThread(Message message) {
		SomeArgs args = (SomeArgs) message.obj;
		final int accessibilityViewId = args.argi1;
		final int virtualDescendantId = args.argi2;
		final int interactionId = args.argi3;
		final IAccessibilityInteractionConnectionCallback callback =
		(IAccessibilityInteractionConnectionCallback) args.arg1;
		final MagnificationSpec spec = (MagnificationSpec) args.arg2;
		final Region interactiveRegion = (Region) args.arg3;
		args.recycle();

		boolean succeeded = false;
		Point point = mTempPoint;
		try {
		if (mViewRootImpl.mView == null \|\| mViewRootImpl.mAttachInfo == null) {
		return;
		}
		View target = null;
		if (accessibilityViewId != AccessibilityNodeInfo.UNDEFINED_ITEM_ID) {
		target = findViewByAccessibilityId(accessibilityViewId);
		} else {
		target = mViewRootImpl.mView;
		}
		if (target != null && isShown(target)) {
		AccessibilityNodeProvider provider = target.getAccessibilityNodeProvider();
		if (provider != null) {
		// For virtual views just use the center of the bounds in screen.
		AccessibilityNodeInfo node = null;
		if (virtualDescendantId != AccessibilityNodeInfo.UNDEFINED_ITEM_ID) {
		node = provider.createAccessibilityNodeInfo(virtualDescendantId);
		} else {
		node = provider.createAccessibilityNodeInfo(
		AccessibilityNodeProvider.HOST_VIEW_ID);
		}
		if (node != null) {
		succeeded = true;
		Rect boundsInScreen = mTempRect;
		node.getBoundsInScreen(boundsInScreen);
		point.set(boundsInScreen.centerX(), boundsInScreen.centerY());
		}
		} else if (virtualDescendantId == AccessibilityNodeInfo.UNDEFINED_ITEM_ID) {
		// For a real view, ask the view to compute the click point.
		succeeded = target.computeClickPointInScreenForAccessibility(
		interactiveRegion, point);
		}
		}
		} finally {
		try {
		Point result = null;
		if (succeeded) {
		applyAppScaleAndMagnificationSpecIfNeeded(point, spec);
		result = point;
		}
		callback.setComputeClickPointInScreenActionResult(result, interactionId);
		} catch (RemoteException re) {
		/* ignore - the other side will time out */
		}
		}
		}

		private View findViewByAccessibilityId(int accessibilityId) {
		View root = mViewRootImpl.mView;
		if (root == null) {
		@@ -688,6 +776,26 @@ final class AccessibilityInteractionController {
		}
		}

		private void applyAppScaleAndMagnificationSpecIfNeeded(Point point,
		MagnificationSpec spec) {
		final float applicationScale = mViewRootImpl.mAttachInfo.mApplicationScale;
		if (!shouldApplyAppScaleAndMagnificationSpec(applicationScale, spec)) {
		return;
		}

		if (applicationScale != 1.0f) {
		point.x *= applicationScale;
		point.y *= applicationScale;
		}

		if (spec != null) {
		point.x *= spec.scale;
		point.y *= spec.scale;
		point.x += (int) spec.offsetX;
		point.y += (int) spec.offsetY;
		}
		}

		private void applyAppScaleAndMagnificationSpecIfNeeded(AccessibilityNodeInfo info,
		MagnificationSpec spec) {
		if (info == null) {
		@@ -1080,11 +1188,12 @@ final class AccessibilityInteractionController {

		private class PrivateHandler extends Handler {
		private final static int MSG_PERFORM_ACCESSIBILITY_ACTION = 1;
		private final static int MSG_FIND_ACCESSIBLITY_NODE_INFO_BY_ACCESSIBILITY_ID = 2;
		private final static int MSG_FIND_ACCESSIBLITY_NODE_INFOS_BY_VIEW_ID = 3;
		private final static int MSG_FIND_ACCESSIBLITY_NODE_INFO_BY_TEXT = 4;
		private final static int MSG_FIND_ACCESSIBILITY_NODE_INFO_BY_ACCESSIBILITY_ID = 2;
		private final static int MSG_FIND_ACCESSIBILITY_NODE_INFOS_BY_VIEW_ID = 3;
		private final static int MSG_FIND_ACCESSIBILITY_NODE_INFO_BY_TEXT = 4;
		private final static int MSG_FIND_FOCUS = 5;
		private final static int MSG_FOCUS_SEARCH = 6;
		private final static int MSG_COMPUTE_CLICK_POINT_IN_SCREEN = 7;

		public PrivateHandler(Looper looper) {
		super(looper);
		@@ -1096,16 +1205,18 @@ final class AccessibilityInteractionController {
		switch (type) {
		case MSG_PERFORM_ACCESSIBILITY_ACTION:
		return "MSG_PERFORM_ACCESSIBILITY_ACTION";
		case MSG_FIND_ACCESSIBLITY_NODE_INFO_BY_ACCESSIBILITY_ID:
		return "MSG_FIND_ACCESSIBLITY_NODE_INFO_BY_ACCESSIBILITY_ID";
		case MSG_FIND_ACCESSIBLITY_NODE_INFOS_BY_VIEW_ID:
		return "MSG_FIND_ACCESSIBLITY_NODE_INFOS_BY_VIEW_ID";
		case MSG_FIND_ACCESSIBLITY_NODE_INFO_BY_TEXT:
		return "MSG_FIND_ACCESSIBLITY_NODE_INFO_BY_TEXT";
		case MSG_FIND_ACCESSIBILITY_NODE_INFO_BY_ACCESSIBILITY_ID:
		return "MSG_FIND_ACCESSIBILITY_NODE_INFO_BY_ACCESSIBILITY_ID";
		case MSG_FIND_ACCESSIBILITY_NODE_INFOS_BY_VIEW_ID:
		return "MSG_FIND_ACCESSIBILITY_NODE_INFOS_BY_VIEW_ID";
		case MSG_FIND_ACCESSIBILITY_NODE_INFO_BY_TEXT:
		return "MSG_FIND_ACCESSIBILITY_NODE_INFO_BY_TEXT";
		case MSG_FIND_FOCUS:
		return "MSG_FIND_FOCUS";
		case MSG_FOCUS_SEARCH:
		return "MSG_FOCUS_SEARCH";
		case MSG_COMPUTE_CLICK_POINT_IN_SCREEN:
		return "MSG_COMPUTE_CLICK_POINT_IN_SCREEN";
		default:
		throw new IllegalArgumentException("Unknown message type: " + type);
		}
		@@ -1115,16 +1226,16 @@ final class AccessibilityInteractionController {
		public void handleMessage(Message message) {
		final int type = message.what;
		switch (type) {
		case MSG_FIND_ACCESSIBLITY_NODE_INFO_BY_ACCESSIBILITY_ID: {
		case MSG_FIND_ACCESSIBILITY_NODE_INFO_BY_ACCESSIBILITY_ID: {
		findAccessibilityNodeInfoByAccessibilityIdUiThread(message);
		} break;
		case MSG_PERFORM_ACCESSIBILITY_ACTION: {
		perfromAccessibilityActionUiThread(message);
		} break;
		case MSG_FIND_ACCESSIBLITY_NODE_INFOS_BY_VIEW_ID: {
		case MSG_FIND_ACCESSIBILITY_NODE_INFOS_BY_VIEW_ID: {
		findAccessibilityNodeInfosByViewIdUiThread(message);
		} break;
		case MSG_FIND_ACCESSIBLITY_NODE_INFO_BY_TEXT: {
		case MSG_FIND_ACCESSIBILITY_NODE_INFO_BY_TEXT: {
		findAccessibilityNodeInfosByTextUiThread(message);
		} break;
		case MSG_FIND_FOCUS: {
		@@ -1133,6 +1244,9 @@ final class AccessibilityInteractionController {
		case MSG_FOCUS_SEARCH: {
		focusSearchUiThread(message);
		} break;
		case MSG_COMPUTE_CLICK_POINT_IN_SCREEN: {
		computeClickPointInScreenUiThread(message);
		} break;
		default:
		throw new IllegalArgumentException("Unknown message type: " + type);
		}

core/java/android/view/View.java

+142 −0

Original line number	Diff line number	Diff line
		@@ -35,6 +35,8 @@ import android.graphics.LinearGradient;
		import android.graphics.Matrix;
		import android.graphics.Outline;
		import android.graphics.Paint;
		import android.graphics.Path;
		import android.graphics.PathMeasure;
		import android.graphics.PixelFormat;
		import android.graphics.Point;
		import android.graphics.PorterDuff;
		@@ -5730,6 +5732,136 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
		return false;
		}

		/**
		* Computes a point on which a sequence of a down/up event can be sent to
		* trigger clicking this view. This method is for the exclusive use by the
		* accessibility layer to determine where to send a click event in explore
		* by touch mode.
		*
		* @param interactiveRegion The interactive portion of this window.
		* @param outPoint The point to populate.
		* @return True of such a point exists.
		*/
		boolean computeClickPointInScreenForAccessibility(Region interactiveRegion,
		Point outPoint) {
		// Since the interactive portion of the view is a region but as a view
		// may have a transformation matrix which cannot be applied to a
		// region we compute the view bounds rectangle and all interactive
		// predecessor's and sibling's (siblings of predecessors included)
		// rectangles that intersect the view bounds. At the
		// end if the view was partially covered by another interactive
		// view we compute the view's interactive region and pick a point
		// on its boundary path as regions do not offer APIs to get inner
		// points. Note that the the code is optimized to fail early and
		// avoid unnecessary allocations plus computations.

		// The current approach has edge cases that may produce false
		// positives or false negatives. For example, a portion of the
		// view may be covered by an interactive descendant of a
		// predecessor, which we do not compute. Also a view may be handling
		// raw touch events instead registering click listeners, which
		// we cannot compute. Despite these limitations this approach will
		// work most of the time and it is a huge improvement over just
		// blindly sending the down and up events in the center of the
		// view.

		// Cannot click on an unattached view.
		if (mAttachInfo == null) {
		return false;
		}

		// Attached to an invisible window means this view is not visible.
		if (mAttachInfo.mWindowVisibility != View.VISIBLE) {
		return false;
		}

		RectF bounds = mAttachInfo.mTmpTransformRect;
		bounds.set(0, 0, getWidth(), getHeight());
		List<RectF> intersections = mAttachInfo.mTmpRectList;
		intersections.clear();

		if (mParent instanceof ViewGroup) {
		ViewGroup parentGroup = (ViewGroup) mParent;
		if (!parentGroup.translateBoundsAndIntersectionsInWindowCoordinates(
		this, bounds, intersections)) {
		intersections.clear();
		return false;
		}
		}

		// Take into account the window location.
		final int dx = mAttachInfo.mWindowLeft;
		final int dy = mAttachInfo.mWindowTop;
		bounds.offset(dx, dy);
		offsetRects(intersections, dx, dy);

		if (intersections.isEmpty() && interactiveRegion == null) {
		outPoint.set((int) bounds.centerX(), (int) bounds.centerY());
		} else {
		// This view is partially covered by other views, then compute
		// the not covered region and pick a point on its boundary.
		Region region = new Region();
		region.set((int) bounds.left, (int) bounds.top,
		(int) bounds.right, (int) bounds.bottom);

		final int intersectionCount = intersections.size();
		for (int i = intersectionCount - 1; i >= 0; i--) {
		RectF intersection = intersections.remove(i);
		region.op((int) intersection.left, (int) intersection.top,
		(int) intersection.right, (int) intersection.bottom,
		Region.Op.DIFFERENCE);
		}

		// If the view is completely covered, done.
		if (region.isEmpty()) {
		return false;
		}

		// Take into account the interactive portion of the window
		// as the rest is covered by other windows. If no such a region
		// then the whole window is interactive.
		if (interactiveRegion != null) {
		region.op(interactiveRegion, Region.Op.INTERSECT);
		}

		// If the view is completely covered, done.
		if (region.isEmpty()) {
		return false;
		}

		// Try a shortcut here.
		if (region.isRect()) {
		Rect regionBounds = mAttachInfo.mTmpInvalRect;
		region.getBounds(regionBounds);
		outPoint.set(regionBounds.centerX(), regionBounds.centerY());
		return true;
		}

		// Get the a point on the region boundary path.
		Path path = region.getBoundaryPath();
		PathMeasure pathMeasure = new PathMeasure(path, false);
		final float[] coordinates = mAttachInfo.mTmpTransformLocation;

		// Without loss of generality pick a point.
		final float point = pathMeasure.getLength() * 0.01f;
		if (!pathMeasure.getPosTan(point, coordinates, null)) {
		return false;
		}

		outPoint.set(Math.round(coordinates[0]), Math.round(coordinates[1]));
		}

		return true;
		}

		static void offsetRects(List<RectF> rects, float offsetX, float offsetY) {
		final int rectCount = rects.size();
		for (int i = 0; i < rectCount; i++) {
		RectF intersection = rects.get(i);
		intersection.offset(offsetX, offsetY);
		}
		}

		/**
		* Returns the delegate for implementing accessibility support via
		* composition. For more details see {@link AccessibilityDelegate}.
		@@ -20141,6 +20273,16 @@ public class View implements Drawable.Callback, KeyEvent.Callback,
		*/
		final RectF mTmpTransformRect = new RectF();

		/**
		* Temporary for use in computing hit areas with transformed views
		*/
		final RectF mTmpTransformRect1 = new RectF();

		/**
		* Temporary list of rectanges.
		*/
		final List<RectF> mTmpRectList = new ArrayList<>();

		/**
		* Temporary for use in transforming invalidation rect
		*/

core/java/android/view/ViewGroup.java

+106 −0

Original line number	Diff line number	Diff line
		@@ -770,6 +770,112 @@ public abstract class ViewGroup extends View implements ViewParent, ViewManager
		return true;
		}

		/**
		* Translates the given bounds and intersections from child coordinates to
		* local coordinates. In case any interactive sibling of the calling child
		* covers the latter, a new intersections is added to the intersection list.
		* This method is for the exclusive use by the accessibility layer to compute
		* a point where a sequence of down and up events would click on a view.
		*
		* @param child The child making the call.
		* @param bounds The bounds to translate in child coordinates.
		* @param intersections The intersections of interactive views covering the child.
		* @return True if the bounds and intersections were computed, false otherwise.
		*/
		boolean translateBoundsAndIntersectionsInWindowCoordinates(View child,
		RectF bounds, List<RectF> intersections) {
		// Not attached, done.
		if (mAttachInfo == null) {
		return false;
		}

		if (getAlpha() <= 0 \|\| getTransitionAlpha() <= 0 \|\|
		getVisibility() != VISIBLE) {
		// Cannot click on a view with an invisible predecessor.
		return false;
		}

		// Compensate for the child transformation.
		if (!child.hasIdentityMatrix()) {
		Matrix matrix = child.getMatrix();
		matrix.mapRect(bounds);
		final int intersectionCount = intersections.size();
		for (int i = 0; i < intersectionCount; i++) {
		RectF intersection = intersections.get(i);
		matrix.mapRect(intersection);
		}
		}

		// Translate the bounds from child to parent coordinates.
		final int dx = child.mLeft - mScrollX;
		final int dy = child.mTop - mScrollY;
		bounds.offset(dx, dy);
		offsetRects(intersections, dx, dy);

		// If the bounds do not intersect our bounds, done.
		if (!bounds.intersects(0, 0, getWidth(), getHeight())) {
		return false;
		}

		// Check whether any clickable siblings cover the child
		// view and if so keep track of the intersections. Also
		// respect Z ordering when iterating over children.
		ArrayList<View> orderedList = buildOrderedChildList();
		final boolean useCustomOrder = orderedList == null
		&& isChildrenDrawingOrderEnabled();

		final int childCount = mChildrenCount;
		for (int i = childCount - 1; i >= 0; i--) {
		final int childIndex = useCustomOrder
		? getChildDrawingOrder(childCount, i) : i;
		final View sibling = (orderedList == null)
		? mChildren[childIndex] : orderedList.get(childIndex);

		// We care only about siblings over the child.
		if (sibling == child) {
		break;
		}

		// If sibling is not interactive we do not care.
		if (!sibling.isClickable() && !sibling.isLongClickable()) {
		continue;
		}

		// Compute the sibling bounds in its coordinates.
		RectF siblingBounds = mAttachInfo.mTmpTransformRect1;
		siblingBounds.set(0, 0, sibling.getWidth(), sibling.getHeight());

		// Take into account the sibling transformation matrix.
		if (!sibling.hasIdentityMatrix()) {
		sibling.getMatrix().mapRect(siblingBounds);
		}

		// Offset the sibling to our coordinates.
		final int siblingDx = sibling.mLeft - mScrollX;
		final int siblingDy = sibling.mTop - mScrollY;
		siblingBounds.offset(siblingDx, siblingDy);

		// Compute the intersection between the child and the sibling.
		if (siblingBounds.intersect(bounds)) {
		// If an interactive sibling completely covers the child, done.
		if (siblingBounds.equals(bounds)) {
		return false;
		}
		// Keep track of the intersection rectangle.
		RectF intersection = new RectF(siblingBounds);
		intersections.add(intersection);
		}
		}

		if (mParent instanceof ViewGroup) {
		ViewGroup parentGroup = (ViewGroup) mParent;
		return parentGroup.translateBoundsAndIntersectionsInWindowCoordinates(
		this, bounds, intersections);
		}

		return true;
		}

		/**
		* Called when a child view has changed whether or not it is tracking transient state.
		*/

core/java/android/view/ViewRootImpl.java

+22 −2

Original line number	Diff line number	Diff line
		@@ -6679,12 +6679,12 @@ public final class ViewRootImpl implements ViewParent,
		public void performAccessibilityAction(long accessibilityNodeId, int action,
		Bundle arguments, int interactionId,
		IAccessibilityInteractionConnectionCallback callback, int flags,
		int interogatingPid, long interrogatingTid) {
		int interrogatingPid, long interrogatingTid) {
		ViewRootImpl viewRootImpl = mViewRootImpl.get();
		if (viewRootImpl != null && viewRootImpl.mView != null) {
		viewRootImpl.getAccessibilityInteractionController()
		.performAccessibilityActionClientThread(accessibilityNodeId, action, arguments,
		interactionId, callback, flags, interogatingPid, interrogatingTid);
		interactionId, callback, flags, interrogatingPid, interrogatingTid);
		} else {
		// We cannot make the call and notify the caller so it does not wait.
		try {
		@@ -6695,6 +6695,26 @@ public final class ViewRootImpl implements ViewParent,
		}
		}

		@Override
		public void computeClickPointInScreen(long accessibilityNodeId, Region interactiveRegion,
		int interactionId, IAccessibilityInteractionConnectionCallback callback,
		int interrogatingPid, long interrogatingTid, MagnificationSpec spec) {
		ViewRootImpl viewRootImpl = mViewRootImpl.get();
		if (viewRootImpl != null && viewRootImpl.mView != null) {
		viewRootImpl.getAccessibilityInteractionController()
		.computeClickPointInScreenClientThread(accessibilityNodeId,
		interactiveRegion, interactionId, callback, interrogatingPid,
		interrogatingTid, spec);
		} else {
		// We cannot make the call and notify the caller so it does not wait.
		try {
		callback.setComputeClickPointInScreenActionResult(null, interactionId);
		} catch (RemoteException re) {
		/* best effort - ignore */
		}
		}
		}

		@Override
		public void findAccessibilityNodeInfosByViewId(long accessibilityNodeId,
		String viewId, Region interactiveRegion, int interactionId,