Merge "Introduce A11yService#takeScreenshotOfWindow()" (9650e7a3) · Commits · e / os / android_frameworks_base

core/api/current.txt

+3 −0

Original line number	Diff line number	Diff line
		@@ -3112,10 +3112,13 @@ package android.accessibilityservice {
		method public final void setServiceInfo(android.accessibilityservice.AccessibilityServiceInfo);
		method public void setTouchExplorationPassthroughRegion(int, @NonNull android.graphics.Region);
		method public void takeScreenshot(int, @NonNull java.util.concurrent.Executor, @NonNull android.accessibilityservice.AccessibilityService.TakeScreenshotCallback);
		method public void takeScreenshotOfWindow(int, @NonNull java.util.concurrent.Executor, @NonNull android.accessibilityservice.AccessibilityService.TakeScreenshotCallback);
		field public static final int ERROR_TAKE_SCREENSHOT_INTERNAL_ERROR = 1; // 0x1
		field public static final int ERROR_TAKE_SCREENSHOT_INTERVAL_TIME_SHORT = 3; // 0x3
		field public static final int ERROR_TAKE_SCREENSHOT_INVALID_DISPLAY = 4; // 0x4
		field public static final int ERROR_TAKE_SCREENSHOT_INVALID_WINDOW = 5; // 0x5
		field public static final int ERROR_TAKE_SCREENSHOT_NO_ACCESSIBILITY_ACCESS = 2; // 0x2
		field public static final int ERROR_TAKE_SCREENSHOT_SECURE_WINDOW = 6; // 0x6
		field public static final int GESTURE_2_FINGER_DOUBLE_TAP = 20; // 0x14
		field public static final int GESTURE_2_FINGER_DOUBLE_TAP_AND_HOLD = 40; // 0x28
		field public static final int GESTURE_2_FINGER_SINGLE_TAP = 19; // 0x13

core/java/android/accessibilityservice/AccessibilityService.java

+50 −3

Original line number	Diff line number	Diff line
		@@ -696,7 +696,8 @@ public abstract class AccessibilityService extends Service {
		ERROR_TAKE_SCREENSHOT_INTERNAL_ERROR,
		ERROR_TAKE_SCREENSHOT_NO_ACCESSIBILITY_ACCESS,
		ERROR_TAKE_SCREENSHOT_INTERVAL_TIME_SHORT,
		ERROR_TAKE_SCREENSHOT_INVALID_DISPLAY
		ERROR_TAKE_SCREENSHOT_INVALID_DISPLAY,
		ERROR_TAKE_SCREENSHOT_INVALID_WINDOW
		})
		public @interface ScreenshotErrorCode {}

		@@ -727,6 +728,18 @@ public abstract class AccessibilityService extends Service {
		*/
		public static final int ERROR_TAKE_SCREENSHOT_INVALID_DISPLAY = 4;

		/**
		* The status of taking screenshot is failure and the reason is invalid accessibility window Id.
		*/
		public static final int ERROR_TAKE_SCREENSHOT_INVALID_WINDOW = 5;

		/**
		* The status of taking screenshot is failure and the reason is the window contains secure
		* content.
		* @see WindowManager.LayoutParams#FLAG_SECURE
		*/
		public static final int ERROR_TAKE_SCREENSHOT_SECURE_WINDOW = 6;

		/**
		* The interval time of calling
		* {@link AccessibilityService#takeScreenshot(int, Executor, Consumer)} API.
		@@ -2568,6 +2581,7 @@ public abstract class AccessibilityService extends Service {
		* @param executor Executor on which to run the callback.
		* @param callback The callback invoked when taking screenshot has succeeded or failed.
		* See {@link TakeScreenshotCallback} for details.
		* @see #takeScreenshotOfWindow
		*/
		public void takeScreenshot(int displayId, @NonNull @CallbackExecutor Executor executor,
		@NonNull TakeScreenshotCallback callback) {
		@@ -2589,7 +2603,8 @@ public abstract class AccessibilityService extends Service {
		final HardwareBuffer hardwareBuffer =
		result.getParcelable(KEY_ACCESSIBILITY_SCREENSHOT_HARDWAREBUFFER, android.hardware.HardwareBuffer.class);
		final ParcelableColorSpace colorSpace =
		result.getParcelable(KEY_ACCESSIBILITY_SCREENSHOT_COLORSPACE, android.graphics.ParcelableColorSpace.class);
		result.getParcelable(KEY_ACCESSIBILITY_SCREENSHOT_COLORSPACE,
		android.graphics.ParcelableColorSpace.class);
		final ScreenshotResult screenshot = new ScreenshotResult(hardwareBuffer,
		colorSpace.getColorSpace(),
		result.getLong(KEY_ACCESSIBILITY_SCREENSHOT_TIMESTAMP));
		@@ -2600,6 +2615,37 @@ public abstract class AccessibilityService extends Service {
		}
		}

		/**
		* Takes a screenshot of the specified window and returns it via an
		* {@link AccessibilityService.ScreenshotResult}. You can use {@link Bitmap#wrapHardwareBuffer}
		* to construct the bitmap from the ScreenshotResult's payload.
		* <p>
		* <strong>Note:</strong> In order to take screenshots your service has
		* to declare the capability to take screenshot by setting the
		* {@link android.R.styleable#AccessibilityService_canTakeScreenshot}
		* property in its meta-data. For details refer to {@link #SERVICE_META_DATA}.
		* </p>
		* <p>
		* Both this method and {@link #takeScreenshot} can be used for machine learning-based visual
		* screen understanding. Use <code>takeScreenshotOfWindow</code> if your target window might be
		* visually underneath an accessibility overlay (from your or another accessibility service) in
		* order to capture the window contents without the screenshot being covered by the overlay
		* contents drawn on the screen.
		* </p>
		*
		* @param accessibilityWindowId The window id, from {@link AccessibilityWindowInfo#getId()}.
		* @param executor Executor on which to run the callback.
		* @param callback The callback invoked when taking screenshot has succeeded or failed.
		* See {@link TakeScreenshotCallback} for details.
		* @see #takeScreenshot
		*/
		public void takeScreenshotOfWindow(int accessibilityWindowId,
		@NonNull @CallbackExecutor Executor executor,
		@NonNull TakeScreenshotCallback callback) {
		AccessibilityInteractionClient.getInstance(this).takeScreenshotOfWindow(
		mConnectionId, accessibilityWindowId, executor, callback);
		}

		/**
		* Sets the strokeWidth and color of the accessibility focus rectangle.
		* <p>
		@@ -3113,7 +3159,8 @@ public abstract class AccessibilityService extends Service {
		private final @NonNull ColorSpace mColorSpace;
		private final long mTimestamp;

		private ScreenshotResult(@NonNull HardwareBuffer hardwareBuffer,
		/** @hide */
		public ScreenshotResult(@NonNull HardwareBuffer hardwareBuffer,
		@NonNull ColorSpace colorSpace, long timestamp) {
		Preconditions.checkNotNull(hardwareBuffer, "hardwareBuffer cannot be null");
		Preconditions.checkNotNull(colorSpace, "colorSpace cannot be null");

core/java/android/accessibilityservice/IAccessibilityServiceConnection.aidl

+5 −0

Original line number	Diff line number	Diff line
		@@ -30,6 +30,7 @@ import android.view.accessibility.AccessibilityNodeInfo.AccessibilityAction;
		import android.view.accessibility.IAccessibilityInteractionConnectionCallback;
		import android.view.accessibility.AccessibilityWindowInfo;
		import java.util.List;
		import android.window.ScreenCapture;

		/**
		* Interface given to an AccessibilitySerivce to talk to the AccessibilityManagerService.
		@@ -122,6 +123,10 @@ interface IAccessibilityServiceConnection {

		void takeScreenshot(int displayId, in RemoteCallback callback);

		void takeScreenshotOfWindow(int accessibilityWindowId, int interactionId,
		in ScreenCapture.ScreenCaptureListener listener,
		IAccessibilityInteractionConnectionCallback callback);

		void setGestureDetectionPassthroughRegion(int displayId, in Region region);

		void setTouchExplorationPassthroughRegion(int displayId, in Region region);

core/java/android/view/AccessibilityInteractionController.java

+40 −0

Original line number	Diff line number	Diff line
		@@ -21,6 +21,7 @@ import static android.view.accessibility.AccessibilityNodeInfo.ACTION_ARGUMENT_A
		import static android.view.accessibility.AccessibilityNodeInfo.EXTRA_DATA_REQUESTED_KEY;
		import static android.view.accessibility.AccessibilityNodeInfo.EXTRA_DATA_TEXT_CHARACTER_LOCATION_KEY;

		import android.accessibilityservice.AccessibilityService;
		import android.annotation.NonNull;
		import android.graphics.Matrix;
		import android.graphics.Rect;
		@@ -46,11 +47,13 @@ import android.view.accessibility.AccessibilityNodeInfo.AccessibilityAction;
		import android.view.accessibility.AccessibilityNodeProvider;
		import android.view.accessibility.AccessibilityRequestPreparer;
		import android.view.accessibility.IAccessibilityInteractionConnectionCallback;
		import android.window.ScreenCapture;

		import com.android.internal.R;
		import com.android.internal.annotations.GuardedBy;
		import com.android.internal.annotations.VisibleForTesting;
		import com.android.internal.os.SomeArgs;
		import com.android.internal.util.function.pooled.PooledLambda;

		import java.util.ArrayDeque;
		import java.util.ArrayList;
		@@ -588,6 +591,43 @@ public final class AccessibilityInteractionController {
		}
		}

		/**
		* Take a screenshot using {@link ScreenCapture} of this {@link ViewRootImpl}'s {@link
		* SurfaceControl}.
		*/
		public void takeScreenshotOfWindowClientThread(int interactionId,
		ScreenCapture.ScreenCaptureListener listener,
		IAccessibilityInteractionConnectionCallback callback) {
		Message message = PooledLambda.obtainMessage(
		AccessibilityInteractionController::takeScreenshotOfWindowUiThread,
		this, interactionId, listener, callback);

		// Screenshot results are returned to the service asynchronously, so the same-thread
		// message wait logic from #scheduleMessage() is not needed.
		mHandler.sendMessage(message);
		}

		private void takeScreenshotOfWindowUiThread(int interactionId,
		ScreenCapture.ScreenCaptureListener listener,
		IAccessibilityInteractionConnectionCallback callback) {
		try {
		if ((mViewRootImpl.getWindowFlags() & WindowManager.LayoutParams.FLAG_SECURE) != 0) {
		callback.sendTakeScreenshotOfWindowError(
		AccessibilityService.ERROR_TAKE_SCREENSHOT_SECURE_WINDOW, interactionId);
		return;
		}
		final ScreenCapture.LayerCaptureArgs captureArgs =
		new ScreenCapture.LayerCaptureArgs.Builder(mViewRootImpl.getSurfaceControl())
		.setChildrenOnly(false).setUid(Process.myUid()).build();
		if (ScreenCapture.captureLayers(captureArgs, listener) != 0) {
		callback.sendTakeScreenshotOfWindowError(
		AccessibilityService.ERROR_TAKE_SCREENSHOT_INTERNAL_ERROR, interactionId);
		}
		} catch (RemoteException re) {
		/* ignore - the other side will time out */
		}
		}

		public void findFocusClientThread(long accessibilityNodeId, int focusType,
		Region interactiveRegion, int interactionId,
		IAccessibilityInteractionConnectionCallback callback, int flags, int interrogatingPid,

core/java/android/view/ViewRootImpl.java

+21 −0

Original line number	Diff line number	Diff line
		@@ -88,6 +88,7 @@ import static android.view.inputmethod.InputMethodEditorTraceProto.InputMethodCl
		import static android.view.inputmethod.InputMethodEditorTraceProto.InputMethodClientsTraceProto.ClientSideProto.INSETS_CONTROLLER;

		import android.Manifest;
		import android.accessibilityservice.AccessibilityService;
		import android.animation.AnimationHandler;
		import android.animation.LayoutTransition;
		import android.annotation.AnyThread;
		@@ -199,6 +200,7 @@ import android.window.ClientWindowFrames;
		import android.window.CompatOnBackInvokedCallback;
		import android.window.OnBackInvokedCallback;
		import android.window.OnBackInvokedDispatcher;
		import android.window.ScreenCapture;
		import android.window.SurfaceSyncGroup;
		import android.window.WindowOnBackInvokedDispatcher;

		@@ -10707,6 +10709,25 @@ public final class ViewRootImpl implements ViewParent,
		.notifyOutsideTouchClientThread();
		}
		}

		@Override
		public void takeScreenshotOfWindow(int interactionId,
		ScreenCapture.ScreenCaptureListener listener,
		IAccessibilityInteractionConnectionCallback callback) {
		ViewRootImpl viewRootImpl = mViewRootImpl.get();
		if (viewRootImpl != null && viewRootImpl.mView != null) {
		viewRootImpl.getAccessibilityInteractionController()
		.takeScreenshotOfWindowClientThread(interactionId, listener, callback);
		} else {
		try {
		callback.sendTakeScreenshotOfWindowError(
		AccessibilityService.ERROR_TAKE_SCREENSHOT_INTERNAL_ERROR,
		interactionId);
		} catch (RemoteException re) {
		/* best effort - ignore */
		}
		}
		}
		}

		/**