Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 32afd293 authored by Daniel Norman's avatar Daniel Norman
Browse files

Introduce A11yService#takeScreenshotOfWindow() 

This enables accessibility services to take a screenshot of
a window without any overlay windows drawn on top, which is
beneficial for ML-based understanding of the visual content
in the window.

Bug: 249830826
Test: atest AccessibilityTakeScreenshotTest
Test: use in an a11y service, observe screenshots
      contain only content from the requested windows
      and no overlays
Change-Id: Ic886aeace9cb2984a472273858281b4f2c3a0e46
parent b091fb85

core/api/current.txt

+3 −0
Original line number Diff line number Diff line
@@ -3112,10 +3112,13 @@ package android.accessibilityservice { 
    method public final void setServiceInfo(android.accessibilityservice.AccessibilityServiceInfo);

    method public void setTouchExplorationPassthroughRegion(int, @NonNull android.graphics.Region);

    method public void takeScreenshot(int, @NonNull java.util.concurrent.Executor, @NonNull android.accessibilityservice.AccessibilityService.TakeScreenshotCallback);

    method public void takeScreenshotOfWindow(int, @NonNull java.util.concurrent.Executor, @NonNull android.accessibilityservice.AccessibilityService.TakeScreenshotCallback);

    field public static final int ERROR_TAKE_SCREENSHOT_INTERNAL_ERROR = 1; // 0x1

    field public static final int ERROR_TAKE_SCREENSHOT_INTERVAL_TIME_SHORT = 3; // 0x3

    field public static final int ERROR_TAKE_SCREENSHOT_INVALID_DISPLAY = 4; // 0x4

    field public static final int ERROR_TAKE_SCREENSHOT_INVALID_WINDOW = 5; // 0x5

    field public static final int ERROR_TAKE_SCREENSHOT_NO_ACCESSIBILITY_ACCESS = 2; // 0x2

    field public static final int ERROR_TAKE_SCREENSHOT_SECURE_WINDOW = 6; // 0x6

    field public static final int GESTURE_2_FINGER_DOUBLE_TAP = 20; // 0x14

    field public static final int GESTURE_2_FINGER_DOUBLE_TAP_AND_HOLD = 40; // 0x28

    field public static final int GESTURE_2_FINGER_SINGLE_TAP = 19; // 0x13

core/java/android/accessibilityservice/AccessibilityService.java

+50 −3
Original line number Diff line number Diff line
@@ -696,7 +696,8 @@ public abstract class AccessibilityService extends Service { 
            ERROR_TAKE_SCREENSHOT_INTERNAL_ERROR,

            ERROR_TAKE_SCREENSHOT_NO_ACCESSIBILITY_ACCESS,

            ERROR_TAKE_SCREENSHOT_INTERVAL_TIME_SHORT,

            ERROR_TAKE_SCREENSHOT_INVALID_DISPLAY

            ERROR_TAKE_SCREENSHOT_INVALID_DISPLAY,

            ERROR_TAKE_SCREENSHOT_INVALID_WINDOW

    })

    public @interface ScreenshotErrorCode {}
@@ -727,6 +728,18 @@ public abstract class AccessibilityService extends Service { 
     */

    public static final int ERROR_TAKE_SCREENSHOT_INVALID_DISPLAY = 4;



    /**

     * The status of taking screenshot is failure and the reason is invalid accessibility window Id.

     */

    public static final int ERROR_TAKE_SCREENSHOT_INVALID_WINDOW = 5;



    /**

     * The status of taking screenshot is failure and the reason is the window contains secure

     * content.

     * @see WindowManager.LayoutParams#FLAG_SECURE

     */

    public static final int ERROR_TAKE_SCREENSHOT_SECURE_WINDOW = 6;



    /**

     * The interval time of calling

     * {@link AccessibilityService#takeScreenshot(int, Executor, Consumer)} API.
@@ -2568,6 +2581,7 @@ public abstract class AccessibilityService extends Service { 
     * @param executor Executor on which to run the callback.

     * @param callback The callback invoked when taking screenshot has succeeded or failed.

     *                 See {@link TakeScreenshotCallback} for details.

     * @see #takeScreenshotOfWindow

     */

    public void takeScreenshot(int displayId, @NonNull @CallbackExecutor Executor executor,

            @NonNull TakeScreenshotCallback callback) {
@@ -2589,7 +2603,8 @@ public abstract class AccessibilityService extends Service { 
                final HardwareBuffer hardwareBuffer =

                        result.getParcelable(KEY_ACCESSIBILITY_SCREENSHOT_HARDWAREBUFFER, android.hardware.HardwareBuffer.class);

                final ParcelableColorSpace colorSpace =

                        result.getParcelable(KEY_ACCESSIBILITY_SCREENSHOT_COLORSPACE, android.graphics.ParcelableColorSpace.class);

                        result.getParcelable(KEY_ACCESSIBILITY_SCREENSHOT_COLORSPACE,

                                android.graphics.ParcelableColorSpace.class);

                final ScreenshotResult screenshot = new ScreenshotResult(hardwareBuffer,

                        colorSpace.getColorSpace(),

                        result.getLong(KEY_ACCESSIBILITY_SCREENSHOT_TIMESTAMP));
@@ -2600,6 +2615,37 @@ public abstract class AccessibilityService extends Service { 
        }

    }



    /**

     * Takes a screenshot of the specified window and returns it via an

     * {@link AccessibilityService.ScreenshotResult}. You can use {@link Bitmap#wrapHardwareBuffer}

     * to construct the bitmap from the ScreenshotResult's payload.

     * <p>

     * <strong>Note:</strong> In order to take screenshots your service has

     * to declare the capability to take screenshot by setting the

     * {@link android.R.styleable#AccessibilityService_canTakeScreenshot}

     * property in its meta-data. For details refer to {@link #SERVICE_META_DATA}.

     * </p>

     * <p>

     * Both this method and {@link #takeScreenshot} can be used for machine learning-based visual

     * screen understanding. Use <code>takeScreenshotOfWindow</code> if your target window might be

     * visually underneath an accessibility overlay (from your or another accessibility service) in

     * order to capture the window contents without the screenshot being covered by the overlay

     * contents drawn on the screen.

     * </p>

     *

     * @param accessibilityWindowId The window id, from {@link AccessibilityWindowInfo#getId()}.

     * @param executor Executor on which to run the callback.

     * @param callback The callback invoked when taking screenshot has succeeded or failed.

     *                 See {@link TakeScreenshotCallback} for details.

     * @see #takeScreenshot

     */

    public void takeScreenshotOfWindow(int accessibilityWindowId,

            @NonNull @CallbackExecutor Executor executor,

            @NonNull TakeScreenshotCallback callback) {

        AccessibilityInteractionClient.getInstance(this).takeScreenshotOfWindow(

                        mConnectionId, accessibilityWindowId, executor, callback);

    }



    /**

     * Sets the strokeWidth and color of the accessibility focus rectangle.

     * <p>
@@ -3113,7 +3159,8 @@ public abstract class AccessibilityService extends Service { 
        private final @NonNull ColorSpace mColorSpace;

        private final long mTimestamp;



        private ScreenshotResult(@NonNull HardwareBuffer hardwareBuffer,

        /** @hide */

        public ScreenshotResult(@NonNull HardwareBuffer hardwareBuffer,

                @NonNull ColorSpace colorSpace, long timestamp) {

            Preconditions.checkNotNull(hardwareBuffer, "hardwareBuffer cannot be null");

            Preconditions.checkNotNull(colorSpace, "colorSpace cannot be null");

core/java/android/accessibilityservice/IAccessibilityServiceConnection.aidl

+5 −0
Original line number Diff line number Diff line
@@ -30,6 +30,7 @@ import android.view.accessibility.AccessibilityNodeInfo.AccessibilityAction; 
import android.view.accessibility.IAccessibilityInteractionConnectionCallback;

import android.view.accessibility.AccessibilityWindowInfo;

import java.util.List;

import android.window.ScreenCapture;



/**

 * Interface given to an AccessibilitySerivce to talk to the AccessibilityManagerService.
@@ -122,6 +123,10 @@ interface IAccessibilityServiceConnection { 


    void takeScreenshot(int displayId, in RemoteCallback callback);



    void takeScreenshotOfWindow(int accessibilityWindowId, int interactionId,

        in ScreenCapture.ScreenCaptureListener listener,

        IAccessibilityInteractionConnectionCallback callback);



    void setGestureDetectionPassthroughRegion(int displayId, in Region region);



    void setTouchExplorationPassthroughRegion(int displayId, in Region region);

core/java/android/view/AccessibilityInteractionController.java

+40 −0
Original line number Diff line number Diff line
@@ -21,6 +21,7 @@ import static android.view.accessibility.AccessibilityNodeInfo.ACTION_ARGUMENT_A 
import static android.view.accessibility.AccessibilityNodeInfo.EXTRA_DATA_REQUESTED_KEY;

import static android.view.accessibility.AccessibilityNodeInfo.EXTRA_DATA_TEXT_CHARACTER_LOCATION_KEY;



import android.accessibilityservice.AccessibilityService;

import android.annotation.NonNull;

import android.graphics.Matrix;

import android.graphics.Rect;
@@ -46,11 +47,13 @@ import android.view.accessibility.AccessibilityNodeInfo.AccessibilityAction; 
import android.view.accessibility.AccessibilityNodeProvider;

import android.view.accessibility.AccessibilityRequestPreparer;

import android.view.accessibility.IAccessibilityInteractionConnectionCallback;

import android.window.ScreenCapture;



import com.android.internal.R;

import com.android.internal.annotations.GuardedBy;

import com.android.internal.annotations.VisibleForTesting;

import com.android.internal.os.SomeArgs;

import com.android.internal.util.function.pooled.PooledLambda;



import java.util.ArrayDeque;

import java.util.ArrayList;
@@ -588,6 +591,43 @@ public final class AccessibilityInteractionController { 
        }

    }



    /**

     * Take a screenshot using {@link ScreenCapture} of this {@link ViewRootImpl}'s {@link

     * SurfaceControl}.

     */

    public void takeScreenshotOfWindowClientThread(int interactionId,

            ScreenCapture.ScreenCaptureListener listener,

            IAccessibilityInteractionConnectionCallback callback) {

        Message message = PooledLambda.obtainMessage(

                AccessibilityInteractionController::takeScreenshotOfWindowUiThread,

                this, interactionId, listener, callback);



        // Screenshot results are returned to the service asynchronously, so the same-thread

        // message wait logic from #scheduleMessage() is not needed.

        mHandler.sendMessage(message);

    }



    private void takeScreenshotOfWindowUiThread(int interactionId,

            ScreenCapture.ScreenCaptureListener listener,

            IAccessibilityInteractionConnectionCallback callback) {

        try {

            if ((mViewRootImpl.getWindowFlags() & WindowManager.LayoutParams.FLAG_SECURE) != 0) {

                callback.sendTakeScreenshotOfWindowError(

                        AccessibilityService.ERROR_TAKE_SCREENSHOT_SECURE_WINDOW, interactionId);

                return;

            }

            final ScreenCapture.LayerCaptureArgs captureArgs =

                    new ScreenCapture.LayerCaptureArgs.Builder(mViewRootImpl.getSurfaceControl())

                            .setChildrenOnly(false).setUid(Process.myUid()).build();

            if (ScreenCapture.captureLayers(captureArgs, listener) != 0) {

                callback.sendTakeScreenshotOfWindowError(

                        AccessibilityService.ERROR_TAKE_SCREENSHOT_INTERNAL_ERROR, interactionId);

            }

        } catch (RemoteException re) {

            /* ignore - the other side will time out */

        }

    }



    public void findFocusClientThread(long accessibilityNodeId, int focusType,

            Region interactiveRegion, int interactionId,

            IAccessibilityInteractionConnectionCallback callback, int flags, int interrogatingPid,

core/java/android/view/ViewRootImpl.java

+21 −0
Original line number Diff line number Diff line
@@ -88,6 +88,7 @@ import static android.view.inputmethod.InputMethodEditorTraceProto.InputMethodCl 
import static android.view.inputmethod.InputMethodEditorTraceProto.InputMethodClientsTraceProto.ClientSideProto.INSETS_CONTROLLER;



import android.Manifest;

import android.accessibilityservice.AccessibilityService;

import android.animation.AnimationHandler;

import android.animation.LayoutTransition;

import android.annotation.AnyThread;
@@ -198,6 +199,7 @@ import android.window.ClientWindowFrames; 
import android.window.CompatOnBackInvokedCallback;

import android.window.OnBackInvokedCallback;

import android.window.OnBackInvokedDispatcher;

import android.window.ScreenCapture;

import android.window.SurfaceSyncGroup;

import android.window.WindowOnBackInvokedDispatcher;
@@ -10691,6 +10693,25 @@ public final class ViewRootImpl implements ViewParent, 
                        .notifyOutsideTouchClientThread();

            }

        }



        @Override

        public void takeScreenshotOfWindow(int interactionId,

                ScreenCapture.ScreenCaptureListener listener,

                IAccessibilityInteractionConnectionCallback callback) {

            ViewRootImpl viewRootImpl = mViewRootImpl.get();

            if (viewRootImpl != null && viewRootImpl.mView != null) {

                viewRootImpl.getAccessibilityInteractionController()

                        .takeScreenshotOfWindowClientThread(interactionId, listener, callback);

            } else {

                try {

                    callback.sendTakeScreenshotOfWindowError(

                            AccessibilityService.ERROR_TAKE_SCREENSHOT_INTERNAL_ERROR,

                            interactionId);

                } catch (RemoteException re) {

                    /* best effort - ignore */

                }

            }

        }

    }



    /**