Merge "Snapshot Task with the app window crop for Recents" (418346a8) · Commits · e / os / android_frameworks_base

core/java/android/view/SurfaceControl.java

+20 −8

Original line number	Diff line number	Diff line
		@@ -56,7 +56,9 @@ public class SurfaceControl {
		Rect sourceCrop, int width, int height, int minLayer, int maxLayer,
		boolean allLayers, boolean useIdentityTransform);
		private static native void nativeCaptureLayers(IBinder layerHandleToken, Surface consumer,
		int rotation);
		Rect sourceCrop, float frameScale);
		private static native GraphicBuffer nativeCaptureLayers(IBinder layerHandleToken,
		Rect sourceCrop, float frameScale);

		private static native long nativeCreateTransaction();
		private static native long nativeGetNativeTransactionFinalizer();
		@@ -1178,13 +1180,23 @@ public class SurfaceControl {
		*
		* @param layerHandleToken The root layer to capture.
		* @param consumer The {@link Surface} to capture the layer into.
		* @param rotation Apply a custom clockwise rotation to the screenshot, i.e.
		* Surface.ROTATION_0,90,180,270. Surfaceflinger will always capture in its
		* native portrait orientation by default, so this is useful for returning
		* captures that are independent of device orientation.
		* @param sourceCrop The portion of the root surface to capture; caller may pass in 'new
		* Rect()' or null if no cropping is desired.
		* @param frameScale The desired scale of the returned buffer; the raw
		* screen will be scaled up/down.
		*/
		public static void captureLayers(IBinder layerHandleToken, Surface consumer, Rect sourceCrop,
		float frameScale) {
		nativeCaptureLayers(layerHandleToken, consumer, sourceCrop, frameScale);
		}

		/**
		* Same as {@link #captureLayers(IBinder, Surface, Rect, float)} except this
		* captures to a {@link GraphicBuffer} instead of a {@link Surface}.
		*/
		public static void captureLayers(IBinder layerHandleToken, Surface consumer, int rotation) {
		nativeCaptureLayers(layerHandleToken, consumer, rotation);
		public static GraphicBuffer captureLayersToBuffer(IBinder layerHandleToken, Rect sourceCrop,
		float frameScale) {
		return nativeCaptureLayers(layerHandleToken, sourceCrop, frameScale);
		}

		public static class Transaction implements Closeable {

core/jni/android_view_SurfaceControl.cpp

+40 −3

Original line number	Diff line number	Diff line
		@@ -291,7 +291,7 @@ static void nativeScreenshot(JNIEnv* env, jclass clazz, jobject displayTokenObj,
		}

		static void nativeCaptureLayers(JNIEnv* env, jclass clazz, jobject layerHandleToken,
		jobject surfaceObj, int rotation) {
		jobject surfaceObj, jobject sourceCropObj, jfloat frameScale) {

		sp<IBinder> layerHandle = ibinderForJavaObject(env, layerHandleToken);
		if (layerHandle == NULL) {
		@@ -303,7 +303,42 @@ static void nativeCaptureLayers(JNIEnv* env, jclass clazz, jobject layerHandleTo
		return;
		}

		ScreenshotClient::captureLayers(layerHandle, consumer->getIGraphicBufferProducer(), rotation);
		Rect sourceCrop;
		if (sourceCropObj != NULL) {
		sourceCrop = rectFromObj(env, sourceCropObj);
		}

		ScreenshotClient::captureLayers(layerHandle, consumer->getIGraphicBufferProducer(), sourceCrop,
		frameScale);
		}

		static jobject nativeCaptureLayersToBuffer(JNIEnv* env, jclass clazz, jobject layerHandleToken,
		jobject sourceCropObj, jfloat frameScale) {

		sp<IBinder> layerHandle = ibinderForJavaObject(env, layerHandleToken);
		if (layerHandle == NULL) {
		return NULL;
		}

		Rect sourceCrop;
		if (sourceCropObj != NULL) {
		sourceCrop = rectFromObj(env, sourceCropObj);
		}

		sp<GraphicBuffer> buffer;
		status_t res = ScreenshotClient::captureLayersToBuffer(layerHandle, sourceCrop, frameScale,
		&buffer);
		if (res != NO_ERROR) {
		return NULL;
		}

		return env->CallStaticObjectMethod(gGraphicBufferClassInfo.clazz,
		gGraphicBufferClassInfo.builder,
		buffer->getWidth(),
		buffer->getHeight(),
		buffer->getPixelFormat(),
		(jint)buffer->getUsage(),
		(jlong)buffer.get());
		}

		static void nativeApplyTransaction(JNIEnv* env, jclass clazz, jlong transactionObj, jboolean sync) {
		@@ -975,8 +1010,10 @@ static const JNINativeMethod sSurfaceControlMethods[] = {
		{"nativeScreenshotToBuffer",
		"(Landroid/os/IBinder;Landroid/graphics/Rect;IIIIZZI)Landroid/graphics/GraphicBuffer;",
		(void*)nativeScreenshotToBuffer },
		{"nativeCaptureLayers", "(Landroid/os/IBinder;Landroid/view/Surface;I)V",
		{"nativeCaptureLayers", "(Landroid/os/IBinder;Landroid/view/Surface;Landroid/graphics/Rect;F)V",
		(void*)nativeCaptureLayers },
		{"nativeCaptureLayers", "(Landroid/os/IBinder;Landroid/graphics/Rect;F)Landroid/graphics/GraphicBuffer;",
		(void*)nativeCaptureLayersToBuffer },
		};

		int register_android_view_SurfaceControl(JNIEnv* env)

services/core/java/com/android/server/wm/DisplayContent.java

+41 −238

Original line number	Diff line number	Diff line
		@@ -2969,227 +2969,30 @@ class DisplayContent extends WindowContainer<DisplayContent.DisplayChildWindowCo
		* Takes a snapshot of the display. In landscape mode this grabs the whole screen.
		* In portrait mode, it grabs the full screenshot.
		*
		* @param width the width of the target bitmap
		* @param height the height of the target bitmap
		* @param includeFullDisplay true if the screen should not be cropped before capture
		* @param frameScale the scale to apply to the frame, only used when width = -1 and height = -1
		* @param config of the output bitmap
		* @param wallpaperOnly true if only the wallpaper layer should be included in the screenshot
		* @param includeDecor whether to include window decors, like the status or navigation bar
		* background of the window
		*/
		Bitmap screenshotApplications(IBinder appToken, int width, int height,
		boolean includeFullDisplay, float frameScale, Bitmap.Config config,
		boolean wallpaperOnly, boolean includeDecor) {
		Bitmap bitmap = screenshotApplications(appToken, width, height, includeFullDisplay,
		frameScale, wallpaperOnly, includeDecor, SurfaceControl::screenshot);
		if (bitmap == null) {
		return null;
		}

		if (DEBUG_SCREENSHOT) {
		// TEST IF IT's ALL BLACK
		int[] buffer = new int[bitmap.getWidth() * bitmap.getHeight()];
		bitmap.getPixels(buffer, 0, bitmap.getWidth(), 0, 0, bitmap.getWidth(),
		bitmap.getHeight());
		boolean allBlack = true;
		final int firstColor = buffer[0];
		for (int i = 0; i < buffer.length; i++) {
		if (buffer[i] != firstColor) {
		allBlack = false;
		break;
		}
		}
		if (allBlack) {
		final WindowState appWin = mScreenshotApplicationState.appWin;
		final int maxLayer = mScreenshotApplicationState.maxLayer;
		final int minLayer = mScreenshotApplicationState.minLayer;
		Slog.i(TAG_WM, "Screenshot " + appWin + " was monochrome(" +
		Integer.toHexString(firstColor) + ")! mSurfaceLayer=" +
		(appWin != null ?
		appWin.mWinAnimator.mSurfaceController.getLayer() : "null") +
		" minLayer=" + minLayer + " maxLayer=" + maxLayer);
		}
		}

		// Create a copy of the screenshot that is immutable and backed in ashmem.
		// This greatly reduces the overhead of passing the bitmap between processes.
		Bitmap ret = bitmap.createAshmemBitmap(config);
		bitmap.recycle();
		return ret;
		}

		GraphicBuffer screenshotApplicationsToBuffer(IBinder appToken, int width, int height,
		boolean includeFullDisplay, float frameScale, boolean wallpaperOnly,
		boolean includeDecor) {
		return screenshotApplications(appToken, width, height, includeFullDisplay, frameScale,
		wallpaperOnly, includeDecor, SurfaceControl::screenshotToBuffer);
		}

		private <E> E screenshotApplications(IBinder appToken, int width, int height,
		boolean includeFullDisplay, float frameScale, boolean wallpaperOnly,
		boolean includeDecor, Screenshoter<E> screenshoter) {
		int dw = mDisplayInfo.logicalWidth;
		int dh = mDisplayInfo.logicalHeight;
		if (dw == 0 \|\| dh == 0) {
		if (DEBUG_SCREENSHOT) Slog.i(TAG_WM, "Screenshot of " + appToken
		+ ": returning null. logical widthxheight=" + dw + "x" + dh);
		return null;
		}

		E bitmap;

		mScreenshotApplicationState.reset(appToken == null && !wallpaperOnly);
		final Rect frame = new Rect();
		final Rect stackBounds = new Rect();

		final int aboveAppLayer = (mService.mPolicy.getWindowLayerFromTypeLw(TYPE_APPLICATION) + 1)
		* TYPE_LAYER_MULTIPLIER + TYPE_LAYER_OFFSET;
		final MutableBoolean mutableIncludeFullDisplay = new MutableBoolean(includeFullDisplay);
		Bitmap screenshotDisplay(Bitmap.Config config, boolean wallpaperOnly) {
		synchronized (mService.mWindowMap) {
		if (!mService.mPolicy.isScreenOn()) {
		if (DEBUG_SCREENSHOT) Slog.i(TAG_WM, "Attempted to take screenshot while display"
		+ " was off.");
		return null;
		}
		// Figure out the part of the screen that is actually the app.
		mScreenshotApplicationState.appWin = null;
		forAllWindows(w -> {
		if (!w.mHasSurface) {
		return false;
		}
		if (w.mLayer >= aboveAppLayer) {
		return false;
		}
		if (wallpaperOnly && !w.mIsWallpaper) {
		return false;
		}
		if (w.mIsImWindow) {
		return false;
		} else if (w.mIsWallpaper) {
		// If this is the wallpaper layer and we're only looking for the wallpaper layer
		// then the target window state is this one.
		if (wallpaperOnly) {
		mScreenshotApplicationState.appWin = w;
		}

		if (mScreenshotApplicationState.appWin == null) {
		// We have not ran across the target window yet, so it is probably behind
		// the wallpaper. This can happen when the keyguard is up and all windows
		// are moved behind the wallpaper. We don't want to include the wallpaper
		// layer in the screenshot as it will cover-up the layer of the target
		// window.
		return false;
		}
		// Fall through. The target window is in front of the wallpaper. For this
		// case we want to include the wallpaper layer in the screenshot because
		// the target window might have some transparent areas.
		} else if (appToken != null) {
		if (w.mAppToken == null \|\| w.mAppToken.token != appToken) {
		// This app window is of no interest if it is not associated with the
		// screenshot app.
		return false;
		}
		mScreenshotApplicationState.appWin = w;
		}

		// Include this window.

		final WindowStateAnimator winAnim = w.mWinAnimator;

		// Don't include wallpaper in bounds calculation
		if (!w.mIsWallpaper && !mutableIncludeFullDisplay.value) {
		if (includeDecor) {
		final Task task = w.getTask();
		if (task != null) {
		task.getBounds(frame);
		} else {

		// No task bounds? Too bad! Ain't no screenshot then.
		return true;
		}
		} else {
		final Rect wf = w.mFrame;
		final Rect cr = w.mContentInsets;
		int left = wf.left + cr.left;
		int top = wf.top + cr.top;
		int right = wf.right - cr.right;
		int bottom = wf.bottom - cr.bottom;
		frame.union(left, top, right, bottom);
		w.getVisibleBounds(stackBounds);
		if (!Rect.intersects(frame, stackBounds)) {
		// Set frame empty if there's no intersection.
		frame.setEmpty();
		}
		}
		}

		final boolean foundTargetWs =
		(w.mAppToken != null && w.mAppToken.token == appToken)
		\|\| (mScreenshotApplicationState.appWin != null && wallpaperOnly);
		if (foundTargetWs && winAnim.getShown() && winAnim.mLastAlpha > 0f) {
		mScreenshotApplicationState.screenshotReady = true;
		}

		if (w.isObscuringDisplay()){
		return true;
		if (DEBUG_SCREENSHOT) {
		Slog.i(TAG_WM, "Attempted to take screenshot while display was off.");
		}
		return false;
		}, true /* traverseTopToBottom */);

		final WindowState appWin = mScreenshotApplicationState.appWin;
		final boolean screenshotReady = mScreenshotApplicationState.screenshotReady;

		if (appToken != null && appWin == null) {
		// Can't find a window to snapshot.
		if (DEBUG_SCREENSHOT) Slog.i(TAG_WM,
		"Screenshot: Couldn't find a surface matching " + appToken);
		return null;
		}

		if (!screenshotReady) {
		Slog.i(TAG_WM, "Failed to capture screenshot of " + appToken +
		" appWin=" + (appWin == null ? "null" : (appWin + " drawState=" +
		appWin.mWinAnimator.mDrawState)));
		if (wallpaperOnly && !shouldScreenshotWallpaper()) {
		return null;
		}

		// Screenshot is ready to be taken. Everything from here below will continue
		// through the bottom of the loop and return a value. We only stay in the loop
		// because we don't want to release the mWindowMap lock until the screenshot is
		// taken.

		int dw = mDisplayInfo.logicalWidth;
		int dh = mDisplayInfo.logicalHeight;

		if (!mutableIncludeFullDisplay.value) {
		// Constrain frame to the screen size.
		if (!frame.intersect(0, 0, dw, dh)) {
		frame.setEmpty();
		}
		} else {
		// Caller just wants entire display.
		frame.set(0, 0, dw, dh);
		}
		if (frame.isEmpty()) {
		if (dw <= 0 \|\| dh <= 0) {
		return null;
		}

		if (width < 0) {
		width = (int) (frame.width() * frameScale);
		}
		if (height < 0) {
		height = (int) (frame.height() * frameScale);
		}

		// Tell surface flinger what part of the image to crop. Take the top
		// right part of the application, and crop the larger dimension to fit.
		Rect crop = new Rect(frame);
		if (width / (float) frame.width() < height / (float) frame.height()) {
		int cropWidth = (int)((float)width / (float)height * frame.height());
		crop.right = crop.left + cropWidth;
		} else {
		int cropHeight = (int)((float)height / (float)width * frame.width());
		crop.bottom = crop.top + cropHeight;
		}
		final Rect frame = new Rect(0, 0, dw, dh);

		// The screenshot API does not apply the current screen rotation.
		int rot = mDisplay.getRotation();
		@@ -3198,43 +3001,52 @@ class DisplayContent extends WindowContainer<DisplayContent.DisplayChildWindowCo
		rot = (rot == ROTATION_90) ? ROTATION_270 : ROTATION_90;
		}

		// Surfaceflinger is not aware of orientation, so convert our logical
		// crop to surfaceflinger's portrait orientation.
		convertCropForSurfaceFlinger(crop, rot, dw, dh);

		if (DEBUG_SCREENSHOT) {
		forAllWindows(w -> {
		final WindowSurfaceController controller = w.mWinAnimator.mSurfaceController;
		Slog.i(TAG_WM, w + ": " + w.mLayer
		+ " animLayer=" + w.mWinAnimator.mAnimLayer
		+ " surfaceLayer=" + ((controller == null)
		? "null" : controller.getLayer()));
		}, false /* traverseTopToBottom */);
		}
		// SurfaceFlinger is not aware of orientation, so convert our logical
		// crop to SurfaceFlinger's portrait orientation.
		convertCropForSurfaceFlinger(frame, rot, dw, dh);

		final ScreenRotationAnimation screenRotationAnimation =
		mService.mAnimator.getScreenRotationAnimationLocked(DEFAULT_DISPLAY);
		final boolean inRotation = screenRotationAnimation != null &&
		screenRotationAnimation.isAnimating();
		if (DEBUG_SCREENSHOT && inRotation) Slog.v(TAG_WM,
		"Taking screenshot while rotating");

		// We force pending transactions to flush before taking
		// the screenshot by pushing an empty synchronous transaction.
		SurfaceControl.openTransaction();
		SurfaceControl.closeTransactionSync();
		if (DEBUG_SCREENSHOT && inRotation) Slog.v(TAG_WM, "Taking screenshot while rotating");

		// TODO(b/68392460): We should screenshot Task controls directly
		// but it's difficult at the moment as the Task doesn't have the
		// correct size set.
		bitmap = screenshoter.screenshot(crop, width, height, 0, 1,
		inRotation, rot);
		final Bitmap bitmap = SurfaceControl.screenshot(frame, dw, dh, 0, 1, inRotation, rot);
		if (bitmap == null) {
		Slog.w(TAG_WM, "Failed to take screenshot");
		return null;
		}

		// Create a copy of the screenshot that is immutable and backed in ashmem.
		// This greatly reduces the overhead of passing the bitmap between processes.
		final Bitmap ret = bitmap.createAshmemBitmap(config);
		bitmap.recycle();
		return ret;
		}
		return bitmap;
		}

		private boolean shouldScreenshotWallpaper() {
		MutableBoolean screenshotReady = new MutableBoolean(false);

		forAllWindows(w -> {
		if (!w.mIsWallpaper) {
		return false;
		}

		// Found the wallpaper window
		final WindowStateAnimator winAnim = w.mWinAnimator;

		if (winAnim.getShown() && winAnim.mLastAlpha > 0f) {
		screenshotReady.value = true;
		}

		return true;
		}, true /* traverseTopToBottom */);

		return screenshotReady.value;
		}

		// TODO: Can this use createRotationMatrix()?
		@@ -3848,15 +3660,6 @@ class DisplayContent extends WindowContainer<DisplayContent.DisplayChildWindowCo
		}
		}

		/**
		* Interface to screenshot into various types, i.e. {@link Bitmap} and {@link GraphicBuffer}.
		*/
		@FunctionalInterface
		private interface Screenshoter<E> {
		E screenshot(Rect sourceCrop, int width, int height, int minLayer, int maxLayer,
		boolean useIdentityTransform, int rotation);
		}

		SurfaceControl.Builder makeSurface(SurfaceSession s) {
		return mService.makeSurfaceBuilder(s)
		.setParent(mWindowingLayer);

services/core/java/com/android/server/wm/TaskSnapshotController.java

+21 −3

Original line number	Diff line number	Diff line
		@@ -18,12 +18,12 @@ package com.android.server.wm;

		import static com.android.server.wm.TaskSnapshotPersister.DISABLE_FULL_SIZED_BITMAPS;
		import static com.android.server.wm.TaskSnapshotPersister.REDUCED_SCALE;
		import static com.android.server.wm.WindowManagerDebugConfig.DEBUG_SCREENSHOT;
		import static com.android.server.wm.WindowManagerDebugConfig.TAG_WITH_CLASS_NAME;
		import static com.android.server.wm.WindowManagerDebugConfig.TAG_WM;

		import android.annotation.Nullable;
		import android.app.ActivityManager;
		import android.app.ActivityManager.StackId;
		import android.app.ActivityManager.TaskSnapshot;
		import android.content.pm.PackageManager;
		import android.graphics.Bitmap;
		@@ -35,6 +35,7 @@ import android.util.ArraySet;
		import android.util.Slog;
		import android.view.DisplayListCanvas;
		import android.view.RenderNode;
		import android.view.SurfaceControl;
		import android.view.ThreadedRenderer;
		import android.view.WindowManager.LayoutParams;

		@@ -210,11 +211,28 @@ class TaskSnapshotController {
		if (mainWindow == null) {
		return null;
		}
		if (!mService.mPolicy.isScreenOn()) {
		if (DEBUG_SCREENSHOT) {
		Slog.i(TAG_WM, "Attempted to take screenshot while display was off.");
		}
		return null;
		}
		if (task.getSurfaceControl() == null) {
		return null;
		}

		final boolean isLowRamDevice = ActivityManager.isLowRamDeviceStatic();
		final float scaleFraction = isLowRamDevice ? REDUCED_SCALE : 1f;
		final GraphicBuffer buffer = top.mDisplayContent.screenshotApplicationsToBuffer(top.token,
		-1, -1, false, scaleFraction, false, true);
		final Rect taskFrame = new Rect();
		task.getBounds(taskFrame);

		final GraphicBuffer buffer = SurfaceControl.captureLayersToBuffer(
		task.getSurfaceControl().getHandle(), taskFrame, scaleFraction);

		if (buffer == null \|\| buffer.getWidth() <= 1 \|\| buffer.getHeight() <= 1) {
		if (DEBUG_SCREENSHOT) {
		Slog.w(TAG_WM, "Failed to take screenshot");
		}
		return null;
		}
		return new TaskSnapshot(buffer, top.getConfiguration().orientation,

services/core/java/com/android/server/wm/WindowManagerService.java

+9 −20

Original line number	Diff line number	Diff line
		@@ -3691,9 +3691,8 @@ public class WindowManagerService extends IWindowManager.Stub
		}
		try {
		Trace.traceBegin(TRACE_TAG_WINDOW_MANAGER, "screenshotWallpaper");
		return screenshotApplications(null /* appToken /, DEFAULT_DISPLAY, -1 / width */,
		-1 /* height /, true / includeFullDisplay /, 1f / frameScale */,
		Bitmap.Config.ARGB_8888, true /* wallpaperOnly /, false / includeDecor */);
		return screenshotApplications(DEFAULT_DISPLAY, Bitmap.Config.ARGB_8888,
		true /* wallpaperOnly */);
		} finally {
		Trace.traceEnd(TRACE_TAG_WINDOW_MANAGER);
		}
		@@ -3712,10 +3711,8 @@ public class WindowManagerService extends IWindowManager.Stub
		}

		FgThread.getHandler().post(() -> {
		Bitmap bm = screenshotApplications(null /* appToken */, DEFAULT_DISPLAY,
		-1 /* width /, -1 / height /, true / includeFullDisplay */,
		1f /* frameScale /, Bitmap.Config.ARGB_8888, false / wallpaperOnly */,
		false /* includeDecor */);
		Bitmap bm = screenshotApplications(DEFAULT_DISPLAY, Bitmap.Config.ARGB_8888,
		false /* wallpaperOnly */);
		try {
		receiver.onHandleAssistScreenshot(bm);
		} catch (RemoteException e) {
		@@ -3749,29 +3746,21 @@ public class WindowManagerService extends IWindowManager.Stub
		* In portrait mode, it grabs the full screenshot.
		*
		* @param displayId the Display to take a screenshot of.
		* @param width the width of the target bitmap
		* @param height the height of the target bitmap
		* @param includeFullDisplay true if the screen should not be cropped before capture
		* @param frameScale the scale to apply to the frame, only used when width = -1 and height = -1
		* @param config of the output bitmap
		* @param wallpaperOnly true if only the wallpaper layer should be included in the screenshot
		* @param includeDecor whether to include window decors, like the status or navigation bar
		* background of the window
		*/
		private Bitmap screenshotApplications(IBinder appToken, int displayId, int width,
		int height, boolean includeFullDisplay, float frameScale, Bitmap.Config config,
		boolean wallpaperOnly, boolean includeDecor) {
		private Bitmap screenshotApplications(int displayId, Bitmap.Config config,
		boolean wallpaperOnly) {
		final DisplayContent displayContent;
		synchronized(mWindowMap) {
		displayContent = mRoot.getDisplayContentOrCreate(displayId);
		if (displayContent == null) {
		if (DEBUG_SCREENSHOT) Slog.i(TAG_WM, "Screenshot of " + appToken
		+ ": returning null. No Display for displayId=" + displayId);
		if (DEBUG_SCREENSHOT) Slog.i(TAG_WM, "Screenshot returning null. No Display for "
		+ "displayId=" + displayId);
		return null;
		}
		}
		return displayContent.screenshotApplications(appToken, width, height,
		includeFullDisplay, frameScale, config, wallpaperOnly, includeDecor);
		return displayContent.screenshotDisplay(config, wallpaperOnly);
		}

		/**