Merge "Adds a model & view layer for long screenshot tiles" (29099129) · Commits · e / os / android_frameworks_base

packages/SystemUI/src/com/android/systemui/screenshot/ImageTile.java

0 → 100644

+113 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2020 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/
		package com.android.systemui.screenshot;

		import static android.graphics.ColorSpace.Named.SRGB;

		import static java.util.Objects.requireNonNull;

		import android.graphics.Bitmap;
		import android.graphics.ColorSpace;
		import android.graphics.RecordingCanvas;
		import android.graphics.Rect;
		import android.graphics.RenderNode;
		import android.media.Image;

		/**
		* Holds a hardware image, coordinates and render node to draw the tile. The tile manages clipping
		* and dimensions. The tile must be drawn translated to the correct target position:
		* <pre>
		* ImageTile tile = getTile();
		* canvas.save();
		* canvas.translate(tile.getLeft(), tile.getTop());
		* canvas.drawRenderNode(tile.getDisplayList());
		* canvas.restore();
		* </pre>
		*/
		class ImageTile implements AutoCloseable {
		private final Image mImage;
		private final Rect mLocation;
		private RenderNode mNode;

		private static final ColorSpace COLOR_SPACE = ColorSpace.get(SRGB);

		/**
		* Create an image tile from the given image.
		*
		* @param image an image containing a hardware buffer
		* @param location the captured area represented by image tile (virtual coordinates)
		*/
		ImageTile(Image image, Rect location) {
		mImage = requireNonNull(image, "image");
		mLocation = location;

		requireNonNull(mImage.getHardwareBuffer(), "image must be a hardware image");
		}

		RenderNode getDisplayList() {
		if (mNode == null) {
		mNode = new RenderNode("Tile{" + Integer.toHexString(mImage.hashCode()) + "}");
		}
		if (mNode.hasDisplayList()) {
		return mNode;
		}
		final int w = Math.min(mImage.getWidth(), mLocation.width());
		final int h = Math.min(mImage.getHeight(), mLocation.height());
		mNode.setPosition(0, 0, w, h);

		RecordingCanvas canvas = mNode.beginRecording(w, h);
		Rect rect = new Rect(0, 0, w, h);
		canvas.save();
		canvas.clipRect(0, 0, mLocation.right, mLocation.bottom);
		canvas.drawBitmap(Bitmap.wrapHardwareBuffer(mImage.getHardwareBuffer(), COLOR_SPACE),
		0, 0, null);
		canvas.restore();
		mNode.endRecording();
		return mNode;
		}

		Rect getLocation() {
		return mLocation;
		}

		int getLeft() {
		return mLocation.left;
		}

		int getTop() {
		return mLocation.top;
		}

		int getRight() {
		return mLocation.right;
		}

		int getBottom() {
		return mLocation.bottom;
		}

		@Override
		public void close() {
		mImage.close();
		mNode.discardDisplayList();
		}

		@Override
		public String toString() {
		return "{location=" + mLocation + ", source=" + mImage
		+ ", buffer=" + mImage.getHardwareBuffer() + "}";
		}
		}

packages/SystemUI/src/com/android/systemui/screenshot/ImageTileSet.java

0 → 100644

+163 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2020 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/
		package com.android.systemui.screenshot;

		import android.graphics.Bitmap;
		import android.graphics.HardwareRenderer;
		import android.graphics.RecordingCanvas;
		import android.graphics.Rect;
		import android.graphics.RenderNode;
		import android.graphics.drawable.Drawable;

		import androidx.annotation.UiThread;

		import java.util.ArrayList;
		import java.util.List;

		/**
		* Owns a series of partial screen captures (tiles).
		* <p>
		* To display on-screen, use {@link #getDrawable()}.
		*/
		@UiThread
		class ImageTileSet {

		private static final String TAG = "ImageTileSet";

		interface OnBoundsChangedListener {
		/**
		* Reports an update to the bounding box that contains all active tiles. These are virtual
		* (capture) coordinates which can be either negative or positive.
		*/
		void onBoundsChanged(int left, int top, int right, int bottom);
		}

		interface OnContentChangedListener {
		/**
		* Mark as dirty and rebuild display list.
		*/
		void onContentChanged();
		}

		private final List<ImageTile> mTiles = new ArrayList<>();
		private final Rect mBounds = new Rect();

		private OnContentChangedListener mOnContentChangedListener;
		private OnBoundsChangedListener mOnBoundsChangedListener;

		void setOnBoundsChangedListener(OnBoundsChangedListener listener) {
		mOnBoundsChangedListener = listener;
		}

		void setOnContentChangedListener(OnContentChangedListener listener) {
		mOnContentChangedListener = listener;
		}

		void addTile(ImageTile tile) {
		final Rect newBounds = new Rect(mBounds);
		final Rect newRect = tile.getLocation();
		mTiles.add(tile);
		newBounds.union(newRect);
		if (!newBounds.equals(mBounds)) {
		mBounds.set(newBounds);
		if (mOnBoundsChangedListener != null) {
		mOnBoundsChangedListener.onBoundsChanged(
		newBounds.left, newBounds.top, newBounds.right, newBounds.bottom);
		}
		}
		if (mOnContentChangedListener != null) {
		mOnContentChangedListener.onContentChanged();
		}
		}

		/**
		* Returns a drawable to paint the combined contents of the tiles. Drawable dimensions are
		* zero-based and map directly to {@link #getLeft()}, {@link #getTop()}, {@link #getRight()},
		* and {@link #getBottom()} which are dimensions relative to the capture start position
		* (positive or negative).
		*
		* @return a drawable to display the image content
		*/
		Drawable getDrawable() {
		return new TiledImageDrawable(this);
		}

		boolean isEmpty() {
		return mTiles.isEmpty();
		}

		int size() {
		return mTiles.size();
		}

		ImageTile get(int i) {
		return mTiles.get(i);
		}

		Bitmap toBitmap() {
		if (mTiles.isEmpty()) {
		return null;
		}
		final RenderNode output = new RenderNode("Bitmap Export");
		output.setPosition(0, 0, getWidth(), getHeight());
		RecordingCanvas canvas = output.beginRecording();
		canvas.translate(-getLeft(), -getTop());
		for (ImageTile tile : mTiles) {
		canvas.save();
		canvas.translate(tile.getLeft(), tile.getTop());
		canvas.drawRenderNode(tile.getDisplayList());
		canvas.restore();
		}
		output.endRecording();
		return HardwareRenderer.createHardwareBitmap(output, getWidth(), getHeight());
		}

		int getLeft() {
		return mBounds.left;
		}

		int getTop() {
		return mBounds.top;
		}

		int getRight() {
		return mBounds.right;
		}

		int getBottom() {
		return mBounds.bottom;
		}

		int getWidth() {
		return mBounds.width();
		}

		int getHeight() {
		return mBounds.height();
		}

		void clear() {
		mBounds.set(0, 0, 0, 0);
		mTiles.forEach(ImageTile::close);
		mTiles.clear();
		if (mOnBoundsChangedListener != null) {
		mOnBoundsChangedListener.onBoundsChanged(0, 0, 0, 0);
		}
		if (mOnContentChangedListener != null) {
		mOnContentChangedListener.onContentChanged();
		}
		}
		}

packages/SystemUI/src/com/android/systemui/screenshot/ScrollCaptureClient.java

+64 −33

Original line number	Diff line number	Diff line
		@@ -18,6 +18,7 @@ package com.android.systemui.screenshot;

		import static com.android.systemui.screenshot.LogConfig.DEBUG_SCROLL;

		import static java.lang.Math.min;
		import static java.util.Objects.requireNonNull;

		import android.annotation.UiContext;
		@@ -44,15 +45,20 @@ import java.util.function.Consumer;
		import javax.inject.Inject;

		/**
		* High level interface to scroll capture API.
		* High(er) level interface to scroll capture API.
		*/
		public class ScrollCaptureClient {
		private static final int TILE_SIZE_PX_MAX = 4 * (1024 * 1024);
		private static final int TILES_PER_PAGE = 2; // increase once b/174571735 is addressed
		private static final int MAX_PAGES = 5;
		private static final int MAX_IMAGE_COUNT = MAX_PAGES * TILES_PER_PAGE;

		@VisibleForTesting
		static final int MATCH_ANY_TASK = ActivityTaskManager.INVALID_TASK_ID;

		private static final String TAG = LogConfig.logTag(ScrollCaptureClient.class);


		/**
		* A connection to a remote window. Starts a capture session.
		*/
		@@ -60,13 +66,10 @@ public class ScrollCaptureClient {
		/**
		* Session start should be deferred until UI is active because of resource allocation and
		* potential visible side effects in the target window.
		*
		* @param maxBuffers the maximum number of buffers (tiles) that may be in use at one
		* time, tiles are not cached anywhere so set this to a large enough
		* number to retain offscreen content until it is no longer needed

		* @param sessionConsumer listener to receive the session once active
		*/
		void start(int maxBuffers, Consumer<Session> sessionConsumer);
		void start(Consumer<Session> sessionConsumer);

		/**
		* Close the connection.
		@@ -100,26 +103,33 @@ public class ScrollCaptureClient {
		*/
		interface Session {
		/**
		* Request the given horizontal strip. Values are y-coordinates in captured space, relative
		* to start position.
		* Request an image tile at the given position, from top, to top + {@link #getTileHeight()},
		* and from left 0, to {@link #getPageWidth()}
		*
		* @param contentRect the area to capture, in content rect space, relative to scroll-bounds
		* @param top the top (y) position of the tile to capture, in content rect space
		* @param consumer listener to be informed of the result
		*/
		void requestTile(Rect contentRect, Consumer<CaptureResult> consumer);
		void requestTile(int top, Consumer<CaptureResult> consumer);

		/**
		* End the capture session, return the target app to original state. The returned
		* stage must be waited for to complete to allow the target app a chance to restore to
		* original state before becoming visible.
		* Returns the maximum number of tiles which may be requested and retained without
		* being {@link Image#close() closed}.
		*
		* @return a stage presenting the session shutdown
		* @return the maximum number of open tiles allowed
		*/
		void end(Runnable listener);
		int getMaxTiles();

		int getTileHeight();

		int getPageHeight();

		int getMaxTileHeight();
		int getPageWidth();

		int getMaxTileWidth();
		/**
		* End the capture session, return the target app to original state. The listener
		* will be called when the target app is ready to before visible and interactive.
		*/
		void end(Runnable listener);
		}

		private final IWindowManager mWindowManagerService;
		@@ -131,6 +141,12 @@ public class ScrollCaptureClient {
		mWindowManagerService = windowManagerService;
		}

		/**
		* Set the window token for the screenshot window/ This is required to avoid targeting our
		* window or any above it.
		*
		* @param token the windowToken of the screenshot window
		*/
		public void setHostWindowToken(IBinder token) {
		mHostWindowToken = token;
		}
		@@ -176,6 +192,8 @@ public class ScrollCaptureClient {

		private ImageReader mReader;
		private Rect mScrollBounds;
		private int mTileHeight;
		private int mTileWidth;
		private Rect mRequestRect;
		private boolean mStarted;

		@@ -197,6 +215,15 @@ public class ScrollCaptureClient {
		mScrollBounds = scrollBounds;
		mConnectionConsumer.accept(this);
		mConnectionConsumer = null;

		int pxPerPage = mScrollBounds.width() * mScrollBounds.height();
		int pxPerTile = min(TILE_SIZE_PX_MAX, (pxPerPage / TILES_PER_PAGE));
		mTileWidth = mScrollBounds.width();
		mTileHeight = pxPerTile / mScrollBounds.width();
		if (DEBUG_SCROLL) {
		Log.d(TAG, "scrollBounds: " + mScrollBounds);
		Log.d(TAG, "tile dimen: " + mTileWidth + "x" + mTileHeight);
		}
		}

		@Override
		@@ -257,24 +284,19 @@ public class ScrollCaptureClient {

		// ScrollCaptureController.Connection

		// -> Error handling: BiConsumer<Session, Throwable> ?
		@Override
		public void start(int maxBufferCount, Consumer<Session> sessionConsumer) {
		public void start(Consumer<Session> sessionConsumer) {
		if (DEBUG_SCROLL) {
		Log.d(TAG, "start(maxBufferCount=" + maxBufferCount
		+ ", sessionConsumer=" + sessionConsumer + ")");
		Log.d(TAG, "start(sessionConsumer=" + sessionConsumer + ")");
		}
		mReader = ImageReader.newInstance(mScrollBounds.width(), mScrollBounds.height(),
		PixelFormat.RGBA_8888, maxBufferCount, HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE);
		mReader = ImageReader.newInstance(mTileWidth, mTileHeight, PixelFormat.RGBA_8888,
		MAX_IMAGE_COUNT, HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE);
		mSessionConsumer = sessionConsumer;
		try {
		mConnection.startCapture(mReader.getSurface());
		mStarted = true;
		} catch (RemoteException e) {
		Log.w(TAG, "should not be happening :-(");
		// ?
		//mSessionListener.onError(e);
		//mSessionListener = null;
		Log.w(TAG, "Failed to start", e);
		}
		}

		@@ -307,27 +329,36 @@ public class ScrollCaptureClient {
		}

		@Override
		public int getMaxTileHeight() {
		public int getPageHeight() {
		return mScrollBounds.height();
		}

		@Override
		public int getMaxTileWidth() {
		public int getPageWidth() {
		return mScrollBounds.width();
		}

		@Override
		public void requestTile(Rect contentRect, Consumer<CaptureResult> consumer) {
		public int getTileHeight() {
		return mTileHeight;
		}

		@Override
		public int getMaxTiles() {
		return MAX_IMAGE_COUNT;
		}

		@Override
		public void requestTile(int top, Consumer<CaptureResult> consumer) {
		if (DEBUG_SCROLL) {
		Log.d(TAG, "requestTile(contentRect=" + contentRect + "consumer=" + consumer + ")");
		Log.d(TAG, "requestTile(top=" + top + ", consumer=" + consumer + ")");
		}
		mRequestRect = new Rect(contentRect);
		mRequestRect = new Rect(0, top, mTileWidth, top + mTileHeight);
		mResultConsumer = consumer;
		try {
		mConnection.requestImage(mRequestRect);
		} catch (RemoteException e) {
		Log.e(TAG, "Caught remote exception from requestImage", e);
		// ?
		}
		}

packages/SystemUI/src/com/android/systemui/screenshot/ScrollCaptureController.java

+18 −57

Original line number	Diff line number	Diff line
		@@ -16,16 +16,9 @@

		package com.android.systemui.screenshot;

		import static android.graphics.ColorSpace.Named.SRGB;

		import android.content.Context;
		import android.content.Intent;
		import android.graphics.Bitmap;
		import android.graphics.Canvas;
		import android.graphics.ColorSpace;
		import android.graphics.Picture;
		import android.graphics.Rect;
		import android.media.Image;
		import android.net.Uri;
		import android.os.UserHandle;
		import android.util.Log;
		@@ -46,6 +39,8 @@ import java.util.function.Consumer;
		public class ScrollCaptureController {
		private static final String TAG = "ScrollCaptureController";

		private static final boolean USE_TILED_IMAGE = false;

		public static final int MAX_PAGES = 5;
		public static final int MAX_HEIGHT = 12000;

		@@ -55,7 +50,7 @@ public class ScrollCaptureController {
		private final Executor mUiExecutor;
		private final Executor mBgExecutor;
		private final ImageExporter mImageExporter;
		private Picture mPicture;
		private final ImageTileSet mImageTileSet;

		public ScrollCaptureController(Context context, Connection connection, Executor uiExecutor,
		Executor bgExecutor, ImageExporter exporter) {
		@@ -64,6 +59,7 @@ public class ScrollCaptureController {
		mUiExecutor = uiExecutor;
		mBgExecutor = bgExecutor;
		mImageExporter = exporter;
		mImageTileSet = new ImageTileSet();
		}

		/**
		@@ -72,50 +68,50 @@ public class ScrollCaptureController {
		* @param after action to take after the flow is complete
		*/
		public void run(final Runnable after) {
		mConnection.start(MAX_PAGES, (session) -> startCapture(session, after));
		mConnection.start((session) -> startCapture(session, after));
		}

		private void startCapture(Session session, final Runnable onDismiss) {
		Rect requestRect = new Rect(0, 0,
		session.getMaxTileWidth(), session.getMaxTileHeight());
		Consumer<ScrollCaptureClient.CaptureResult> consumer =
		new Consumer<ScrollCaptureClient.CaptureResult>() {

		int mFrameCount = 0;
		int mTop = 0;

		@Override
		public void accept(ScrollCaptureClient.CaptureResult result) {
		mFrameCount++;

		boolean emptyFrame = result.captured.height() == 0;
		if (!emptyFrame) {
		mPicture = stackBelow(mPicture, result.image, result.captured.width(),
		result.captured.height());
		mImageTileSet.addTile(new ImageTile(result.image, result.captured));
		}

		if (emptyFrame \|\| mFrameCount >= MAX_PAGES
		\|\| requestRect.bottom > MAX_HEIGHT) {
		if (mPicture != null) {
		exportToFile(mPicture, session, onDismiss);
		\|\| mTop + session.getTileHeight() > MAX_HEIGHT) {
		if (!mImageTileSet.isEmpty()) {
		exportToFile(mImageTileSet.toBitmap(), session, onDismiss);
		mImageTileSet.clear();
		} else {
		session.end(onDismiss);
		}
		return;
		}
		requestRect.offset(0, session.getMaxTileHeight());
		session.requestTile(requestRect, /* consumer */ this);
		mTop += result.captured.height();
		session.requestTile(mTop, /* consumer */ this);
		}
		};

		// fire it up!
		session.requestTile(requestRect, consumer);
		session.requestTile(0, consumer);
		};

		void exportToFile(Picture picture, Session session, Runnable afterEnd) {
		void exportToFile(Bitmap bitmap, Session session, Runnable afterEnd) {
		mImageExporter.setFormat(Bitmap.CompressFormat.PNG);
		mImageExporter.setQuality(6);
		ListenableFuture<Uri> future =
		mImageExporter.export(mBgExecutor, Bitmap.createBitmap(picture));
		mImageExporter.export(mBgExecutor, bitmap);
		future.addListener(() -> {
		picture.close(); // release resources
		try {
		launchViewer(future.get());
		} catch (InterruptedException \| ExecutionException e) {
		@@ -126,41 +122,6 @@ public class ScrollCaptureController {
		}, mUiExecutor);
		}

		/**
		* Combine the top {@link Picture} with an {@link Image} by appending the image directly
		* below, creating a result that is the combined height of both.
		* <p>
		* Note: no pixel data is transferred here, only a record of drawing commands. Backing
		* hardware buffers must not be modified/recycled until the picture is
		* {@link Picture#close closed}.
		*
		* @param top the existing picture
		* @param below the image to append below
		* @param cropWidth the width of the pixel data to use from the image
		* @param cropHeight the height of the pixel data to use from the image
		*
		* @return a new Picture which draws the previous picture with the image below it
		*/
		private static Picture stackBelow(Picture top, Image below, int cropWidth, int cropHeight) {
		int width = cropWidth;
		int height = cropHeight;
		if (top != null) {
		height += top.getHeight();
		width = Math.max(width, top.getWidth());
		}
		Picture combined = new Picture();
		Canvas canvas = combined.beginRecording(width, height);
		int y = 0;
		if (top != null) {
		canvas.drawPicture(top, new Rect(0, 0, top.getWidth(), top.getHeight()));
		y += top.getHeight();
		}
		canvas.drawBitmap(Bitmap.wrapHardwareBuffer(
		below.getHardwareBuffer(), ColorSpace.get(SRGB)), 0, y, null);
		combined.endRecording();
		return combined;
		}

		void launchViewer(Uri uri) {
		Intent editIntent = new Intent(Intent.ACTION_VIEW);
		editIntent.setType("image/png");

packages/SystemUI/src/com/android/systemui/screenshot/TiledImageDrawable.java

0 → 100644

+117 −0

File added.

Preview size limit exceeded, changes collapsed.