Merge "ImageReader/Writer: Add ImageWriter and Opaque ImageReader" (960a285a) · Commits · e / os / android_frameworks_base

api/current.txt

+17 −0

Original line number	Diff line number	Diff line
		@@ -14748,7 +14748,9 @@ package android.media {
		method public abstract android.media.Image.Plane[] getPlanes();
		method public abstract long getTimestamp();
		method public abstract int getWidth();
		method public boolean isOpaque();
		method public void setCropRect(android.graphics.Rect);
		method public void setTimestamp(long);
		}

		public static abstract class Image.Plane {
		@@ -14766,7 +14768,9 @@ package android.media {
		method public int getMaxImages();
		method public android.view.Surface getSurface();
		method public int getWidth();
		method public boolean isOpaque();
		method public static android.media.ImageReader newInstance(int, int, int, int);
		method public static android.media.ImageReader newOpaqueInstance(int, int, int);
		method public void setOnImageAvailableListener(android.media.ImageReader.OnImageAvailableListener, android.os.Handler);
		}

		@@ -14774,6 +14778,19 @@ package android.media {
		method public abstract void onImageAvailable(android.media.ImageReader);
		}

		public class ImageWriter implements java.lang.AutoCloseable {
		method public void close();
		method public android.media.Image dequeueInputImage();
		method public int getMaxImages();
		method public static android.media.ImageWriter newInstance(android.view.Surface, int);
		method public void queueInputImage(android.media.Image);
		method public void setImageListener(android.media.ImageWriter.ImageListener, android.os.Handler);
		}

		public static abstract interface ImageWriter.ImageListener {
		method public abstract void onInputImageReleased(android.media.ImageWriter);
		}

		public class JetPlayer {
		method public boolean clearQueue();
		method public java.lang.Object clone() throws java.lang.CloneNotSupportedException;

api/system-current.txt

+17 −0

Original line number	Diff line number	Diff line
		@@ -15938,7 +15938,9 @@ package android.media {
		method public abstract android.media.Image.Plane[] getPlanes();
		method public abstract long getTimestamp();
		method public abstract int getWidth();
		method public boolean isOpaque();
		method public void setCropRect(android.graphics.Rect);
		method public void setTimestamp(long);
		}

		public static abstract class Image.Plane {
		@@ -15956,7 +15958,9 @@ package android.media {
		method public int getMaxImages();
		method public android.view.Surface getSurface();
		method public int getWidth();
		method public boolean isOpaque();
		method public static android.media.ImageReader newInstance(int, int, int, int);
		method public static android.media.ImageReader newOpaqueInstance(int, int, int);
		method public void setOnImageAvailableListener(android.media.ImageReader.OnImageAvailableListener, android.os.Handler);
		}

		@@ -15964,6 +15968,19 @@ package android.media {
		method public abstract void onImageAvailable(android.media.ImageReader);
		}

		public class ImageWriter implements java.lang.AutoCloseable {
		method public void close();
		method public android.media.Image dequeueInputImage();
		method public int getMaxImages();
		method public static android.media.ImageWriter newInstance(android.view.Surface, int);
		method public void queueInputImage(android.media.Image);
		method public void setImageListener(android.media.ImageWriter.ImageListener, android.os.Handler);
		}

		public static abstract interface ImageWriter.ImageListener {
		method public abstract void onInputImageReleased(android.media.ImageWriter);
		}

		public class JetPlayer {
		method public boolean clearQueue();
		method public java.lang.Object clone() throws java.lang.CloneNotSupportedException;

media/java/android/media/Image.java

+85 −7

Original line number	Diff line number	Diff line
		@@ -115,14 +115,49 @@ public abstract class Image implements AutoCloseable {
		/**
		* Get the timestamp associated with this frame.
		* <p>
		* The timestamp is measured in nanoseconds, and is monotonically
		* increasing. However, the zero point and whether the timestamp can be
		* compared against other sources of time or images depend on the source of
		* this image.
		* The timestamp is measured in nanoseconds, and is normally monotonically
		* increasing. However, the behavior of the timestamp depends on the source
		* of this image. See {@link android.hardware.Camera Camera},
		* {@link android.hardware.camera2.CameraDevice CameraDevice}, {@link MediaPlayer} and
		* {@link MediaCodec} for more details.
		* </p>
		*/
		public abstract long getTimestamp();

		/**
		* Set the timestamp associated with this frame.
		* <p>
		* The timestamp is measured in nanoseconds, and is normally monotonically
		* increasing. However, However, the behavior of the timestamp depends on
		* the destination of this image. See {@link android.hardware.Camera Camera}
		* , {@link android.hardware.camera2.CameraDevice CameraDevice},
		* {@link MediaPlayer} and {@link MediaCodec} for more details.
		* </p>
		* <p>
		* For images dequeued from {@link ImageWriter} via
		* {@link ImageWriter#dequeueInputImage()}, it's up to the application to
		* set the timestamps correctly before sending them back to the
		* {@link ImageWriter}, or the timestamp will be generated automatically when
		* {@link ImageWriter#queueInputImage queueInputImage()} is called.
		* </p>
		*
		* @param timestamp The timestamp to be set for this image.
		*/
		public void setTimestamp(long timestamp) {
		return;
		}

		/**
		* <p>Check if the image is opaque.</p>
		*
		* <p>The pixel data of opaque images are not accessible to the application,
		* and therefore {@link #getPlanes} will return an empty array for an opaque image.
		* </p>
		*/
		public boolean isOpaque() {
		return false;
		}

		private Rect mCropRect;

		/**
		@@ -155,7 +190,10 @@ public abstract class Image implements AutoCloseable {

		/**
		* Get the array of pixel planes for this Image. The number of planes is
		* determined by the format of the Image.
		* determined by the format of the Image. The application will get an
		* empty array if the image is opaque because the opaque image pixel data
		* is not directly accessible. The application can check if an image is
		* opaque by calling {@link Image#isOpaque}.
		*/
		public abstract Plane[] getPlanes();

		@@ -164,13 +202,53 @@ public abstract class Image implements AutoCloseable {
		* <p>
		* After calling this method, calling any methods on this {@code Image} will
		* result in an {@link IllegalStateException}, and attempting to read from
		* {@link ByteBuffer ByteBuffers} returned by an earlier
		* {@link Plane#getBuffer} call will have undefined behavior.
		* or write to {@link ByteBuffer ByteBuffers} returned by an earlier
		* {@link Plane#getBuffer} call will have undefined behavior. If the image
		* was obtained from {@link ImageWriter} via
		* {@link ImageWriter#dequeueInputImage()}, after calling this method, any
		* image data filled by the application will be lost and the image will be
		* returned to {@link ImageWriter} for reuse. Images given to
		* {@link ImageWriter#queueInputImage queueInputImage()} are automatically
		* closed.
		* </p>
		*/
		@Override
		public abstract void close();

		/**
		* <p>
		* Check if the image can be attached to a new owner (e.g. {@link ImageWriter}).
		* </p>
		* <p>
		* This is a package private method that is only used internally.
		* </p>
		*
		* @return true if the image is attachable to a new owner, false if the image is still attached
		* to its current owner, or the image is a stand-alone image and is not attachable to
		* a new owner.
		*/
		boolean isAttachable() {
		return false;
		}

		/**
		* <p>
		* Get the owner of the {@link Image}.
		* </p>
		* <p>
		* The owner of an {@link Image} could be {@link ImageReader}, {@link ImageWriter},
		* {@link MediaCodec} etc. This method returns the owner that produces this image, or null
		* if the image is stand-alone image or the owner is unknown.
		* </p>
		* <p>
		* This is a package private method that is only used internally.
		* </p>
		*
		* @return The owner of the Image.
		*/
		Object getOwner() {
		return null;
		}
		/**
		* <p>A single color plane of image data.</p>
		*

media/java/android/media/ImageReader.java

+167 −2

Original line number	Diff line number	Diff line
		@@ -27,6 +27,7 @@ import java.lang.ref.WeakReference;
		import java.nio.ByteBuffer;
		import java.nio.ByteOrder;
		import java.nio.NioUtils;
		import java.util.concurrent.atomic.AtomicBoolean;

		/**
		* <p>The ImageReader class allows direct application access to image data
		@@ -34,7 +35,7 @@ import java.nio.NioUtils;
		*
		* <p>Several Android media API classes accept Surface objects as targets to
		* render to, including {@link MediaPlayer}, {@link MediaCodec},
		* {@link android.hardware.camera2.CameraDevice}, and
		* {@link android.hardware.camera2.CameraDevice}, {@link ImageWriter} and
		* {@link android.renderscript.Allocation RenderScript Allocations}. The image
		* sizes and formats that can be used with each source vary, and should be
		* checked in the documentation for the specific API.</p>
		@@ -97,9 +98,59 @@ public class ImageReader implements AutoCloseable {
		* @see Image
		*/
		public static ImageReader newInstance(int width, int height, int format, int maxImages) {
		if (format == PixelFormat.OPAQUE) {
		throw new IllegalArgumentException("To obtain an opaque ImageReader, please use"
		+ " newOpaqueInstance rather than newInstance");
		}
		return new ImageReader(width, height, format, maxImages);
		}

		/**
		* <p>
		* Create a new opaque reader for images of the desired size.
		* </p>
		* <p>
		* An opaque {@link ImageReader} produces images that are not directly
		* accessible by the application. The application can still acquire images
		* from an opaque image reader, and send them to the
		* {@link android.hardware.camera2.CameraDevice camera} for reprocessing via
		* {@link ImageWriter} interface. However, the {@link Image#getPlanes()
		* getPlanes()} will return an empty array for opaque images. The
		* application can check if an existing reader is an opaque reader by
		* calling {@link #isOpaque()}.
		* </p>
		* <p>
		* The {@code maxImages} parameter determines the maximum number of
		* {@link Image} objects that can be be acquired from the
		* {@code ImageReader} simultaneously. Requesting more buffers will use up
		* more memory, so it is important to use only the minimum number necessary.
		* </p>
		* <p>
		* The valid sizes and formats depend on the source of the image data.
		* </p>
		* <p>
		* Opaque ImageReaders are more efficient to use when application access to
		* image data is not necessary, comparing to ImageReaders using a non-opaque
		* format such as {@link ImageFormat#YUV_420_888 YUV_420_888}.
		* </p>
		*
		* @param width The default width in pixels of the Images that this reader
		* will produce.
		* @param height The default height in pixels of the Images that this reader
		* will produce.
		* @param maxImages The maximum number of images the user will want to
		* access simultaneously. This should be as small as possible to
		* limit memory use. Once maxImages Images are obtained by the
		* user, one of them has to be released before a new Image will
		* become available for access through
		* {@link #acquireLatestImage()} or {@link #acquireNextImage()}.
		* Must be greater than 0.
		* @see Image
		*/
		public static ImageReader newOpaqueInstance(int width, int height, int maxImages) {
		return new ImageReader(width, height, PixelFormat.OPAQUE, maxImages);
		}

		/**
		* @hide
		*/
		@@ -196,6 +247,23 @@ public class ImageReader implements AutoCloseable {
		return mMaxImages;
		}

		/**
		* <p>
		* Check if the {@link ImageReader} is an opaque reader.
		* </p>
		* <p>
		* An opaque image reader produces opaque images, see {@link Image#isOpaque}
		* for more details.
		* </p>
		*
		* @return true if the ImageReader is opaque.
		* @see Image#isOpaque
		* @see ImageReader#newOpaqueInstance
		*/
		public boolean isOpaque() {
		return mFormat == PixelFormat.OPAQUE;
		}

		/**
		* <p>Get a {@link Surface} that can be used to produce {@link Image Images} for this
		* {@code ImageReader}.</p>
		@@ -456,6 +524,58 @@ public class ImageReader implements AutoCloseable {
		}
		}

		/**
		* <p>
		* Remove the ownership of this image from the ImageReader.
		* </p>
		* <p>
		* After this call, the ImageReader no longer owns this image, and the image
		* ownership can be transfered to another entity like {@link ImageWriter}
		* via {@link ImageWriter#queueInputImage}. It's up to the new owner to
		* release the resources held by this image. For example, if the ownership
		* of this image is transfered to an {@link ImageWriter}, the image will be
		* freed by the ImageWriter after the image data consumption is done.
		* </p>
		* <p>
		* This method can be used to achieve zero buffer copy for use cases like
		* {@link android.hardware.camera2.CameraDevice Camera2 API} OPAQUE and YUV
		* reprocessing, where the application can select an output image from
		* {@link ImageReader} and transfer this image directly to
		* {@link ImageWriter}, where this image can be consumed by camera directly.
		* For OPAQUE reprocessing, this is the only way to send input buffers to
		* the {@link android.hardware.camera2.CameraDevice camera} for
		* reprocessing.
		* </p>
		* <p>
		* This is a package private method that is only used internally.
		* </p>
		*
		* @param image The image to be detached from this ImageReader.
		* @throws IllegalStateException If the ImageReader or image have been
		* closed, or the has been detached, or has not yet been
		* acquired.
		*/
		void detachImage(Image image) {
		if (image == null) {
		throw new IllegalArgumentException("input image must not be null");
		}
		if (!isImageOwnedbyMe(image)) {
		throw new IllegalArgumentException("Trying to detach an image that is not owned by"
		+ " this ImageReader");
		}

		SurfaceImage si = (SurfaceImage) image;
		if (!si.isImageValid()) {
		throw new IllegalStateException("Image is no longer valid");
		}
		if (si.isAttachable()) {
		throw new IllegalStateException("Image was already detached from this ImageReader");
		}

		nativeDetachImage(image);
		si.setDetached(true);
		}

		/**
		* Only a subset of the formats defined in
		* {@link android.graphics.ImageFormat ImageFormat} and
		@@ -487,12 +607,22 @@ public class ImageReader implements AutoCloseable {
		case ImageFormat.DEPTH16:
		case ImageFormat.DEPTH_POINT_CLOUD:
		return 1;
		case PixelFormat.OPAQUE:
		return 0;
		default:
		throw new UnsupportedOperationException(
		String.format("Invalid format specified %d", mFormat));
		}
		}

		private boolean isImageOwnedbyMe(Image image) {
		if (!(image instanceof SurfaceImage)) {
		return false;
		}
		SurfaceImage si = (SurfaceImage) image;
		return si.getReader() == this;
		}

		/**
		* Called from Native code when an Event happens.
		*
		@@ -561,9 +691,13 @@ public class ImageReader implements AutoCloseable {
		@Override
		public void close() {
		if (mIsImageValid) {
		if (!mIsDetached.get()) {
		// For detached images, the new owner is responsible for
		// releasing the resources
		ImageReader.this.releaseImage(this);
		}
		}
		}

		public ImageReader getReader() {
		return ImageReader.this;
		@@ -613,6 +747,15 @@ public class ImageReader implements AutoCloseable {
		}
		}

		@Override
		public void setTimestamp(long timestampNs) {
		if (mIsImageValid) {
		mTimestamp = timestampNs;
		} else {
		throw new IllegalStateException("Image is already released");
		}
		}

		@Override
		public Plane[] getPlanes() {
		if (mIsImageValid) {
		@@ -623,6 +766,11 @@ public class ImageReader implements AutoCloseable {
		}
		}

		@Override
		public boolean isOpaque() {
		return mFormat == PixelFormat.OPAQUE;
		}

		@Override
		protected final void finalize() throws Throwable {
		try {
		@@ -632,6 +780,20 @@ public class ImageReader implements AutoCloseable {
		}
		}

		@Override
		boolean isAttachable() {
		return mIsDetached.get();
		}

		@Override
		ImageReader getOwner() {
		return ImageReader.this;
		}

		private void setDetached(boolean detached) {
		mIsDetached.getAndSet(detached);
		}

		private void setImageValid(boolean isValid) {
		mIsImageValid = isValid;
		}
		@@ -734,6 +896,8 @@ public class ImageReader implements AutoCloseable {
		private boolean mIsImageValid;
		private int mHeight = -1;
		private int mWidth = -1;
		// If this image is detached from the ImageReader.
		private AtomicBoolean mIsDetached = new AtomicBoolean(false);

		private synchronized native ByteBuffer nativeImageGetBuffer(int idx, int readerFormat);
		private synchronized native SurfacePlane nativeCreatePlane(int idx, int readerFormat);
		@@ -746,6 +910,7 @@ public class ImageReader implements AutoCloseable {
		private synchronized native void nativeClose();
		private synchronized native void nativeReleaseImage(Image i);
		private synchronized native Surface nativeGetSurface();
		private synchronized native void nativeDetachImage(Image i);

		/**
		* @return A return code {@code ACQUIRE_*}

media/java/android/media/ImageUtils.java

0 → 100644

+120 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright 2015 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package android.media;

		import android.graphics.ImageFormat;
		import android.graphics.PixelFormat;
		import android.media.Image.Plane;
		import android.util.Size;

		import java.nio.ByteBuffer;

		/**
		* Package private utility class for hosting commonly used Image related methods.
		*/
		class ImageUtils {

		/**
		* Only a subset of the formats defined in
		* {@link android.graphics.ImageFormat ImageFormat} and
		* {@link android.graphics.PixelFormat PixelFormat} are supported by
		* ImageReader. When reading RGB data from a surface, the formats defined in
		* {@link android.graphics.PixelFormat PixelFormat} can be used; when
		* reading YUV, JPEG or raw sensor data (for example, from the camera or video
		* decoder), formats from {@link android.graphics.ImageFormat ImageFormat}
		* are used.
		*/
		public static int getNumPlanesForFormat(int format) {
		switch (format) {
		case ImageFormat.YV12:
		case ImageFormat.YUV_420_888:
		case ImageFormat.NV21:
		return 3;
		case ImageFormat.NV16:
		return 2;
		case PixelFormat.RGB_565:
		case PixelFormat.RGBA_8888:
		case PixelFormat.RGBX_8888:
		case PixelFormat.RGB_888:
		case ImageFormat.JPEG:
		case ImageFormat.YUY2:
		case ImageFormat.Y8:
		case ImageFormat.Y16:
		case ImageFormat.RAW_SENSOR:
		case ImageFormat.RAW10:
		return 1;
		case PixelFormat.OPAQUE:
		return 0;
		default:
		throw new UnsupportedOperationException(
		String.format("Invalid format specified %d", format));
		}
		}

		/**
		* <p>
		* Copy source image data to destination Image.
		* </p>
		* <p>
		* Only support the copy between two non-opaque images with same properties
		* (format, size, etc.). The data from the source image will be copied to
		* the byteBuffers from the destination Image starting from position zero,
		* and the destination image will be rewound to zero after copy is done.
		* </p>
		*
		* @param src The source image to be copied from.
		* @param dst The destination image to be copied to.
		* @throws IllegalArgumentException If the source and destination images
		* have different format, or one of the images is not copyable.
		*/
		public static void imageCopy(Image src, Image dst) {
		if (src == null \|\| dst == null) {
		throw new IllegalArgumentException("Images should be non-null");
		}
		if (src.getFormat() != dst.getFormat()) {
		throw new IllegalArgumentException("Src and dst images should have the same format");
		}
		if (src.isOpaque() \|\| dst.isOpaque()) {
		throw new IllegalArgumentException("Opaque image is not copyable");
		}
		if (!(dst.getOwner() instanceof ImageWriter)) {
		throw new IllegalArgumentException("Destination image is not from ImageWriter. Only"
		+ " the images from ImageWriter are writable");
		}
		Size srcSize = new Size(src.getWidth(), src.getHeight());
		Size dstSize = new Size(dst.getWidth(), dst.getHeight());
		if (!srcSize.equals(dstSize)) {
		throw new IllegalArgumentException("source image size " + srcSize + " is different"
		+ " with " + "destination image size " + dstSize);
		}

		Plane[] srcPlanes = src.getPlanes();
		Plane[] dstPlanes = dst.getPlanes();
		ByteBuffer srcBuffer = null;
		ByteBuffer dstBuffer = null;
		for (int i = 0; i < srcPlanes.length; i++) {
		srcBuffer = srcPlanes[i].getBuffer();
		int srcPos = srcBuffer.position();
		srcBuffer.rewind();
		dstBuffer = dstPlanes[i].getBuffer();
		dstBuffer.rewind();
		dstBuffer.put(srcBuffer);
		srcBuffer.position(srcPos);
		dstBuffer.rewind();
		}
		}
		}