ImageReader/Writer: implement opaque format operations (ce9d6f9c) · Commits · e / os / android_frameworks_base

api/current.txt

+1 −0

Original line number	Diff line number	Diff line
		@@ -14912,6 +14912,7 @@ package android.media {
		public class ImageWriter implements java.lang.AutoCloseable {
		method public void close();
		method public android.media.Image dequeueInputImage();
		method public int getFormat();
		method public int getMaxImages();
		method public static android.media.ImageWriter newInstance(android.view.Surface, int);
		method public void queueInputImage(android.media.Image);

api/system-current.txt

+1 −0

Original line number	Diff line number	Diff line
		@@ -16120,6 +16120,7 @@ package android.media {
		public class ImageWriter implements java.lang.AutoCloseable {
		method public void close();
		method public android.media.Image dequeueInputImage();
		method public int getFormat();
		method public int getMaxImages();
		method public static android.media.ImageWriter newInstance(android.view.Surface, int);
		method public void queueInputImage(android.media.Image);

media/java/android/media/Image.java

+16 −0

Original line number	Diff line number	Diff line
		@@ -249,6 +249,22 @@ public abstract class Image implements AutoCloseable {
		Object getOwner() {
		return null;
		}

		/**
		* Get native context (buffer pointer) associated with this image.
		* <p>
		* This is a package private method that is only used internally. It can be
		* used to get the native buffer pointer and passed to native, which may be
		* passed to {@link ImageWriter#attachAndQueueInputImage} to avoid a reverse
		* JNI call.
		* </p>
		*
		* @return native context associated with this Image.
		*/
		long getNativeContext() {
		return 0;
		}

		/**
		* <p>A single color plane of image data.</p>
		*

media/java/android/media/ImageReader.java

+57 −72

Original line number	Diff line number	Diff line
		@@ -98,7 +98,7 @@ public class ImageReader implements AutoCloseable {
		* @see Image
		*/
		public static ImageReader newInstance(int width, int height, int format, int maxImages) {
		if (format == PixelFormat.OPAQUE) {
		if (format == ImageFormat.PRIVATE) {
		throw new IllegalArgumentException("To obtain an opaque ImageReader, please use"
		+ " newOpaqueInstance rather than newInstance");
		}
		@@ -148,7 +148,7 @@ public class ImageReader implements AutoCloseable {
		* @see Image
		*/
		public static ImageReader newOpaqueInstance(int width, int height, int maxImages) {
		return new ImageReader(width, height, PixelFormat.OPAQUE, maxImages);
		return new ImageReader(width, height, ImageFormat.PRIVATE, maxImages);
		}

		/**
		@@ -261,7 +261,7 @@ public class ImageReader implements AutoCloseable {
		* @see ImageReader#newOpaqueInstance
		*/
		public boolean isOpaque() {
		return mFormat == PixelFormat.OPAQUE;
		return mFormat == ImageFormat.PRIVATE;
		}

		/**
		@@ -343,7 +343,7 @@ public class ImageReader implements AutoCloseable {
		* @hide
		*/
		public Image acquireNextImageNoThrowISE() {
		SurfaceImage si = new SurfaceImage();
		SurfaceImage si = new SurfaceImage(mFormat);
		return acquireNextSurfaceImage(si) == ACQUIRE_SUCCESS ? si : null;
		}

		@@ -408,7 +408,9 @@ public class ImageReader implements AutoCloseable {
		* @see #acquireLatestImage
		*/
		public Image acquireNextImage() {
		SurfaceImage si = new SurfaceImage();
		// Initialize with reader format, but can be overwritten by native if the image
		// format is different from the reader format.
		SurfaceImage si = new SurfaceImage(mFormat);
		int status = acquireNextSurfaceImage(si);

		switch (status) {
		@@ -565,9 +567,8 @@ public class ImageReader implements AutoCloseable {
		}

		SurfaceImage si = (SurfaceImage) image;
		if (!si.isImageValid()) {
		throw new IllegalStateException("Image is no longer valid");
		}
		si.throwISEIfImageIsInvalid();

		if (si.isAttachable()) {
		throw new IllegalStateException("Image was already detached from this ImageReader");
		}
		@@ -607,7 +608,7 @@ public class ImageReader implements AutoCloseable {
		case ImageFormat.DEPTH16:
		case ImageFormat.DEPTH_POINT_CLOUD:
		return 1;
		case PixelFormat.OPAQUE:
		case ImageFormat.PRIVATE:
		return 0;
		default:
		throw new UnsupportedOperationException(
		@@ -684,20 +685,17 @@ public class ImageReader implements AutoCloseable {
		}

		private class SurfaceImage extends android.media.Image {
		public SurfaceImage() {
		public SurfaceImage(int format) {
		mIsImageValid = false;
		mFormat = format;
		}

		@Override
		public void close() {
		if (mIsImageValid) {
		if (!mIsDetached.get()) {
		// For detached images, the new owner is responsible for
		// releasing the resources
		ImageReader.this.releaseImage(this);
		}
		}
		}

		public ImageReader getReader() {
		return ImageReader.this;
		@@ -705,70 +703,53 @@ public class ImageReader implements AutoCloseable {

		@Override
		public int getFormat() {
		if (mIsImageValid) {
		return ImageReader.this.mFormat;
		} else {
		throw new IllegalStateException("Image is already released");
		}
		throwISEIfImageIsInvalid();
		return mFormat;
		}

		@Override
		public int getWidth() {
		if (mIsImageValid) {
		throwISEIfImageIsInvalid();
		if (mWidth == -1) {
		mWidth = (getFormat() == ImageFormat.JPEG) ? ImageReader.this.getWidth() :
		nativeGetWidth();
		nativeGetWidth(mFormat);
		}
		return mWidth;
		} else {
		throw new IllegalStateException("Image is already released");
		}
		}

		@Override
		public int getHeight() {
		if (mIsImageValid) {
		throwISEIfImageIsInvalid();
		if (mHeight == -1) {
		mHeight = (getFormat() == ImageFormat.JPEG) ? ImageReader.this.getHeight() :
		nativeGetHeight();
		nativeGetHeight(mFormat);
		}
		return mHeight;
		} else {
		throw new IllegalStateException("Image is already released");
		}
		}

		@Override
		public long getTimestamp() {
		if (mIsImageValid) {
		throwISEIfImageIsInvalid();
		return mTimestamp;
		} else {
		throw new IllegalStateException("Image is already released");
		}
		}

		@Override
		public void setTimestamp(long timestampNs) {
		if (mIsImageValid) {
		throwISEIfImageIsInvalid();
		mTimestamp = timestampNs;
		} else {
		throw new IllegalStateException("Image is already released");
		}
		}

		@Override
		public Plane[] getPlanes() {
		if (mIsImageValid) {
		throwISEIfImageIsInvalid();
		// Shallow copy is fine.
		return mPlanes.clone();
		} else {
		throw new IllegalStateException("Image is already released");
		}
		}

		@Override
		public boolean isOpaque() {
		return mFormat == PixelFormat.OPAQUE;
		throwISEIfImageIsInvalid();
		return mFormat == ImageFormat.PRIVATE;
		}

		@Override
		@@ -782,15 +763,24 @@ public class ImageReader implements AutoCloseable {

		@Override
		boolean isAttachable() {
		throwISEIfImageIsInvalid();
		return mIsDetached.get();
		}

		@Override
		ImageReader getOwner() {
		throwISEIfImageIsInvalid();
		return ImageReader.this;
		}

		@Override
		long getNativeContext() {
		throwISEIfImageIsInvalid();
		return mNativeBuffer;
		}

		private void setDetached(boolean detached) {
		throwISEIfImageIsInvalid();
		mIsDetached.getAndSet(detached);
		}

		@@ -798,8 +788,10 @@ public class ImageReader implements AutoCloseable {
		mIsImageValid = isValid;
		}

		private boolean isImageValid() {
		return mIsImageValid;
		private void throwISEIfImageIsInvalid() {
		if (!mIsImageValid) {
		throw new IllegalStateException("Image is already closed");
		}
		}

		private void clearSurfacePlanes() {
		@@ -829,9 +821,7 @@ public class ImageReader implements AutoCloseable {

		@Override
		public ByteBuffer getBuffer() {
		if (SurfaceImage.this.isImageValid() == false) {
		throw new IllegalStateException("Image is already released");
		}
		SurfaceImage.this.throwISEIfImageIsInvalid();
		if (mBuffer != null) {
		return mBuffer;
		} else {
		@@ -845,20 +835,14 @@ public class ImageReader implements AutoCloseable {

		@Override
		public int getPixelStride() {
		if (SurfaceImage.this.isImageValid()) {
		SurfaceImage.this.throwISEIfImageIsInvalid();
		return mPixelStride;
		} else {
		throw new IllegalStateException("Image is already released");
		}
		}

		@Override
		public int getRowStride() {
		if (SurfaceImage.this.isImageValid()) {
		SurfaceImage.this.throwISEIfImageIsInvalid();
		return mRowStride;
		} else {
		throw new IllegalStateException("Image is already released");
		}
		}

		private void clearBuffer() {
		@@ -885,7 +869,7 @@ public class ImageReader implements AutoCloseable {
		* This field is used to keep track of native object and used by native code only.
		* Don't modify.
		*/
		private long mLockedBuffer;
		private long mNativeBuffer;

		/**
		* This field is set by native code during nativeImageSetup().
		@@ -896,13 +880,14 @@ public class ImageReader implements AutoCloseable {
		private boolean mIsImageValid;
		private int mHeight = -1;
		private int mWidth = -1;
		private int mFormat = ImageFormat.UNKNOWN;
		// If this image is detached from the ImageReader.
		private AtomicBoolean mIsDetached = new AtomicBoolean(false);

		private synchronized native ByteBuffer nativeImageGetBuffer(int idx, int readerFormat);
		private synchronized native SurfacePlane nativeCreatePlane(int idx, int readerFormat);
		private synchronized native int nativeGetWidth();
		private synchronized native int nativeGetHeight();
		private synchronized native int nativeGetWidth(int format);
		private synchronized native int nativeGetHeight(int format);
		}

		private synchronized native void nativeInit(Object weakSelf, int w, int h,
		@@ -910,7 +895,7 @@ public class ImageReader implements AutoCloseable {
		private synchronized native void nativeClose();
		private synchronized native void nativeReleaseImage(Image i);
		private synchronized native Surface nativeGetSurface();
		private synchronized native void nativeDetachImage(Image i);
		private synchronized native int nativeDetachImage(Image i);

		/**
		* @return A return code {@code ACQUIRE_*}

media/java/android/media/ImageWriter.java

+110 −75

Original line number	Diff line number	Diff line
		@@ -16,7 +16,7 @@

		package android.media;

		import android.graphics.PixelFormat;
		import android.graphics.ImageFormat;
		import android.graphics.Rect;
		import android.os.Handler;
		import android.os.Looper;
		@@ -77,9 +77,7 @@ public class ImageWriter implements AutoCloseable {
		private int mWriterFormat;

		private final int mMaxImages;
		// Keep track of the currently attached Image; or an attached Image that is
		// released will be removed from this list.
		private List<Image> mAttachedImages = new ArrayList<Image>();
		// Keep track of the currently dequeued Image.
		private List<Image> mDequeuedImages = new ArrayList<Image>();

		/**
		@@ -168,21 +166,38 @@ public class ImageWriter implements AutoCloseable {
		* {@link Image#close()}.
		* </p>
		* <p>
		* This call will block if all available input images have been filled by
		* This call will block if all available input images have been queued by
		* the application and the downstream consumer has not yet consumed any.
		* When an Image is consumed by the downstream consumer, an
		* When an Image is consumed by the downstream consumer and released, an
		* {@link ImageListener#onInputImageReleased} callback will be fired, which
		* indicates that there is one input Image available. It is recommended to
		* dequeue next Image only after this callback is fired, in the steady state.
		* indicates that there is one input Image available. For non-opaque formats
		* (({@link ImageWriter#getFormat()} != {@link ImageFormat#PRIVATE})), it is
		* recommended to dequeue the next Image only after this callback is fired,
		* in the steady state.
		* </p>
		* <p>
		* If the ImageWriter is opaque ({@link ImageWriter#getFormat()} ==
		* {@link ImageFormat#PRIVATE}), the image buffer is inaccessible to
		* the application, and calling this method will result in an
		* {@link IllegalStateException}. Instead, the application should acquire
		* opaque images from some other component (e.g. an opaque
		* {@link ImageReader}), and queue them directly to this ImageWriter via the
		* {@link ImageWriter#queueInputImage queueInputImage()} method.
		* </p>
		*
		* @return The next available input Image from this ImageWriter.
		* @throws IllegalStateException if {@code maxImages} Images are currently
		* dequeued.
		* dequeued, or the ImageWriter is opaque.
		* @see #queueInputImage
		* @see Image#close
		*/
		public Image dequeueInputImage() {
		if (mWriterFormat == ImageFormat.PRIVATE) {
		throw new IllegalStateException(
		"Opaque ImageWriter doesn't support this operation since opaque images are"
		+ " inaccessible to the application!");
		}

		if (mDequeuedImages.size() >= mMaxImages) {
		throw new IllegalStateException("Already dequeued max number of Images " + mMaxImages);
		}
		@@ -214,12 +229,19 @@ public class ImageWriter implements AutoCloseable {
		* capture time.
		* </p>
		* <p>
		* Passing in a non-opaque Image may result in a memory copy, which also
		* requires a free input Image from this ImageWriter as the destination. In
		* this case, this call will block, as {@link #dequeueInputImage} does, if
		* there are no free Images available. To be safe, the application should ensure
		* that there is at least one free Image available in this ImageWriter before calling
		* this method.
		* After this method is called and the downstream consumer consumes and
		* releases the Image, an {@link ImageListener#onInputImageReleased
		* onInputImageReleased()} callback will fire. The application can use this
		* callback to avoid sending Images faster than the downstream consumer
		* processing rate in steady state.
		* </p>
		* <p>
		* Passing in an Image from some other component (e.g. an
		* {@link ImageReader}) requires a free input Image from this ImageWriter as
		* the destination. In this case, this call will block, as
		* {@link #dequeueInputImage} does, if there are no free Images available.
		* To avoid blocking, the application should ensure that there is at least
		* one free Image available in this ImageWriter before calling this method.
		* </p>
		* <p>
		* After this call, the input Image is no longer valid for further access,
		@@ -252,10 +274,17 @@ public class ImageWriter implements AutoCloseable {
		ImageReader prevOwner = (ImageReader) image.getOwner();
		// Only do the image attach for opaque images for now. Do the image
		// copy for other formats. TODO: use attach for other formats to
		// improve the performance, and fall back to copy when attach/detach fails.
		// improve the performance, and fall back to copy when attach/detach
		// fails. Right now, detach is guaranteed to fail as the buffer is
		// locked when ImageReader#acquireNextImage is called. See bug 19962027.
		if (image.isOpaque()) {
		prevOwner.detachImage(image);
		attachInputImage(image);
		attachAndQueueInputImage(image);
		// This clears the native reference held by the original owner.
		// When this Image is detached later by this ImageWriter, the
		// native memory won't be leaked.
		image.close();
		return;
		} else {
		Image inputImage = dequeueInputImage();
		inputImage.setTimestamp(image.getTimestamp());
		@@ -273,24 +302,34 @@ public class ImageWriter implements AutoCloseable {

		/**
		* Only remove and cleanup the Images that are owned by this
		* ImageWriter. Images detached from other owners are only
		* temporarily owned by this ImageWriter and will be detached immediately
		* after they are released by downstream consumers, so there is no need to
		* keep track of them in mDequeuedImages.
		* ImageWriter. Images detached from other owners are only temporarily
		* owned by this ImageWriter and will be detached immediately after they
		* are released by downstream consumers, so there is no need to keep
		* track of them in mDequeuedImages.
		*/
		if (ownedByMe) {
		mDequeuedImages.remove(image);
		// Do not call close here, as close is essentially cancel image.
		WriterSurfaceImage wi = (WriterSurfaceImage) image;
		wi.clearSurfacePlanes();
		wi.setImageValid(false);
		} else {
		// This clears the native reference held by the original owner. When
		// this Image is detached later by this ImageWriter, the native
		// memory won't be leaked.
		image.close();
		}
		}

		/**
		* Get the ImageWriter format.
		* <p>
		* This format may be different than the Image format returned by
		* {@link Image#getFormat()}. However, if the ImageWriter is opaque (format
		* == {@link ImageFormat#PRIVATE}) , the images from it will also be opaque.
		* </p>
		*
		* @return The ImageWriter format.
		*/
		public int getFormat() {
		return mWriterFormat;
		}

		/**
		* ImageWriter callback interface, used to to asynchronously notify the
		* application of various ImageWriter events.
		@@ -302,27 +341,33 @@ public class ImageWriter implements AutoCloseable {
		* ImageWriter after the data consumption.
		* </p>
		* <p>
		* The client can use this callback to indicate either an input Image is
		* available to fill data into, or the input Image is returned and freed
		* if it was attached from other components (e.g. an
		* {@link ImageReader}). For the latter case, the ownership of the Image
		* will be automatically removed by ImageWriter right before this
		* callback is fired.
		* The client can use this callback to be notified that an input Image
		* has been consumed and released by the downstream consumer. More
		* specifically, this callback will be fired for below cases:
		* <li>The application dequeues an input Image via the
		* {@link ImageWriter#dequeueInputImage dequeueInputImage()} method,
		* uses it, and then queues it back to this ImageWriter via the
		* {@link ImageWriter#queueInputImage queueInputImage()} method. After
		* the downstream consumer uses and releases this image to this
		* ImageWriter, this callback will be fired. This image will be
		* available to be dequeued after this callback.</li>
		* <li>The application obtains an Image from some other component (e.g.
		* an {@link ImageReader}), uses it, and then queues it to this
		* ImageWriter via {@link ImageWriter#queueInputImage queueInputImage()}.
		* After the downstream consumer uses and releases this image to this
		* ImageWriter, this callback will be fired.</li>
		* </p>
		*
		* @param writer the ImageWriter the callback is associated with.
		* @see ImageWriter
		* @see Image
		*/
		// TODO: the semantics is confusion, does't tell which buffer is
		// released if an application is doing queueInputImage with a mix of
		// buffers from dequeueInputImage and from an ImageReader. see b/19872821
		void onInputImageReleased(ImageWriter writer);
		}

		/**
		* Register a listener to be invoked when an input Image is returned to
		* the ImageWriter.
		* Register a listener to be invoked when an input Image is returned to the
		* ImageWriter.
		*
		* @param listener The listener that will be run.
		* @param handler The handler on which the listener should be invoked, or
		@@ -383,38 +428,25 @@ public class ImageWriter implements AutoCloseable {
		}

		/**
		* Get the ImageWriter format.
		* <p>
		* This format may be different than the Image format returned by
		* {@link Image#getFormat()}
		* Attach and queue input Image to this ImageWriter.
		* </p>
		*
		* @return The ImageWriter format.
		*/
		int getFormat() {
		return mWriterFormat;
		}


		/**
		* <p>
		* Attach input Image to this ImageWriter.
		* </p>
		* <p>
		* When an Image is from an opaque source (e.g. an opaque ImageReader created
		* by {@link ImageReader#newOpaqueInstance}), or the source Image is so large
		* that copying its data is too expensive, this method can be used to
		* migrate the source Image into ImageWriter without a data copy. The source
		* Image must be detached from its previous owner already, or this call will
		* throw an {@link IllegalStateException}.
		* When an Image is from an opaque source (e.g. an opaque ImageReader
		* created by {@link ImageReader#newOpaqueInstance}), or the source Image is
		* so large that copying its data is too expensive, this method can be used
		* to migrate the source Image into ImageWriter without a data copy, and
		* then queue it to this ImageWriter. The source Image must be detached from
		* its previous owner already, or this call will throw an
		* {@link IllegalStateException}.
		* </p>
		* <p>
		* After this call, the ImageWriter takes ownership of this Image.
		* This ownership will be automatically removed from this writer after the
		* After this call, the ImageWriter takes ownership of this Image. This
		* ownership will automatically be removed from this writer after the
		* consumer releases this Image, that is, after
		* {@link ImageListener#onInputImageReleased}. The caller is
		* responsible for closing this Image through {@link Image#close()} to free up
		* the resources held by this Image.
		* {@link ImageListener#onInputImageReleased}. The caller is responsible for
		* closing this Image through {@link Image#close()} to free up the resources
		* held by this Image.
		* </p>
		*
		* @param image The source Image to be attached and queued into this
		@@ -423,7 +455,7 @@ public class ImageWriter implements AutoCloseable {
		* previous owner, or the Image is already attached to this
		* ImageWriter, or the source Image is invalid.
		*/
		private void attachInputImage(Image image) {
		private void attachAndQueueInputImage(Image image) {
		if (image == null) {
		throw new IllegalArgumentException("image shouldn't be null");
		}
		@@ -441,15 +473,13 @@ public class ImageWriter implements AutoCloseable {
		throw new IllegalStateException("Image was not detached from last owner, or image "
		+ " is not detachable");
		}
		if (mAttachedImages.contains(image)) {
		throw new IllegalStateException("Image was already attached to ImageWritter");
		}

		// TODO: what if attach failed, throw RTE or detach a slot then attach?
		// need do some cleanup to make sure no orphaned
		// buffer caused leak.
		nativeAttachImage(mNativeContext, image);
		mAttachedImages.add(image);
		Rect crop = image.getCropRect();
		nativeAttachAndQueueImage(mNativeContext, image.getNativeContext(), image.getFormat(),
		image.getTimestamp(), crop.left, crop.top, crop.right, crop.bottom);
		}

		/**
		@@ -467,8 +497,6 @@ public class ImageWriter implements AutoCloseable {
		synchronized (mListenerLock) {
		listener = mListener;
		}
		// TODO: detach Image from ImageWriter and remove the Image from
		// mAttachedImage list.
		if (listener != null) {
		listener.onInputImageReleased(ImageWriter.this);
		}
		@@ -635,7 +663,7 @@ public class ImageWriter implements AutoCloseable {
		throw new IllegalStateException("Image is already released");
		}

		return getFormat() == PixelFormat.OPAQUE;
		return getFormat() == ImageFormat.PRIVATE;
		}

		@Override
		@@ -667,6 +695,11 @@ public class ImageWriter implements AutoCloseable {
		return mOwner;
		}

		@Override
		long getNativeContext() {
		return mNativeBuffer;
		}

		@Override
		public void close() {
		if (mIsImageValid.get()) {
		@@ -776,13 +809,15 @@ public class ImageWriter implements AutoCloseable {

		private synchronized native void nativeClose(long nativeCtx);

		private synchronized native void nativeAttachImage(long nativeCtx, Image image);

		private synchronized native void nativeDequeueInputImage(long nativeCtx, Image wi);

		private synchronized native void nativeQueueInputImage(long nativeCtx, Image image,
		long timestampNs, int left, int top, int right, int bottom);

		private synchronized native int nativeAttachAndQueueImage(long nativeCtx,
		long imageNativeBuffer, int imageFormat, long timestampNs, int left,
		int top, int right, int bottom);

		private synchronized native void cancelImage(long nativeCtx, Image image);

		/**