ImageReader/Writer: add usage flag support (916d8ac6) · Commits · e / os / android_frameworks_base

media/java/android/media/ImageReader.java

+128 −5

Original line number	Diff line number	Diff line
		@@ -17,6 +17,8 @@
		package android.media;

		import android.graphics.ImageFormat;
		import android.graphics.PixelFormat;
		import android.hardware.HardwareBuffer;
		import android.os.Handler;
		import android.os.Looper;
		import android.os.Message;
		@@ -71,6 +73,12 @@ public class ImageReader implements AutoCloseable {
		*/
		private static final int ACQUIRE_MAX_IMAGES = 2;

		/**
		* Invalid consumer buffer usage flag. This usage flag will be ignored
		* by the {@code ImageReader} instance is constructed with this value.
		*/
		private static final long BUFFER_USAGE_UNKNOWN = 0;

		/**
		* <p>
		* Create a new reader for images of the desired size and format.
		@@ -121,13 +129,104 @@ public class ImageReader implements AutoCloseable {
		* @see Image
		*/
		public static ImageReader newInstance(int width, int height, int format, int maxImages) {
		return new ImageReader(width, height, format, maxImages);
		return new ImageReader(width, height, format, maxImages, BUFFER_USAGE_UNKNOWN);
		}

		/**
		* <p>
		* Create a new reader for images of the desired size, format and consumer usage flag.
		* </p>
		* <p>
		* The {@code maxImages} parameter determines the maximum number of {@link Image} objects that
		* can be be acquired from the {@code ImageReader} simultaneously. Requesting more buffers will
		* use up more memory, so it is important to use only the minimum number necessary for the use
		* case.
		* </p>
		* <p>
		* The valid sizes and formats depend on the source of the image data.
		* </p>
		* <p>
		* The format and usage flag combination describes how the buffer will be used by
		* consumer end-points. For example, if the application intends to send the images to
		* {@link android.media.MediaCodec} or {@link android.media.MediaRecorder} for hardware video
		* encoding, the format and usage flag combination needs to be
		* {@link ImageFormat#PRIVATE PRIVATE} and {@link HardwareBuffer#USAGE0_VIDEO_ENCODE}. When an
		* {@link ImageReader} object is created with a valid size and such format/usage flag
		* combination, the application can send the {@link Image images} to an {@link ImageWriter} that
		* is created with the input {@link android.view.Surface} provided by the
		* {@link android.media.MediaCodec} or {@link android.media.MediaRecorder}.
		* </p>
		* <p>
		* If the {@code format} is {@link ImageFormat#PRIVATE PRIVATE}, the created {@link ImageReader}
		* will produce images that are not directly accessible by the application. The application can
		* still acquire images from this {@link ImageReader}, and send them to the
		* {@link android.hardware.camera2.CameraDevice camera} for reprocessing, or to the
		* {@link android.media.MediaCodec} / {@link android.media.MediaRecorder} for hardware video
		* encoding via {@link ImageWriter} interface. However, the {@link Image#getPlanes()
		* getPlanes()} will return an empty array for {@link ImageFormat#PRIVATE PRIVATE} format
		* images. The application can check if an existing reader's format by calling
		* {@link #getImageFormat()}.
		* </p>
		* <p>
		* {@link ImageFormat#PRIVATE PRIVATE} format {@link ImageReader ImageReaders} are more
		* efficient to use when application access to image data is not necessary, compared to
		* ImageReaders using other format such as {@link ImageFormat#YUV_420_888 YUV_420_888}.
		* </p>
		* <p>
		* Note that not all format and usage flag combination is supported by the
		* {@link ImageReader}. Below are the supported combinations by the {@link ImageReader}
		* (assuming the consumer end-points support the such image consumption, e.g., hardware video
		* encoding).
		* <table>
		* <tr>
		* <th>Format</th>
		* <th>Compatible usage flags</th>
		* </tr>
		* <tr>
		* <td>non-{@link android.graphics.ImageFormat#PRIVATE PRIVATE} formats defined by
		* {@link android.graphics.ImageFormat ImageFormat} or
		* {@link android.graphics.PixelFormat PixelFormat}</td>
		* <td>{@link HardwareBuffer#USAGE0_CPU_READ} or
		* {@link HardwareBuffer#USAGE0_CPU_READ_OFTEN}</td>
		* </tr>
		* <tr>
		* <td>{@link android.graphics.ImageFormat#PRIVATE}</td>
		* <td>{@link HardwareBuffer#USAGE0_VIDEO_ENCODE} or
		* {@link HardwareBuffer#USAGE0_GPU_SAMPLED_IMAGE}, or combined</td>
		* </tr>
		* </table>
		* Using other combinations may result in {@link IllegalArgumentException}.
		* </p>
		* @param width The default width in pixels of the Images that this reader will produce.
		* @param height The default height in pixels of the Images that this reader will produce.
		* @param format The format of the Image that this reader will produce. This must be one of the
		* {@link android.graphics.ImageFormat} or {@link android.graphics.PixelFormat}
		* constants. Note that not all formats are supported, like ImageFormat.NV21.
		* @param maxImages The maximum number of images the user will want to access simultaneously.
		* This should be as small as possible to limit memory use. Once maxImages Images are
		* obtained by the user, one of them has to be released before a new Image will
		* become available for access through {@link #acquireLatestImage()} or
		* {@link #acquireNextImage()}. Must be greater than 0.
		* @param usage The intended usage of the images produced by this ImageReader. It needs
		* to be one of the Usage0 defined by {@link HardwareBuffer}, or an
		* {@link IllegalArgumentException} will be thrown.
		* @see Image
		* @see HardwareBuffer
		* @hide
		*/
		protected ImageReader(int width, int height, int format, int maxImages) {
		public static ImageReader newInstance(int width, int height, int format, int maxImages,
		long usage) {
		if (!isFormatUsageCombinationAllowed(format, usage)) {
		throw new IllegalArgumentException("Format usage combination is not supported:"
		+ " format = " + format + ", usage = " + usage);
		}
		return new ImageReader(width, height, format, maxImages, usage);
		}

		/**
		* @hide
		*/
		protected ImageReader(int width, int height, int format, int maxImages, long usage) {
		mWidth = width;
		mHeight = height;
		mFormat = format;
		@@ -149,7 +248,7 @@ public class ImageReader implements AutoCloseable {

		mNumPlanes = ImageUtils.getNumPlanesForFormat(mFormat);

		nativeInit(new WeakReference<ImageReader>(this), width, height, format, maxImages);
		nativeInit(new WeakReference<>(this), width, height, format, maxImages, usage);

		mSurface = nativeGetSurface();

		@@ -617,6 +716,30 @@ public class ImageReader implements AutoCloseable {
		return si.getReader() == this;
		}

		private static boolean isFormatUsageCombinationAllowed(int format, long usage) {
		if (!ImageFormat.isPublicFormat(format) && !PixelFormat.isPublicFormat(format)) {
		return false;
		}

		// Valid usage needs to be provided.
		if (usage == BUFFER_USAGE_UNKNOWN) {
		return false;
		}

		if (format == ImageFormat.PRIVATE) {
		// Usage need to be either USAGE0_GPU_SAMPLED_IMAGE or USAGE0_VIDEO_ENCODE or combined.
		boolean isAllowed = (usage == HardwareBuffer.USAGE0_GPU_SAMPLED_IMAGE);
		isAllowed = isAllowed \|\| (usage == HardwareBuffer.USAGE0_VIDEO_ENCODE);
		isAllowed = isAllowed \|\| (usage ==
		(HardwareBuffer.USAGE0_VIDEO_ENCODE \| HardwareBuffer.USAGE0_GPU_SAMPLED_IMAGE));
		return isAllowed;
		} else {
		// Usage need to make the buffer CPU readable for explicit format.
		return ((usage == HardwareBuffer.USAGE0_CPU_READ) \|\|
		(usage == HardwareBuffer.USAGE0_CPU_READ_OFTEN));
		}
		}

		/**
		* Called from Native code when an Event happens.
		*
		@@ -655,7 +778,7 @@ public class ImageReader implements AutoCloseable {
		private ListenerHandler mListenerHandler;
		// Keep track of the successfully acquired Images. This need to be thread safe as the images
		// could be closed by different threads (e.g., application thread and GC thread).
		private List<Image> mAcquiredImages = new CopyOnWriteArrayList<Image>();
		private List<Image> mAcquiredImages = new CopyOnWriteArrayList<>();

		/**
		* This field is used by native code, do not access or modify.
		@@ -896,7 +1019,7 @@ public class ImageReader implements AutoCloseable {
		}

		private synchronized native void nativeInit(Object weakSelf, int w, int h,
		int fmt, int maxImgs);
		int fmt, int maxImgs, long consumerUsage);
		private synchronized native void nativeClose();
		private synchronized native void nativeReleaseImage(Image i);
		private synchronized native Surface nativeGetSurface();

media/java/android/media/ImageWriter.java

+60 −6

Original line number	Diff line number	Diff line
		@@ -17,6 +17,7 @@
		package android.media;

		import android.graphics.ImageFormat;
		import android.graphics.PixelFormat;
		import android.graphics.Rect;
		import android.hardware.camera2.utils.SurfaceUtils;
		import android.os.Handler;
		@@ -89,7 +90,7 @@ public class ImageWriter implements AutoCloseable {
		private final int mMaxImages;
		// Keep track of the currently dequeued Image. This need to be thread safe as the images
		// could be closed by different threads (e.g., application thread and GC thread).
		private List<Image> mDequeuedImages = new CopyOnWriteArrayList<Image>();
		private List<Image> mDequeuedImages = new CopyOnWriteArrayList<>();
		private int mEstimatedNativeAllocBytes;

		/**
		@@ -118,22 +119,75 @@ public class ImageWriter implements AutoCloseable {
		* @return a new ImageWriter instance.
		*/
		public static ImageWriter newInstance(Surface surface, int maxImages) {
		return new ImageWriter(surface, maxImages);
		return new ImageWriter(surface, maxImages, ImageFormat.UNKNOWN);
		}

		/**
		* <p>
		* Create a new ImageWriter with given number of max Images and format.
		* </p>
		* <p>
		* The {@code maxImages} parameter determines the maximum number of
		* {@link Image} objects that can be be dequeued from the
		* {@code ImageWriter} simultaneously. Requesting more buffers will use up
		* more memory, so it is important to use only the minimum number necessary.
		* </p>
		* <p>
		* The format specifies the image format of this ImageWriter. The format
		* from the {@code surface} will be overridden with this format. For example,
		* if the surface is obtained from a {@link android.graphics.SurfaceTexture}, the default
		* format may be {@link PixelFormat#RGBA_8888}. If the application creates an ImageWriter
		* with this surface and {@link ImageFormat#PRIVATE}, this ImageWriter will be able to operate
		* with {@link ImageFormat#PRIVATE} Images.
		* </p>
		* <p>
		* Note that the consumer end-point may or may not be able to support Images with different
		* format, for such case, the application should only use this method if the consumer is able
		* to consume such images.
		* </p>
		* <p>
		* The input Image size depends on the Surface that is provided by
		* the downstream consumer end-point.
		* </p>
		*
		* @param surface The destination Surface this writer produces Image data
		* into.
		* @param maxImages The maximum number of Images the user will want to
		* access simultaneously for producing Image data. This should be
		* as small as possible to limit memory use. Once maxImages
		* Images are dequeued by the user, one of them has to be queued
		* back before a new Image can be dequeued for access via
		* {@link #dequeueInputImage()}.
		* @param format The format of this ImageWriter. It can be any valid format specified by
		* {@link ImageFormat} or {@link PixelFormat}.
		*
		* @return a new ImageWriter instance.
		* @hide
		*/
		protected ImageWriter(Surface surface, int maxImages) {
		public static ImageWriter newInstance(Surface surface, int maxImages, int format) {
		if (!ImageFormat.isPublicFormat(format) && !PixelFormat.isPublicFormat(format)) {
		throw new IllegalArgumentException("Invalid format is specified: " + format);
		}
		return new ImageWriter(surface, maxImages, format);
		}

		/**
		* @hide
		*/
		protected ImageWriter(Surface surface, int maxImages, int format) {
		if (surface == null \|\| maxImages < 1) {
		throw new IllegalArgumentException("Illegal input argument: surface " + surface
		+ ", maxImages: " + maxImages);
		}

		mMaxImages = maxImages;

		if (format == ImageFormat.UNKNOWN) {
		format = SurfaceUtils.getSurfaceFormat(surface);
		}
		// Note that the underlying BufferQueue is working in synchronous mode
		// to avoid dropping any buffers.
		mNativeContext = nativeInit(new WeakReference<ImageWriter>(this), surface, maxImages);
		mNativeContext = nativeInit(new WeakReference<>(this), surface, maxImages, format);

		// Estimate the native buffer allocation size and register it so it gets accounted for
		// during GC. Note that this doesn't include the buffers required by the buffer queue
		@@ -142,7 +196,6 @@ public class ImageWriter implements AutoCloseable {
		// complex, and 1 buffer is enough for the VM to treat the ImageWriter as being of some
		// size.
		Size surfSize = SurfaceUtils.getSurfaceSize(surface);
		int format = SurfaceUtils.getSurfaceFormat(surface);
		mEstimatedNativeAllocBytes =
		ImageUtils.getEstimatedNativeAllocBytes(surfSize.getWidth(),surfSize.getHeight(),
		format, /buffer count/ 1);
		@@ -809,7 +862,8 @@ public class ImageWriter implements AutoCloseable {
		}

		// Native implemented ImageWriter methods.
		private synchronized native long nativeInit(Object weakSelf, Surface surface, int maxImgs);
		private synchronized native long nativeInit(Object weakSelf, Surface surface, int maxImgs,
		int format);

		private synchronized native void nativeClose(long nativeCtx);

media/jni/Android.mk

+1 −0

Original line number	Diff line number	Diff line
		@@ -51,6 +51,7 @@ LOCAL_SHARED_LIBRARIES := \
		libandroidfw

		LOCAL_STATIC_LIBRARIES := \
		libgrallocusage \

		LOCAL_C_INCLUDES += \
		external/libexif/ \

media/jni/android_media_ImageReader.cpp

+19 −4

Original line number	Diff line number	Diff line
		@@ -31,6 +31,8 @@

		#include <android_runtime/AndroidRuntime.h>
		#include <android_runtime/android_view_Surface.h>
		#include <android_runtime/android_hardware_HardwareBuffer.h>
		#include <grallocusage/GrallocUsageConversion.h>

		#include <jni.h>
		#include <JNIHelp.h>
		@@ -42,6 +44,7 @@
		#define ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID "mNativeBuffer"
		#define ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID "mTimestamp"

		#define CONSUMER_BUFFER_USAGE_UNKNOWN 0;
		// ----------------------------------------------------------------------------

		using namespace android;
		@@ -327,8 +330,8 @@ static void ImageReader_classInit(JNIEnv* env, jclass clazz)
		"Can not find SurfacePlane constructor");
		}

		static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz,
		jint width, jint height, jint format, jint maxImages)
		static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz, jint width, jint height,
		jint format, jint maxImages, jlong ndkUsage)
		{
		status_t res;
		int nativeFormat;
		@@ -358,17 +361,29 @@ static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz,
		width, height, format, maxImages, getpid(),
		createProcessUniqueId());
		uint32_t consumerUsage = GRALLOC_USAGE_SW_READ_OFTEN;
		bool needUsageOverride = ndkUsage != CONSUMER_BUFFER_USAGE_UNKNOWN;
		uint64_t outProducerUsage = 0;
		uint64_t outConsumerUsage = 0;
		android_hardware_HardwareBuffer_convertToGrallocUsageBits(&outProducerUsage, &outConsumerUsage,
		ndkUsage, 0);

		if (isFormatOpaque(nativeFormat)) {
		// Use the SW_READ_NEVER usage to tell producer that this format is not for preview or video
		// encoding. The only possibility will be ZSL output.
		consumerUsage = GRALLOC_USAGE_SW_READ_NEVER;
		if (needUsageOverride) {
		consumerUsage = android_convertGralloc1To0Usage(0, outConsumerUsage);
		}
		} else if (needUsageOverride) {
		ALOGW("Consumer usage override for non-opaque format is not implemented yet, "
		"ignore the provided usage from the application");
		}
		bufferConsumer = new BufferItemConsumer(gbConsumer, consumerUsage, maxImages,
		/controlledByApp/true);
		if (bufferConsumer == nullptr) {
		jniThrowExceptionFmt(env, "java/lang/RuntimeException",
		"Failed to allocate native buffer consumer for format 0x%x", nativeFormat);
		"Failed to allocate native buffer consumer for format 0x%x and usage 0x%x",
		nativeFormat, consumerUsage);
		return;
		}
		ctx->setBufferConsumer(bufferConsumer);
		@@ -788,7 +803,7 @@ static jint Image_getFormat(JNIEnv* env, jobject thiz, jint readerFormat)

		static const JNINativeMethod gImageReaderMethods[] = {
		{"nativeClassInit", "()V", (void*)ImageReader_classInit },
		{"nativeInit", "(Ljava/lang/Object;IIII)V", (void*)ImageReader_init },
		{"nativeInit", "(Ljava/lang/Object;IIIIJ)V", (void*)ImageReader_init },
		{"nativeClose", "()V", (void*)ImageReader_close },
		{"nativeReleaseImage", "(Landroid/media/Image;)V", (void*)ImageReader_imageRelease },
		{"nativeImageSetup", "(Landroid/media/Image;)I", (void*)ImageReader_imageSetup },

media/jni/android_media_ImageWriter.cpp

+19 −13

Original line number	Diff line number	Diff line
		@@ -33,7 +33,7 @@
		#include <inttypes.h>

		#define IMAGE_BUFFER_JNI_ID "mNativeBuffer"

		#define IMAGE_FORMAT_UNKNOWN 0 // This is the same value as ImageFormat#UNKNOWN.
		// ----------------------------------------------------------------------------

		using namespace android;
		@@ -222,7 +222,7 @@ static void ImageWriter_classInit(JNIEnv* env, jclass clazz) {
		}

		static jlong ImageWriter_init(JNIEnv* env, jobject thiz, jobject weakThiz, jobject jsurface,
		jint maxImages) {
		jint maxImages, jint userFormat) {
		status_t res;

		ALOGV("%s: maxImages:%d", __FUNCTION__, maxImages);
		@@ -255,7 +255,7 @@ static jlong ImageWriter_init(JNIEnv* env, jobject thiz, jobject weakThiz, jobje

		// Get the dimension and format of the producer.
		sp<ANativeWindow> anw = producer;
		int32_t width, height, format;
		int32_t width, height, surfaceFormat;
		if ((res = anw->query(anw.get(), NATIVE_WINDOW_WIDTH, &width)) != OK) {
		ALOGE("%s: Query Surface width failed: %s (%d)", __FUNCTION__, strerror(-res), res);
		jniThrowRuntimeException(env, "Failed to query Surface width");
		@@ -270,21 +270,27 @@ static jlong ImageWriter_init(JNIEnv* env, jobject thiz, jobject weakThiz, jobje
		}
		ctx->setBufferHeight(height);

		if ((res = anw->query(anw.get(), NATIVE_WINDOW_FORMAT, &format)) != OK) {
		// Query surface format if no valid user format is specified, otherwise, override surface format
		// with user format.
		if (userFormat == IMAGE_FORMAT_UNKNOWN) {
		if ((res = anw->query(anw.get(), NATIVE_WINDOW_FORMAT, &surfaceFormat)) != OK) {
		ALOGE("%s: Query Surface format failed: %s (%d)", __FUNCTION__, strerror(-res), res);
		jniThrowRuntimeException(env, "Failed to query Surface format");
		return 0;
		}
		ctx->setBufferFormat(format);
		env->SetIntField(thiz, gImageWriterClassInfo.mWriterFormat, reinterpret_cast<jint>(format));

		} else {
		surfaceFormat = userFormat;
		}
		ctx->setBufferFormat(surfaceFormat);
		env->SetIntField(thiz,
		gImageWriterClassInfo.mWriterFormat, reinterpret_cast<jint>(surfaceFormat));

		if (!isFormatOpaque(format)) {
		if (!isFormatOpaque(surfaceFormat)) {
		res = native_window_set_usage(anw.get(), GRALLOC_USAGE_SW_WRITE_OFTEN);
		if (res != OK) {
		ALOGE("%s: Configure usage %08x for format %08x failed: %s (%d)",
		__FUNCTION__, static_cast<unsigned int>(GRALLOC_USAGE_SW_WRITE_OFTEN),
		format, strerror(-res), res);
		surfaceFormat, strerror(-res), res);
		jniThrowRuntimeException(env, "Failed to SW_WRITE_OFTEN configure usage");
		return 0;
		}
		@@ -784,7 +790,7 @@ static jobjectArray Image_createSurfacePlanes(JNIEnv* env, jobject thiz,

		static JNINativeMethod gImageWriterMethods[] = {
		{"nativeClassInit", "()V", (void*)ImageWriter_classInit },
		{"nativeInit", "(Ljava/lang/Object;Landroid/view/Surface;I)J",
		{"nativeInit", "(Ljava/lang/Object;Landroid/view/Surface;II)J",
		(void*)ImageWriter_init },
		{"nativeClose", "(J)V", (void*)ImageWriter_close },
		{"nativeAttachAndQueueImage", "(JJIJIIII)I", (void*)ImageWriter_attachAndQueueImage },