Adding MediaCodec APIs for Large audio frames (206270e9) · Commits · e / os / android_frameworks_base

core/api/current.txt

+2 −0

Original line number	Diff line number	Diff line
		@@ -22573,6 +22573,7 @@ package android.media {
		method @NonNull public java.util.List<java.lang.String> getSupportedVendorParameters();
		method @Nullable public static android.media.Image mapHardwareBuffer(@NonNull android.hardware.HardwareBuffer);
		method public void queueInputBuffer(int, int, int, long, int) throws android.media.MediaCodec.CryptoException;
		method @FlaggedApi("com.android.media.codec.flags.large_audio_frame") public void queueInputBuffers(int, @NonNull java.util.ArrayDeque<android.media.MediaCodec.BufferInfo>);
		method public void queueSecureInputBuffer(int, int, @NonNull android.media.MediaCodec.CryptoInfo, long, int) throws android.media.MediaCodec.CryptoException;
		method public void release();
		method public void releaseOutputBuffer(int, boolean);
		@@ -22634,6 +22635,7 @@ package android.media {
		method public abstract void onError(@NonNull android.media.MediaCodec, @NonNull android.media.MediaCodec.CodecException);
		method public abstract void onInputBufferAvailable(@NonNull android.media.MediaCodec, int);
		method public abstract void onOutputBufferAvailable(@NonNull android.media.MediaCodec, int, @NonNull android.media.MediaCodec.BufferInfo);
		method @FlaggedApi("com.android.media.codec.flags.large_audio_frame") public void onOutputBuffersAvailable(@NonNull android.media.MediaCodec, int, @NonNull java.util.ArrayDeque<android.media.MediaCodec.BufferInfo>);
		method public abstract void onOutputFormatChanged(@NonNull android.media.MediaCodec, @NonNull android.media.MediaFormat);
		}

media/java/android/media/MediaCodec.java

+137 −0

Original line number	Diff line number	Diff line
		@@ -17,6 +17,7 @@
		package android.media;

		import android.Manifest;
		import android.annotation.FlaggedApi;
		import android.annotation.IntDef;
		import android.annotation.NonNull;
		import android.annotation.Nullable;
		@@ -43,21 +44,25 @@ import java.lang.annotation.RetentionPolicy;
		import java.nio.ByteBuffer;
		import java.nio.ByteOrder;
		import java.nio.ReadOnlyBufferException;
		import java.util.ArrayDeque;
		import java.util.ArrayList;
		import java.util.Arrays;
		import java.util.BitSet;
		import java.util.Collections;
		import java.util.HashMap;
		import java.util.HashSet;
		import java.util.Iterator;
		import java.util.List;
		import java.util.Map;
		import java.util.Objects;
		import java.util.Optional;
		import java.util.Set;
		import java.util.concurrent.BlockingQueue;
		import java.util.concurrent.LinkedBlockingQueue;
		import java.util.concurrent.locks.Lock;
		import java.util.concurrent.locks.ReentrantLock;

		import static com.android.media.codec.flags.Flags.FLAG_LARGE_AUDIO_FRAME;
		/**
		MediaCodec class can be used to access low-level media codecs, i.e. encoder/decoder components.
		It is part of the Android low-level multimedia support infrastructure (normally used together
		@@ -1824,6 +1829,7 @@ final public class MediaCodec {
		private static final String EOS_AND_DECODE_ONLY_ERROR_MESSAGE = "An input buffer cannot have "
		+ "both BUFFER_FLAG_END_OF_STREAM and BUFFER_FLAG_DECODE_ONLY flags";
		private static final int CB_CRYPTO_ERROR = 6;
		private static final int CB_LARGE_FRAME_OUTPUT_AVAILABLE = 7;

		private class EventHandler extends Handler {
		private MediaCodec mCodec;
		@@ -1945,6 +1951,29 @@ final public class MediaCodec {
		break;
		}

		case CB_LARGE_FRAME_OUTPUT_AVAILABLE:
		{
		int index = msg.arg2;
		ArrayDeque<BufferInfo> infos = (ArrayDeque<BufferInfo>)msg.obj;
		synchronized(mBufferLock) {
		switch (mBufferMode) {
		case BUFFER_MODE_LEGACY:
		validateOutputByteBuffersLocked(mCachedOutputBuffers,
		index, infos);
		break;
		case BUFFER_MODE_BLOCK:
		// TODO
		default:
		throw new IllegalArgumentException(
		"Unrecognized buffer mode: for large frame audio");
		}
		}
		mCallback.onOutputBuffersAvailable(
		mCodec, index, infos);

		break;
		}

		case CB_ERROR:
		{
		mCallback.onError(mCodec, (MediaCodec.CodecException) msg.obj);
		@@ -2836,11 +2865,72 @@ final public class MediaCodec {
		}
		}

		/**
		* Submit multiple access units to the codec along with multiple
		* {@link MediaCodec.BufferInfo} describing the contents of the buffer. This method
		* is supported only in asynchronous mode. While this method can be used for all codecs,
		* it is meant for buffer batching, which is only supported by codecs that advertise
		* FEATURE_MultipleFrames. Other codecs will not output large output buffers via
		* onOutputBuffersAvailable, and instead will output single-access-unit output via
		* onOutputBufferAvailable.
		* <p>
		* Output buffer size can be configured using the following MediaFormat keys.
		* {@link MediaFormat#KEY_BUFFER_BATCH_MAX_OUTPUT_SIZE} and
		* {@link MediaFormat#KEY_BUFFER_BATCH_THRESHOLD_OUTPUT_SIZE}.
		* Details for each access unit present in the buffer should be described using
		* {@link MediaCodec.BufferInfo}. Access units must be laid out contiguously (without any gaps)
		* and in order. Multiple access units in the output if present, will be available in
		* {@link Callback#onOutputBuffersAvailable} or {@link Callback#onOutputBufferAvailable}
		* in case of single-access-unit output or when output does not contain any buffers,
		* such as flags.
		* <p>
		* All other details for populating {@link MediaCodec.BufferInfo} is the same as described in
		* {@link #queueInputBuffer}.
		*
		* @param index The index of a client-owned input buffer previously returned
		* in a call to {@link #dequeueInputBuffer}.
		* @param bufferInfos ArrayDeque of {@link MediaCodec.BufferInfo} that describes the
		* contents in the buffer. The ArrayDeque and the BufferInfo objects provided
		* can be recycled by the caller for re-use.
		* @throws IllegalStateException if not in the Executing state or not in asynchronous mode.
		* @throws MediaCodec.CodecException upon codec error.
		* @throws IllegalArgumentException upon if bufferInfos is empty, contains null, or if the
		* access units are not contiguous.
		* @throws CryptoException if a crypto object has been specified in
		* {@link #configure}
		*/
		@FlaggedApi(FLAG_LARGE_AUDIO_FRAME)
		public final void queueInputBuffers(
		int index,
		@NonNull ArrayDeque<BufferInfo> bufferInfos) {
		synchronized(mBufferLock) {
		if (mBufferMode == BUFFER_MODE_BLOCK) {
		throw new IncompatibleWithBlockModelException("queueInputBuffers() "
		+ "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. "
		+ "Please use getQueueRequest() to queue buffers");
		}
		invalidateByteBufferLocked(mCachedInputBuffers, index, true /* input */);
		mDequeuedInputBuffers.remove(index);
		}
		try {
		native_queueInputBuffers(
		index, bufferInfos.toArray());
		} catch (CryptoException \| IllegalStateException \| IllegalArgumentException e) {
		revalidateByteBuffer(mCachedInputBuffers, index, true /* input */);
		throw e;
		}
		}

		private native final void native_queueInputBuffer(
		int index,
		int offset, int size, long presentationTimeUs, int flags)
		throws CryptoException;

		private native final void native_queueInputBuffers(
		int index,
		@NonNull Object[] infos)
		throws CryptoException, CodecException;

		public static final int CRYPTO_MODE_UNENCRYPTED = 0;
		public static final int CRYPTO_MODE_AES_CTR = 1;
		public static final int CRYPTO_MODE_AES_CBC = 2;
		@@ -4048,6 +4138,27 @@ final public class MediaCodec {
		}
		}

		private void validateOutputByteBuffersLocked(
		@Nullable ByteBuffer[] buffers, int index, @NonNull ArrayDeque<BufferInfo> infoDeque) {
		Optional<BufferInfo> minInfo = infoDeque.stream().min(
		(info1, info2) -> Integer.compare(info1.offset, info2.offset));
		Optional<BufferInfo> maxInfo = infoDeque.stream().max(
		(info1, info2) -> Integer.compare(info1.offset, info2.offset));
		if (buffers == null) {
		if (index >= 0) {
		mValidOutputIndices.set(index);
		}
		} else if (index >= 0 && index < buffers.length) {
		ByteBuffer buffer = buffers[index];
		if (buffer != null && minInfo.isPresent() && maxInfo.isPresent()) {
		buffer.setAccessible(true);
		buffer.limit(maxInfo.get().offset + maxInfo.get().size);
		buffer.position(minInfo.get().offset);
		}
		}

		}

		private void validateOutputByteBufferLocked(
		@Nullable ByteBuffer[] buffers, int index, @NonNull BufferInfo info) {
		if (buffers == null) {
		@@ -5169,6 +5280,32 @@ final public class MediaCodec {
		public abstract void onOutputBufferAvailable(
		@NonNull MediaCodec codec, int index, @NonNull BufferInfo info);

		/**
		* Called when multiple access-units are available in the output.
		*
		* @param codec The MediaCodec object.
		* @param index The index of the available output buffer.
		* @param infos Infos describing the available output buffer {@link MediaCodec.BufferInfo}.
		* Access units present in the output buffer are laid out contiguously
		* without gaps and in order.
		*/
		@FlaggedApi(FLAG_LARGE_AUDIO_FRAME)
		public void onOutputBuffersAvailable(
		@NonNull MediaCodec codec, int index, @NonNull ArrayDeque<BufferInfo> infos) {
		/*
		* This callback returns multiple BufferInfos when codecs are configured to operate on
		* large audio frame. Since at this point, we have a single large buffer, returning
		* each BufferInfo using
		* {@link Callback#onOutputBufferAvailable onOutputBufferAvailable} may cause the
		* index to be released to the codec using {@link MediaCodec#releaseOutputBuffer}
		* before all BuffersInfos can be returned to the client.
		* Hence this callback is required to be implemented or else an exception is thrown.
		*/
		throw new IllegalStateException(
		"Client must override onOutputBuffersAvailable when codec is " +
		"configured to operate with multiple access units");
		}

		/**
		* Called when the MediaCodec encountered an error
		*

media/jni/android_media_MediaCodec.cpp

+180 −0

Original line number	Diff line number	Diff line
		@@ -160,6 +160,13 @@ static struct {
		jmethodID addId;
		} gArrayListInfo;

		static struct {
		jclass clazz;
		jmethodID ctorId;
		jmethodID sizeId;
		jmethodID addId;
		} gArrayDequeInfo;

		static struct {
		jclass clazz;
		jmethodID ctorId;
		@@ -202,6 +209,11 @@ struct fields_t {
		jfieldID outputFrameHardwareBufferID;
		jfieldID outputFrameChangedKeysID;
		jfieldID outputFrameFormatID;
		jfieldID bufferInfoFlags;
		jfieldID bufferInfoOffset;
		jfieldID bufferInfoSize;
		jfieldID bufferInfoPresentationTimeUs;

		};

		static fields_t gFields;
		@@ -412,6 +424,22 @@ status_t JMediaCodec::queueInputBuffer(
		index, offset, size, timeUs, flags, errorDetailMsg);
		}

		status_t JMediaCodec::queueInputBuffers(
		size_t index,
		size_t offset,
		size_t size,
		const sp<RefBase> &infos,
		AString *errorDetailMsg) {

		sp<BufferInfosWrapper> auInfo((BufferInfosWrapper *)infos.get());
		return mCodec->queueInputBuffers(
		index,
		offset,
		size,
		auInfo,
		errorDetailMsg);
		}

		status_t JMediaCodec::queueSecureInputBuffer(
		size_t index,
		size_t offset,
		@@ -1250,6 +1278,7 @@ static jthrowable createCryptoException(JNIEnv *env, status_t err,
		void JMediaCodec::handleCallback(const sp<AMessage> &msg) {
		int32_t arg1, arg2 = 0;
		jobject obj = NULL;
		std::vector<jobject> jObjectInfos;
		CHECK(msg->findInt32("callbackID", &arg1));
		JNIEnv *env = AndroidRuntime::getJNIEnv();

		@@ -1287,6 +1316,35 @@ void JMediaCodec::handleCallback(const sp<AMessage> &msg) {
		break;
		}

		case MediaCodec::CB_LARGE_FRAME_OUTPUT_AVAILABLE:
		{
		sp<RefBase> spobj = nullptr;
		CHECK(msg->findInt32("index", &arg2));
		CHECK(msg->findObject("accessUnitInfo", &spobj));
		if (spobj != nullptr) {
		sp<BufferInfosWrapper> bufferInfoParamsWrapper {
		(BufferInfosWrapper *)spobj.get()};
		std::vector<AccessUnitInfo> &bufferInfoParams =
		bufferInfoParamsWrapper.get()->value;
		obj = env->NewObject(gArrayDequeInfo.clazz, gArrayDequeInfo.ctorId);
		jint offset = 0;
		for (int i = 0 ; i < bufferInfoParams.size(); i++) {
		jobject bufferInfo = env->NewObject(gBufferInfo.clazz, gBufferInfo.ctorId);
		if (bufferInfo != NULL) {
		env->CallVoidMethod(bufferInfo, gBufferInfo.setId,
		offset,
		(jint)(bufferInfoParams)[i].mSize,
		(bufferInfoParams)[i].mTimestamp,
		(bufferInfoParams)[i].mFlags);
		(void)env->CallBooleanMethod(obj, gArrayDequeInfo.addId, bufferInfo);
		offset += (bufferInfoParams)[i].mSize;
		jObjectInfos.push_back(bufferInfo);
		}
		}
		}
		break;
		}

		case MediaCodec::CB_CRYPTO_ERROR:
		{
		int32_t err, actionCode;
		@@ -1346,6 +1404,9 @@ void JMediaCodec::handleCallback(const sp<AMessage> &msg) {
		arg2,
		obj);

		for (int i = 0; i < jObjectInfos.size(); i++) {
		env->DeleteLocalRef(jObjectInfos[i]);
		}
		env->DeleteLocalRef(obj);
		}

		@@ -1913,6 +1974,103 @@ static void android_media_MediaCodec_queueInputBuffer(
		codec->getExceptionMessage(errorDetailMsg.c_str()).c_str());
		}

		static status_t extractInfosFromObject(
		JNIEnv * const env,
		jint * const initialOffset,
		jint * const totalSize,
		std::vector<AccessUnitInfo> * const infos,
		const jobjectArray &objArray,
		AString * const errorDetailMsg) {
		if (totalSize == nullptr
		\|\| initialOffset == nullptr
		\|\| infos == nullptr) {
		if (errorDetailMsg) {
		*errorDetailMsg = "Error: Null arguments provided for extracting Access unit info";
		}
		return BAD_VALUE;
		}
		const jsize numEntries = env->GetArrayLength(objArray);
		if (numEntries <= 0) {
		if (errorDetailMsg) {
		*errorDetailMsg = "Error: No BufferInfo found while queuing for large frame input";
		}
		return BAD_VALUE;
		}
		*initialOffset = 0;
		*totalSize = 0;
		for (jsize i = 0; i < numEntries; i++) {
		jobject param = env->GetObjectArrayElement(objArray, i);
		if (param == NULL) {
		if (errorDetailMsg) {
		*errorDetailMsg = "Error: Queuing a null BufferInfo";
		}
		return BAD_VALUE;
		}
		size_t offset = static_cast<size_t>(env->GetIntField(param, gFields.bufferInfoOffset));
		size_t size = static_cast<size_t>(env->GetIntField(param, gFields.bufferInfoSize));
		uint32_t flags = static_cast<uint32_t>(env->GetIntField(param, gFields.bufferInfoFlags));
		if (flags == 0 && size == 0) {
		if (errorDetailMsg) {
		*errorDetailMsg = "Error: Queuing an empty BufferInfo";
		}
		return BAD_VALUE;
		}
		if (i == 0) {
		*initialOffset = offset;
		}
		if (CC_UNLIKELY((offset > UINT32_MAX)
		\|\| ((long)(offset + size) > UINT32_MAX)
		\|\| ((offset - initialOffset) != totalSize))) {
		if (errorDetailMsg) {
		*errorDetailMsg = "Error: offset/size in BufferInfo";
		}
		return BAD_VALUE;
		}
		infos->emplace_back(
		flags,
		size,
		env->GetLongField(param, gFields.bufferInfoPresentationTimeUs));
		*totalSize += size;
		}
		return OK;
		}

		static void android_media_MediaCodec_queueInputBuffers(
		JNIEnv *env,
		jobject thiz,
		jint index,
		jobjectArray objArray) {
		ALOGV("android_media_MediaCodec_queueInputBuffers");
		sp<JMediaCodec> codec = getMediaCodec(env, thiz);
		if (codec == NULL \|\| codec->initCheck() != OK \|\| objArray == NULL) {
		throwExceptionAsNecessary(env, INVALID_OPERATION, codec);
		return;
		}
		sp<BufferInfosWrapper> infoObj =
		new BufferInfosWrapper{decltype(infoObj->value)()};
		AString errorDetailMsg;
		jint initialOffset = 0;
		jint totalSize = 0;
		status_t err = extractInfosFromObject(
		env,
		&initialOffset,
		&totalSize,
		&infoObj->value,
		objArray,
		&errorDetailMsg);
		if (err == OK) {
		err = codec->queueInputBuffers(
		index,
		initialOffset,
		totalSize,
		infoObj,
		&errorDetailMsg);
		}
		throwExceptionAsNecessary(
		env, err, ACTION_CODE_FATAL,
		codec->getExceptionMessage(errorDetailMsg.c_str()).c_str());
		}

		struct NativeCryptoInfo {
		NativeCryptoInfo(JNIEnv *env, jobject cryptoInfoObj)
		: mEnv{env},
		@@ -3401,6 +3559,19 @@ static void android_media_MediaCodec_native_init(JNIEnv *env, jclass) {
		gArrayListInfo.addId = env->GetMethodID(clazz.get(), "add", "(Ljava/lang/Object;)Z");
		CHECK(gArrayListInfo.addId != NULL);

		clazz.reset(env->FindClass("java/util/ArrayDeque"));
		CHECK(clazz.get() != NULL);
		gArrayDequeInfo.clazz = (jclass)env->NewGlobalRef(clazz.get());

		gArrayDequeInfo.ctorId = env->GetMethodID(clazz.get(), "<init>", "()V");
		CHECK(gArrayDequeInfo.ctorId != NULL);

		gArrayDequeInfo.sizeId = env->GetMethodID(clazz.get(), "size", "()I");
		CHECK(gArrayDequeInfo.sizeId != NULL);

		gArrayDequeInfo.addId = env->GetMethodID(clazz.get(), "add", "(Ljava/lang/Object;)Z");
		CHECK(gArrayDequeInfo.addId != NULL);

		clazz.reset(env->FindClass("android/media/MediaCodec$LinearBlock"));
		CHECK(clazz.get() != NULL);

		@@ -3444,6 +3615,12 @@ static void android_media_MediaCodec_native_init(JNIEnv *env, jclass) {

		gBufferInfo.setId = env->GetMethodID(clazz.get(), "set", "(IIJI)V");
		CHECK(gBufferInfo.setId != NULL);

		gFields.bufferInfoSize = env->GetFieldID(clazz.get(), "size", "I");
		gFields.bufferInfoFlags = env->GetFieldID(clazz.get(), "flags", "I");
		gFields.bufferInfoOffset = env->GetFieldID(clazz.get(), "offset", "I");
		gFields.bufferInfoPresentationTimeUs =
		env->GetFieldID(clazz.get(), "presentationTimeUs", "J");
		}

		static void android_media_MediaCodec_native_setup(
		@@ -3701,6 +3878,9 @@ static const JNINativeMethod gMethods[] = {
		{ "native_queueInputBuffer", "(IIIJI)V",
		(void *)android_media_MediaCodec_queueInputBuffer },

		{ "native_queueInputBuffers", "(I[Ljava/lang/Object;)V",
		(void *)android_media_MediaCodec_queueInputBuffers },

		{ "native_queueSecureInputBuffer", "(IILandroid/media/MediaCodec$CryptoInfo;JI)V",
		(void *)android_media_MediaCodec_queueSecureInputBuffer },

media/jni/android_media_MediaCodec.h

+8 −0

Original line number	Diff line number	Diff line
		@@ -35,6 +35,7 @@ class C2Buffer;
		namespace android {

		struct ABuffer;
		struct AccessUnitInfo;
		struct ALooper;
		struct AMessage;
		struct AString;
		@@ -93,6 +94,13 @@ struct JMediaCodec : public AHandler {
		size_t offset, size_t size, int64_t timeUs, uint32_t flags,
		AString *errorDetailMsg);

		status_t queueInputBuffers(
		size_t index,
		size_t offset,
		size_t size,
		const sp<RefBase> &auInfo,
		AString *errorDetailMsg = NULL);

		status_t queueSecureInputBuffer(
		size_t index,
		size_t offset,