Enabling large audio for BLOCK_MODEL (8d6a41a9) · Commits · e / os / android_frameworks_base

core/api/current.txt

+2 −0

Original line number	Diff line number	Diff line
		@@ -22723,6 +22723,7 @@ package android.media {
		}

		public static final class MediaCodec.OutputFrame {
		method @FlaggedApi("com.android.media.codec.flags.large_audio_frame") @NonNull public java.util.ArrayDeque<android.media.MediaCodec.BufferInfo> getBufferInfos();
		method @NonNull public java.util.Set<java.lang.String> getChangedKeys();
		method public int getFlags();
		method @NonNull public android.media.MediaFormat getFormat();
		@@ -22738,6 +22739,7 @@ package android.media {

		public final class MediaCodec.QueueRequest {
		method public void queue();
		method @FlaggedApi("com.android.media.codec.flags.large_audio_frame") @NonNull public android.media.MediaCodec.QueueRequest setBufferInfos(@NonNull java.util.ArrayDeque<android.media.MediaCodec.BufferInfo>);
		method @NonNull public android.media.MediaCodec.QueueRequest setByteBufferParameter(@NonNull String, @NonNull java.nio.ByteBuffer);
		method @NonNull public android.media.MediaCodec.QueueRequest setEncryptedLinearBlock(@NonNull android.media.MediaCodec.LinearBlock, int, int, @NonNull android.media.MediaCodec.CryptoInfo);
		method @NonNull public android.media.MediaCodec.QueueRequest setFlags(int);

media/java/android/media/MediaCodec.java

+68 −8

Original line number	Diff line number	Diff line
		@@ -1962,10 +1962,20 @@ final public class MediaCodec {
		index, infos);
		break;
		case BUFFER_MODE_BLOCK:
		// TODO
		while (mOutputFrames.size() <= index) {
		mOutputFrames.add(null);
		}
		OutputFrame frame = mOutputFrames.get(index);
		if (frame == null) {
		frame = new OutputFrame(index);
		mOutputFrames.set(index, frame);
		}
		frame.setBufferInfos(infos);
		frame.setAccessible(true);
		break;
		default:
		throw new IllegalArgumentException(
		"Unrecognized buffer mode: for large frame audio");
		"Unrecognized buffer mode: for large frame output");
		}
		}
		mCallback.onOutputBuffersAvailable(
		@@ -3551,6 +3561,26 @@ final public class MediaCodec {
		return this;
		}

		/**
		* Sets MediaCodec.BufferInfo objects describing the access units
		* contained in this queue request. Access units must be laid out
		* contiguously without gaps and in order.
		*
		* @param infos Represents {@link MediaCodec.BufferInfo} objects to mark
		* individual access-unit boundaries and the timestamps associated with it.
		* The buffer is expected to contain the data in a continuous manner.
		* @return this object
		*/
		@FlaggedApi(FLAG_LARGE_AUDIO_FRAME)
		public @NonNull QueueRequest setBufferInfos(@NonNull ArrayDeque<BufferInfo> infos) {
		if (!isAccessible()) {
		throw new IllegalStateException("The request is stale");
		}
		mBufferInfos.clear();
		mBufferInfos.addAll(infos);
		return this;
		}

		/**
		* Add an integer parameter.
		* See {@link MediaFormat} for an exhaustive list of supported keys with
		@@ -3667,10 +3697,18 @@ final public class MediaCodec {
		throw new IllegalStateException("No block is set");
		}
		setAccessible(false);
		if (mBufferInfos.isEmpty()) {
		BufferInfo info = new BufferInfo();
		info.size = mSize;
		info.offset = mOffset;
		info.presentationTimeUs = mPresentationTimeUs;
		info.flags = mFlags;
		mBufferInfos.add(info);
		}
		if (mLinearBlock != null) {
		mCodec.native_queueLinearBlock(
		mIndex, mLinearBlock, mOffset, mSize, mCryptoInfo,
		mPresentationTimeUs, mFlags,
		mIndex, mLinearBlock, mCryptoInfo,
		mBufferInfos.toArray(),
		mTuningKeys, mTuningValues);
		} else if (mHardwareBuffer != null) {
		mCodec.native_queueHardwareBuffer(
		@@ -3688,6 +3726,7 @@ final public class MediaCodec {
		mHardwareBuffer = null;
		mPresentationTimeUs = 0;
		mFlags = 0;
		mBufferInfos.clear();
		mTuningKeys.clear();
		mTuningValues.clear();
		return this;
		@@ -3711,6 +3750,7 @@ final public class MediaCodec {
		private HardwareBuffer mHardwareBuffer = null;
		private long mPresentationTimeUs = 0;
		private @BufferFlag int mFlags = 0;
		private final ArrayDeque<BufferInfo> mBufferInfos = new ArrayDeque<>();
		private final ArrayList<String> mTuningKeys = new ArrayList<>();
		private final ArrayList<Object> mTuningValues = new ArrayList<>();

		@@ -3720,11 +3760,8 @@ final public class MediaCodec {
		private native void native_queueLinearBlock(
		int index,
		@NonNull LinearBlock block,
		int offset,
		int size,
		@Nullable CryptoInfo cryptoInfo,
		long presentationTimeUs,
		int flags,
		@NonNull Object[] bufferInfos,
		@NonNull ArrayList<String> keys,
		@NonNull ArrayList<Object> values);

		@@ -4516,6 +4553,22 @@ final public class MediaCodec {
		return mFlags;
		}

		/*
		* Returns the BufferInfos associated with this OutputFrame. These BufferInfos
		* describes the access units present in the OutputFrame. Access units are laid
		* out contiguously without gaps and in order.
		*/
		@FlaggedApi(FLAG_LARGE_AUDIO_FRAME)
		public @NonNull ArrayDeque<BufferInfo> getBufferInfos() {
		if (mBufferInfos.isEmpty()) {
		// single BufferInfo could be present.
		BufferInfo bufferInfo = new BufferInfo();
		bufferInfo.set(0, 0, mPresentationTimeUs, mFlags);
		mBufferInfos.add(bufferInfo);
		}
		return mBufferInfos;
		}

		/**
		* Returns a read-only {@link MediaFormat} for this frame. The returned
		* object is valid only until the client calls {@link MediaCodec#releaseOutputBuffer}.
		@@ -4541,6 +4594,7 @@ final public class MediaCodec {
		mLinearBlock = null;
		mHardwareBuffer = null;
		mFormat = null;
		mBufferInfos.clear();
		mChangedKeys.clear();
		mKeySet.clear();
		mLoaded = false;
		@@ -4559,6 +4613,11 @@ final public class MediaCodec {
		mFlags = info.flags;
		}

		void setBufferInfos(ArrayDeque<BufferInfo> infos) {
		mBufferInfos.clear();
		mBufferInfos.addAll(infos);
		}

		boolean isLoaded() {
		return mLoaded;
		}
		@@ -4573,6 +4632,7 @@ final public class MediaCodec {
		private long mPresentationTimeUs = 0;
		private @BufferFlag int mFlags = 0;
		private MediaFormat mFormat = null;
		private final ArrayDeque<BufferInfo> mBufferInfos = new ArrayDeque<>();
		private final ArrayList<String> mChangedKeys = new ArrayList<>();
		private final Set<String> mKeySet = new HashSet<>();
		private boolean mAccessible = false;

media/jni/android_media_MediaCodec.cpp

+87 −21

Original line number	Diff line number	Diff line
		@@ -207,6 +207,7 @@ struct fields_t {
		jfieldID queueRequestIndexID;
		jfieldID outputFrameLinearBlockID;
		jfieldID outputFrameHardwareBufferID;
		jfieldID outputFramebufferInfosID;
		jfieldID outputFrameChangedKeysID;
		jfieldID outputFrameFormatID;
		jfieldID bufferInfoFlags;
		@@ -458,10 +459,11 @@ status_t JMediaCodec::queueSecureInputBuffer(
		}

		status_t JMediaCodec::queueBuffer(
		size_t index, const std::shared_ptr<C2Buffer> &buffer, int64_t timeUs,
		uint32_t flags, const sp<AMessage> &tunings, AString *errorDetailMsg) {
		size_t index, const std::shared_ptr<C2Buffer> &buffer,
		const sp<RefBase> &infos, const sp<AMessage> &tunings, AString *errorDetailMsg) {
		sp<BufferInfosWrapper> auInfo((BufferInfosWrapper *)infos.get());
		return mCodec->queueBuffer(
		index, buffer, timeUs, flags, tunings, errorDetailMsg);
		index, buffer, auInfo, tunings, errorDetailMsg);
		}

		status_t JMediaCodec::queueEncryptedLinearBlock(
		@@ -474,13 +476,13 @@ status_t JMediaCodec::queueEncryptedLinearBlock(
		const uint8_t iv[16],
		CryptoPlugin::Mode mode,
		const CryptoPlugin::Pattern &pattern,
		int64_t presentationTimeUs,
		uint32_t flags,
		const sp<RefBase> &infos,
		const sp<AMessage> &tunings,
		AString *errorDetailMsg) {
		sp<BufferInfosWrapper> auInfo((BufferInfosWrapper *)infos.get());
		return mCodec->queueEncryptedBuffer(
		index, buffer, offset, subSamples, numSubSamples, key, iv, mode, pattern,
		presentationTimeUs, flags, tunings, errorDetailMsg);
		auInfo, tunings, errorDetailMsg);
		}

		status_t JMediaCodec::dequeueInputBuffer(size_t *index, int64_t timeoutUs) {
		@@ -750,6 +752,42 @@ status_t JMediaCodec::getImage(
		return OK;
		}

		void maybeSetBufferInfos(JNIEnv *env, jobject &frame, const sp<BufferInfosWrapper> &bufInfos) {
		if (!bufInfos) {
		return;
		}
		std::vector<AccessUnitInfo> &infos = bufInfos.get()->value;
		if (infos.empty()) {
		return;
		}
		ScopedLocalRef<jobject> dequeObj{env, env->NewObject(
		gArrayDequeInfo.clazz, gArrayDequeInfo.ctorId)};
		jint offset = 0;
		std::vector<jobject> jObjectInfos;
		for (int i = 0 ; i < infos.size(); i++) {
		jobject bufferInfo = env->NewObject(
		gBufferInfo.clazz, gBufferInfo.ctorId);
		if (bufferInfo != NULL) {
		env->CallVoidMethod(bufferInfo, gBufferInfo.setId,
		offset,
		(jint)(infos)[i].mSize,
		(infos)[i].mTimestamp,
		(infos)[i].mFlags);
		(void)env->CallBooleanMethod(
		dequeObj.get(), gArrayDequeInfo.addId, bufferInfo);
		offset += (infos)[i].mSize;
		jObjectInfos.push_back(bufferInfo);
		}
		}
		env->SetObjectField(
		frame,
		gFields.outputFramebufferInfosID,
		dequeObj.get());
		for (int i = 0; i < jObjectInfos.size(); i++) {
		env->DeleteLocalRef(jObjectInfos[i]);
		}
		}

		status_t JMediaCodec::getOutputFrame(
		JNIEnv *env, jobject frame, size_t index) const {
		sp<MediaCodecBuffer> buffer;
		@@ -760,6 +798,11 @@ status_t JMediaCodec::getOutputFrame(
		}

		if (buffer->size() > 0) {
		sp<RefBase> obj;
		sp<BufferInfosWrapper> bufInfos;
		if (buffer->meta()->findObject("accessUnitInfo", &obj)) {
		bufInfos = std::move(((decltype(bufInfos.get()))obj.get()));
		}
		std::shared_ptr<C2Buffer> c2Buffer = buffer->asC2Buffer();
		if (c2Buffer) {
		switch (c2Buffer->data().type()) {
		@@ -775,6 +818,7 @@ status_t JMediaCodec::getOutputFrame(
		(jlong)context.release(),
		true);
		env->SetObjectField(frame, gFields.outputFrameLinearBlockID, linearBlock.get());
		maybeSetBufferInfos(env, frame, bufInfos);
		break;
		}
		case C2BufferData::GRAPHIC: {
		@@ -815,6 +859,7 @@ status_t JMediaCodec::getOutputFrame(
		(jlong)context.release(),
		true);
		env->SetObjectField(frame, gFields.outputFrameLinearBlockID, linearBlock.get());
		maybeSetBufferInfos(env, frame, bufInfos);
		} else {
		// No-op.
		}
		@@ -2717,8 +2762,7 @@ static void extractBufferFromContext(

		static void android_media_MediaCodec_native_queueLinearBlock(
		JNIEnv *env, jobject thiz, jint index, jobject bufferObj,
		jint offset, jint size, jobject cryptoInfoObj,
		jlong presentationTimeUs, jint flags, jobject keys, jobject values) {
		jobject cryptoInfoObj, jobjectArray objArray, jobject keys, jobject values) {
		ALOGV("android_media_MediaCodec_native_queueLinearBlock");

		sp<JMediaCodec> codec = getMediaCodec(env, thiz);
		@@ -2736,7 +2780,24 @@ static void android_media_MediaCodec_native_queueLinearBlock(
		"error occurred while converting tunings from Java to native");
		return;
		}

		jint totalSize;
		jint initialOffset;
		std::vector<AccessUnitInfo> infoVec;
		AString errorDetailMsg;
		err = extractInfosFromObject(env,
		&initialOffset,
		&totalSize,
		&infoVec,
		objArray,
		&errorDetailMsg);
		if (err != OK) {
		throwExceptionAsNecessary(
		env, INVALID_OPERATION, ACTION_CODE_FATAL,
		codec->getExceptionMessage(errorDetailMsg.c_str()).c_str());
		return;
		}
		sp<BufferInfosWrapper> infos =
		new BufferInfosWrapper{std::move(infoVec)};
		std::shared_ptr<C2Buffer> buffer;
		sp<hardware::HidlMemory> memory;
		ScopedLocalRef<jobject> lock{env, env->GetObjectField(bufferObj, gLinearBlockInfo.lockId)};
		@@ -2745,10 +2806,10 @@ static void android_media_MediaCodec_native_queueLinearBlock(
		JMediaCodecLinearBlock *context =
		(JMediaCodecLinearBlock *)env->GetLongField(bufferObj, gLinearBlockInfo.contextId);
		if (codec->hasCryptoOrDescrambler()) {
		extractMemoryFromContext(context, offset, size, &memory);
		offset += context->mHidlMemoryOffset;
		extractMemoryFromContext(context, initialOffset, totalSize, &memory);
		initialOffset += context->mHidlMemoryOffset;
		} else {
		extractBufferFromContext(context, offset, size, &buffer);
		extractBufferFromContext(context, initialOffset, totalSize, &buffer);
		}
		}
		env->MonitorExit(lock.get());
		@@ -2759,7 +2820,6 @@ static void android_media_MediaCodec_native_queueLinearBlock(
		return;
		}

		AString errorDetailMsg;
		if (codec->hasCryptoOrDescrambler()) {
		if (!memory) {
		// It means there was an unexpected failure in extractMemoryFromContext above
		@@ -2773,7 +2833,7 @@ static void android_media_MediaCodec_native_queueLinearBlock(
		return;
		}
		auto cryptoInfo =
		cryptoInfoObj ? NativeCryptoInfo{env, cryptoInfoObj} : NativeCryptoInfo{size};
		cryptoInfoObj ? NativeCryptoInfo{env, cryptoInfoObj} : NativeCryptoInfo{totalSize};
		if (env->ExceptionCheck()) {
		// Creation of cryptoInfo failed. Let the exception bubble up.
		return;
		@@ -2781,13 +2841,12 @@ static void android_media_MediaCodec_native_queueLinearBlock(
		err = codec->queueEncryptedLinearBlock(
		index,
		memory,
		offset,
		initialOffset,
		cryptoInfo.mSubSamples, cryptoInfo.mNumSubSamples,
		(const uint8_t )cryptoInfo.mKey, (const uint8_t )cryptoInfo.mIv,
		cryptoInfo.mMode,
		cryptoInfo.mPattern,
		presentationTimeUs,
		flags,
		infos,
		tunings,
		&errorDetailMsg);
		ALOGI_IF(err != OK, "queueEncryptedLinearBlock returned err = %d", err);
		@@ -2804,7 +2863,7 @@ static void android_media_MediaCodec_native_queueLinearBlock(
		return;
		}
		err = codec->queueBuffer(
		index, buffer, presentationTimeUs, flags, tunings, &errorDetailMsg);
		index, buffer, infos, tunings, &errorDetailMsg);
		}
		throwExceptionAsNecessary(
		env, err, ACTION_CODE_FATAL,
		@@ -2862,8 +2921,11 @@ static void android_media_MediaCodec_native_queueHardwareBuffer(
		std::shared_ptr<C2Buffer> buffer = C2Buffer::CreateGraphicBuffer(block->share(
		block->crop(), C2Fence{}));
		AString errorDetailMsg;
		sp<BufferInfosWrapper> infos =
		new BufferInfosWrapper{decltype(infos->value)()};
		infos->value.emplace_back(flags, 0 /not used/, presentationTimeUs);
		err = codec->queueBuffer(
		index, buffer, presentationTimeUs, flags, tunings, &errorDetailMsg);
		index, buffer, infos, tunings, &errorDetailMsg);
		throwExceptionAsNecessary(
		env, err, ACTION_CODE_FATAL,
		codec->getExceptionMessage(errorDetailMsg.c_str()).c_str());
		@@ -3372,6 +3434,10 @@ static void android_media_MediaCodec_native_init(JNIEnv *env, jclass) {
		env->GetFieldID(clazz.get(), "mLinearBlock", "Landroid/media/MediaCodec$LinearBlock;");
		CHECK(gFields.outputFrameLinearBlockID != NULL);

		gFields.outputFramebufferInfosID =
		env->GetFieldID(clazz.get(), "mBufferInfos", "Ljava/util/ArrayDeque;");
		CHECK(gFields.outputFramebufferInfosID != NULL);

		gFields.outputFrameHardwareBufferID =
		env->GetFieldID(clazz.get(), "mHardwareBuffer", "Landroid/hardware/HardwareBuffer;");
		CHECK(gFields.outputFrameHardwareBufferID != NULL);
		@@ -3891,8 +3957,8 @@ static const JNINativeMethod gMethods[] = {
		{ "native_closeMediaImage", "(J)V", (void *)android_media_MediaCodec_closeMediaImage },

		{ "native_queueLinearBlock",
		"(ILandroid/media/MediaCodec$LinearBlock;IILandroid/media/MediaCodec$CryptoInfo;JI"
		"Ljava/util/ArrayList;Ljava/util/ArrayList;)V",
		"(ILandroid/media/MediaCodec$LinearBlock;Landroid/media/MediaCodec$CryptoInfo;"
		"[Ljava/lang/Object;Ljava/util/ArrayList;Ljava/util/ArrayList;)V",
		(void *)android_media_MediaCodec_native_queueLinearBlock },

		{ "native_queueHardwareBuffer",

media/jni/android_media_MediaCodec.h

+2 −3

Original line number	Diff line number	Diff line
		@@ -116,7 +116,7 @@ struct JMediaCodec : public AHandler {

		status_t queueBuffer(
		size_t index, const std::shared_ptr<C2Buffer> &buffer,
		int64_t timeUs, uint32_t flags, const sp<AMessage> &tunings,
		const sp<RefBase> &infos, const sp<AMessage> &tunings,
		AString *errorDetailMsg);

		status_t queueEncryptedLinearBlock(
		@@ -129,8 +129,7 @@ struct JMediaCodec : public AHandler {
		const uint8_t iv[16],
		CryptoPlugin::Mode mode,
		const CryptoPlugin::Pattern &pattern,
		int64_t presentationTimeUs,
		uint32_t flags,
		const sp<RefBase> &infos,
		const sp<AMessage> &tunings,
		AString *errorDetailMsg);