Large audio frames in MediaCodec (f4a81f7f) · Commits · e / os / android_frameworks_av

media/codec2/sfplugin/CCodecBufferChannel.cpp

+74 −1

Original line number	Diff line number	Diff line
		@@ -90,6 +90,28 @@ static bool areRenderMetricsEnabled() {
		return v == "true";
		}

		// Flags can come with individual BufferInfos
		// when used with large frame audio
		constexpr static std::initializer_list<std::pair<uint32_t, uint32_t>> flagList = {
		{BUFFER_FLAG_CODEC_CONFIG, C2FrameData::FLAG_CODEC_CONFIG},
		{BUFFER_FLAG_END_OF_STREAM, C2FrameData::FLAG_END_OF_STREAM},
		{BUFFER_FLAG_DECODE_ONLY, C2FrameData::FLAG_DROP_FRAME}
		};

		static uint32_t convertFlags(uint32_t flags, bool toC2) {
		return std::transform_reduce(
		flagList.begin(), flagList.end(),
		0u,
		std::bit_or{},
		[flags, toC2](const std::pair<uint32_t, uint32_t> &entry) {
		if (toC2) {
		return (flags & entry.first) ? entry.second : 0;
		} else {
		return (flags & entry.second) ? entry.first : 0;
		}
		});
		}

		} // namespace

		CCodecBufferChannel::QueueGuard::QueueGuard(
		@@ -245,7 +267,8 @@ status_t CCodecBufferChannel::queueInputBufferInternal(
		if (buffer->meta()->findInt32("decode-only", &tmp) && tmp) {
		flags \|= C2FrameData::FLAG_DROP_FRAME;
		}
		ALOGV("[%s] queueInputBuffer: buffer->size() = %zu", mName, buffer->size());
		ALOGV("[%s] queueInputBuffer: buffer->size() = %zu time: %lld",
		mName, buffer->size(), (long long)timeUs);
		std::list<std::unique_ptr<C2Work>> items;
		std::unique_ptr<C2Work> work(new C2Work);
		work->input.ordinal.timestamp = timeUs;
		@@ -296,6 +319,34 @@ status_t CCodecBufferChannel::queueInputBufferInternal(
		uint64_t frameIndex = work->input.ordinal.frameIndex.peeku();
		output->rotation[frameIndex] = rotation;
		}
		sp<RefBase> obj;
		if (buffer->meta()->findObject("accessUnitInfo", &obj)) {
		ALOGV("Filling C2Info from multiple access units");
		sp<WrapperObject<std::vector<AccessUnitInfo>>> infos{
		(decltype(infos.get()))obj.get()};
		std::vector<AccessUnitInfo> &accessUnitInfoVec = infos->value;
		std::vector<C2AccessUnitInfosStruct> multipleAccessUnitInfos;
		uint32_t outFlags = 0;
		for (int i = 0; i < accessUnitInfoVec.size(); i++) {
		outFlags = 0;
		outFlags = convertFlags(accessUnitInfoVec[i].mFlags, true);
		if (eos && (outFlags & C2FrameData::FLAG_END_OF_STREAM)) {
		outFlags &= (~C2FrameData::FLAG_END_OF_STREAM);
		}
		multipleAccessUnitInfos.emplace_back(
		outFlags,
		accessUnitInfoVec[i].mSize,
		accessUnitInfoVec[i].mTimestamp);
		ALOGV("%d) flags: %d, size: %d, time: %llu",
		i, outFlags, accessUnitInfoVec[i].mSize,
		(long long)accessUnitInfoVec[i].mTimestamp);

		}
		const std::shared_ptr<C2AccessUnitInfos::input> c2AccessUnitInfos =
		C2AccessUnitInfos::input::AllocShared(
		multipleAccessUnitInfos.size(), 0u, multipleAccessUnitInfos);
		c2buffer->setInfo(c2AccessUnitInfos);
		}
		work->input.buffers.push_back(c2buffer);
		if (encryptedBlock) {
		work->input.infoBuffers.emplace_back(C2InfoBuffer::CreateLinearBuffer(
		@@ -2265,12 +2316,34 @@ void CCodecBufferChannel::sendOutputBuffers() {
		case OutputBuffers::DISCARD:
		break;
		case OutputBuffers::NOTIFY_CLIENT:
		{
		// TRICKY: we want popped buffers reported in order, so sending
		// the callback while holding the lock here. This assumes that
		// onOutputBufferAvailable() does not block. onOutputBufferAvailable()
		// callbacks are always sent with the Output lock held.
		if (c2Buffer) {
		std::shared_ptr<const C2AccessUnitInfos::output> bufferMetadata =
		std::static_pointer_cast<const C2AccessUnitInfos::output>(
		c2Buffer->getInfo(C2AccessUnitInfos::output::PARAM_TYPE));
		if (bufferMetadata && bufferMetadata->flexCount() > 0) {
		uint32_t flag = 0;
		std::vector<AccessUnitInfo> accessUnitInfos;
		for (int nMeta = 0; nMeta < bufferMetadata->flexCount(); nMeta++) {
		const C2AccessUnitInfosStruct &bufferMetadataStruct =
		bufferMetadata->m.values[nMeta];
		flag = convertFlags(bufferMetadataStruct.flags, false);
		accessUnitInfos.emplace_back(flag,
		static_cast<size_t>(bufferMetadataStruct.size),
		static_cast<size_t>(bufferMetadataStruct.timestamp));
		}
		sp<WrapperObject<std::vector<AccessUnitInfo>>> obj{
		new WrapperObject<std::vector<AccessUnitInfo>>{accessUnitInfos}};
		outBuffer->meta()->setObject("accessUnitInfo", obj);
		}
		}
		mCallback->onOutputBufferAvailable(index, outBuffer);
		break;
		}
		case OutputBuffers::REALLOCATE:
		if (++reallocTryNum > kMaxReallocTry) {
		output.unlock();

media/codec2/sfplugin/CCodecBuffers.cpp

+182 −2

Original line number	Diff line number	Diff line
		@@ -18,11 +18,14 @@
		#define LOG_TAG "CCodecBuffers"
		#include <utils/Log.h>

		#include <numeric>

		#include <C2AllocatorGralloc.h>
		#include <C2PlatformSupport.h>

		#include <media/stagefright/foundation/ADebug.h>
		#include <media/stagefright/foundation/MediaDefs.h>
		#include <media/stagefright/CodecBase.h>
		#include <media/stagefright/MediaCodecConstants.h>
		#include <media/stagefright/SkipCutBuffer.h>
		#include <mediadrm/ICrypto.h>
		@@ -147,6 +150,165 @@ sp<Codec2Buffer> InputBuffers::cloneAndReleaseBuffer(const sp<MediaCodecBuffer>
		return copy;
		}

		// MultiAccessUnitSkipCutBuffer for buffer and bufferInfos

		class MultiAccessUnitSkipCutBuffer : public SkipCutBuffer {

		public:
		explicit MultiAccessUnitSkipCutBuffer(
		int32_t skip, int32_t cut, size_t num16BitChannels):
		SkipCutBuffer(skip, cut, num16BitChannels),
		mFrontPaddingDelay(0), mSize(0) {
		}

		virtual ~MultiAccessUnitSkipCutBuffer() {

		}

		void submitMultiAccessUnits(
		const sp<MediaCodecBuffer>& buffer,
		int32_t sampleRate, size_t num16BitChannels,
		std::shared_ptr<const C2AccessUnitInfos::output> &infos) {
		if (infos == nullptr) {
		// there is nothing to do more.
		SkipCutBuffer::submit(buffer);
		return;
		}
		typedef WrapperObject<std::vector<AccessUnitInfo>> BufferInfosWrapper;
		CHECK_EQ(mSize, SkipCutBuffer::size());
		sp<BufferInfosWrapper> bufferInfos{new BufferInfosWrapper(decltype(bufferInfos->value)())};
		uint32_t availableSize = buffer->size() + SkipCutBuffer::size();
		uint32_t frontPadding = mFrontPadding;
		int32_t lastEmptyAccessUnitIndex = -1;
		int64_t byteInUs = 0;
		if (sampleRate > 0 && num16BitChannels > 0) {
		byteInUs = (1000000u / (sampleRate * num16BitChannels * 2));
		}
		if (frontPadding > 0) {
		mInfos.clear();
		mSize = 0;
		}
		for (int i = 0 ; i < infos->flexCount() && frontPadding > 0; i++) {
		uint32_t flagsInPadding = 0;
		int64_t timeInPadding = 0;
		if (infos->m.values[i].size <= frontPadding) {
		// we have more front padding so this buffer is not going to be used.
		int32_t consumed = infos->m.values[i].size;
		frontPadding -= consumed;
		mFrontPaddingDelay += byteInUs * (consumed);
		availableSize -= consumed;
		flagsInPadding \|= toMediaCodecFlags(infos->m.values[i].flags);
		timeInPadding = infos->m.values[i].timestamp;
		} else {
		C2AccessUnitInfosStruct info = infos->m.values[i];
		mFrontPaddingDelay += byteInUs * (frontPadding);
		info.size -= frontPadding;
		info.timestamp -= mFrontPaddingDelay;
		availableSize -= frontPadding;
		flagsInPadding \|= toMediaCodecFlags(infos->m.values[i].flags);
		timeInPadding = infos->m.values[i].timestamp;
		frontPadding = 0;
		mInfos.push_back(info);
		mSize += info.size;
		}
		if (flagsInPadding != 0) {
		bufferInfos->value.emplace_back(
		flagsInPadding, 0, timeInPadding);
		}
		lastEmptyAccessUnitIndex = i;
		}
		if (frontPadding <= 0) {
		// process what's already in the buffer first
		auto it = mInfos.begin();
		while (it != mInfos.end() && availableSize > mBackPadding) {
		// we have samples to send out.
		if ((availableSize - it->size) >= mBackPadding) {
		// this is totally used here.
		int32_t consumed = it->size;
		bufferInfos->value.emplace_back(
		toMediaCodecFlags(it->flags), consumed, it->timestamp);
		availableSize -= consumed;
		mSize -= consumed;
		it = mInfos.erase(it);
		} else {
		int32_t consumed = availableSize - mBackPadding;
		bufferInfos->value.emplace_back(
		toMediaCodecFlags(it->flags),
		consumed,
		it->timestamp);
		it->size -= consumed;
		it->timestamp += consumed * byteInUs;
		availableSize -= consumed;
		mSize -= consumed;
		it++;
		}
		}
		// if buffer has more process all of it and keep the remaining info.
		for (int i = (lastEmptyAccessUnitIndex + 1) ; i < infos->flexCount() ; i++) {
		// upddate updatedInfo and mInfos
		if (availableSize > mBackPadding) {
		// we have to take data from the new buffer.
		if (availableSize - infos->m.values[i].size >= mBackPadding) {
		// we are using this info
		int32_t consumed = infos->m.values[i].size;
		bufferInfos->value.emplace_back(
		toMediaCodecFlags(infos->m.values[i].flags),
		consumed,
		infos->m.values[i].timestamp - mFrontPaddingDelay);
		availableSize -= consumed;
		} else {
		// if we need to update the size
		C2AccessUnitInfosStruct info = infos->m.values[i];
		int32_t consumed = availableSize - mBackPadding;
		bufferInfos->value.emplace_back(
		toMediaCodecFlags(infos->m.values[i].flags),
		consumed,
		infos->m.values[i].timestamp - mFrontPaddingDelay);
		info.size -= consumed;
		info.timestamp = info.timestamp - mFrontPaddingDelay +
		consumed * byteInUs;
		mInfos.push_back(info);
		availableSize -= consumed;
		mSize += info.size;
		}
		} else {
		// we have to maintain infos
		C2AccessUnitInfosStruct info = infos->m.values[i];
		info.timestamp -= mFrontPaddingDelay;
		mInfos.push_back(info);
		mSize += info.size;
		}
		}
		}
		SkipCutBuffer::submit(buffer);
		infos = nullptr;
		if (!bufferInfos->value.empty()) {
		buffer->meta()->setObject("accessUnitInfo", bufferInfos);
		}
		}
		protected:
		// Flags can come with individual BufferInfos
		// when used with large frame audio
		constexpr static std::initializer_list<std::pair<uint32_t, uint32_t>> flagList = {
		{BUFFER_FLAG_CODEC_CONFIG, C2FrameData::FLAG_CODEC_CONFIG},
		{BUFFER_FLAG_END_OF_STREAM, C2FrameData::FLAG_END_OF_STREAM},
		{BUFFER_FLAG_DECODE_ONLY, C2FrameData::FLAG_DROP_FRAME}
		};

		static uint32_t toMediaCodecFlags(uint32_t flags) {
		return std::transform_reduce(
		flagList.begin(), flagList.end(),
		0u,
		std::bit_or{},
		[flags](const std::pair<uint32_t, uint32_t> &entry) {
		return (flags & entry.second) ? entry.first : 0;
		});
		}
		std::list<C2AccessUnitInfosStruct> mInfos;
		int64_t mFrontPaddingDelay;
		size_t mSize;
		};

		// OutputBuffers

		OutputBuffers::OutputBuffers(const char componentName, const char name)
		@@ -201,6 +363,15 @@ void OutputBuffers::submit(const sp<MediaCodecBuffer> &buffer) {
		}
		}

		bool OutputBuffers::submit(const sp<MediaCodecBuffer> &buffer, int32_t sampleRate,
		int32_t channelCount, std::shared_ptr<const C2AccessUnitInfos::output> &infos) {
		if (mSkipCutBuffer == nullptr) {
		return false;
		}
		mSkipCutBuffer->submitMultiAccessUnits(buffer, sampleRate, channelCount, infos);
		return true;
		}

		void OutputBuffers::setSkipCutBuffer(int32_t skip, int32_t cut) {
		if (mSkipCutBuffer != nullptr) {
		size_t prevSize = mSkipCutBuffer->size();
		@@ -208,7 +379,7 @@ void OutputBuffers::setSkipCutBuffer(int32_t skip, int32_t cut) {
		ALOGD("[%s] Replacing SkipCutBuffer holding %zu bytes", mName, prevSize);
		}
		}
		mSkipCutBuffer = new SkipCutBuffer(skip, cut, mChannelCount);
		mSkipCutBuffer = new MultiAccessUnitSkipCutBuffer(skip, cut, mChannelCount);
		}

		bool OutputBuffers::convert(
		@@ -1160,7 +1331,16 @@ status_t OutputBuffersArray::registerBuffer(
		ALOGD("[%s] copy buffer failed", mName);
		return WOULD_BLOCK;
		}
		if (buffer && buffer->hasInfo(C2AccessUnitInfos::output::PARAM_TYPE)) {
		std::shared_ptr<const C2AccessUnitInfos::output> bufferMetadata =
		std::static_pointer_cast<const C2AccessUnitInfos::output>(
		buffer->getInfo(C2AccessUnitInfos::output::PARAM_TYPE));
		if (submit(c2Buffer, mSampleRate, mChannelCount, bufferMetadata)) {
		buffer->removeInfo(C2AccessUnitInfos::output::PARAM_TYPE);
		}
		} else {
		submit(c2Buffer);
		}
		handleImageData(c2Buffer);
		*clientBuffer = c2Buffer;
		ALOGV("[%s] grabbed buffer %zu", mName, *index);

media/codec2/sfplugin/CCodecBuffers.h

+8 −1

Original line number	Diff line number	Diff line
		@@ -20,6 +20,7 @@

		#include <optional>
		#include <string>
		#include <vector>

		#include <C2Config.h>
		#include <DataConverter.h>
		@@ -33,6 +34,8 @@ namespace android {
		struct ICrypto;
		class MemoryDealer;
		class SkipCutBuffer;
		class MultiAccessUnitSkipCutBuffer;
		struct AccessUnitInfo;

		constexpr size_t kLinearBufferSize = 1048576;
		// This can fit an 8K frame.
		@@ -382,13 +385,17 @@ public:
		sp<MediaCodecBuffer>* outBuffer);

		protected:
		sp<SkipCutBuffer> mSkipCutBuffer;

		sp<MultiAccessUnitSkipCutBuffer> mSkipCutBuffer;

		/**
		* Update the SkipCutBuffer object. No-op if it's never initialized.
		*/
		void updateSkipCutBuffer(int32_t sampleRate, int32_t channelCount);

		bool submit(const sp<MediaCodecBuffer> &buffer, int32_t sampleRate,
		int32_t channelCount, std::shared_ptr<const C2AccessUnitInfos::output> &infos);

		/**
		* Submit buffer to SkipCutBuffer object, if initialized.
		*/

media/codec2/sfplugin/CCodecConfig.cpp

+9 −0

Original line number	Diff line number	Diff line
		@@ -402,10 +402,19 @@ void CCodecConfig::initializeStandardParams() {

		add(ConfigMapper(KEY_MAX_INPUT_SIZE, C2_PARAMKEY_INPUT_MAX_BUFFER_SIZE, "value")
		.limitTo(D::INPUT));

		// remove when codecs switch to PARAMKEY
		deprecated(ConfigMapper(KEY_MAX_INPUT_SIZE, "coded.max-frame-size", "value")
		.limitTo(D::INPUT));

		// large frame params
		add(ConfigMapper(KEY_BUFFER_BATCH_MAX_OUTPUT_SIZE,
		C2_PARAMKEY_OUTPUT_LARGE_FRAME, "max-size")
		.limitTo(D::AUDIO & D::OUTPUT));
		add(ConfigMapper(KEY_BUFFER_BATCH_THRESHOLD_OUTPUT_SIZE,
		C2_PARAMKEY_OUTPUT_LARGE_FRAME, "threshold-size")
		.limitTo(D::AUDIO & D::OUTPUT));

		// Rotation
		// Note: SDK rotation is clock-wise, while C2 rotation is counter-clock-wise
		add(ConfigMapper(KEY_ROTATION, C2_PARAMKEY_VUI_ROTATION, "value")

media/libstagefright/Android.bp

+1 −0

Original line number	Diff line number	Diff line
		@@ -315,6 +315,7 @@ cc_library {
		"libaudioclient_aidl_conversion",
		"packagemanager_aidl-cpp",
		"server_configurable_flags",
		"aconfig_mediacodec_flags_c_lib",
		],

		static_libs: [