Merge "CCodec: support audio encoders requesting frame size" (9b5bc7fc) · Commits · e / os / android_frameworks_av

media/codec2/sfplugin/Android.bp

+1 −0

Original line number	Original line	Diff line number	Diff line
	@@ -11,6 +11,7 @@ cc_library_shared {
	"CCodecConfig.cpp",		"CCodecConfig.cpp",
	"Codec2Buffer.cpp",		"Codec2Buffer.cpp",
	"Codec2InfoBuilder.cpp",		"Codec2InfoBuilder.cpp",
			"FrameReassembler.cpp",
	"PipelineWatcher.cpp",		"PipelineWatcher.cpp",
	"ReflectedParamUpdater.cpp",		"ReflectedParamUpdater.cpp",
	],		],

media/codec2/sfplugin/CCodecBufferChannel.cpp

+79 −41

Original line number	Original line	Diff line number	Diff line
	@@ -213,6 +213,7 @@ status_t CCodecBufferChannel::queueInputBufferInternal(
	flags \|= C2FrameData::FLAG_CODEC_CONFIG;		flags \|= C2FrameData::FLAG_CODEC_CONFIG;
	}		}
	ALOGV("[%s] queueInputBuffer: buffer->size() = %zu", mName, buffer->size());		ALOGV("[%s] queueInputBuffer: buffer->size() = %zu", mName, buffer->size());
			std::list<std::unique_ptr<C2Work>> items;
	std::unique_ptr<C2Work> work(new C2Work);		std::unique_ptr<C2Work> work(new C2Work);
	work->input.ordinal.timestamp = timeUs;		work->input.ordinal.timestamp = timeUs;
	work->input.ordinal.frameIndex = mFrameIndex++;		work->input.ordinal.frameIndex = mFrameIndex++;
	@@ -222,9 +223,8 @@ status_t CCodecBufferChannel::queueInputBufferInternal(
	work->input.ordinal.customOrdinal = timeUs;		work->input.ordinal.customOrdinal = timeUs;
	work->input.buffers.clear();		work->input.buffers.clear();

	uint64_t queuedFrameIndex = work->input.ordinal.frameIndex.peeku();
	std::vector<std::shared_ptr<C2Buffer>> queuedBuffers;
	sp<Codec2Buffer> copy;		sp<Codec2Buffer> copy;
			bool usesFrameReassembler = false;

	if (buffer->size() > 0u) {		if (buffer->size() > 0u) {
	Mutexed<Input>::Locked input(mInput);		Mutexed<Input>::Locked input(mInput);
	@@ -249,16 +249,26 @@ status_t CCodecBufferChannel::queueInputBufferInternal(
	"buffer starvation on component.", mName);		"buffer starvation on component.", mName);
	}		}
	}		}
			if (input->frameReassembler) {
			usesFrameReassembler = true;
			input->frameReassembler.process(buffer, &items);
			} else {
	work->input.buffers.push_back(c2buffer);		work->input.buffers.push_back(c2buffer);
	if (encryptedBlock) {		if (encryptedBlock) {
	work->input.infoBuffers.emplace_back(C2InfoBuffer::CreateLinearBuffer(		work->input.infoBuffers.emplace_back(C2InfoBuffer::CreateLinearBuffer(
	kParamIndexEncryptedBuffer,		kParamIndexEncryptedBuffer,
	encryptedBlock->share(0, blockSize, C2Fence())));		encryptedBlock->share(0, blockSize, C2Fence())));
	}		}
	queuedBuffers.push_back(c2buffer);		}
	} else if (eos) {		} else if (eos) {
	flags \|= C2FrameData::FLAG_END_OF_STREAM;		flags \|= C2FrameData::FLAG_END_OF_STREAM;
	}		}
			if (usesFrameReassembler) {
			if (!items.empty()) {
			items.front()->input.configUpdate = std::move(mParamsToBeSet);
			mFrameIndex = (items.back()->input.ordinal.frameIndex + 1).peek();
			}
			} else {
	work->input.flags = (C2FrameData::flags_t)flags;		work->input.flags = (C2FrameData::flags_t)flags;
	// TODO: fill info's		// TODO: fill info's

	@@ -266,18 +276,11 @@ status_t CCodecBufferChannel::queueInputBufferInternal(
	work->worklets.clear();		work->worklets.clear();
	work->worklets.emplace_back(new C2Worklet);		work->worklets.emplace_back(new C2Worklet);

	std::list<std::unique_ptr<C2Work>> items;
	items.push_back(std::move(work));		items.push_back(std::move(work));
	mPipelineWatcher.lock()->onWorkQueued(
	queuedFrameIndex,
	std::move(queuedBuffers),
	PipelineWatcher::Clock::now());
	c2_status_t err = mComponent->queue(&items);
	if (err != C2_OK) {
	mPipelineWatcher.lock()->onWorkDone(queuedFrameIndex);
	}

	if (err == C2_OK && eos && buffer->size() > 0u) {		eos = eos && buffer->size() > 0u;
			}
			if (eos) {
	work.reset(new C2Work);		work.reset(new C2Work);
	work->input.ordinal.timestamp = timeUs;		work->input.ordinal.timestamp = timeUs;
	work->input.ordinal.frameIndex = mFrameIndex++;		work->input.ordinal.frameIndex = mFrameIndex++;
	@@ -286,23 +289,28 @@ status_t CCodecBufferChannel::queueInputBufferInternal(
	work->input.buffers.clear();		work->input.buffers.clear();
	work->input.flags = C2FrameData::FLAG_END_OF_STREAM;		work->input.flags = C2FrameData::FLAG_END_OF_STREAM;
	work->worklets.emplace_back(new C2Worklet);		work->worklets.emplace_back(new C2Worklet);

	queuedFrameIndex = work->input.ordinal.frameIndex.peeku();
	queuedBuffers.clear();

	items.clear();
	items.push_back(std::move(work));		items.push_back(std::move(work));
			}
	mPipelineWatcher.lock()->onWorkQueued(		c2_status_t err = C2_OK;
	queuedFrameIndex,		if (!items.empty()) {
	std::move(queuedBuffers),		{
	PipelineWatcher::Clock::now());		Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
			PipelineWatcher::Clock::time_point now = PipelineWatcher::Clock::now();
			for (const std::unique_ptr<C2Work> &work : items) {
			watcher->onWorkQueued(
			work->input.ordinal.frameIndex.peeku(),
			std::vector(work->input.buffers),
			now);
			}
			}
	err = mComponent->queue(&items);		err = mComponent->queue(&items);
	if (err != C2_OK) {
	mPipelineWatcher.lock()->onWorkDone(queuedFrameIndex);
	}		}
			if (err != C2_OK) {
			Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
			for (const std::unique_ptr<C2Work> &work : items) {
			watcher->onWorkDone(work->input.ordinal.frameIndex.peeku());
	}		}
	if (err == C2_OK) {		} else {
	Mutexed<Input>::Locked input(mInput);		Mutexed<Input>::Locked input(mInput);
	bool released = false;		bool released = false;
	if (buffer) {		if (buffer) {
	@@ -926,6 +934,7 @@ status_t CCodecBufferChannel::start(
	bool buffersBoundToCodec) {		bool buffersBoundToCodec) {
	C2StreamBufferTypeSetting::input iStreamFormat(0u);		C2StreamBufferTypeSetting::input iStreamFormat(0u);
	C2StreamBufferTypeSetting::output oStreamFormat(0u);		C2StreamBufferTypeSetting::output oStreamFormat(0u);
			C2ComponentKindSetting kind;
	C2PortReorderBufferDepthTuning::output reorderDepth;		C2PortReorderBufferDepthTuning::output reorderDepth;
	C2PortReorderKeySetting::output reorderKey;		C2PortReorderKeySetting::output reorderKey;
	C2PortActualDelayTuning::input inputDelay(0);		C2PortActualDelayTuning::input inputDelay(0);
	@@ -937,6 +946,7 @@ status_t CCodecBufferChannel::start(
	{		{
	&iStreamFormat,		&iStreamFormat,
	&oStreamFormat,		&oStreamFormat,
			&kind,
	&reorderDepth,		&reorderDepth,
	&reorderKey,		&reorderKey,
	&inputDelay,		&inputDelay,
	@@ -948,7 +958,7 @@ status_t CCodecBufferChannel::start(
	C2_DONT_BLOCK,		C2_DONT_BLOCK,
	nullptr);		nullptr);
	if (err == C2_BAD_INDEX) {		if (err == C2_BAD_INDEX) {
	if (!iStreamFormat \|\| !oStreamFormat) {		if (!iStreamFormat \|\| !oStreamFormat \|\| !kind) {
	return UNKNOWN_ERROR;		return UNKNOWN_ERROR;
	}		}
	} else if (err != C2_OK) {		} else if (err != C2_OK) {
	@@ -974,12 +984,17 @@ status_t CCodecBufferChannel::start(

	if (inputFormat != nullptr) {		if (inputFormat != nullptr) {
	bool graphic = (iStreamFormat.value == C2BufferData::GRAPHIC);		bool graphic = (iStreamFormat.value == C2BufferData::GRAPHIC);
			bool audioEncoder = !graphic && (kind.value == C2Component::KIND_ENCODER);
	C2Config::api_feature_t apiFeatures = C2Config::api_feature_t(		C2Config::api_feature_t apiFeatures = C2Config::api_feature_t(
	API_REFLECTION \|		API_REFLECTION \|
	API_VALUES \|		API_VALUES \|
	API_CURRENT_VALUES \|		API_CURRENT_VALUES \|
	API_DEPENDENCY \|		API_DEPENDENCY \|
	API_SAME_INPUT_BUFFER);		API_SAME_INPUT_BUFFER);
			C2StreamAudioFrameSizeInfo::input encoderFrameSize(0u);
			C2StreamSampleRateInfo::input sampleRate(0u);
			C2StreamChannelCountInfo::input channelCount(0u);
			C2StreamPcmEncodingInfo::input pcmEncoding(0u);
	std::shared_ptr<C2BlockPool> pool;		std::shared_ptr<C2BlockPool> pool;
	{		{
	Mutexed<BlockPools>::Locked pools(mBlockPools);		Mutexed<BlockPools>::Locked pools(mBlockPools);
	@@ -992,7 +1007,19 @@ status_t CCodecBufferChannel::start(
	// from component, create the input block pool with given ID. Otherwise, use default IDs.		// from component, create the input block pool with given ID. Otherwise, use default IDs.
	std::vector<std::unique_ptr<C2Param>> params;		std::vector<std::unique_ptr<C2Param>> params;
	C2ApiFeaturesSetting featuresSetting{apiFeatures};		C2ApiFeaturesSetting featuresSetting{apiFeatures};
	err = mComponent->query({ &featuresSetting },		std::vector<C2Param *> stackParams({&featuresSetting});
			if (audioEncoder) {
			stackParams.push_back(&encoderFrameSize);
			stackParams.push_back(&sampleRate);
			stackParams.push_back(&channelCount);
			stackParams.push_back(&pcmEncoding);
			} else {
			encoderFrameSize.invalidate();
			sampleRate.invalidate();
			channelCount.invalidate();
			pcmEncoding.invalidate();
			}
			err = mComponent->query(stackParams,
	{ C2PortAllocatorsTuning::input::PARAM_TYPE },		{ C2PortAllocatorsTuning::input::PARAM_TYPE },
	C2_DONT_BLOCK,		C2_DONT_BLOCK,
	&params);		&params);
	@@ -1050,10 +1077,21 @@ status_t CCodecBufferChannel::start(
	input->numSlots = numInputSlots;		input->numSlots = numInputSlots;
	input->extraBuffers.flush();		input->extraBuffers.flush();
	input->numExtraSlots = 0u;		input->numExtraSlots = 0u;
			if (audioEncoder && encoderFrameSize && sampleRate && channelCount) {
			input->frameReassembler.init(
			pool,
			{C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE},
			encoderFrameSize.value,
			sampleRate.value,
			channelCount.value,
			pcmEncoding ? pcmEncoding.value : C2Config::PCM_16);
			}
	bool conforming = (apiFeatures & API_SAME_INPUT_BUFFER);		bool conforming = (apiFeatures & API_SAME_INPUT_BUFFER);
	// For encrypted content, framework decrypts source buffer (ashmem) into		// For encrypted content, framework decrypts source buffer (ashmem) into
	// C2Buffers. Thus non-conforming codecs can process these.		// C2Buffers. Thus non-conforming codecs can process these.
	if (!buffersBoundToCodec && (hasCryptoOrDescrambler() \|\| conforming)) {		if (!buffersBoundToCodec
			&& !input->frameReassembler
			&& (hasCryptoOrDescrambler() \|\| conforming)) {
	input->buffers.reset(new SlotInputBuffers(mName));		input->buffers.reset(new SlotInputBuffers(mName));
	} else if (graphic) {		} else if (graphic) {
	if (mInputSurface) {		if (mInputSurface) {

media/codec2/sfplugin/CCodecBufferChannel.h

+3 −0

Original line number	Original line	Diff line number	Diff line
	@@ -31,6 +31,7 @@
	#include <media/stagefright/CodecBase.h>		#include <media/stagefright/CodecBase.h>

	#include "CCodecBuffers.h"		#include "CCodecBuffers.h"
			#include "FrameReassembler.h"
	#include "InputSurfaceWrapper.h"		#include "InputSurfaceWrapper.h"
	#include "PipelineWatcher.h"		#include "PipelineWatcher.h"

	@@ -271,6 +272,8 @@ private:
	size_t numExtraSlots;		size_t numExtraSlots;
	uint32_t inputDelay;		uint32_t inputDelay;
	uint32_t pipelineDelay;		uint32_t pipelineDelay;

			FrameReassembler frameReassembler;
	};		};
	Mutexed<Input> mInput;		Mutexed<Input> mInput;
	struct Output {		struct Output {

media/codec2/sfplugin/FrameReassembler.cpp

0 → 100644

+226 −0

Original line number	Original line	Diff line number	Diff line
			/*
			* Copyright 2019 The Android Open Source Project
			*
			* Licensed under the Apache License, Version 2.0 (the "License");
			* you may not use this file except in compliance with the License.
			* You may obtain a copy of the License at
			*
			* http://www.apache.org/licenses/LICENSE-2.0
			*
			* Unless required by applicable law or agreed to in writing, software
			* distributed under the License is distributed on an "AS IS" BASIS,
			* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
			* See the License for the specific language governing permissions and
			* limitations under the License.
			*/

			//#define LOG_NDEBUG 0
			#define LOG_TAG "FrameReassembler"

			#include <log/log.h>

			#include <media/stagefright/foundation/AMessage.h>

			#include "FrameReassembler.h"

			namespace android {

			static constexpr uint64_t kToleranceUs = 1000; // 1ms

			FrameReassembler::FrameReassembler()
			: mUsage{0, 0},
			mSampleRate(0u),
			mChannelCount(0u),
			mEncoding(C2Config::PCM_16),
			mCurrentOrdinal({0, 0, 0}) {
			}

			void FrameReassembler::init(
			const std::shared_ptr<C2BlockPool> &pool,
			C2MemoryUsage usage,
			uint32_t frameSize,
			uint32_t sampleRate,
			uint32_t channelCount,
			C2Config::pcm_encoding_t encoding) {
			mBlockPool = pool;
			mUsage = usage;
			mFrameSize = frameSize;
			mSampleRate = sampleRate;
			mChannelCount = channelCount;
			mEncoding = encoding;
			}

			void FrameReassembler::updateFrameSize(uint32_t frameSize) {
			finishCurrentBlock(&mPendingWork);
			mFrameSize = frameSize;
			}

			void FrameReassembler::updateSampleRate(uint32_t sampleRate) {
			finishCurrentBlock(&mPendingWork);
			mSampleRate = sampleRate;
			}

			void FrameReassembler::updateChannelCount(uint32_t channelCount) {
			finishCurrentBlock(&mPendingWork);
			mChannelCount = channelCount;
			}

			void FrameReassembler::updatePcmEncoding(C2Config::pcm_encoding_t encoding) {
			finishCurrentBlock(&mPendingWork);
			mEncoding = encoding;
			}

			void FrameReassembler::reset() {
			flush();
			mCurrentOrdinal = {0, 0, 0};
			mBlockPool.reset();
			mFrameSize.reset();
			mSampleRate = 0u;
			mChannelCount = 0u;
			mEncoding = C2Config::PCM_16;
			}

			FrameReassembler::operator bool() const {
			return mFrameSize.has_value();
			}

			c2_status_t FrameReassembler::process(
			const sp<MediaCodecBuffer> &buffer,
			std::list<std::unique_ptr<C2Work>> *items) {
			int64_t timeUs;
			if (buffer->size() == 0u
			\|\| !buffer->meta()->findInt64("timeUs", &timeUs)) {
			return C2_BAD_VALUE;
			}

			items->splice(items->end(), mPendingWork);

			// Fill mCurrentBlock
			if (mCurrentBlock) {
			// First check the timestamp
			c2_cntr64_t endTimestampUs = mCurrentOrdinal.timestamp;
			endTimestampUs += bytesToSamples(mWriteView->size()) * 1000000 / mSampleRate;
			if (timeUs < endTimestampUs.peek()) {
			uint64_t diffUs = (endTimestampUs - timeUs).peeku();
			if (diffUs > kToleranceUs) {
			// The timestamp is going back in time in large amount.
			// TODO: b/145702136
			ALOGW("timestamp going back in time! from %lld to %lld",
			endTimestampUs.peekll(), (long long)timeUs);
			}
			} else { // timeUs >= endTimestampUs.peek()
			uint64_t diffUs = (timeUs - endTimestampUs).peeku();
			if (diffUs > kToleranceUs) {
			// The timestamp is going forward; add silence as necessary.
			size_t gapSamples = usToSamples(diffUs);
			size_t remainingSamples =
			(mWriteView->capacity() - mWriteView->size())
			/ mChannelCount / bytesPerSample();
			if (gapSamples < remainingSamples) {
			size_t gapBytes = gapSamples * mChannelCount * bytesPerSample();
			memset(mWriteView->base() + mWriteView->size(), 0u, gapBytes);
			mWriteView->setSize(mWriteView->size() + gapBytes);
			} else {
			finishCurrentBlock(items);
			}
			}
			}
			}

			if (mCurrentBlock) {
			// Append the data at the end of the current block
			size_t copySize = std::min(
			buffer->size(),
			size_t(mWriteView->capacity() - mWriteView->size()));
			memcpy(mWriteView->base() + mWriteView->size(), buffer->data(), copySize);
			buffer->setRange(buffer->offset() + copySize, buffer->size() - copySize);
			mWriteView->setSize(mWriteView->size() + copySize);
			if (mWriteView->size() == mWriteView->capacity()) {
			finishCurrentBlock(items);
			}
			timeUs += bytesToSamples(copySize) * 1000000 / mSampleRate;
			}

			if (buffer->size() > 0) {
			mCurrentOrdinal.timestamp = timeUs;
			}

			size_t frameSizeBytes = mFrameSize.value() * mChannelCount * bytesPerSample();
			while (buffer->size() > 0) {
			LOG_ALWAYS_FATAL_IF(
			mCurrentBlock,
			"There's remaining data but the pending block is not filled & finished");
			std::unique_ptr<C2Work> work(new C2Work);
			c2_status_t err = mBlockPool->fetchLinearBlock(frameSizeBytes, mUsage, &mCurrentBlock);
			if (err != C2_OK) {
			return err;
			}
			size_t copySize = std::min(buffer->size(), frameSizeBytes);
			mWriteView = mCurrentBlock->map().get();
			if (mWriteView->error() != C2_OK) {
			return mWriteView->error();
			}
			ALOGV("buffer={offset=%zu size=%zu) copySize=%zu",
			buffer->offset(), buffer->size(), copySize);
			memcpy(mWriteView->base(), buffer->data(), copySize);
			mWriteView->setOffset(0u);
			mWriteView->setSize(copySize);
			buffer->setRange(buffer->offset() + copySize, buffer->size() - copySize);
			if (copySize == frameSizeBytes) {
			finishCurrentBlock(items);
			}
			}

			int32_t eos = 0;
			if (buffer->meta()->findInt32("eos", &eos) && eos) {
			finishCurrentBlock(items);
			}

			return C2_OK;
			}

			void FrameReassembler::flush() {
			mPendingWork.clear();
			mWriteView.reset();
			mCurrentBlock.reset();
			}

			uint64_t FrameReassembler::bytesToSamples(size_t numBytes) const {
			return numBytes / mChannelCount / bytesPerSample();
			}

			size_t FrameReassembler::usToSamples(uint64_t us) const {
			return (us * mChannelCount * mSampleRate / 1000000);
			}

			uint32_t FrameReassembler::bytesPerSample() const {
			return (mEncoding == C2Config::PCM_8) ? 1
			: (mEncoding == C2Config::PCM_16) ? 2
			: (mEncoding == C2Config::PCM_FLOAT) ? 4 : 0;
			}

			void FrameReassembler::finishCurrentBlock(std::list<std::unique_ptr<C2Work>> *items) {
			if (!mCurrentBlock) {
			// No-op
			return;
			}
			if (mWriteView->size() < mWriteView->capacity()) {
			memset(mWriteView->base() + mWriteView->size(), 0u,
			mWriteView->capacity() - mWriteView->size());
			mWriteView->setSize(mWriteView->capacity());
			}
			std::unique_ptr<C2Work> work{std::make_unique<C2Work>()};
			work->input.ordinal = mCurrentOrdinal;
			work->input.buffers.push_back(C2Buffer::CreateLinearBuffer(
			mCurrentBlock->share(0, mCurrentBlock->capacity(), C2Fence())));
			work->worklets.clear();
			work->worklets.emplace_back(new C2Worklet);
			items->push_back(std::move(work));

			++mCurrentOrdinal.frameIndex;
			mCurrentOrdinal.timestamp += mFrameSize.value() * 1000000 / mSampleRate;
			mCurrentBlock.reset();
			mWriteView.reset();
			}

			} // namespace android

media/codec2/sfplugin/FrameReassembler.h

0 → 100644

+75 −0

Original line number	Original line	Diff line number	Diff line
			/*
			* Copyright 2019 The Android Open Source Project
			*
			* Licensed under the Apache License, Version 2.0 (the "License");
			* you may not use this file except in compliance with the License.
			* You may obtain a copy of the License at
			*
			* http://www.apache.org/licenses/LICENSE-2.0
			*
			* Unless required by applicable law or agreed to in writing, software
			* distributed under the License is distributed on an "AS IS" BASIS,
			* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
			* See the License for the specific language governing permissions and
			* limitations under the License.
			*/

			#ifndef FRAME_REASSEMBLER_H_
			#define FRAME_REASSEMBLER_H_

			#include <set>
			#include <memory>

			#include <media/MediaCodecBuffer.h>

			#include <C2Config.h>
			#include <C2Work.h>

			namespace android {

			class FrameReassembler {
			public:
			FrameReassembler();

			void init(
			const std::shared_ptr<C2BlockPool> &pool,
			C2MemoryUsage usage,
			uint32_t frameSize,
			uint32_t sampleRate,
			uint32_t channelCount,
			C2Config::pcm_encoding_t encoding);
			void updateFrameSize(uint32_t frameSize);
			void updateSampleRate(uint32_t sampleRate);
			void updateChannelCount(uint32_t channelCount);
			void updatePcmEncoding(C2Config::pcm_encoding_t encoding);
			void reset();
			void flush();

			explicit operator bool() const;

			c2_status_t process(
			const sp<MediaCodecBuffer> &buffer,
			std::list<std::unique_ptr<C2Work>> *items);

			private:
			std::shared_ptr<C2BlockPool> mBlockPool;
			C2MemoryUsage mUsage;
			std::optional<uint32_t> mFrameSize;
			uint32_t mSampleRate;
			uint32_t mChannelCount;
			C2Config::pcm_encoding_t mEncoding;
			std::list<std::unique_ptr<C2Work>> mPendingWork;
			C2WorkOrdinalStruct mCurrentOrdinal;
			std::shared_ptr<C2LinearBlock> mCurrentBlock;
			std::optional<C2WriteView> mWriteView;

			uint64_t bytesToSamples(size_t numBytes) const;
			size_t usToSamples(uint64_t us) const;
			uint32_t bytesPerSample() const;

			void finishCurrentBlock(std::list<std::unique_ptr<C2Work>> *items);
			};

			} // namespace android

			#endif // FRAME_REASSEMBLER_H_