Merge "CCodec: support audio encoders requesting frame size" am: 9b5bc7fc75 (321062ed) · Commits · e / os / android_frameworks_av

media/codec2/sfplugin/Android.bp

+1 −0

Original line number	Diff line number	Diff line
		@@ -11,6 +11,7 @@ cc_library_shared {
		"CCodecConfig.cpp",
		"Codec2Buffer.cpp",
		"Codec2InfoBuilder.cpp",
		"FrameReassembler.cpp",
		"PipelineWatcher.cpp",
		"ReflectedParamUpdater.cpp",
		],

media/codec2/sfplugin/CCodecBufferChannel.cpp

+79 −41

Original line number	Diff line number	Diff line
		@@ -213,6 +213,7 @@ status_t CCodecBufferChannel::queueInputBufferInternal(
		flags \|= C2FrameData::FLAG_CODEC_CONFIG;
		}
		ALOGV("[%s] queueInputBuffer: buffer->size() = %zu", mName, buffer->size());
		std::list<std::unique_ptr<C2Work>> items;
		std::unique_ptr<C2Work> work(new C2Work);
		work->input.ordinal.timestamp = timeUs;
		work->input.ordinal.frameIndex = mFrameIndex++;
		@@ -222,9 +223,8 @@ status_t CCodecBufferChannel::queueInputBufferInternal(
		work->input.ordinal.customOrdinal = timeUs;
		work->input.buffers.clear();

		uint64_t queuedFrameIndex = work->input.ordinal.frameIndex.peeku();
		std::vector<std::shared_ptr<C2Buffer>> queuedBuffers;
		sp<Codec2Buffer> copy;
		bool usesFrameReassembler = false;

		if (buffer->size() > 0u) {
		Mutexed<Input>::Locked input(mInput);
		@@ -249,16 +249,26 @@ status_t CCodecBufferChannel::queueInputBufferInternal(
		"buffer starvation on component.", mName);
		}
		}
		if (input->frameReassembler) {
		usesFrameReassembler = true;
		input->frameReassembler.process(buffer, &items);
		} else {
		work->input.buffers.push_back(c2buffer);
		if (encryptedBlock) {
		work->input.infoBuffers.emplace_back(C2InfoBuffer::CreateLinearBuffer(
		kParamIndexEncryptedBuffer,
		encryptedBlock->share(0, blockSize, C2Fence())));
		}
		queuedBuffers.push_back(c2buffer);
		}
		} else if (eos) {
		flags \|= C2FrameData::FLAG_END_OF_STREAM;
		}
		if (usesFrameReassembler) {
		if (!items.empty()) {
		items.front()->input.configUpdate = std::move(mParamsToBeSet);
		mFrameIndex = (items.back()->input.ordinal.frameIndex + 1).peek();
		}
		} else {
		work->input.flags = (C2FrameData::flags_t)flags;
		// TODO: fill info's

		@@ -266,18 +276,11 @@ status_t CCodecBufferChannel::queueInputBufferInternal(
		work->worklets.clear();
		work->worklets.emplace_back(new C2Worklet);

		std::list<std::unique_ptr<C2Work>> items;
		items.push_back(std::move(work));
		mPipelineWatcher.lock()->onWorkQueued(
		queuedFrameIndex,
		std::move(queuedBuffers),
		PipelineWatcher::Clock::now());
		c2_status_t err = mComponent->queue(&items);
		if (err != C2_OK) {
		mPipelineWatcher.lock()->onWorkDone(queuedFrameIndex);
		}

		if (err == C2_OK && eos && buffer->size() > 0u) {
		eos = eos && buffer->size() > 0u;
		}
		if (eos) {
		work.reset(new C2Work);
		work->input.ordinal.timestamp = timeUs;
		work->input.ordinal.frameIndex = mFrameIndex++;
		@@ -286,23 +289,28 @@ status_t CCodecBufferChannel::queueInputBufferInternal(
		work->input.buffers.clear();
		work->input.flags = C2FrameData::FLAG_END_OF_STREAM;
		work->worklets.emplace_back(new C2Worklet);

		queuedFrameIndex = work->input.ordinal.frameIndex.peeku();
		queuedBuffers.clear();

		items.clear();
		items.push_back(std::move(work));

		mPipelineWatcher.lock()->onWorkQueued(
		queuedFrameIndex,
		std::move(queuedBuffers),
		PipelineWatcher::Clock::now());
		}
		c2_status_t err = C2_OK;
		if (!items.empty()) {
		{
		Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
		PipelineWatcher::Clock::time_point now = PipelineWatcher::Clock::now();
		for (const std::unique_ptr<C2Work> &work : items) {
		watcher->onWorkQueued(
		work->input.ordinal.frameIndex.peeku(),
		std::vector(work->input.buffers),
		now);
		}
		}
		err = mComponent->queue(&items);
		if (err != C2_OK) {
		mPipelineWatcher.lock()->onWorkDone(queuedFrameIndex);
		}
		if (err != C2_OK) {
		Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
		for (const std::unique_ptr<C2Work> &work : items) {
		watcher->onWorkDone(work->input.ordinal.frameIndex.peeku());
		}
		if (err == C2_OK) {
		} else {
		Mutexed<Input>::Locked input(mInput);
		bool released = false;
		if (buffer) {
		@@ -926,6 +934,7 @@ status_t CCodecBufferChannel::start(
		bool buffersBoundToCodec) {
		C2StreamBufferTypeSetting::input iStreamFormat(0u);
		C2StreamBufferTypeSetting::output oStreamFormat(0u);
		C2ComponentKindSetting kind;
		C2PortReorderBufferDepthTuning::output reorderDepth;
		C2PortReorderKeySetting::output reorderKey;
		C2PortActualDelayTuning::input inputDelay(0);
		@@ -937,6 +946,7 @@ status_t CCodecBufferChannel::start(
		{
		&iStreamFormat,
		&oStreamFormat,
		&kind,
		&reorderDepth,
		&reorderKey,
		&inputDelay,
		@@ -948,7 +958,7 @@ status_t CCodecBufferChannel::start(
		C2_DONT_BLOCK,
		nullptr);
		if (err == C2_BAD_INDEX) {
		if (!iStreamFormat \|\| !oStreamFormat) {
		if (!iStreamFormat \|\| !oStreamFormat \|\| !kind) {
		return UNKNOWN_ERROR;
		}
		} else if (err != C2_OK) {
		@@ -974,12 +984,17 @@ status_t CCodecBufferChannel::start(

		if (inputFormat != nullptr) {
		bool graphic = (iStreamFormat.value == C2BufferData::GRAPHIC);
		bool audioEncoder = !graphic && (kind.value == C2Component::KIND_ENCODER);
		C2Config::api_feature_t apiFeatures = C2Config::api_feature_t(
		API_REFLECTION \|
		API_VALUES \|
		API_CURRENT_VALUES \|
		API_DEPENDENCY \|
		API_SAME_INPUT_BUFFER);
		C2StreamAudioFrameSizeInfo::input encoderFrameSize(0u);
		C2StreamSampleRateInfo::input sampleRate(0u);
		C2StreamChannelCountInfo::input channelCount(0u);
		C2StreamPcmEncodingInfo::input pcmEncoding(0u);
		std::shared_ptr<C2BlockPool> pool;
		{
		Mutexed<BlockPools>::Locked pools(mBlockPools);
		@@ -992,7 +1007,19 @@ status_t CCodecBufferChannel::start(
		// from component, create the input block pool with given ID. Otherwise, use default IDs.
		std::vector<std::unique_ptr<C2Param>> params;
		C2ApiFeaturesSetting featuresSetting{apiFeatures};
		err = mComponent->query({ &featuresSetting },
		std::vector<C2Param *> stackParams({&featuresSetting});
		if (audioEncoder) {
		stackParams.push_back(&encoderFrameSize);
		stackParams.push_back(&sampleRate);
		stackParams.push_back(&channelCount);
		stackParams.push_back(&pcmEncoding);
		} else {
		encoderFrameSize.invalidate();
		sampleRate.invalidate();
		channelCount.invalidate();
		pcmEncoding.invalidate();
		}
		err = mComponent->query(stackParams,
		{ C2PortAllocatorsTuning::input::PARAM_TYPE },
		C2_DONT_BLOCK,
		&params);
		@@ -1050,10 +1077,21 @@ status_t CCodecBufferChannel::start(
		input->numSlots = numInputSlots;
		input->extraBuffers.flush();
		input->numExtraSlots = 0u;
		if (audioEncoder && encoderFrameSize && sampleRate && channelCount) {
		input->frameReassembler.init(
		pool,
		{C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE},
		encoderFrameSize.value,
		sampleRate.value,
		channelCount.value,
		pcmEncoding ? pcmEncoding.value : C2Config::PCM_16);
		}
		bool conforming = (apiFeatures & API_SAME_INPUT_BUFFER);
		// For encrypted content, framework decrypts source buffer (ashmem) into
		// C2Buffers. Thus non-conforming codecs can process these.
		if (!buffersBoundToCodec && (hasCryptoOrDescrambler() \|\| conforming)) {
		if (!buffersBoundToCodec
		&& !input->frameReassembler
		&& (hasCryptoOrDescrambler() \|\| conforming)) {
		input->buffers.reset(new SlotInputBuffers(mName));
		} else if (graphic) {
		if (mInputSurface) {

media/codec2/sfplugin/CCodecBufferChannel.h

+3 −0

Original line number	Diff line number	Diff line
		@@ -31,6 +31,7 @@
		#include <media/stagefright/CodecBase.h>

		#include "CCodecBuffers.h"
		#include "FrameReassembler.h"
		#include "InputSurfaceWrapper.h"
		#include "PipelineWatcher.h"

		@@ -271,6 +272,8 @@ private:
		size_t numExtraSlots;
		uint32_t inputDelay;
		uint32_t pipelineDelay;

		FrameReassembler frameReassembler;
		};
		Mutexed<Input> mInput;
		struct Output {

media/codec2/sfplugin/FrameReassembler.cpp

0 → 100644

+226 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright 2019 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		//#define LOG_NDEBUG 0
		#define LOG_TAG "FrameReassembler"

		#include <log/log.h>

		#include <media/stagefright/foundation/AMessage.h>

		#include "FrameReassembler.h"

		namespace android {

		static constexpr uint64_t kToleranceUs = 1000; // 1ms

		FrameReassembler::FrameReassembler()
		: mUsage{0, 0},
		mSampleRate(0u),
		mChannelCount(0u),
		mEncoding(C2Config::PCM_16),
		mCurrentOrdinal({0, 0, 0}) {
		}

		void FrameReassembler::init(
		const std::shared_ptr<C2BlockPool> &pool,
		C2MemoryUsage usage,
		uint32_t frameSize,
		uint32_t sampleRate,
		uint32_t channelCount,
		C2Config::pcm_encoding_t encoding) {
		mBlockPool = pool;
		mUsage = usage;
		mFrameSize = frameSize;
		mSampleRate = sampleRate;
		mChannelCount = channelCount;
		mEncoding = encoding;
		}

		void FrameReassembler::updateFrameSize(uint32_t frameSize) {
		finishCurrentBlock(&mPendingWork);
		mFrameSize = frameSize;
		}

		void FrameReassembler::updateSampleRate(uint32_t sampleRate) {
		finishCurrentBlock(&mPendingWork);
		mSampleRate = sampleRate;
		}

		void FrameReassembler::updateChannelCount(uint32_t channelCount) {
		finishCurrentBlock(&mPendingWork);
		mChannelCount = channelCount;
		}

		void FrameReassembler::updatePcmEncoding(C2Config::pcm_encoding_t encoding) {
		finishCurrentBlock(&mPendingWork);
		mEncoding = encoding;
		}

		void FrameReassembler::reset() {
		flush();
		mCurrentOrdinal = {0, 0, 0};
		mBlockPool.reset();
		mFrameSize.reset();
		mSampleRate = 0u;
		mChannelCount = 0u;
		mEncoding = C2Config::PCM_16;
		}

		FrameReassembler::operator bool() const {
		return mFrameSize.has_value();
		}

		c2_status_t FrameReassembler::process(
		const sp<MediaCodecBuffer> &buffer,
		std::list<std::unique_ptr<C2Work>> *items) {
		int64_t timeUs;
		if (buffer->size() == 0u
		\|\| !buffer->meta()->findInt64("timeUs", &timeUs)) {
		return C2_BAD_VALUE;
		}

		items->splice(items->end(), mPendingWork);

		// Fill mCurrentBlock
		if (mCurrentBlock) {
		// First check the timestamp
		c2_cntr64_t endTimestampUs = mCurrentOrdinal.timestamp;
		endTimestampUs += bytesToSamples(mWriteView->size()) * 1000000 / mSampleRate;
		if (timeUs < endTimestampUs.peek()) {
		uint64_t diffUs = (endTimestampUs - timeUs).peeku();
		if (diffUs > kToleranceUs) {
		// The timestamp is going back in time in large amount.
		// TODO: b/145702136
		ALOGW("timestamp going back in time! from %lld to %lld",
		endTimestampUs.peekll(), (long long)timeUs);
		}
		} else { // timeUs >= endTimestampUs.peek()
		uint64_t diffUs = (timeUs - endTimestampUs).peeku();
		if (diffUs > kToleranceUs) {
		// The timestamp is going forward; add silence as necessary.
		size_t gapSamples = usToSamples(diffUs);
		size_t remainingSamples =
		(mWriteView->capacity() - mWriteView->size())
		/ mChannelCount / bytesPerSample();
		if (gapSamples < remainingSamples) {
		size_t gapBytes = gapSamples * mChannelCount * bytesPerSample();
		memset(mWriteView->base() + mWriteView->size(), 0u, gapBytes);
		mWriteView->setSize(mWriteView->size() + gapBytes);
		} else {
		finishCurrentBlock(items);
		}
		}
		}
		}

		if (mCurrentBlock) {
		// Append the data at the end of the current block
		size_t copySize = std::min(
		buffer->size(),
		size_t(mWriteView->capacity() - mWriteView->size()));
		memcpy(mWriteView->base() + mWriteView->size(), buffer->data(), copySize);
		buffer->setRange(buffer->offset() + copySize, buffer->size() - copySize);
		mWriteView->setSize(mWriteView->size() + copySize);
		if (mWriteView->size() == mWriteView->capacity()) {
		finishCurrentBlock(items);
		}
		timeUs += bytesToSamples(copySize) * 1000000 / mSampleRate;
		}

		if (buffer->size() > 0) {
		mCurrentOrdinal.timestamp = timeUs;
		}

		size_t frameSizeBytes = mFrameSize.value() * mChannelCount * bytesPerSample();
		while (buffer->size() > 0) {
		LOG_ALWAYS_FATAL_IF(
		mCurrentBlock,
		"There's remaining data but the pending block is not filled & finished");
		std::unique_ptr<C2Work> work(new C2Work);
		c2_status_t err = mBlockPool->fetchLinearBlock(frameSizeBytes, mUsage, &mCurrentBlock);
		if (err != C2_OK) {
		return err;
		}
		size_t copySize = std::min(buffer->size(), frameSizeBytes);
		mWriteView = mCurrentBlock->map().get();
		if (mWriteView->error() != C2_OK) {
		return mWriteView->error();
		}
		ALOGV("buffer={offset=%zu size=%zu) copySize=%zu",
		buffer->offset(), buffer->size(), copySize);
		memcpy(mWriteView->base(), buffer->data(), copySize);
		mWriteView->setOffset(0u);
		mWriteView->setSize(copySize);
		buffer->setRange(buffer->offset() + copySize, buffer->size() - copySize);
		if (copySize == frameSizeBytes) {
		finishCurrentBlock(items);
		}
		}

		int32_t eos = 0;
		if (buffer->meta()->findInt32("eos", &eos) && eos) {
		finishCurrentBlock(items);
		}

		return C2_OK;
		}

		void FrameReassembler::flush() {
		mPendingWork.clear();
		mWriteView.reset();
		mCurrentBlock.reset();
		}

		uint64_t FrameReassembler::bytesToSamples(size_t numBytes) const {
		return numBytes / mChannelCount / bytesPerSample();
		}

		size_t FrameReassembler::usToSamples(uint64_t us) const {
		return (us * mChannelCount * mSampleRate / 1000000);
		}

		uint32_t FrameReassembler::bytesPerSample() const {
		return (mEncoding == C2Config::PCM_8) ? 1
		: (mEncoding == C2Config::PCM_16) ? 2
		: (mEncoding == C2Config::PCM_FLOAT) ? 4 : 0;
		}

		void FrameReassembler::finishCurrentBlock(std::list<std::unique_ptr<C2Work>> *items) {
		if (!mCurrentBlock) {
		// No-op
		return;
		}
		if (mWriteView->size() < mWriteView->capacity()) {
		memset(mWriteView->base() + mWriteView->size(), 0u,
		mWriteView->capacity() - mWriteView->size());
		mWriteView->setSize(mWriteView->capacity());
		}
		std::unique_ptr<C2Work> work{std::make_unique<C2Work>()};
		work->input.ordinal = mCurrentOrdinal;
		work->input.buffers.push_back(C2Buffer::CreateLinearBuffer(
		mCurrentBlock->share(0, mCurrentBlock->capacity(), C2Fence())));
		work->worklets.clear();
		work->worklets.emplace_back(new C2Worklet);
		items->push_back(std::move(work));

		++mCurrentOrdinal.frameIndex;
		mCurrentOrdinal.timestamp += mFrameSize.value() * 1000000 / mSampleRate;
		mCurrentBlock.reset();
		mWriteView.reset();
		}

		} // namespace android

media/codec2/sfplugin/FrameReassembler.h

0 → 100644

+75 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright 2019 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#ifndef FRAME_REASSEMBLER_H_
		#define FRAME_REASSEMBLER_H_

		#include <set>
		#include <memory>

		#include <media/MediaCodecBuffer.h>

		#include <C2Config.h>
		#include <C2Work.h>

		namespace android {

		class FrameReassembler {
		public:
		FrameReassembler();

		void init(
		const std::shared_ptr<C2BlockPool> &pool,
		C2MemoryUsage usage,
		uint32_t frameSize,
		uint32_t sampleRate,
		uint32_t channelCount,
		C2Config::pcm_encoding_t encoding);
		void updateFrameSize(uint32_t frameSize);
		void updateSampleRate(uint32_t sampleRate);
		void updateChannelCount(uint32_t channelCount);
		void updatePcmEncoding(C2Config::pcm_encoding_t encoding);
		void reset();
		void flush();

		explicit operator bool() const;

		c2_status_t process(
		const sp<MediaCodecBuffer> &buffer,
		std::list<std::unique_ptr<C2Work>> *items);

		private:
		std::shared_ptr<C2BlockPool> mBlockPool;
		C2MemoryUsage mUsage;
		std::optional<uint32_t> mFrameSize;
		uint32_t mSampleRate;
		uint32_t mChannelCount;
		C2Config::pcm_encoding_t mEncoding;
		std::list<std::unique_ptr<C2Work>> mPendingWork;
		C2WorkOrdinalStruct mCurrentOrdinal;
		std::shared_ptr<C2LinearBlock> mCurrentBlock;
		std::optional<C2WriteView> mWriteView;

		uint64_t bytesToSamples(size_t numBytes) const;
		size_t usToSamples(uint64_t us) const;
		uint32_t bytesPerSample() const;

		void finishCurrentBlock(std::list<std::unique_ptr<C2Work>> *items);
		};

		} // namespace android

		#endif // FRAME_REASSEMBLER_H_