CCodec: isolate input surface buffer passing from CCodec (0fa28dcc) · Commits · e / os / android_frameworks_av

media/libstagefright/Android.bp

+4 −1

Original line number	Diff line number	Diff line
		@@ -130,7 +130,6 @@ cc_library_shared {
		"libui",
		"libutils",
		"libmedia_helper",
		"libstagefright_bufferqueue_helper",
		"libstagefright_codec2",
		"libstagefright_codec2_vndk",
		"libstagefright_foundation",
		@@ -149,6 +148,10 @@ cc_library_shared {
		"android.hardware.media.omx@1.0",
		"android.hardware.graphics.allocator@2.0",
		"android.hardware.graphics.mapper@2.0",

		// TODO: do not link directly with impl
		"android.hardware.media.c2@1.0-service-impl",
		"libstagefright_bufferqueue_helper",
		],

		static_libs: [

media/libstagefright/CCodec.cpp

+15 −12

Original line number	Diff line number	Diff line
		@@ -24,17 +24,20 @@
		#include <C2PlatformSupport.h>
		#include <C2V4l2Support.h>

		#include <gui/bufferqueue/1.0/H2BGraphicBufferProducer.h>
		#include <gui/Surface.h>
		#include <media/stagefright/codec2/1.0/InputSurface.h>
		#include <media/stagefright/BufferProducerWrapper.h>
		#include <media/stagefright/CCodec.h>
		#include <media/stagefright/PersistentSurface.h>

		#include "include/CCodecBufferChannel.h"

		using namespace std::chrono_literals;

		namespace android {

		using namespace std::chrono_literals;
		using ::android::hardware::graphics::bufferqueue::V1_0::utils::H2BGraphicBufferProducer;

		namespace {

		class CCodecWatchdog : public AHandler {
		@@ -300,18 +303,18 @@ void CCodec::initiateCreateInputSurface() {
		}

		void CCodec::createInputSurface() {
		sp<IGraphicBufferProducer> producer;
		sp<GraphicBufferSource> source(new GraphicBufferSource);
		// TODO: get this from codec process
		sp<InputSurface> surface(InputSurface::Create());

		status_t err = source->initCheck();
		// TODO: get proper error code.
		status_t err = (surface == nullptr) ? UNKNOWN_ERROR : OK;
		if (err != OK) {
		ALOGE("Failed to initialize graphic buffer source: %d", err);
		ALOGE("Failed to initialize input surface: %d", err);
		mCallback->onInputSurfaceCreationFailed(err);
		return;
		}
		producer = source->getIGraphicBufferProducer();

		err = setupInputSurface(source);
		err = setupInputSurface(surface);
		if (err != OK) {
		ALOGE("Failed to set up input surface: %d", err);
		mCallback->onInputSurfaceCreationFailed(err);
		@@ -328,16 +331,16 @@ void CCodec::createInputSurface() {
		mCallback->onInputSurfaceCreated(
		inputFormat,
		outputFormat,
		new BufferProducerWrapper(producer));
		new BufferProducerWrapper(new H2BGraphicBufferProducer(surface)));
		}

		status_t CCodec::setupInputSurface(const sp<GraphicBufferSource> &source) {
		status_t err = mChannel->setGraphicBufferSource(source);
		status_t CCodec::setupInputSurface(const sp<InputSurface> &surface) {
		status_t err = mChannel->setInputSurface(surface);
		if (err != OK) {
		return err;
		}

		// TODO: configure \|source\| with other settings.
		// TODO: configure \|surface\| with other settings.
		return OK;
		}

media/libstagefright/CCodecBufferChannel.cpp

+41 −257

Original line number	Diff line number	Diff line
		@@ -97,15 +97,6 @@ public:
		*/
		virtual bool requestNewBuffer(size_t index, sp<MediaCodecBuffer> buffer) = 0;

		/**
		* Release the buffer obtained from requestNewBuffer() and get the
		* associated C2Buffer object back. Returns empty shared_ptr if the
		* buffer is not on file.
		*
		* XXX: this is a quick hack to be removed
		*/
		virtual std::shared_ptr<C2Buffer> releaseBufferIndex(size_t /* index */) { return nullptr; }

		/**
		* Release the buffer obtained from requestNewBuffer() and get the
		* associated C2Buffer object back. Returns empty shared_ptr if the
		@@ -134,47 +125,6 @@ private:
		DISALLOW_EVIL_CONSTRUCTORS(InputBuffers);
		};

		class CCodecBufferChannel::InputBufferClient {
		public:
		explicit InputBufferClient(
		const std::shared_ptr<CCodecBufferChannel> &channel) : mChannel(channel) {}
		virtual ~InputBufferClient() = default;

		virtual void onInputBufferAdded(size_t index, const sp<MediaCodecBuffer> &buffer) {
		std::shared_ptr<CCodecBufferChannel> channel = mChannel.lock();
		if (!channel) {
		return;
		}
		channel->mCallback->onInputBufferAvailable(index, buffer);
		}

		virtual void onStart() {
		// no-op
		}

		virtual void onStop() {
		// no-op
		}

		virtual void onRelease() {
		// no-op
		}

		virtual void onInputBufferAvailable(size_t index, const sp<MediaCodecBuffer> &buffer) {
		std::shared_ptr<CCodecBufferChannel> channel = mChannel.lock();
		if (!channel) {
		return;
		}
		channel->mCallback->onInputBufferAvailable(index, buffer);
		}

		protected:
		InputBufferClient() = default;
		std::weak_ptr<CCodecBufferChannel> mChannel;

		DISALLOW_EVIL_CONSTRUCTORS(InputBufferClient);
		};

		class CCodecBufferChannel::OutputBuffers : public CCodecBufferChannel::Buffers {
		public:
		OutputBuffers() = default;
		@@ -227,8 +177,6 @@ private:

		namespace {

		constexpr int32_t kMaskI32 = ~0;

		// TODO: get this info from component
		const static size_t kMinBufferArraySize = 16;
		const static size_t kLinearBufferSize = 524288;
		@@ -422,23 +370,8 @@ class GraphicInputBuffers : public CCodecBufferChannel::InputBuffers {
		public:
		GraphicInputBuffers() = default;

		bool requestNewBuffer(size_t index, sp<MediaCodecBuffer> buffer) override {
		*buffer = nullptr;
		for (size_t i = 0; i < mAvailable.size(); ++i) {
		if (mAvailable[i]) {
		*index = i;
		mAvailable[i] = false;
		return true;
		}
		}
		*index = mAvailable.size();
		mAvailable.push_back(false);
		return true;
		}

		std::shared_ptr<C2Buffer> releaseBufferIndex(size_t index) override {
		mAvailable[index] = true;
		return nullptr;
		bool requestNewBuffer(size_t , sp<MediaCodecBuffer> ) override {
		return false;
		}

		std::shared_ptr<C2Buffer> releaseBuffer(const sp<MediaCodecBuffer> &) override {
		@@ -452,8 +385,11 @@ public:
		return nullptr;
		}

		private:
		std::vector<bool> mAvailable;
		bool isArrayMode() const final { return true; }

		void getArray(Vector<sp<MediaCodecBuffer>> *array) const final {
		array->clear();
		}
		};

		class OutputBuffersArray : public CCodecBufferChannel::OutputBuffers {
		@@ -753,164 +689,8 @@ public:
		}
		};

		class BufferQueueClient : public CCodecBufferChannel::InputBufferClient {
		public:
		explicit BufferQueueClient(const sp<GraphicBufferSource> &source) : mSource(source) {}
		virtual ~BufferQueueClient() = default;

		void onInputBufferAdded(size_t index, const sp<MediaCodecBuffer> &buffer) override {
		(void)buffer;
		mSource->onInputBufferAdded(index & kMaskI32);
		}

		void onStart() override {
		mSource->start();
		}

		void onStop() override {
		mSource->stop();
		}

		void onRelease() override {
		mSource->release();
		}

		void onInputBufferAvailable(size_t index, const sp<MediaCodecBuffer> &buffer) override {
		ALOGV("onInputBufferEmptied index = %zu", index);
		(void)buffer;
		// TODO: can we really ignore fence here?
		mSource->onInputBufferEmptied(index & kMaskI32, -1 /* fenceFd */);
		}

		private:
		sp<GraphicBufferSource> mSource;
		};

		class GraphicBlock : public C2GraphicBlock {
		using C2GraphicBlock::C2GraphicBlock;
		friend class ::android::CCodecBufferChannel;
		};

		} // namespace

		class CCodecBufferChannel::C2ComponentWrapper : public ComponentWrapper {
		public:
		explicit C2ComponentWrapper(
		const std::shared_ptr<CCodecBufferChannel> &channel)
		: mChannel(channel), mLastTimestamp(0) {}

		virtual ~C2ComponentWrapper() {
		for (const std::pair<int32_t, C2Handle *> &entry : mHandles) {
		native_handle_delete(entry.second);
		}
		}

		status_t submitBuffer(
		int32_t bufferId, const sp<GraphicBuffer> &buffer,
		int64_t timestamp, int fenceFd) override {
		ALOGV("submitBuffer bufferId = %d", bufferId);
		// TODO: Use fd to construct fence
		(void)fenceFd;

		std::shared_ptr<CCodecBufferChannel> channel = mChannel.lock();
		if (!channel) {
		return NO_INIT;
		}

		std::shared_ptr<C2Allocator> allocator = mAllocator.lock();
		if (!allocator) {
		c2_status_t err = GetCodec2PlatformAllocatorStore()->fetchAllocator(
		C2PlatformAllocatorStore::GRALLOC,
		&allocator);
		if (err != OK) {
		return UNKNOWN_ERROR;
		}
		mAllocator = allocator;
		}

		std::shared_ptr<C2GraphicAllocation> alloc;
		C2Handle *handle = WrapNativeCodec2GrallocHandle(
		buffer->handle, buffer->width, buffer->height,
		buffer->format, buffer->usage, buffer->stride);
		c2_status_t err = allocator->priorGraphicAllocation(handle, &alloc);
		if (err != OK) {
		return UNKNOWN_ERROR;
		}

		std::shared_ptr<C2GraphicBlock> block = _C2BlockFactory::CreateGraphicBlock(alloc);

		std::unique_ptr<C2Work> work(new C2Work);
		work->input.flags = (C2FrameData::flags_t)0;
		work->input.ordinal.timestamp = timestamp;
		work->input.ordinal.frameIndex = channel->mFrameIndex++;
		work->input.buffers.clear();
		work->input.buffers.emplace_back(new Buffer2D(
		// TODO: fence
		block->share(C2Rect(block->width(), block->height()), ::android::C2Fence())));
		work->worklets.clear();
		work->worklets.emplace_back(new C2Worklet);
		std::list<std::unique_ptr<C2Work>> items;
		items.push_back(std::move(work));

		err = channel->mComponent->queue_nb(&items);
		if (err != OK) {
		native_handle_delete(handle);
		return UNKNOWN_ERROR;
		}

		mLastTimestamp = timestamp;
		if (mHandles.count(bufferId) > 0) {
		native_handle_delete(mHandles[bufferId]);
		}
		mHandles[bufferId] = handle;

		Mutexed<std::unique_ptr<InputBuffers>>::Locked buffers(channel->mInputBuffers);
		ALOGV("releaseBufferIndex index = %d", bufferId);
		(*buffers)->releaseBufferIndex(bufferId);

		return OK;
		}

		status_t submitEos(int32_t bufferId) override {
		std::shared_ptr<CCodecBufferChannel> channel = mChannel.lock();
		if (!channel) {
		return NO_INIT;
		}

		std::unique_ptr<C2Work> work(new C2Work);
		work->input.flags = C2FrameData::FLAG_END_OF_STREAM;
		work->input.ordinal.timestamp = mLastTimestamp;
		work->input.ordinal.frameIndex = channel->mFrameIndex++;
		work->input.buffers.clear();
		work->input.buffers.push_back(nullptr);
		work->worklets.clear();
		work->worklets.emplace_back(new C2Worklet);
		std::list<std::unique_ptr<C2Work>> items;
		items.push_back(std::move(work));

		c2_status_t err = channel->mComponent->queue_nb(&items);

		Mutexed<std::unique_ptr<InputBuffers>>::Locked buffers(channel->mInputBuffers);
		(*buffers)->releaseBufferIndex(bufferId);

		return (err == C2_OK) ? OK : UNKNOWN_ERROR;
		}

		void dispatchDataSpaceChanged(
		int32_t dataSpace, int32_t aspects, int32_t pixelFormat) override {
		// TODO
		(void)dataSpace;
		(void)aspects;
		(void)pixelFormat;
		}

		private:
		std::weak_ptr<CCodecBufferChannel> mChannel;
		std::map<int32_t, C2Handle *> mHandles;
		int64_t mLastTimestamp;
		std::weak_ptr<C2Allocator> mAllocator;
		};

		CCodecBufferChannel::QueueGuard::QueueGuard(
		CCodecBufferChannel::QueueSync &sync) : mSync(sync) {
		std::unique_lock<std::mutex> l(mSync.mMutex);
		@@ -967,13 +747,10 @@ CCodecBufferChannel::~CCodecBufferChannel() {
		if (mCrypto != nullptr && mDealer != nullptr && mHeapSeqNum >= 0) {
		mCrypto->unsetHeap(mHeapSeqNum);
		}
		// TODO: is this the right place?
		mInputClient->onRelease();
		}

		void CCodecBufferChannel::setComponent(const std::shared_ptr<C2Component> &component) {
		mComponent = component;
		mInputClient.reset(new InputBufferClient(shared_from_this()));

		C2StreamFormatConfig::input inputFormat(0u);
		C2StreamFormatConfig::output outputFormat(0u);
		@@ -1026,16 +803,10 @@ void CCodecBufferChannel::setComponent(const std::shared_ptr<C2Component> &compo
		}
		}

		status_t CCodecBufferChannel::setGraphicBufferSource(
		const sp<GraphicBufferSource> &source) {
		ALOGV("setGraphicBufferSource");
		mInputClient.reset(new BufferQueueClient(source));

		// TODO: proper color aspect & dataspace
		android_dataspace dataSpace = HAL_DATASPACE_BT709;
		// TODO: read settings properly from the interface
		return source->configure(new C2ComponentWrapper(
		shared_from_this()), dataSpace, 16, 1080, 1920, GRALLOC_USAGE_SW_READ_OFTEN);
		status_t CCodecBufferChannel::setInputSurface(const sp<InputSurface> &surface) {
		ALOGV("setInputSurface");
		mInputSurface = surface;
		return OK;
		}

		status_t CCodecBufferChannel::queueInputBuffer(const sp<MediaCodecBuffer> &buffer) {
		@@ -1100,12 +871,13 @@ void CCodecBufferChannel::feedInputBufferIfAvailable() {
		{
		Mutexed<std::unique_ptr<InputBuffers>>::Locked buffers(mInputBuffers);
		if (!(*buffers)->requestNewBuffer(&index, &inBuffer)) {
		ALOGW("no new buffer available");
		ALOGV("no new buffer available");
		inBuffer = nullptr;
		return;
		}
		}
		ALOGV("new input index = %zu", index);
		mInputClient->onInputBufferAvailable(index, inBuffer);
		mCallback->onInputBufferAvailable(index, inBuffer);
		}

		status_t CCodecBufferChannel::renderOutputBuffer(
		@@ -1182,7 +954,9 @@ status_t CCodecBufferChannel::discardBuffer(const sp<MediaCodecBuffer> &buffer)
		{
		Mutexed<std::unique_ptr<OutputBuffers>>::Locked buffers(mOutputBuffers);
		if((*buffers)->releaseBuffer(buffer)) {
		buffers.unlock();
		feedInputBufferIfAvailable();
		buffers.lock();
		}
		}
		return OK;
		@@ -1221,6 +995,7 @@ void CCodecBufferChannel::start(const sp<AMessage> &inputFormat, const sp<AMessa
		}

		mSync.start();
		if (mInputSurface == nullptr) {
		// TODO: use proper buffer depth instead of this random value
		for (size_t i = 0; i < kMinBufferArraySize; ++i) {
		size_t index;
		@@ -1228,22 +1003,31 @@ void CCodecBufferChannel::start(const sp<AMessage> &inputFormat, const sp<AMessa
		{
		Mutexed<std::unique_ptr<InputBuffers>>::Locked buffers(mInputBuffers);
		if (!(*buffers)->requestNewBuffer(&index, &buffer)) {
		if (i == 0) {
		ALOGE("start: cannot allocate memory at all");
		buffers.unlock();
		ALOGE("start: cannot allocate memory");
		mOnError(NO_MEMORY, ACTION_CODE_FATAL);
		buffers.lock();
		return;
		} else {
		ALOGV("start: cannot allocate memory, only %zu buffers allocated", i);
		}
		break;
		}
		}
		mCallback->onInputBufferAvailable(index, buffer);
		}
		mInputClient->onInputBufferAdded(index, buffer);
		} else {
		mInputSurfaceConnection = mInputSurface->connectToComponent(mComponent);
		}
		mInputClient->onStart();
		}

		void CCodecBufferChannel::stop() {
		mSync.stop();
		mFirstValidFrameIndex = mFrameIndex.load();
		mInputClient->onStop();
		if (mInputSurfaceConnection != nullptr) {
		mInputSurfaceConnection->disconnect();
		mInputSurfaceConnection.clear();
		}
		}

		void CCodecBufferChannel::flush(const std::list<std::unique_ptr<C2Work>> &flushedWork) {

media/libstagefright/bqhelper/Android.bp

+1 −0

Original line number	Diff line number	Diff line
		@@ -43,6 +43,7 @@ cc_library_shared {

		export_shared_lib_headers: [
		"libstagefright_foundation",
		"libhidlmemory",
		],

		cflags: [

media/libstagefright/bqhelper/include/media/stagefright/bqhelper/WGraphicBufferProducer.h

+9 −1

Original line number	Diff line number	Diff line
		@@ -20,7 +20,6 @@
		#include <hidl/MQDescriptor.h>
		#include <hidl/Status.h>

		#include <android-base/logging.h>
		#include <binder/Binder.h>
		#include <gui/IGraphicBufferProducer.h>
		#include <gui/IProducerListener.h>
		@@ -52,6 +51,15 @@ typedef ::android::IGraphicBufferProducer BGraphicBufferProducer;
		typedef ::android::IProducerListener BProducerListener;
		using ::android::BnGraphicBufferProducer;

		#ifndef LOG
		struct LOG_dummy {
		template <typename T>
		LOG_dummy& operator<< (const T&) { return *this; }
		};

		#define LOG(x) LOG_dummy()
		#endif

		// Instantiate only if HGraphicBufferProducer is base of BASE.
		template <typename BASE,
		typename = typename std::enable_if<std::is_base_of<HGraphicBufferProducer, BASE>::value>::type>