Merge "MediaCodec: implement detach surface" into main am: bd125c1808 (ba7cf56c) · Commits · e / os / android_frameworks_av

media/libstagefright/Android.bp

+1 −0

Original line number	Diff line number	Diff line
		@@ -319,6 +319,7 @@ cc_library {
		],

		static_libs: [
		"android.media.codec-aconfig-cc",
		"libstagefright_esds",
		"libstagefright_color_conversion",
		"libyuv_static",

media/libstagefright/MediaCodec.cpp

+149 −28

Original line number	Diff line number	Diff line
		@@ -30,6 +30,8 @@

		#include "include/SoftwareRenderer.h"

		#include <android_media_codec.h>

		#include <android/api-level.h>
		#include <android/content/pm/IPackageManagerNative.h>
		#include <android/hardware/cas/native/1.0/IDescrambler.h>
		@@ -3017,6 +3019,13 @@ status_t MediaCodec::setInputSurface(
		return PostAndAwaitResponse(msg, &response);
		}

		status_t MediaCodec::detachOutputSurface() {
		sp<AMessage> msg = new AMessage(kWhatDetachSurface, this);

		sp<AMessage> response;
		return PostAndAwaitResponse(msg, &response);
		}

		status_t MediaCodec::setSurface(const sp<Surface> &surface) {
		sp<AMessage> msg = new AMessage(kWhatSetSurface, this);
		msg->setObject("surface", surface);
		@@ -4676,7 +4685,7 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) {
		}

		mResourceManagerProxy->removeClient();
		mReleaseSurface.reset();
		mDetachedSurface.reset();

		if (mReplyID != nullptr) {
		postPendingRepliesAndDeferredMessages("kWhatReleaseCompleted");
		@@ -4849,6 +4858,23 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) {
		mFlags \|= kFlagPushBlankBuffersOnShutdown;
		}

		uint32_t flags;
		CHECK(msg->findInt32("flags", (int32_t *)&flags));

		if (android::media::codec::provider_->null_output_surface_support()) {
		if (obj == nullptr
		&& (flags & CONFIGURE_FLAG_DETACHED_SURFACE)
		&& !(flags & CONFIGURE_FLAG_ENCODE)) {
		sp<Surface> surface = getOrCreateDetachedSurface();
		if (surface == nullptr) {
		mErrorLog.log(
		LOG_TAG, "Detached surface mode is not supported by this codec");
		PostReplyWithError(replyID, INVALID_OPERATION);
		}
		obj = surface;
		}
		}

		if (obj != NULL) {
		if (!format->findInt32(KEY_ALLOW_FRAME_DROP, &mAllowFrameDroppingBySurface)) {
		// allow frame dropping by surface by default
		@@ -4872,8 +4898,6 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) {

		mApiUsageMetrics.isUsingOutputSurface = true;

		uint32_t flags;
		CHECK(msg->findInt32("flags", (int32_t *)&flags));
		if (flags & CONFIGURE_FLAG_USE_BLOCK_MODEL \|\|
		flags & CONFIGURE_FLAG_USE_CRYPTO_ASYNC) {
		if (!(mFlags & kFlagIsAsync)) {
		@@ -4994,6 +5018,23 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) {
		break;
		}

		case kWhatDetachSurface:
		{
		// detach surface is equivalent to setSurface(mDetachedSurface)
		sp<Surface> surface = getOrCreateDetachedSurface();

		if (surface == nullptr) {
		sp<AReplyToken> replyID;
		CHECK(msg->senderAwaitsResponse(&replyID));
		mErrorLog.log(LOG_TAG, "Detaching surface is not supported by the codec.");
		PostReplyWithError(replyID, INVALID_OPERATION);
		break;
		}

		msg->setObject("surface", surface);
		}
		[[fallthrough]];

		case kWhatSetSurface:
		{
		sp<AReplyToken> replyID;
		@@ -5011,14 +5052,17 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) {
		sp<Surface> surface = static_cast<Surface *>(obj.get());
		if (mSurface == NULL) {
		// do not support setting surface if it was not set
		mErrorLog.log(LOG_TAG,
		"Cannot set surface if the codec is not configured with "
		"a surface already");
		mErrorLog.log(LOG_TAG, base::StringPrintf(
		"Cannot %s surface if the codec is not configured with "
		"a surface already",
		msg->what() == kWhatDetachSurface ? "detach" : "set"));
		err = INVALID_OPERATION;
		} else if (obj == NULL) {
		// do not support unsetting surface
		mErrorLog.log(LOG_TAG, "Unsetting surface is not supported");
		err = BAD_VALUE;
		} else if (android::media::codec::provider_->null_output_surface_support()) {
		err = handleSetSurface(surface, true /* callCodec */);
		} else {
		uint32_t generation;
		err = connectToSurface(surface, &generation);
		@@ -5052,7 +5096,8 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) {

		default:
		mErrorLog.log(LOG_TAG, base::StringPrintf(
		"setSurface() is valid only at Executing states; currently %s",
		"%sSurface() is valid only at Executing states; currently %s",
		msg->what() == kWhatDetachSurface ? "detach" : "set",
		apiStateString().c_str()));
		err = INVALID_OPERATION;
		break;
		@@ -5273,24 +5318,33 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) {

		bool forceSync = false;
		if (asyncNotify != nullptr && mSurface != NULL) {
		if (!mReleaseSurface) {
		if (android::media::codec::provider_->null_output_surface_support()) {
		if (handleSetSurface(getOrCreateDetachedSurface(), true /* callCodec */,
		true /* onShutDown */) != OK) {
		// We were not able to detach the surface, so force
		// synchronous release.
		forceSync = true;
		}
		} else {
		if (!mDetachedSurface) {
		uint64_t usage = 0;
		if (mSurface->getConsumerUsage(&usage) != OK) {
		usage = 0;
		}
		mReleaseSurface.reset(new ReleaseSurface(usage));
		mDetachedSurface.reset(new ReleaseSurface(usage));
		}
		if (mSurface != mReleaseSurface->getSurface()) {
		if (mSurface != mDetachedSurface->getSurface()) {
		uint32_t generation;
		status_t err = connectToSurface(mReleaseSurface->getSurface(), &generation);
		status_t err =
		connectToSurface(mDetachedSurface->getSurface(), &generation);
		ALOGW_IF(err != OK, "error connecting to release surface: err = %d", err);
		if (err == OK && !(mFlags & kFlagUsesSoftwareRenderer)) {
		err = mCodec->setSurface(mReleaseSurface->getSurface(), generation);
		err = mCodec->setSurface(mDetachedSurface->getSurface(), generation);
		ALOGW_IF(err != OK, "error setting release surface: err = %d", err);
		}
		if (err == OK) {
		(void)disconnectFromSurface();
		mSurface = mReleaseSurface->getSurface();
		mSurface = mDetachedSurface->getSurface();
		mSurfaceGeneration = generation;
		} else {
		// We were not able to switch the surface, so force
		@@ -5299,6 +5353,7 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) {
		}
		}
		}
		}

		if (mReplyID) {
		// State transition replies are handled above, so this reply
		@@ -6612,6 +6667,23 @@ ssize_t MediaCodec::dequeuePortBuffer(int32_t portIndex) {
		return index;
		}

		sp<Surface> MediaCodec::getOrCreateDetachedSurface() {
		if (mDomain != DOMAIN_VIDEO \|\| (mFlags & kFlagIsEncoder)) {
		return nullptr;
		}

		if (!mDetachedSurface) {
		uint64_t usage = 0;
		if (!mSurface \|\| mSurface->getConsumerUsage(&usage) != OK) {
		// TODO: should we use a/the default consumer usage?
		usage = 0;
		}
		mDetachedSurface.reset(new ReleaseSurface(usage));
		}

		return mDetachedSurface->getSurface();
		}

		status_t MediaCodec::connectToSurface(const sp<Surface> &surface, uint32_t *generation) {
		status_t err = OK;
		if (surface != NULL) {
		@@ -6685,7 +6757,56 @@ status_t MediaCodec::disconnectFromSurface() {
		return err;
		}

		status_t MediaCodec::handleSetSurface(const sp<Surface> &surface, bool callCodec, bool onShutDown) {
		uint32_t generation;
		status_t err = OK;
		if (surface != nullptr) {
		err = connectToSurface(surface, &generation);
		if (err == ALREADY_EXISTS) {
		// reconnecting to same surface
		return OK;
		}

		if (err == OK && callCodec) {
		if (mFlags & kFlagUsesSoftwareRenderer) {
		if (mSoftRenderer != NULL
		&& (mFlags & kFlagPushBlankBuffersOnShutdown)) {
		pushBlankBuffersToNativeWindow(mSurface.get());
		}
		// do not create a new software renderer on shutdown (release)
		// as it will not be used anyway
		if (!onShutDown) {
		surface->setDequeueTimeout(-1);
		mSoftRenderer = new SoftwareRenderer(surface);
		// TODO: check if this was successful
		}
		} else {
		err = mCodec->setSurface(surface, generation);
		}

		mReliabilityContextMetrics.setOutputSurfaceCount++;
		}
		}

		if (err == OK) {
		if (mSurface != NULL) {
		(void)disconnectFromSurface();
		}

		if (surface != NULL) {
		mSurface = surface;
		mSurfaceGeneration = generation;
		}
		}

		return err;
		}

		status_t MediaCodec::handleSetSurface(const sp<Surface> &surface) {
		if (android::media::codec::provider_->null_output_surface_support()) {
		return handleSetSurface(surface, false /* callCodec */);
		}

		status_t err = OK;
		if (mSurface != NULL) {
		(void)disconnectFromSurface();

media/libstagefright/include/media/stagefright/MediaCodec.h

+16 −1

Original line number	Diff line number	Diff line
		@@ -96,6 +96,7 @@ struct MediaCodec : public AHandler {
		CONFIGURE_FLAG_ENCODE = 1,
		CONFIGURE_FLAG_USE_BLOCK_MODEL = 2,
		CONFIGURE_FLAG_USE_CRYPTO_ASYNC = 4,
		CONFIGURE_FLAG_DETACHED_SURFACE = 8,
		};

		enum BufferFlags {
		@@ -274,6 +275,8 @@ struct MediaCodec : public AHandler {

		status_t setSurface(const sp<Surface> &nativeWindow);

		status_t detachOutputSurface();

		status_t requestIDRFrame();

		// Notification will be posted once there "is something to do", i.e.
		@@ -368,6 +371,7 @@ private:
		kWhatInit = 'init',
		kWhatConfigure = 'conf',
		kWhatSetSurface = 'sSur',
		kWhatDetachSurface = 'dSur',
		kWhatCreateInputSurface = 'cisf',
		kWhatSetInputSurface = 'sisf',
		kWhatStart = 'strt',
		@@ -474,6 +478,10 @@ private:
		uint32_t mSurfaceGeneration = 0;
		SoftwareRenderer *mSoftRenderer;

		// Get the detached BufferQueue surface for a video decoder, and create it
		// if it did not yet exist.
		sp<Surface> getOrCreateDetachedSurface();

		Mutex mMetricsLock;
		mediametrics_handle_t mMetricsHandle = 0;
		bool mMetricsToUpload = false;
		@@ -642,6 +650,13 @@ private:
		status_t queueCSDInputBuffer(size_t bufferIndex);

		status_t handleSetSurface(const sp<Surface> &surface);

		// Common reimplementation of changing the output surface.
		// Handles setting null surface, which is used during configure and init.
		// Set \|callCodec\| to true if the codec needs to be notified (e.g. during executing state).
		// Setting \|onShutdown\| to true will avoid extra work, if this is used for detaching on
		// delayed release.
		status_t handleSetSurface(const sp<Surface> &surface, bool callCodec, bool onShutdown = false);
		status_t connectToSurface(const sp<Surface> &surface, uint32_t *generation);
		status_t disconnectFromSurface();

		@@ -714,7 +729,7 @@ private:
		sp<AMessage> mMsgPollForRenderedBuffers;

		class ReleaseSurface;
		std::unique_ptr<ReleaseSurface> mReleaseSurface;
		std::unique_ptr<ReleaseSurface> mDetachedSurface;

		std::list<sp<AMessage>> mLeftover;
		status_t handleLeftover(size_t index);