Merge "NuPlayer: tunnel decoder with renderer for data buffer passing." into lmp-mr1-dev (addfc718) · Commits · e / os / android_frameworks_av

media/libmediaplayerservice/nuplayer/NuPlayer.cpp

+16 −74

Original line number	Diff line number	Diff line
		@@ -165,8 +165,6 @@ NuPlayer::NuPlayer()
		mTimeDiscontinuityPending(false),
		mFlushingAudio(NONE),
		mFlushingVideo(NONE),
		mSkipRenderingAudioUntilMediaTimeUs(-1ll),
		mSkipRenderingVideoUntilMediaTimeUs(-1ll),
		mNumFramesTotal(0ll),
		mNumFramesDropped(0ll),
		mVideoScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW),
		@@ -701,19 +699,14 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {

		handleFlushComplete(audio, true /* isDecoder */);
		finishFlushIfPossible();
		} else if (what == Decoder::kWhatOutputFormatChanged) {
		} else if (what == Decoder::kWhatVideoSizeChanged) {
		sp<AMessage> format;
		CHECK(msg->findMessage("format", &format));

		if (audio) {
		openAudioSink(format, false /offloadOnly/);
		} else {
		// video
		sp<AMessage> inputFormat =
		mSource->getFormat(false /* audio */);

		updateVideoSize(inputFormat, format);
		}
		} else if (what == Decoder::kWhatShutdownCompleted) {
		ALOGV("%s shutdown completed", audio ? "audio" : "video");
		if (audio) {
		@@ -779,7 +772,7 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
		break; // Finish anyways.
		}
		notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err);
		} else if (what == Decoder::kWhatDrainThisBuffer) {
		} else if (what == Decoder::kWhatRenderBufferTime) {
		renderBuffer(audio, msg);
		} else {
		ALOGV("Unhandled decoder notification %d '%c%c%c%c'.",
		@@ -967,8 +960,6 @@ void NuPlayer::onStart() {
		mOffloadAudio = false;
		mAudioEOS = false;
		mVideoEOS = false;
		mSkipRenderingAudioUntilMediaTimeUs = -1;
		mSkipRenderingVideoUntilMediaTimeUs = -1;
		mNumFramesTotal = 0;
		mNumFramesDropped = 0;
		mStarted = true;
		@@ -1024,6 +1015,13 @@ void NuPlayer::onStart() {
		mRenderer->setVideoFrameRate(rate);
		}

		if (mVideoDecoder != NULL) {
		mVideoDecoder->setRenderer(mRenderer);
		}
		if (mAudioDecoder != NULL) {
		mAudioDecoder->setRenderer(mRenderer);
		}

		postScanSources();
		}

		@@ -1182,16 +1180,16 @@ status_t NuPlayer::instantiateDecoder(bool audio, sp<Decoder> *decoder) {
		notify->setInt32("generation", mAudioDecoderGeneration);

		if (mOffloadAudio) {
		*decoder = new DecoderPassThrough(notify);
		*decoder = new DecoderPassThrough(notify, mSource, mRenderer);
		} else {
		*decoder = new Decoder(notify);
		*decoder = new Decoder(notify, mSource, mRenderer);
		}
		} else {
		sp<AMessage> notify = new AMessage(kWhatVideoNotify, id());
		++mVideoDecoderGeneration;
		notify->setInt32("generation", mVideoDecoderGeneration);

		*decoder = new Decoder(notify, mNativeWindow);
		*decoder = new Decoder(notify, mSource, mRenderer, mNativeWindow);
		}
		(*decoder)->init();
		(*decoder)->configure(format);
		@@ -1280,33 +1278,6 @@ status_t NuPlayer::feedDecoderInputData(bool audio, const sp<AMessage> &msg) {
		ALOGI("%s discontinuity (formatChange=%d, time=%d)",
		audio ? "audio" : "video", formatChange, timeChange);

		if (audio) {
		mSkipRenderingAudioUntilMediaTimeUs = -1;
		} else {
		mSkipRenderingVideoUntilMediaTimeUs = -1;
		}

		if (timeChange) {
		sp<AMessage> extra;
		if (accessUnit->meta()->findMessage("extra", &extra)
		&& extra != NULL) {
		int64_t resumeAtMediaTimeUs;
		if (extra->findInt64(
		"resume-at-mediatimeUs", &resumeAtMediaTimeUs)) {
		ALOGI("suppressing rendering of %s until %lld us",
		audio ? "audio" : "video", resumeAtMediaTimeUs);

		if (audio) {
		mSkipRenderingAudioUntilMediaTimeUs =
		resumeAtMediaTimeUs;
		} else {
		mSkipRenderingVideoUntilMediaTimeUs =
		resumeAtMediaTimeUs;
		}
		}
		}
		}

		mTimeDiscontinuityPending =
		mTimeDiscontinuityPending \|\| timeChange;

		@@ -1447,9 +1418,6 @@ status_t NuPlayer::feedDecoderInputData(bool audio, const sp<AMessage> &msg) {
		void NuPlayer::renderBuffer(bool audio, const sp<AMessage> &msg) {
		// ALOGV("renderBuffer %s", audio ? "audio" : "video");

		sp<AMessage> reply;
		CHECK(msg->findMessage("reply", &reply));

		if ((audio && mFlushingAudio != NONE)
		\|\| (!audio && mFlushingVideo != NONE)) {
		// We're currently attempting to flush the decoder, in order
		@@ -1460,40 +1428,15 @@ void NuPlayer::renderBuffer(bool audio, const sp<AMessage> &msg) {
		ALOGV("we're still flushing the %s decoder, sending its output buffer"
		" right back.", audio ? "audio" : "video");

		reply->post();
		return;
		}

		sp<ABuffer> buffer;
		CHECK(msg->findBuffer("buffer", &buffer));

		int64_t mediaTimeUs;
		CHECK(buffer->meta()->findInt64("timeUs", &mediaTimeUs));

		int64_t &skipUntilMediaTimeUs =
		audio
		? mSkipRenderingAudioUntilMediaTimeUs
		: mSkipRenderingVideoUntilMediaTimeUs;

		if (skipUntilMediaTimeUs >= 0) {

		if (mediaTimeUs < skipUntilMediaTimeUs) {
		ALOGV("dropping %s buffer at time %lld as requested.",
		audio ? "audio" : "video",
		mediaTimeUs);

		reply->post();
		return;
		}

		skipUntilMediaTimeUs = -1;
		}
		CHECK(msg->findInt64("timeUs", &mediaTimeUs));

		if (!audio && mCCDecoder->isSelected()) {
		mCCDecoder->display(mediaTimeUs);
		}

		mRenderer->queueBuffer(audio, buffer, reply);
		}

		void NuPlayer::updateVideoSize(
		@@ -1593,7 +1536,6 @@ void NuPlayer::flushDecoder(
		mScanSourcesPending = false;

		decoder->signalFlush(newFormat);
		mRenderer->flush(audio);

		FlushStatus newStatus =
		needShutdown ? FLUSHING_DECODER_SHUTDOWN : FLUSHING_DECODER;

media/libmediaplayerservice/nuplayer/NuPlayer.h

+0 −3

Original line number	Diff line number	Diff line
		@@ -181,9 +181,6 @@ private:
		FlushStatus mFlushingAudio;
		FlushStatus mFlushingVideo;

		int64_t mSkipRenderingAudioUntilMediaTimeUs;
		int64_t mSkipRenderingVideoUntilMediaTimeUs;

		int64_t mNumFramesTotal, mNumFramesDropped;

		int32_t mVideoScalingMode;

media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp

+99 −20

Original line number	Diff line number	Diff line
		@@ -21,6 +21,9 @@

		#include "NuPlayerDecoder.h"

		#include "NuPlayerRenderer.h"
		#include "NuPlayerSource.h"

		#include <media/ICrypto.h>
		#include <media/stagefright/foundation/ABitReader.h>
		#include <media/stagefright/foundation/ABuffer.h>
		@@ -35,9 +38,14 @@ namespace android {

		NuPlayer::Decoder::Decoder(
		const sp<AMessage> &notify,
		const sp<Source> &source,
		const sp<Renderer> &renderer,
		const sp<NativeWindowWrapper> &nativeWindow)
		: mNotify(notify),
		mNativeWindow(nativeWindow),
		mSource(source),
		mRenderer(renderer),
		mSkipRenderingUntilMediaTimeUs(-1ll),
		mBufferGeneration(0),
		mPaused(true),
		mComponentName("decoder") {
		@@ -169,7 +177,9 @@ void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) {
		mInputBuffers.size(),
		mOutputBuffers.size());

		if (mRenderer != NULL) {
		requestCodecNotification();
		}
		mPaused = false;
		}

		@@ -191,6 +201,7 @@ void NuPlayer::Decoder::releaseAndResetMediaBuffers() {
		}

		mPendingInputMessages.clear();
		mSkipRenderingUntilMediaTimeUs = -1;
		}

		void NuPlayer::Decoder::requestCodecNotification() {
		@@ -217,6 +228,12 @@ void NuPlayer::Decoder::configure(const sp<AMessage> &format) {
		msg->post();
		}

		void NuPlayer::Decoder::setRenderer(const sp<Renderer> &renderer) {
		sp<AMessage> msg = new AMessage(kWhatSetRenderer, id());
		msg->setObject("renderer", renderer);
		msg->post();
		}

		void NuPlayer::Decoder::signalUpdateFormat(const sp<AMessage> &format) {
		sp<AMessage> msg = new AMessage(kWhatUpdateFormat, id());
		msg->setMessage("format", format);
		@@ -342,8 +359,6 @@ bool android::NuPlayer::Decoder::onInputBufferFilled(const sp<AMessage> &msg) {
		}
		}



		if (buffer == NULL /* includes !hasBuffer */) {
		int32_t streamErr = ERROR_END_OF_STREAM;
		CHECK(msg->findInt32("err", &streamErr) \|\| !hasBuffer);
		@@ -375,6 +390,17 @@ bool android::NuPlayer::Decoder::onInputBufferFilled(const sp<AMessage> &msg) {
		handleError(streamErr);
		}
		} else {
		sp<AMessage> extra;
		if (buffer->meta()->findMessage("extra", &extra) && extra != NULL) {
		int64_t resumeAtMediaTimeUs;
		if (extra->findInt64(
		"resume-at-mediaTimeUs", &resumeAtMediaTimeUs)) {
		ALOGI("[%s] suppressing rendering until %lld us",
		mComponentName.c_str(), (long long)resumeAtMediaTimeUs);
		mSkipRenderingUntilMediaTimeUs = resumeAtMediaTimeUs;
		}
		}

		int64_t timeUs = 0;
		uint32_t flags = 0;
		CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
		@@ -454,10 +480,27 @@ bool NuPlayer::Decoder::handleAnOutputBuffer() {
		return false;
		}

		if (isVideo()) {
		sp<AMessage> notify = mNotify->dup();
		notify->setInt32("what", kWhatOutputFormatChanged);
		notify->setInt32("what", kWhatVideoSizeChanged);
		notify->setMessage("format", format);
		notify->post();
		} else if (mRenderer != NULL) {
		uint32_t flags;
		int64_t durationUs;
		bool hasVideo = (mSource->getFormat(false /* audio */) != NULL);
		if (!hasVideo &&
		mSource->getDuration(&durationUs) == OK &&
		durationUs
		> AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US) {
		flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
		} else {
		flags = AUDIO_OUTPUT_FLAG_NONE;
		}

		mRenderer->openAudioSink(
		format, false /* offloadOnly */, hasVideo, flags);
		}
		return true;
		} else if (res == INFO_DISCONTINUITY) {
		// nothing to do
		@@ -485,21 +528,26 @@ bool NuPlayer::Decoder::handleAnOutputBuffer() {
		reply->setSize("buffer-ix", bufferIx);
		reply->setInt32("generation", mBufferGeneration);

		sp<AMessage> notify = mNotify->dup();
		notify->setInt32("what", kWhatDrainThisBuffer);
		notify->setBuffer("buffer", buffer);
		notify->setMessage("reply", reply);
		notify->post();
		if (mSkipRenderingUntilMediaTimeUs >= 0) {
		if (timeUs < mSkipRenderingUntilMediaTimeUs) {
		ALOGV("[%s] dropping buffer at time %lld as requested.",
		mComponentName.c_str(), (long long)timeUs);

		reply->post();
		return true;
		}

		mSkipRenderingUntilMediaTimeUs = -1;
		}

		// FIXME: This should be handled after rendering is complete,
		// but Renderer needs it now
		if (mRenderer != NULL) {
		// send the buffer to renderer.
		mRenderer->queueBuffer(!isVideo(), buffer, reply);
		if (flags & MediaCodec::BUFFER_FLAG_EOS) {
		ALOGV("queueing eos [%s]", mComponentName.c_str());
		sp<AMessage> notify = mNotify->dup();
		notify->setInt32("what", kWhatEOS);
		notify->setInt32("err", ERROR_END_OF_STREAM);
		notify->post();
		mRenderer->queueEOS(!isVideo(), ERROR_END_OF_STREAM);
		}
		}

		return true;
		}

		@@ -508,6 +556,17 @@ void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) {
		int32_t render;
		size_t bufferIx;
		CHECK(msg->findSize("buffer-ix", &bufferIx));

		if (isVideo()) {
		int64_t timeUs;
		sp<ABuffer> buffer = mOutputBuffers[bufferIx];
		buffer->meta()->findInt64("timeUs", &timeUs);
		sp<AMessage> notify = mNotify->dup();
		notify->setInt32("what", kWhatRenderBufferTime);
		notify->setInt64("timeUs", timeUs);
		notify->post();
		}

		if (msg->findInt32("render", &render) && render) {
		int64_t timestampNs;
		CHECK(msg->findInt64("timestampNs", &timestampNs));
		@@ -523,6 +582,10 @@ void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) {
		}

		void NuPlayer::Decoder::onFlush() {
		if (mRenderer != NULL) {
		mRenderer->flush(!isVideo());
		}

		status_t err = OK;
		if (mCodec != NULL) {
		err = mCodec->flush();
		@@ -594,6 +657,18 @@ void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) {
		break;
		}

		case kWhatSetRenderer:
		{
		bool hadNoRenderer = (mRenderer == NULL);
		sp<RefBase> obj;
		CHECK(msg->findObject("renderer", &obj));
		mRenderer = static_cast<Renderer *>(obj.get());
		if (hadNoRenderer && mRenderer != NULL) {
		requestCodecNotification();
		}
		break;
		}

		case kWhatUpdateFormat:
		{
		sp<AMessage> format;
		@@ -772,6 +847,10 @@ bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp<AMessage> &targetF
		return seamless;
		}

		bool NuPlayer::Decoder::isVideo() {
		return mNativeWindow != NULL;
		}

		struct CCData {
		CCData(uint8_t type, uint8_t data1, uint8_t data2)
		: mType(type), mData1(data1), mData2(data2) {

media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h

+14 −3

Original line number	Diff line number	Diff line
		@@ -30,11 +30,15 @@ struct MediaBuffer;

		struct NuPlayer::Decoder : public AHandler {
		Decoder(const sp<AMessage> &notify,
		const sp<Source> &source,
		const sp<Renderer> &renderer = NULL,
		const sp<NativeWindowWrapper> &nativeWindow = NULL);

		virtual void configure(const sp<AMessage> &format);
		virtual void init();

		virtual void setRenderer(const sp<Renderer> &renderer);

		status_t getInputBuffers(Vector<sp<ABuffer> > *dstBuffers) const;
		virtual void signalFlush(const sp<AMessage> &format = NULL);
		virtual void signalUpdateFormat(const sp<AMessage> &format);
		@@ -45,8 +49,8 @@ struct NuPlayer::Decoder : public AHandler {

		enum {
		kWhatFillThisBuffer = 'flTB',
		kWhatDrainThisBuffer = 'drTB',
		kWhatOutputFormatChanged = 'fmtC',
		kWhatRenderBufferTime = 'rnBT',
		kWhatVideoSizeChanged = 'viSC',
		kWhatFlushCompleted = 'flsC',
		kWhatShutdownCompleted = 'shDC',
		kWhatEOS = 'eos ',
		@@ -59,10 +63,10 @@ protected:

		virtual void onMessageReceived(const sp<AMessage> &msg);

		private:
		enum {
		kWhatCodecNotify = 'cdcN',
		kWhatConfigure = 'conf',
		kWhatSetRenderer = 'setR',
		kWhatGetInputBuffers = 'gInB',
		kWhatInputBufferFilled = 'inpF',
		kWhatRenderBuffer = 'rndr',
		@@ -71,9 +75,13 @@ private:
		kWhatUpdateFormat = 'uFmt',
		};

		private:
		sp<AMessage> mNotify;
		sp<NativeWindowWrapper> mNativeWindow;

		sp<Source> mSource;
		sp<Renderer> mRenderer;

		sp<AMessage> mInputFormat;
		sp<AMessage> mOutputFormat;
		sp<MediaCodec> mCodec;
		@@ -89,6 +97,8 @@ private:
		Vector<bool> mInputBufferIsDequeued;
		Vector<MediaBuffer *> mMediaBuffers;

		int64_t mSkipRenderingUntilMediaTimeUs;

		void handleError(int32_t err);
		bool handleAnInputBuffer();
		bool handleAnOutputBuffer();
		@@ -110,6 +120,7 @@ private:

		bool supportsSeamlessAudioFormatChange(const sp<AMessage> &targetFormat) const;
		void rememberCodecSpecificData(const sp<AMessage> &format);
		bool isVideo();

		DISALLOW_EVIL_CONSTRUCTORS(Decoder);
		};

media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.cpp

+67 −18

Original line number	Diff line number	Diff line
		@@ -21,6 +21,9 @@

		#include "NuPlayerDecoderPassThrough.h"

		#include "NuPlayerRenderer.h"
		#include "NuPlayerSource.h"

		#include <media/ICrypto.h>
		#include <media/stagefright/foundation/ABuffer.h>
		#include <media/stagefright/foundation/ADebug.h>
		@@ -36,15 +39,21 @@ static const size_t kMaxCachedBytes = 200000;
		static const size_t kMaxPendingBuffers = 1 + (kMaxCachedBytes / NuPlayer::kAggregateBufferSizeBytes);

		NuPlayer::DecoderPassThrough::DecoderPassThrough(
		const sp<AMessage> &notify)
		: Decoder(notify),
		const sp<AMessage> &notify,
		const sp<Source> &source,
		const sp<Renderer> &renderer)
		: Decoder(notify, source),
		mNotify(notify),
		mSource(source),
		mRenderer(renderer),
		mSkipRenderingUntilMediaTimeUs(-1ll),
		mBufferGeneration(0),
		mReachedEOS(true),
		mPendingBuffersToFill(0),
		mPendingBuffersToDrain(0),
		mCachedBytes(0),
		mComponentName("pass through decoder") {
		ALOGW_IF(renderer == NULL, "expect a non-NULL renderer");
		mDecoderLooper = new ALooper;
		mDecoderLooper->setName("NuPlayerDecoderPassThrough");
		mDecoderLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
		@@ -90,10 +99,17 @@ void NuPlayer::DecoderPassThrough::onConfigure(const sp<AMessage> &format) {

		requestMaxBuffers();

		sp<AMessage> notify = mNotify->dup();
		notify->setInt32("what", kWhatOutputFormatChanged);
		notify->setMessage("format", format);
		notify->post();
		uint32_t flags;
		int64_t durationUs;
		if (mSource->getDuration(&durationUs) == OK &&
		durationUs > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US) {
		flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
		} else {
		flags = AUDIO_OUTPUT_FLAG_NONE;
		}

		mRenderer->openAudioSink(
		format, true /* offloadOnly /, false / hasVideo */, flags);
		}

		bool NuPlayer::DecoderPassThrough::isStaleReply(const sp<AMessage> &msg) {
		@@ -138,25 +154,52 @@ void android::NuPlayer::DecoderPassThrough::onInputBufferFilled(
		msg->findBuffer("buffer", &buffer);
		if (buffer == NULL) {
		mReachedEOS = true;
		if (mRenderer != NULL) {
		mRenderer->queueEOS(true /* audio */, ERROR_END_OF_STREAM);
		}
		return;
		}

		sp<AMessage> notify = mNotify->dup();
		notify->setInt32("what", kWhatEOS);
		notify->setInt32("err", ERROR_END_OF_STREAM);
		notify->post();
		sp<AMessage> extra;
		if (buffer->meta()->findMessage("extra", &extra) && extra != NULL) {
		int64_t resumeAtMediaTimeUs;
		if (extra->findInt64(
		"resume-at-mediatimeUs", &resumeAtMediaTimeUs)) {
		ALOGI("[%s] suppressing rendering until %lld us",
		mComponentName.c_str(), (long long)resumeAtMediaTimeUs);
		mSkipRenderingUntilMediaTimeUs = resumeAtMediaTimeUs;
		}
		}

		int32_t bufferSize = buffer->size();
		mCachedBytes += bufferSize;

		if (mSkipRenderingUntilMediaTimeUs >= 0) {
		int64_t timeUs = 0;
		CHECK(buffer->meta()->findInt64("timeUs", &timeUs));

		if (timeUs < mSkipRenderingUntilMediaTimeUs) {
		ALOGV("[%s] dropping buffer at time %lld as requested.",
		mComponentName.c_str(), (long long)timeUs);

		onBufferConsumed(bufferSize);
		return;
		}

		mCachedBytes += buffer->size();
		mSkipRenderingUntilMediaTimeUs = -1;
		}

		if (mRenderer == NULL) {
		onBufferConsumed(bufferSize);
		return;
		}

		sp<AMessage> reply = new AMessage(kWhatBufferConsumed, id());
		reply->setInt32("generation", mBufferGeneration);
		reply->setInt32("size", buffer->size());
		reply->setInt32("size", bufferSize);

		mRenderer->queueBuffer(true /* audio */, buffer, reply);

		sp<AMessage> notify = mNotify->dup();
		notify->setInt32("what", kWhatDrainThisBuffer);
		notify->setBuffer("buffer", buffer);
		notify->setMessage("reply", reply);
		notify->post();
		++mPendingBuffersToDrain;
		ALOGV("onInputBufferFilled: #ToFill = %zu, #ToDrain = %zu, cachedBytes = %zu",
		mPendingBuffersToFill, mPendingBuffersToDrain, mCachedBytes);
		@@ -172,6 +215,11 @@ void NuPlayer::DecoderPassThrough::onBufferConsumed(int32_t size) {

		void NuPlayer::DecoderPassThrough::onFlush() {
		++mBufferGeneration;
		mSkipRenderingUntilMediaTimeUs = -1;

		if (mRenderer != NULL) {
		mRenderer->flush(true /* audio */);
		}

		sp<AMessage> notify = mNotify->dup();
		notify->setInt32("what", kWhatFlushCompleted);
		@@ -192,6 +240,7 @@ void NuPlayer::DecoderPassThrough::requestMaxBuffers() {

		void NuPlayer::DecoderPassThrough::onShutdown() {
		++mBufferGeneration;
		mSkipRenderingUntilMediaTimeUs = -1;

		sp<AMessage> notify = mNotify->dup();
		notify->setInt32("what", kWhatShutdownCompleted);