Merge "In order to recover from video lagging behind audio, drop avc frames" (c7342fbf) · Commits · e / os / android_frameworks_base

media/libmediaplayerservice/nuplayer/NuPlayer.cpp

+80 −37

Original line number	Diff line number	Diff line
		@@ -34,17 +34,21 @@
		#include <media/stagefright/foundation/ADebug.h>
		#include <media/stagefright/foundation/AMessage.h>
		#include <media/stagefright/ACodec.h>
		#include <media/stagefright/MediaDefs.h>
		#include <media/stagefright/MediaErrors.h>
		#include <media/stagefright/MetaData.h>
		#include <surfaceflinger/Surface.h>
		#include <gui/ISurfaceTexture.h>

		#include "avc_utils.h"

		namespace android {

		////////////////////////////////////////////////////////////////////////////////

		NuPlayer::NuPlayer()
		: mUIDValid(false),
		mVideoIsAVC(false),
		mAudioEOS(false),
		mVideoEOS(false),
		mScanSourcesPending(false),
		@@ -52,7 +56,12 @@ NuPlayer::NuPlayer()
		mFlushingAudio(NONE),
		mFlushingVideo(NONE),
		mResetInProgress(false),
		mResetPostponed(false) {
		mResetPostponed(false),
		mSkipRenderingAudioUntilMediaTimeUs(-1ll),
		mSkipRenderingVideoUntilMediaTimeUs(-1ll),
		mVideoLateByUs(0ll),
		mNumFramesTotal(0ll),
		mNumFramesDropped(0ll) {
		}

		NuPlayer::~NuPlayer() {
		@@ -185,10 +194,14 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
		{
		LOGV("kWhatStart");

		mVideoIsAVC = false;
		mAudioEOS = false;
		mVideoEOS = false;
		mSkipRenderingAudioUntilMediaTimeUs = -1;
		mSkipRenderingVideoUntilMediaTimeUs = -1;
		mVideoLateByUs = 0;
		mNumFramesTotal = 0;
		mNumFramesDropped = 0;

		mSource->start();

		@@ -269,6 +282,8 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
		} else {
		CHECK(IsFlushingState(mFlushingVideo, &needShutdown));
		mFlushingVideo = FLUSHED;

		mVideoLateByUs = 0;
		}

		LOGV("decoder %s flush completed", audio ? "audio" : "video");
		@@ -397,13 +412,18 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
		int64_t positionUs;
		CHECK(msg->findInt64("positionUs", &positionUs));

		CHECK(msg->findInt64("videoLateByUs", &mVideoLateByUs));

		if (mDriver != NULL) {
		sp<NuPlayerDriver> driver = mDriver.promote();
		if (driver != NULL) {
		driver->notifyPosition(positionUs);

		driver->notifyFrameStats(
		mNumFramesTotal, mNumFramesDropped);
		}
		}
		} else {
		} else if (what == Renderer::kWhatFlushComplete) {
		CHECK_EQ(what, (int32_t)Renderer::kWhatFlushComplete);

		int32_t audio;
		@@ -565,6 +585,12 @@ status_t NuPlayer::instantiateDecoder(bool audio, sp<Decoder> *decoder) {
		return -EWOULDBLOCK;
		}

		if (!audio) {
		const char *mime;
		CHECK(meta->findCString(kKeyMIMEType, &mime));
		mVideoIsAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime);
		}

		sp<AMessage> notify =
		new AMessage(audio ? kWhatAudioNotify : kWhatVideoNotify,
		id());
		@@ -598,6 +624,9 @@ status_t NuPlayer::feedDecoderInputData(bool audio, const sp<AMessage> &msg) {
		}

		sp<ABuffer> accessUnit;

		bool dropAccessUnit;
		do {
		status_t err = mSource->dequeueAccessUnit(audio, &accessUnit);

		if (err == -EWOULDBLOCK) {
		@@ -646,6 +675,20 @@ status_t NuPlayer::feedDecoderInputData(bool audio, const sp<AMessage> &msg) {
		return OK;
		}

		if (!audio) {
		++mNumFramesTotal;
		}

		dropAccessUnit = false;
		if (!audio
		&& mVideoLateByUs > 100000ll
		&& mVideoIsAVC
		&& !IsAVCReferenceFrame(accessUnit)) {
		dropAccessUnit = true;
		++mNumFramesDropped;
		}
		} while (dropAccessUnit);

		// LOGV("returned a valid buffer of %s data", audio ? "audio" : "video");

		#if 0

media/libmediaplayerservice/nuplayer/NuPlayer.h

+4 −0

Original line number	Diff line number	Diff line
		@@ -92,6 +92,7 @@ private:
		sp<NativeWindowWrapper> mNativeWindow;
		sp<MediaPlayerBase::AudioSink> mAudioSink;
		sp<Decoder> mVideoDecoder;
		bool mVideoIsAVC;
		sp<Decoder> mAudioDecoder;
		sp<Renderer> mRenderer;

		@@ -119,6 +120,9 @@ private:
		int64_t mSkipRenderingAudioUntilMediaTimeUs;
		int64_t mSkipRenderingVideoUntilMediaTimeUs;

		int64_t mVideoLateByUs;
		int64_t mNumFramesTotal, mNumFramesDropped;

		status_t instantiateDecoder(bool audio, sp<Decoder> *decoder);

		status_t feedDecoderInputData(bool audio, const sp<AMessage> &msg);

media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp

+28 −0

Original line number	Diff line number	Diff line
		@@ -31,6 +31,8 @@ NuPlayerDriver::NuPlayerDriver()
		: mResetInProgress(false),
		mDurationUs(-1),
		mPositionUs(-1),
		mNumFramesTotal(0),
		mNumFramesDropped(0),
		mLooper(new ALooper),
		mState(UNINITIALIZED),
		mStartupSeekTimeUs(-1) {
		@@ -292,4 +294,30 @@ void NuPlayerDriver::notifySeekComplete() {
		sendEvent(MEDIA_SEEK_COMPLETE);
		}

		void NuPlayerDriver::notifyFrameStats(
		int64_t numFramesTotal, int64_t numFramesDropped) {
		Mutex::Autolock autoLock(mLock);
		mNumFramesTotal = numFramesTotal;
		mNumFramesDropped = numFramesDropped;
		}

		status_t NuPlayerDriver::dump(int fd, const Vector<String16> &args) const {
		Mutex::Autolock autoLock(mLock);

		FILE *out = fdopen(dup(fd), "w");

		fprintf(out, " NuPlayer\n");
		fprintf(out, " numFramesTotal(%lld), numFramesDropped(%lld), "
		"percentageDropped(%.2f)\n",
		mNumFramesTotal,
		mNumFramesDropped,
		mNumFramesTotal == 0
		? 0.0 : (double)mNumFramesDropped / mNumFramesTotal);

		fclose(out);
		out = NULL;

		return OK;
		}

		} // namespace android

media/libmediaplayerservice/nuplayer/NuPlayerDriver.h

+6 −1

Original line number	Diff line number	Diff line
		@@ -60,16 +60,19 @@ struct NuPlayerDriver : public MediaPlayerInterface {
		virtual status_t getMetadata(
		const media::Metadata::Filter& ids, Parcel *records);

		virtual status_t dump(int fd, const Vector<String16> &args) const;

		void notifyResetComplete();
		void notifyDuration(int64_t durationUs);
		void notifyPosition(int64_t positionUs);
		void notifySeekComplete();
		void notifyFrameStats(int64_t numFramesTotal, int64_t numFramesDropped);

		protected:
		virtual ~NuPlayerDriver();

		private:
		Mutex mLock;
		mutable Mutex mLock;
		Condition mCondition;

		// The following are protected through "mLock"
		@@ -77,6 +80,8 @@ private:
		bool mResetInProgress;
		int64_t mDurationUs;
		int64_t mPositionUs;
		int64_t mNumFramesTotal;
		int64_t mNumFramesDropped;
		// <<<

		sp<ALooper> mLooper;

media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp

+12 −6

Original line number	Diff line number	Diff line
		@@ -47,7 +47,8 @@ NuPlayer::Renderer::Renderer(
		mHasVideo(false),
		mSyncQueues(false),
		mPaused(false),
		mLastPositionUpdateUs(-1ll) {
		mLastPositionUpdateUs(-1ll),
		mVideoLateByUs(0ll) {
		}

		NuPlayer::Renderer::~Renderer() {
		@@ -357,22 +358,26 @@ void NuPlayer::Renderer::onDrainVideoQueue() {

		mVideoQueue.erase(mVideoQueue.begin());
		entry = NULL;

		mVideoLateByUs = 0ll;

		notifyPosition();
		return;
		}

		#if 0
		int64_t mediaTimeUs;
		CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));

		int64_t realTimeUs = mediaTimeUs - mAnchorTimeMediaUs + mAnchorTimeRealUs;
		int64_t lateByUs = ALooper::GetNowUs() - realTimeUs;
		mVideoLateByUs = ALooper::GetNowUs() - realTimeUs;

		bool tooLate = (mVideoLateByUs > 40000);

		if (lateByUs > 40000) {
		LOGI("video late by %lld us (%.2f secs)", lateByUs, lateByUs / 1E6);
		if (tooLate) {
		LOGV("video late by %lld us (%.2f secs)", lateByUs, lateByUs / 1E6);
		} else {
		LOGV("rendering video at media time %.2f secs", mediaTimeUs / 1E6);
		}
		#endif

		entry->mNotifyConsumed->setInt32("render", true);
		entry->mNotifyConsumed->post();
		@@ -604,6 +609,7 @@ void NuPlayer::Renderer::notifyPosition() {
		sp<AMessage> notify = mNotify->dup();
		notify->setInt32("what", kWhatPosition);
		notify->setInt64("positionUs", positionUs);
		notify->setInt64("videoLateByUs", mVideoLateByUs);
		notify->post();
		}