Merge "VT: Introduce dynamic jitter buffer for RTP source" (d71f5042) · Commits · e / os / android_frameworks_av

media/libmediaplayerservice/nuplayer/RTPSource.cpp

+13 −6

Original line number	Original line	Diff line number	Diff line
	@@ -124,8 +124,16 @@ void NuPlayer::RTPSource::prepareAsync() {
	// index(i) should be started from 1. 0 is reserved for [root]		// index(i) should be started from 1. 0 is reserved for [root]
	mRTPConn->addStream(sockRtp, sockRtcp, desc, i + 1, notify, false);		mRTPConn->addStream(sockRtp, sockRtcp, desc, i + 1, notify, false);
	mRTPConn->setSelfID(info->mSelfID);		mRTPConn->setSelfID(info->mSelfID);
	mRTPConn->setJbTime(		mRTPConn->setStaticJitterTimeMs(info->mJbTimeMs);
	(info->mJbTimeMs <= 3000 && info->mJbTimeMs >= 40) ? info->mJbTimeMs : 300);
			unsigned long PT;
			AString formatDesc, formatParams;
			// index(i) should be started from 1. 0 is reserved for [root]
			desc->getFormatType(i + 1, &PT, &formatDesc, &formatParams);

			int32_t clockRate, numChannels;
			ASessionDescription::ParseFormatDesc(formatDesc.c_str(), &clockRate, &numChannels);
			info->mTimeScale = clockRate;

	info->mRTPSocket = sockRtp;		info->mRTPSocket = sockRtp;
	info->mRTCPSocket = sockRtcp;		info->mRTCPSocket = sockRtcp;
	@@ -146,10 +154,8 @@ void NuPlayer::RTPSource::prepareAsync() {

	if (info->mIsAudio) {		if (info->mIsAudio) {
	mAudioTrack = source;		mAudioTrack = source;
	info->mTimeScale = 16000;
	} else {		} else {
	mVideoTrack = source;		mVideoTrack = source;
	info->mTimeScale = 90000;
	}		}

	info->mSource = source;		info->mSource = source;
	@@ -680,7 +686,7 @@ status_t NuPlayer::RTPSource::setParameter(const String8 &key, const String8 &va
	newTrackInfo.mIsAudio = isAudioKey;		newTrackInfo.mIsAudio = isAudioKey;
	mTracks.push(newTrackInfo);		mTracks.push(newTrackInfo);
	info = &mTracks.editTop();		info = &mTracks.editTop();
	info->mJbTimeMs = 300;		info->mJbTimeMs = kStaticJitterTimeMs;
	}		}

	if (key == "rtp-param-mime-type") {		if (key == "rtp-param-mime-type") {
	@@ -724,7 +730,8 @@ status_t NuPlayer::RTPSource::setParameter(const String8 &key, const String8 &va
	int64_t networkHandle = atoll(value);		int64_t networkHandle = atoll(value);
	setSocketNetwork(networkHandle);		setSocketNetwork(networkHandle);
	} else if (key == "rtp-param-jitter-buffer-time") {		} else if (key == "rtp-param-jitter-buffer-time") {
	info->mJbTimeMs = atoi(value);		// clamping min at 40, max at 3000
			info->mJbTimeMs = std::min(std::max(40, atoi(value)), 3000);
	}		}

	return OK;		return OK;

media/libstagefright/rtsp/AAVCAssembler.cpp

+59 −45

Original line number	Original line	Diff line number	Diff line
	@@ -34,6 +34,8 @@

	namespace android {		namespace android {

			const double JITTER_MULTIPLE = 1.5f;

	// static		// static
	AAVCAssembler::AAVCAssembler(const sp<AMessage> &notify)		AAVCAssembler::AAVCAssembler(const sp<AMessage> &notify)
	: mNotifyMsg(notify),		: mNotifyMsg(notify),
	@@ -123,22 +125,48 @@ ARTPAssembler::AssemblyStatus AAVCAssembler::addNALUnit(

	int64_t rtpTime = findRTPTime(firstRTPTime, buffer);		int64_t rtpTime = findRTPTime(firstRTPTime, buffer);

	int64_t startTime = source->mFirstSysTime / 1000;		const int64_t startTimeMs = source->mFirstSysTime / 1000;
	int64_t nowTime = ALooper::GetNowUs() / 1000;		const int64_t nowTimeMs = ALooper::GetNowUs() / 1000;
	int64_t playedTime = nowTime - startTime;		const int64_t staticJbTimeMs = source->getStaticJitterTimeMs();
			const int64_t dynamicJbTimeMs = source->getDynamicJitterTimeMs();
			const int64_t clockRate = source->mClockRate;

			int64_t playedTimeMs = nowTimeMs - startTimeMs;
			int64_t playedTimeRtp = source->mFirstRtpTime + MsToRtp(playedTimeMs, clockRate);

			/**
			* Based on experience in real commercial network services,
			* 300 ms is a maximum heuristic jitter buffer time for video RTP service.
			*/

			/**
			* The static(base) jitter is a kind of expected propagation time that we desire.
			* We can drop packets if it doesn't meet our standards.
			* If it gets shorter we can get faster response but can lose packets.
			* Expecting range : 50ms ~ 1000ms (But 300 ms would be practical upper bound)
			*/
			const int64_t baseJbTimeRtp = MsToRtp(staticJbTimeMs, clockRate);
			/**
			* Dynamic jitter is a variance of interarrival time as defined in the 6.4.1 of RFC 3550.
			* We can regard this as a tolerance of every moments.
			* Expecting range : 0ms ~ 150ms (Not to over 300 ms practically)
			*/
			const int64_t dynamicJbTimeRtp = // Max 150
			std::min(MsToRtp(dynamicJbTimeMs, clockRate), MsToRtp(150, clockRate));
			const int64_t jitterTimeRtp = baseJbTimeRtp + dynamicJbTimeRtp; // Total jitter time

	int64_t playedTimeRtp = source->mFirstRtpTime + playedTime * (int64_t)source->mClockRate / 1000;		int64_t expiredTimeRtp = rtpTime + jitterTimeRtp; // When does this buffer expire ? (T)
	const int64_t jitterTime = source->mJbTimeMs * (int64_t)source->mClockRate / 1000;		int64_t diffTimeRtp = playedTimeRtp - expiredTimeRtp;
			bool isExpired = (diffTimeRtp >= 0); // It's expired if T is passed away
			bool isFirstLineBroken = (diffTimeRtp > jitterTimeRtp); // (T + jitter) is a standard tolerance

	int64_t expiredTimeInJb = rtpTime + jitterTime;		int64_t finalMargin = dynamicJbTimeRtp * JITTER_MULTIPLE;
	bool isExpired = expiredTimeInJb <= (playedTimeRtp);		bool isSecondLineBroken = (diffTimeRtp > jitterTimeRtp + finalMargin); // The Maginot line
	bool isTooLate200 = expiredTimeInJb < (playedTimeRtp - jitterTime);
	bool isTooLate300 = expiredTimeInJb < (playedTimeRtp - (jitterTime * 3 / 2));

	if (mShowQueue && mShowQueueCnt < 20) {		if (mShowQueue && mShowQueueCnt < 20) {
	showCurrentQueue(queue);		showCurrentQueue(queue);
	printNowTimeUs(startTime, nowTime, playedTime);		printNowTimeMs(startTimeMs, nowTimeMs, playedTimeMs);
	printRTPTime(rtpTime, playedTimeRtp, expiredTimeInJb, isExpired);		printRTPTime(rtpTime, playedTimeRtp, expiredTimeRtp, isExpired);
	mShowQueueCnt++;		mShowQueueCnt++;
	}		}

	@@ -149,17 +177,23 @@ ARTPAssembler::AssemblyStatus AAVCAssembler::addNALUnit(
	return NOT_ENOUGH_DATA;		return NOT_ENOUGH_DATA;
	}		}

	if (isTooLate200) {		if (isFirstLineBroken) {
	ALOGW("=== WARNING === buffer arrived 200ms late. === WARNING === ");		if (isSecondLineBroken) {
	}		ALOGW("buffer too late ... \t Diff in Jb=%lld \t "
			"Seq# %d \t ExpSeq# %d \t"
			"JitterMs %lld + (%lld * %.3f)",
			(long long)(diffTimeRtp),
			buffer->int32Data(), mNextExpectedSeqNo,
			(long long)staticJbTimeMs, (long long)dynamicJbTimeMs, JITTER_MULTIPLE + 1);
			printNowTimeMs(startTimeMs, nowTimeMs, playedTimeMs);
			printRTPTime(rtpTime, playedTimeRtp, expiredTimeRtp, isExpired);

	if (isTooLate300) {		mNextExpectedSeqNo = pickProperSeq(queue, firstRTPTime, playedTimeRtp, jitterTimeRtp);
	ALOGW("buffer arrived after 300ms ... \t Diff in Jb=%lld \t Seq# %d",		} else {
	(long long)(playedTimeRtp - expiredTimeInJb), buffer->int32Data());		ALOGW("=== WARNING === buffer arrived after %lld + %lld = %lld ms === WARNING === ",
	printNowTimeUs(startTime, nowTime, playedTime);		(long long)staticJbTimeMs, (long long)dynamicJbTimeMs,
	printRTPTime(rtpTime, playedTimeRtp, expiredTimeInJb, isExpired);		(long long)RtpToMs(jitterTimeRtp, clockRate));
			}
	mNextExpectedSeqNo = pickProperSeq(queue, firstRTPTime, playedTimeRtp, jitterTime);
	}		}

	if (mNextExpectedSeqNoValid) {		if (mNextExpectedSeqNoValid) {
	@@ -170,6 +204,7 @@ ARTPAssembler::AssemblyStatus AAVCAssembler::addNALUnit(
	source->noticeAbandonBuffer(cntRemove);		source->noticeAbandonBuffer(cntRemove);
	ALOGW("delete %d of %d buffers", cntRemove, size);		ALOGW("delete %d of %d buffers", cntRemove, size);
	}		}

	if (queue->empty()) {		if (queue->empty()) {
	return NOT_ENOUGH_DATA;		return NOT_ENOUGH_DATA;
	}		}
	@@ -565,17 +600,6 @@ void AAVCAssembler::submitAccessUnit() {
	msg->post();		msg->post();
	}		}

	inline int64_t AAVCAssembler::findRTPTime(
	const uint32_t& firstRTPTime, const sp<ABuffer>& buffer) {
	/* If you want to +, -, * rtpTime, recommend to declare rtpTime as int64_t.
	Because rtpTime can be near UINT32_MAX. Beware the overflow. */
	int64_t rtpTime = 0;
	CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));
	// If the first overs 2^31 and rtp unders 2^31, the rtp value is overflowed one.
	int64_t overflowMask = (firstRTPTime & 0x80000000 & ~rtpTime) << 1;
	return rtpTime \| overflowMask;
	}

	int32_t AAVCAssembler::pickProperSeq(const Queue *queue,		int32_t AAVCAssembler::pickProperSeq(const Queue *queue,
	uint32_t first, int64_t play, int64_t jit) {		uint32_t first, int64_t play, int64_t jit) {
	sp<ABuffer> buffer = *(queue->begin());		sp<ABuffer> buffer = *(queue->begin());
	@@ -620,16 +644,6 @@ int32_t AAVCAssembler::deleteUnitUnderSeq(Queue *queue, uint32_t seq) {
	return initSize - queue->size();		return initSize - queue->size();
	}		}

	inline void AAVCAssembler::printNowTimeUs(int64_t start, int64_t now, int64_t play) {
	ALOGD("start=%lld, now=%lld, played=%lld",
	(long long)start, (long long)now, (long long)play);
	}

	inline void AAVCAssembler::printRTPTime(int64_t rtp, int64_t play, int64_t exp, bool isExp) {
	ALOGD("rtp-time(JB)=%lld, played-rtp-time(JB)=%lld, expired-rtp-time(JB)=%lld expired=%d",
	(long long)rtp, (long long)play, (long long)exp, isExp);
	}

	ARTPAssembler::AssemblyStatus AAVCAssembler::assembleMore(		ARTPAssembler::AssemblyStatus AAVCAssembler::assembleMore(
	const sp<ARTPSource> &source) {		const sp<ARTPSource> &source) {
	AssemblyStatus status = addNALUnit(source);		AssemblyStatus status = addNALUnit(source);

media/libstagefright/rtsp/AAVCAssembler.h

+0 −3

Original line number	Original line	Diff line number	Diff line
	@@ -63,13 +63,10 @@ private:

	void submitAccessUnit();		void submitAccessUnit();

	inline int64_t findRTPTime(const uint32_t& firstRTPTime, const sp<ABuffer>& buffer);
	int32_t pickProperSeq(const Queue *q, uint32_t first, int64_t play, int64_t jit);		int32_t pickProperSeq(const Queue *q, uint32_t first, int64_t play, int64_t jit);
	bool recycleUnit(uint32_t start, uint32_t end, uint32_t connected,		bool recycleUnit(uint32_t start, uint32_t end, uint32_t connected,
	size_t avail, float goodRatio);		size_t avail, float goodRatio);
	int32_t deleteUnitUnderSeq(Queue *q, uint32_t seq);		int32_t deleteUnitUnderSeq(Queue *q, uint32_t seq);
	void printNowTimeUs(int64_t start, int64_t now, int64_t play);
	void printRTPTime(int64_t rtp, int64_t play, int64_t exp, bool isExp);

	DISALLOW_EVIL_CONSTRUCTORS(AAVCAssembler);		DISALLOW_EVIL_CONSTRUCTORS(AAVCAssembler);
	};		};

media/libstagefright/rtsp/AHEVCAssembler.cpp

+58 −43

Original line number	Original line	Diff line number	Diff line
	@@ -41,6 +41,8 @@

	namespace android {		namespace android {

			const double JITTER_MULTIPLE = 1.5f;

	// static		// static
	AHEVCAssembler::AHEVCAssembler(const sp<AMessage> &notify)		AHEVCAssembler::AHEVCAssembler(const sp<AMessage> &notify)
	: mNotifyMsg(notify),		: mNotifyMsg(notify),
	@@ -130,23 +132,51 @@ ARTPAssembler::AssemblyStatus AHEVCAssembler::addNALUnit(

	sp<ABuffer> buffer = *queue->begin();		sp<ABuffer> buffer = *queue->begin();
	buffer->meta()->setObject("source", source);		buffer->meta()->setObject("source", source);

	int64_t rtpTime = findRTPTime(firstRTPTime, buffer);		int64_t rtpTime = findRTPTime(firstRTPTime, buffer);

	int64_t startTime = source->mFirstSysTime / 1000;		const int64_t startTimeMs = source->mFirstSysTime / 1000;
	int64_t nowTime = ALooper::GetNowUs() / 1000;		const int64_t nowTimeMs = ALooper::GetNowUs() / 1000;
	int64_t playedTime = nowTime - startTime;		const int64_t staticJbTimeMs = source->getStaticJitterTimeMs();
	int64_t playedTimeRtp = source->mFirstRtpTime + playedTime * (int64_t)source->mClockRate / 1000;		const int64_t dynamicJbTimeMs = source->getDynamicJitterTimeMs();
	const int64_t jitterTime = source->mJbTimeMs * (int64_t)source->mClockRate / 1000;		const int64_t clockRate = source->mClockRate;

			int64_t playedTimeMs = nowTimeMs - startTimeMs;
			int64_t playedTimeRtp = source->mFirstRtpTime + MsToRtp(playedTimeMs, clockRate);

			/**
			* Based on experience in real commercial network services,
			* 300 ms is a maximum heuristic jitter buffer time for video RTP service.
			*/

			/**
			* The static(base) jitter is a kind of expected propagation time that we desire.
			* We can drop packets if it doesn't meet our standards.
			* If it gets shorter we can get faster response but can lose packets.
			* Expecting range : 50ms ~ 1000ms (But 300 ms would be practical upper bound)
			*/
			const int64_t baseJbTimeRtp = MsToRtp(staticJbTimeMs, clockRate);
			/**
			* Dynamic jitter is a variance of interarrival time as defined in the 6.4.1 of RFC 3550.
			* We can regard this as a tolerance of every moments.
			* Expecting range : 0ms ~ 150ms (Not to over 300 ms practically)
			*/
			const int64_t dynamicJbTimeRtp = // Max 150
			std::min(MsToRtp(dynamicJbTimeMs, clockRate), MsToRtp(150, clockRate));
			const int64_t jitterTimeRtp = baseJbTimeRtp + dynamicJbTimeRtp; // Total jitter time

	int64_t expiredTimeInJb = rtpTime + jitterTime;		int64_t expiredTimeRtp = rtpTime + jitterTimeRtp; // When does this buffer expire ? (T)
	bool isExpired = expiredTimeInJb <= (playedTimeRtp);		int64_t diffTimeRtp = playedTimeRtp - expiredTimeRtp;
	bool isTooLate200 = expiredTimeInJb < (playedTimeRtp - jitterTime);		bool isExpired = (diffTimeRtp >= 0); // It's expired if T is passed away
	bool isTooLate300 = expiredTimeInJb < (playedTimeRtp - (jitterTime * 3 / 2));		bool isFirstLineBroken = (diffTimeRtp > jitterTimeRtp); // (T + jitter) is a standard tolerance

			int64_t finalMargin = dynamicJbTimeRtp * JITTER_MULTIPLE;
			bool isSecondLineBroken = (diffTimeRtp > jitterTimeRtp + finalMargin); // The Maginot line

	if (mShowQueueCnt < 20) {		if (mShowQueueCnt < 20) {
	showCurrentQueue(queue);		showCurrentQueue(queue);
	printNowTimeUs(startTime, nowTime, playedTime);		printNowTimeMs(startTimeMs, nowTimeMs, playedTimeMs);
	printRTPTime(rtpTime, playedTimeRtp, expiredTimeInJb, isExpired);		printRTPTime(rtpTime, playedTimeRtp, expiredTimeRtp, isExpired);
	mShowQueueCnt++;		mShowQueueCnt++;
	}		}

	@@ -157,17 +187,23 @@ ARTPAssembler::AssemblyStatus AHEVCAssembler::addNALUnit(
	return NOT_ENOUGH_DATA;		return NOT_ENOUGH_DATA;
	}		}

	if (isTooLate200) {		if (isFirstLineBroken) {
	ALOGW("=== WARNING === buffer arrived 200ms late. === WARNING === ");		if (isSecondLineBroken) {
	}		ALOGW("buffer too late ... \t Diff in Jb=%lld \t "
			"Seq# %d \t ExpSeq# %d \t"
	if (isTooLate300) {		"JitterMs %lld + (%lld * %.3f)",
	ALOGW("buffer arrived after 300ms ... \t Diff in Jb=%lld \t Seq# %d",		(long long)(diffTimeRtp),
	(long long)(playedTimeRtp - expiredTimeInJb), buffer->int32Data());		buffer->int32Data(), mNextExpectedSeqNo,
	printNowTimeUs(startTime, nowTime, playedTime);		(long long)staticJbTimeMs, (long long)dynamicJbTimeMs, JITTER_MULTIPLE + 1);
	printRTPTime(rtpTime, playedTimeRtp, expiredTimeInJb, isExpired);		printNowTimeMs(startTimeMs, nowTimeMs, playedTimeMs);
			printRTPTime(rtpTime, playedTimeRtp, expiredTimeRtp, isExpired);

	mNextExpectedSeqNo = pickProperSeq(queue, firstRTPTime, playedTimeRtp, jitterTime);		mNextExpectedSeqNo = pickProperSeq(queue, firstRTPTime, playedTimeRtp, jitterTimeRtp);
			} else {
			ALOGW("=== WARNING === buffer arrived after %lld + %lld = %lld ms === WARNING === ",
			(long long)staticJbTimeMs, (long long)dynamicJbTimeMs,
			(long long)RtpToMs(jitterTimeRtp, clockRate));
			}
	}		}

	if (mNextExpectedSeqNoValid) {		if (mNextExpectedSeqNoValid) {
	@@ -578,17 +614,6 @@ void AHEVCAssembler::submitAccessUnit() {
	msg->post();		msg->post();
	}		}

	inline int64_t AHEVCAssembler::findRTPTime(
	const uint32_t& firstRTPTime, const sp<ABuffer>& buffer) {
	/* If you want to +, -, * rtpTime, recommend to declare rtpTime as int64_t.
	Because rtpTime can be near UINT32_MAX. Beware the overflow. */
	int64_t rtpTime = 0;
	CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));
	// If the first overs 2^31 and rtp unders 2^31, the rtp value is overflowed one.
	int64_t overflowMask = (firstRTPTime & 0x80000000 & ~rtpTime) << 1;
	return rtpTime \| overflowMask;
	}

	int32_t AHEVCAssembler::pickProperSeq(const Queue *queue,		int32_t AHEVCAssembler::pickProperSeq(const Queue *queue,
	uint32_t first, int64_t play, int64_t jit) {		uint32_t first, int64_t play, int64_t jit) {
	sp<ABuffer> buffer = *(queue->begin());		sp<ABuffer> buffer = *(queue->begin());
	@@ -633,16 +658,6 @@ int32_t AHEVCAssembler::deleteUnitUnderSeq(Queue *queue, uint32_t seq) {
	return initSize - queue->size();		return initSize - queue->size();
	}		}

	inline void AHEVCAssembler::printNowTimeUs(int64_t start, int64_t now, int64_t play) {
	ALOGD("start=%lld, now=%lld, played=%lld",
	(long long)start, (long long)now, (long long)play);
	}

	inline void AHEVCAssembler::printRTPTime(int64_t rtp, int64_t play, int64_t exp, bool isExp) {
	ALOGD("rtp-time(JB)=%lld, played-rtp-time(JB)=%lld, expired-rtp-time(JB)=%lld expired=%d",
	(long long)rtp, (long long)play, (long long)exp, isExp);
	}

	ARTPAssembler::AssemblyStatus AHEVCAssembler::assembleMore(		ARTPAssembler::AssemblyStatus AHEVCAssembler::assembleMore(
	const sp<ARTPSource> &source) {		const sp<ARTPSource> &source) {
	AssemblyStatus status = addNALUnit(source);		AssemblyStatus status = addNALUnit(source);

media/libstagefright/rtsp/AHEVCAssembler.h

+0 −3

Original line number	Original line	Diff line number	Diff line
	@@ -64,13 +64,10 @@ private:

	void submitAccessUnit();		void submitAccessUnit();

	inline int64_t findRTPTime(const uint32_t& firstRTPTime, const sp<ABuffer>& buffer);
	int32_t pickProperSeq(const Queue *q, uint32_t first, int64_t play, int64_t jit);		int32_t pickProperSeq(const Queue *q, uint32_t first, int64_t play, int64_t jit);
	bool recycleUnit(uint32_t start, uint32_t end, uint32_t connected,		bool recycleUnit(uint32_t start, uint32_t end, uint32_t connected,
	size_t avail, float goodRatio);		size_t avail, float goodRatio);
	int32_t deleteUnitUnderSeq(Queue *queue, uint32_t seq);		int32_t deleteUnitUnderSeq(Queue *queue, uint32_t seq);
	void printNowTimeUs(int64_t start, int64_t now, int64_t play);
	void printRTPTime(int64_t rtp, int64_t play, int64_t exp, bool isExp);

	DISALLOW_EVIL_CONSTRUCTORS(AHEVCAssembler);		DISALLOW_EVIL_CONSTRUCTORS(AHEVCAssembler);
	};		};