mediaplayer: smooth out videoplayback based on framerate (c851b5de) · Commits · e / os / android_frameworks_av

media/libmediaplayerservice/VideoFrameScheduler.cpp

+371 −2

Original line number	Diff line number	Diff line
		@@ -34,6 +34,305 @@ namespace android {

		static const nsecs_t kNanosIn1s = 1000000000;

		template<class T>
		inline static const T divRound(const T &nom, const T &den) {
		if ((nom >= 0) ^ (den >= 0)) {
		return (nom - den / 2) / den;
		} else {
		return (nom + den / 2) / den;
		}
		}

		template<class T>
		inline static T abs(const T &a) {
		return a < 0 ? -a : a;
		}

		template<class T>
		inline static const T &min(const T &a, const T &b) {
		return a < b ? a : b;
		}

		template<class T>
		inline static const T &max(const T &a, const T &b) {
		return a > b ? a : b;
		}

		template<class T>
		inline static T periodicError(const T &val, const T &period) {
		T err = abs(val) % period;
		return (err < (period / 2)) ? err : (period - err);
		}

		template<class T>
		static int compare(const T lhs, const T rhs) {
		if (lhs < rhs) {
		return -1;
		} else if (lhs > rhs) {
		return 1;
		} else {
		return 0;
		}
		}

		/* ======================================================================= */
		/* PLL */
		/* ======================================================================= */

		static const size_t kMinSamplesToStartPrime = 3;
		static const size_t kMinSamplesToStopPrime = VideoFrameScheduler::kHistorySize;
		static const size_t kMinSamplesToEstimatePeriod = 3;
		static const size_t kMaxSamplesToEstimatePeriod = VideoFrameScheduler::kHistorySize;

		static const size_t kPrecision = 12;
		static const size_t kErrorThreshold = (1 << (kPrecision * 2)) / 10;
		static const int64_t kMultiplesThresholdDiv = 4; // 25%
		static const int64_t kReFitThresholdDiv = 100; // 1%
		static const nsecs_t kMaxAllowedFrameSkip = kNanosIn1s; // 1 sec
		static const nsecs_t kMinPeriod = kNanosIn1s / 120; // 120Hz
		static const nsecs_t kRefitRefreshPeriod = 10 * kNanosIn1s; // 10 sec

		VideoFrameScheduler::PLL::PLL()
		: mPeriod(-1),
		mPhase(0),
		mPrimed(false),
		mSamplesUsedForPriming(0),
		mLastTime(-1),
		mNumSamples(0) {
		}

		void VideoFrameScheduler::PLL::reset(float fps) {
		//test();

		mSamplesUsedForPriming = 0;
		mLastTime = -1;

		// set up or reset video PLL
		if (fps <= 0.f) {
		mPeriod = -1;
		mPrimed = false;
		} else {
		ALOGV("reset at %.1f fps", fps);
		mPeriod = (nsecs_t)(1e9 / fps + 0.5);
		mPrimed = true;
		}

		restart();
		}

		// reset PLL but keep previous period estimate
		void VideoFrameScheduler::PLL::restart() {
		mNumSamples = 0;
		mPhase = -1;
		}

		#if 0

		void VideoFrameScheduler::PLL::test() {
		nsecs_t period = kNanosIn1s / 60;
		mTimes[0] = 0;
		mTimes[1] = period;
		mTimes[2] = period * 3;
		mTimes[3] = period * 4;
		mTimes[4] = period * 7;
		mTimes[5] = period * 8;
		mTimes[6] = period * 10;
		mTimes[7] = period * 12;
		mNumSamples = 8;
		int64_t a, b, err;
		fit(0, period * 12 / 7, 8, &a, &b, &err);
		// a = 0.8(5)+
		// b = -0.14097(2)+
		// err = 0.2750578(703)+
		ALOGD("a=%lld (%.6f), b=%lld (%.6f), err=%lld (%.6f)",
		(long long)a, (a / (float)(1 << kPrecision)),
		(long long)b, (b / (float)(1 << kPrecision)),
		(long long)err, (err / (float)(1 << (kPrecision * 2))));
		}

		#endif

		void VideoFrameScheduler::PLL::fit(
		nsecs_t phase, nsecs_t period, size_t numSamplesToUse,
		int64_t a, int64_t b, int64_t *err) {
		if (numSamplesToUse > mNumSamples) {
		numSamplesToUse = mNumSamples;
		}

		int64_t sumX = 0;
		int64_t sumXX = 0;
		int64_t sumXY = 0;
		int64_t sumYY = 0;
		int64_t sumY = 0;

		int64_t x = 0; // x usually is in [0..numSamplesToUse)
		nsecs_t lastTime;
		for (size_t i = 0; i < numSamplesToUse; i++) {
		size_t ix = (mNumSamples - numSamplesToUse + i) % kHistorySize;
		nsecs_t time = mTimes[ix];
		if (i > 0) {
		x += divRound(time - lastTime, period);
		}
		// y is usually in [-numSamplesToUse..numSamplesToUse+kRefitRefreshPeriod/kMinPeriod) << kPrecision
		// ideally in [0..numSamplesToUse), but shifted by -numSamplesToUse during
		// priming, and possibly shifted by up to kRefitRefreshPeriod/kMinPeriod
		// while we are not refitting.
		int64_t y = divRound(time - phase, period >> kPrecision);
		sumX += x;
		sumY += y;
		sumXX += x * x;
		sumXY += x * y;
		sumYY += y * y;
		lastTime = time;
		}

		int64_t div = numSamplesToUse * sumXX - sumX * sumX;
		int64_t a_nom = numSamplesToUse * sumXY - sumX * sumY;
		int64_t b_nom = sumXX * sumY - sumX * sumXY;
		*a = divRound(a_nom, div);
		*b = divRound(b_nom, div);
		// don't use a and b directly as the rounding error is significant
		err = sumYY - divRound(a_nom sumXY + b_nom * sumY, div);
		ALOGV("fitting[%zu] a=%lld (%.6f), b=%lld (%.6f), err=%lld (%.6f)",
		numSamplesToUse,
		(long long)a, (a / (float)(1 << kPrecision)),
		(long long)b, (b / (float)(1 << kPrecision)),
		(long long)err, (err / (float)(1 << (kPrecision * 2))));
		}

		void VideoFrameScheduler::PLL::prime(size_t numSamplesToUse) {
		if (numSamplesToUse > mNumSamples) {
		numSamplesToUse = mNumSamples;
		}
		CHECK(numSamplesToUse >= 3); // must have at least 3 samples

		// estimate video framerate from deltas between timestamps, and
		// 2nd order deltas
		Vector<nsecs_t> deltas;
		nsecs_t lastTime, firstTime;
		for (size_t i = 0; i < numSamplesToUse; ++i) {
		size_t index = (mNumSamples - numSamplesToUse + i) % kHistorySize;
		nsecs_t time = mTimes[index];
		if (i > 0) {
		if (time - lastTime > kMinPeriod) {
		//ALOGV("delta: %lld", (long long)(time - lastTime));
		deltas.push(time - lastTime);
		}
		} else {
		firstTime = time;
		}
		lastTime = time;
		}
		deltas.sort(compare<nsecs_t>);
		size_t numDeltas = deltas.size();
		if (numDeltas > 1) {
		nsecs_t deltaMinLimit = min(deltas[0] / kMultiplesThresholdDiv, kMinPeriod);
		nsecs_t deltaMaxLimit = deltas[numDeltas / 2] * kMultiplesThresholdDiv;
		for (size_t i = numDeltas / 2 + 1; i < numDeltas; ++i) {
		if (deltas[i] > deltaMaxLimit) {
		deltas.resize(i);
		numDeltas = i;
		break;
		}
		}
		for (size_t i = 1; i < numDeltas; ++i) {
		nsecs_t delta2nd = deltas[i] - deltas[i - 1];
		if (delta2nd >= deltaMinLimit) {
		//ALOGV("delta2: %lld", (long long)(delta2nd));
		deltas.push(delta2nd);
		}
		}
		}

		// use the one that yields the best match
		int64_t bestScore;
		for (size_t i = 0; i < deltas.size(); ++i) {
		nsecs_t delta = deltas[i];
		int64_t score = 0;
		#if 1
		// simplest score: number of deltas that are near multiples
		size_t matches = 0;
		for (size_t j = 0; j < deltas.size(); ++j) {
		nsecs_t err = periodicError(deltas[j], delta);
		if (err < delta / kMultiplesThresholdDiv) {
		++matches;
		}
		}
		score = matches;
		#if 0
		// could be weighed by the (1 - normalized error)
		if (numSamplesToUse >= kMinSamplesToEstimatePeriod) {
		int64_t a, b, err;
		fit(firstTime, delta, numSamplesToUse, &a, &b, &err);
		err = (1 << (2 * kPrecision)) - err;
		score *= max(0, err);
		}
		#endif
		#else
		// or use the error as a negative score
		if (numSamplesToUse >= kMinSamplesToEstimatePeriod) {
		int64_t a, b, err;
		fit(firstTime, delta, numSamplesToUse, &a, &b, &err);
		score = -delta * err;
		}
		#endif
		if (i == 0 \|\| score > bestScore) {
		bestScore = score;
		mPeriod = delta;
		mPhase = firstTime;
		}
		}
		ALOGV("priming[%zu] phase:%lld period:%lld", numSamplesToUse, mPhase, mPeriod);
		}

		nsecs_t VideoFrameScheduler::PLL::addSample(nsecs_t time) {
		if (mLastTime >= 0
		// if time goes backward, or we skipped rendering
		&& (time > mLastTime + kMaxAllowedFrameSkip \|\| time < mLastTime)) {
		restart();
		}

		mLastTime = time;
		mTimes[mNumSamples % kHistorySize] = time;
		++mNumSamples;

		bool doFit = time > mRefitAt;
		if ((mPeriod <= 0 \|\| !mPrimed) && mNumSamples >= kMinSamplesToStartPrime) {
		prime(kMinSamplesToStopPrime);
		++mSamplesUsedForPriming;
		doFit = true;
		}
		if (mPeriod > 0 && mNumSamples >= kMinSamplesToEstimatePeriod) {
		if (mPhase < 0) {
		// initialize phase to the current render time
		mPhase = time;
		doFit = true;
		} else if (!doFit) {
		int64_t err = periodicError(time - mPhase, mPeriod);
		doFit = err > mPeriod / kReFitThresholdDiv;
		}

		if (doFit) {
		int64_t a, b, err;
		mRefitAt = time + kRefitRefreshPeriod;
		fit(mPhase, mPeriod, kMaxSamplesToEstimatePeriod, &a, &b, &err);
		mPhase += (mPeriod * b) >> kPrecision;
		mPeriod = (mPeriod * a) >> kPrecision;
		ALOGV("new phase:%lld period:%lld", (long long)mPhase, (long long)mPeriod);

		if (err < kErrorThreshold) {
		if (!mPrimed && mSamplesUsedForPriming >= kMinSamplesToStopPrime) {
		mPrimed = true;
		}
		} else {
		mPrimed = false;
		mSamplesUsedForPriming = 0;
		}
		}
		}
		return mPeriod;
		}

		/* ======================================================================= */
		/* Frame Scheduler */
		/* ======================================================================= */
		@@ -44,7 +343,9 @@ static const nsecs_t kVsyncRefreshPeriod = kNanosIn1s; // 1 sec
		VideoFrameScheduler::VideoFrameScheduler()
		: mVsyncTime(0),
		mVsyncPeriod(0),
		mVsyncRefreshAt(0) {
		mVsyncRefreshAt(0),
		mLastVsyncTime(-1),
		mTimeCorrection(0) {
		}

		void VideoFrameScheduler::updateVsync() {
		@@ -75,8 +376,20 @@ void VideoFrameScheduler::updateVsync() {
		}
		}

		void VideoFrameScheduler::init() {
		void VideoFrameScheduler::init(float videoFps) {
		updateVsync();

		mLastVsyncTime = -1;
		mTimeCorrection = 0;

		mPll.reset(videoFps);
		}

		void VideoFrameScheduler::restart() {
		mLastVsyncTime = -1;
		mTimeCorrection = 0;

		mPll.restart();
		}

		nsecs_t VideoFrameScheduler::getVsyncPeriod() {
		@@ -110,6 +423,62 @@ nsecs_t VideoFrameScheduler::schedule(nsecs_t renderTime) {
		// so this effectively becomes a rounding operation (to the _closest_ VSYNC.)
		renderTime -= mVsyncPeriod / 2;

		const nsecs_t videoPeriod = mPll.addSample(origRenderTime);
		if (videoPeriod > 0) {
		// Smooth out rendering
		size_t N = 12;
		nsecs_t fiveSixthDev =
		abs(((videoPeriod * 5 + mVsyncPeriod) % (mVsyncPeriod * 6)) - mVsyncPeriod)
		/ (mVsyncPeriod / 100);
		// use 20 samples if we are doing 5:6 ratio +- 1% (e.g. playing 50Hz on 60Hz)
		if (fiveSixthDev < 12) { /* 12% / 6 = 2% */
		N = 20;
		}

		nsecs_t offset = 0;
		nsecs_t edgeRemainder = 0;
		for (size_t i = 1; i <= N; i++) {
		offset +=
		(renderTime + mTimeCorrection + videoPeriod * i - mVsyncTime) % mVsyncPeriod;
		edgeRemainder += (videoPeriod * i) % mVsyncPeriod;
		}
		mTimeCorrection += mVsyncPeriod / 2 - offset / N;
		renderTime += mTimeCorrection;
		nsecs_t correctionLimit = mVsyncPeriod * 3 / 5;
		edgeRemainder = abs(edgeRemainder / N - mVsyncPeriod / 2);
		if (edgeRemainder <= mVsyncPeriod / 3) {
		correctionLimit /= 2;
		}

		// estimate how many VSYNCs a frame will spend on the display
		nsecs_t nextVsyncTime =
		renderTime + mVsyncPeriod - ((renderTime - mVsyncTime) % mVsyncPeriod);
		if (mLastVsyncTime >= 0) {
		size_t minVsyncsPerFrame = videoPeriod / mVsyncPeriod;
		size_t vsyncsForLastFrame = divRound(nextVsyncTime - mLastVsyncTime, mVsyncPeriod);
		bool vsyncsPerFrameAreNearlyConstant =
		periodicError(videoPeriod, mVsyncPeriod) / (mVsyncPeriod / 20) == 0;

		if (mTimeCorrection > correctionLimit &&
		(vsyncsPerFrameAreNearlyConstant \|\| vsyncsForLastFrame > minVsyncsPerFrame)) {
		// remove a VSYNC
		mTimeCorrection -= mVsyncPeriod / 2;
		renderTime -= mVsyncPeriod / 2;
		nextVsyncTime -= mVsyncPeriod;
		--vsyncsForLastFrame;
		} else if (mTimeCorrection < -correctionLimit &&
		(vsyncsPerFrameAreNearlyConstant \|\| vsyncsForLastFrame == minVsyncsPerFrame)) {
		// add a VSYNC
		mTimeCorrection += mVsyncPeriod / 2;
		renderTime += mVsyncPeriod / 2;
		nextVsyncTime += mVsyncPeriod;
		++vsyncsForLastFrame;
		}
		ATRACE_INT("FRAME_VSYNCS", vsyncsForLastFrame);
		}
		mLastVsyncTime = nextVsyncTime;
		}

		// align rendertime to the center between VSYNC edges
		renderTime -= (renderTime - mVsyncTime) % mVsyncPeriod;
		renderTime += mVsyncPeriod / 2;

media/libmediaplayerservice/VideoFrameScheduler.h

+39 −1

Original line number	Diff line number	Diff line
		@@ -30,7 +30,9 @@ struct VideoFrameScheduler : public RefBase {
		VideoFrameScheduler();

		// (re)initialize scheduler
		void init();
		void init(float videoFps = -1);
		// use in case of video render-time discontinuity, e.g. seek
		void restart();
		// get adjusted nanotime for a video frame render at renderTime
		nsecs_t schedule(nsecs_t renderTime);

		@@ -39,16 +41,52 @@ struct VideoFrameScheduler : public RefBase {

		void release();

		static const size_t kHistorySize = 8;

		protected:
		virtual ~VideoFrameScheduler();

		private:
		struct PLL {
		PLL();

		// reset PLL to new PLL
		void reset(float fps = -1);
		// keep current estimate, but restart phase
		void restart();
		// returns period
		nsecs_t addSample(nsecs_t time);

		private:
		nsecs_t mPeriod;
		nsecs_t mPhase;

		bool mPrimed; // have an estimate for the period
		size_t mSamplesUsedForPriming;

		nsecs_t mLastTime; // last input time
		nsecs_t mRefitAt; // next input time to fit at

		size_t mNumSamples; // can go past kHistorySize
		nsecs_t mTimes[kHistorySize];

		void test();
		void fit(nsecs_t phase, nsecs_t period, size_t numSamples,
		int64_t a, int64_t b, int64_t *err);
		void prime(size_t numSamples);
		};

		void updateVsync();

		nsecs_t mVsyncTime; // vsync timing from display
		nsecs_t mVsyncPeriod;
		nsecs_t mVsyncRefreshAt; // next time to refresh timing info

		nsecs_t mLastVsyncTime; // estimated vsync time for last frame
		nsecs_t mTimeCorrection; // running adjustment

		PLL mPll; // PLL for video frame rate based on render time

		sp<ISurfaceComposer> mComposer;

		DISALLOW_EVIL_CONSTRUCTORS(VideoFrameScheduler);

media/libmediaplayerservice/nuplayer/NuPlayer.cpp

+7 −0

Original line number	Diff line number	Diff line
		@@ -643,6 +643,13 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
		mRendererLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
		mRendererLooper->registerHandler(mRenderer);

		sp<MetaData> meta = getFileMeta();
		int32_t rate;
		if (meta != NULL
		&& meta->findInt32(kKeyFrameRate, &rate) && rate > 0) {
		mRenderer->setVideoFrameRate(rate);
		}

		postScanSources();
		break;
		}

media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp

+25 −0

Original line number	Diff line number	Diff line
		@@ -139,6 +139,12 @@ void NuPlayer::Renderer::resume() {
		(new AMessage(kWhatResume, id()))->post();
		}

		void NuPlayer::Renderer::setVideoFrameRate(float fps) {
		sp<AMessage> msg = new AMessage(kWhatSetVideoFrameRate, id());
		msg->setFloat("frame-rate", fps);
		msg->post();
		}

		void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) {
		switch (msg->what()) {
		case kWhatStopAudioSink:
		@@ -239,6 +245,14 @@ void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) {
		break;
		}

		case kWhatSetVideoFrameRate:
		{
		float fps;
		CHECK(msg->findFloat("frame-rate", &fps));
		onSetVideoFrameRate(fps);
		break;
		}

		case kWhatAudioOffloadTearDown:
		{
		onAudioOffloadTearDown();
		@@ -809,6 +823,10 @@ void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) {
		mDrainVideoQueuePending = false;
		++mVideoQueueGeneration;

		if (mVideoScheduler != NULL) {
		mVideoScheduler->restart();
		}

		prepareForMediaRenderingStart();
		}

		@@ -960,6 +978,13 @@ void NuPlayer::Renderer::onResume() {
		}
		}

		void NuPlayer::Renderer::onSetVideoFrameRate(float fps) {
		if (mVideoScheduler == NULL) {
		mVideoScheduler = new VideoFrameScheduler();
		}
		mVideoScheduler->init(fps);
		}

		// TODO: Remove unnecessary calls to getPlayedOutAudioDurationUs()
		// as it acquires locks and may query the audio driver.
		//

media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h

+4 −0

Original line number	Diff line number	Diff line
		@@ -57,6 +57,8 @@ struct NuPlayer::Renderer : public AHandler {
		void pause();
		void resume();

		void setVideoFrameRate(float fps);

		enum {
		kWhatEOS = 'eos ',
		kWhatFlushComplete = 'fluC',
		@@ -83,6 +85,7 @@ private:
		kWhatResume = 'resm',
		kWhatStopAudioSink = 'stpA',
		kWhatDisableOffloadAudio = 'noOA',
		kWhatSetVideoFrameRate = 'sVFR',
		};

		struct QueueEntry {
		@@ -149,6 +152,7 @@ private:
		void onDisableOffloadAudio();
		void onPause();
		void onResume();
		void onSetVideoFrameRate(float fps);
		void onAudioOffloadTearDown();

		void notifyEOS(bool audio, status_t finalResult, int64_t delayUs = 0);