Merge "Add histogram metrics for video playback judder" into udc-dev (ceed8069) · Commits · e / os / android_frameworks_av

media/libstagefright/MediaCodec.cpp

+12 −0

Original line number	Diff line number	Diff line
		@@ -215,6 +215,11 @@ static const char *kCodecFreezeDistanceAverage = "android.media.mediacodec.freez
		static const char *kCodecFreezeDistanceHistogram =
		"android.media.mediacodec.freeze.distance.histogram";

		static const char *kCodecJudderCount = "android.media.mediacodec.judder.count";
		static const char *kCodecJudderScoreAverage = "android.media.mediacodec.judder.average";
		static const char *kCodecJudderScoreMax = "android.media.mediacodec.judder.max";
		static const char *kCodecJudderScoreHistogram = "android.media.mediacodec.judder.histogram";

		/* -1: shaper disabled
		>=0: number of fields changed */
		static const char *kCodecShapingEnhanced = "android.media.mediacodec.shaped";
		@@ -1135,6 +1140,13 @@ void MediaCodec::updateMediametrics() {
		mediametrics_setInt64(mMetricsHandle, kCodecFreezeDistanceAverage, histogram.getAvg());
		mediametrics_setString(mMetricsHandle, kCodecFreezeDistanceHistogram, histogram.emit());
		}
		if (m.judderScoreHistogram.getCount() >= 1) {
		const MediaHistogram &histogram = m.judderScoreHistogram;
		mediametrics_setInt64(mMetricsHandle, kCodecJudderCount, histogram.getCount());
		mediametrics_setInt64(mMetricsHandle, kCodecJudderScoreAverage, histogram.getAvg());
		mediametrics_setInt64(mMetricsHandle, kCodecJudderScoreMax, histogram.getMax());
		mediametrics_setString(mMetricsHandle, kCodecJudderScoreHistogram, histogram.emit());
		}
		}

		if (mLatencyHist.getCount() != 0 ) {

media/libstagefright/VideoRenderQualityTracker.cpp

+55 −0

Original line number	Diff line number	Diff line
		@@ -55,6 +55,10 @@ VideoRenderQualityTracker::Configuration::Configuration() {
		freezeDurationMsHistogramBuckets = {1, 20, 40, 60, 80, 100, 120, 150, 175, 225, 300, 400, 500};
		freezeDistanceMsHistogramBuckets = {0, 20, 100, 400, 1000, 2000, 3000, 4000, 8000, 15000, 30000,
		60000};

		// Judder configuration
		judderErrorToleranceUs = 2000;
		judderScoreHistogramBuckets = {1, 4, 5, 9, 11, 20, 30, 40, 50, 60, 70, 80};
		}

		VideoRenderQualityTracker::VideoRenderQualityTracker() : mConfiguration(Configuration()) {
		@@ -239,6 +243,14 @@ void VideoRenderQualityTracker::processMetricsForRenderedFrame(int64_t contentTi
		processFreeze(actualRenderTimeUs, mLastRenderTimeUs, mLastFreezeEndTimeUs, mMetrics);
		mLastFreezeEndTimeUs = actualRenderTimeUs;
		}

		// Judder is computed on the prior video frame, not the current video frame
		int64_t judderScore = computePreviousJudderScore(mActualFrameDurationUs,
		mContentFrameDurationUs,
		mConfiguration);
		if (judderScore != 0) {
		mMetrics.judderScoreHistogram.insert(judderScore);
		}
		}

		void VideoRenderQualityTracker::processFreeze(int64_t actualRenderTimeUs, int64_t lastRenderTimeUs,
		@@ -252,10 +264,53 @@ void VideoRenderQualityTracker::processFreeze(int64_t actualRenderTimeUs, int64_
		}
		}

		int64_t VideoRenderQualityTracker::computePreviousJudderScore(
		const FrameDurationUs &actualFrameDurationUs,
		const FrameDurationUs &contentFrameDurationUs,
		const Configuration &c) {
		// If the frame before or after was dropped, then don't generate a judder score, since any
		// problems with frame drops are scored as a freeze instead.
		if (actualFrameDurationUs[0] == -1 \|\| actualFrameDurationUs[1] == -1 \|\|
		actualFrameDurationUs[2] == -1) {
		return 0;
		}

		// Don't score judder for when playback is paused or rebuffering (long frame duration), or if
		// the player is intentionally playing each frame at a slow rate (e.g. half-rate). If the long
		// frame duration was unintentional, it is assumed that this will be coupled with a later frame
		// drop, and be scored as a freeze instead of judder.
		if (actualFrameDurationUs[1] >= 2 * contentFrameDurationUs[1]) {
		return 0;
		}

		// The judder score is based on the error of this frame
		int64_t errorUs = actualFrameDurationUs[1] - contentFrameDurationUs[1];
		// Don't score judder if the previous frame has high error, but this frame has low error
		if (abs(errorUs) < c.judderErrorToleranceUs) {
		return 0;
		}

		// Add a penalty if this frame has judder that amplifies the problem introduced by previous
		// judder, instead of catching up for the previous judder (50, 16, 16, 50) vs (50, 16, 50, 16)
		int64_t previousErrorUs = actualFrameDurationUs[2] - contentFrameDurationUs[2];
		// Don't add the pentalty for errors from the previous frame if the previous frame has low error
		if (abs(previousErrorUs) >= c.judderErrorToleranceUs) {
		errorUs = abs(errorUs) + abs(errorUs + previousErrorUs);
		}

		// Avoid scoring judder for 3:2 pulldown or other minimally-small frame duration errors
		if (abs(errorUs) < contentFrameDurationUs[1] / 4) {
		return 0;
		}

		return abs(errorUs) / 1000; // error in millis to keep numbers small
		}

		void VideoRenderQualityTracker::configureHistograms(VideoRenderQualityMetrics &m,
		const Configuration &c) {
		m.freezeDurationMsHistogram.setup(c.freezeDurationMsHistogramBuckets);
		m.freezeDistanceMsHistogram.setup(c.freezeDistanceMsHistogramBuckets);
		m.judderScoreHistogram.setup(c.judderScoreHistogramBuckets);
		}

		int64_t VideoRenderQualityTracker::nowUs() {

media/libstagefright/include/media/stagefright/VideoRenderQualityTracker.h

+11 −0

Original line number	Diff line number	Diff line
		@@ -63,6 +63,9 @@ struct VideoRenderQualityMetrics {

		// A histogram of the durations between each freeze.
		MediaHistogram freezeDistanceMsHistogram;

		// A histogram of the judder scores.
		MediaHistogram judderScoreHistogram;
		};

		///////////////////////////////////////////////////////
		@@ -113,6 +116,9 @@ public:
		// Freeze configuration
		std::vector<int64_t> freezeDurationMsHistogramBuckets;
		std::vector<int64_t> freezeDistanceMsHistogramBuckets;

		int32_t judderErrorToleranceUs;
		std::vector<int64_t> judderScoreHistogramBuckets;
		};

		VideoRenderQualityTracker();
		@@ -197,6 +203,11 @@ private:
		static void processFreeze(int64_t actualRenderTimeUs, int64_t lastRenderTimeUs,
		int64_t lastFreezeEndTimeUs, VideoRenderQualityMetrics &m);

		// Compute a judder score for the previously-rendered frame.
		static int64_t computePreviousJudderScore(const FrameDurationUs &actualRenderDurationUs,
		const FrameDurationUs &contentRenderDurationUs,
		const Configuration &c);

		// Check to see if a discontinuity has occurred by examining the content time and the
		// app-desired render time. If so, reset some internal state.
		bool resetIfDiscontinuity(int64_t contentTimeUs, int64_t desiredRenderTimeUs);

media/libstagefright/tests/VideoRenderQualityTracker_test.cpp

+206 −2

Original line number	Diff line number	Diff line
		@@ -50,12 +50,13 @@ public:
		}
		}

		void render(int numFrames) {
		void render(int numFrames, float durationMs = -1) {
		int64_t durationUs = durationMs < 0 ? mContentFrameDurationUs : durationMs * 1000;
		for (int i = 0; i < numFrames; ++i) {
		mVideoRenderQualityTracker.onFrameReleased(mMediaTimeUs);
		mVideoRenderQualityTracker.onFrameRendered(mMediaTimeUs, mClockTimeNs);
		mMediaTimeUs += mContentFrameDurationUs;
		mClockTimeNs += mContentFrameDurationUs * 1000;
		mClockTimeNs += durationUs * 1000;
		}
		}

		@@ -291,4 +292,207 @@ TEST_F(VideoRenderQualityTrackerTest, capturesFreezeDistanceHistogram) {
		6 * 17) / 6);
		}

		TEST_F(VideoRenderQualityTrackerTest, when60hz_hasNoJudder) {
		Configuration c;
		Helper h(16.66, c); // ~24Hz
		h.render({16.66, 16.66, 16.66, 16.66, 16.66, 16.66, 16.66});
		EXPECT_LE(h.getMetrics().judderScoreHistogram.getMax(), 0);
		EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 0);
		}

		TEST_F(VideoRenderQualityTrackerTest, whenSmallVariance60hz_hasNoJudder) {
		Configuration c;
		Helper h(16.66, c); // ~24Hz
		h.render({14, 18, 14, 18, 14, 18, 14, 18});
		EXPECT_LE(h.getMetrics().judderScoreHistogram.getMax(), 0);
		EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 0);
		}

		TEST_F(VideoRenderQualityTrackerTest, whenBadSmallVariance60Hz_hasJudder) {
		Configuration c;
		Helper h(16.66, c); // ~24Hz
		h.render({14, 18, 14, /* no 18 between 14s */ 14, 18, 14, 18});
		EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 1);
		}

		TEST_F(VideoRenderQualityTrackerTest, when30Hz_hasNoJudder) {
		Configuration c;
		Helper h(33.33, c);
		h.render({33.33, 33.33, 33.33, 33.33, 33.33, 33.33});
		EXPECT_LE(h.getMetrics().judderScoreHistogram.getMax(), 0);
		EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 0);
		}

		TEST_F(VideoRenderQualityTrackerTest, whenSmallVariance30Hz_hasNoJudder) {
		Configuration c;
		Helper h(33.33, c);
		h.render({29.0, 35.0, 29.0, 35.0, 29.0, 35.0});
		EXPECT_LE(h.getMetrics().judderScoreHistogram.getMax(), 0);
		EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 0);
		}

		TEST_F(VideoRenderQualityTrackerTest, whenBadSmallVariance30Hz_hasJudder) {
		Configuration c;
		Helper h(33.33, c);
		h.render({29.0, 35.0, 29.0, /* no 35 between 29s */ 29.0, 35.0, 29.0, 35.0});
		EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 1);
		}

		TEST_F(VideoRenderQualityTrackerTest, whenBad30HzTo60Hz_hasJudder) {
		Configuration c;
		Helper h(33.33, c);
		h.render({33.33, 33.33, 50.0, /* frame stayed 1 vsync too long */ 16.66, 33.33, 33.33});
		EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 2); // note: 2 counts of judder
		}

		TEST_F(VideoRenderQualityTrackerTest, when24HzTo60Hz_hasNoJudder) {
		Configuration c;
		Helper h(41.66, c);
		h.render({50.0, 33.33, 50.0, 33.33, 50.0, 33.33});
		EXPECT_LE(h.getMetrics().judderScoreHistogram.getMax(), 0);
		EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 0);
		}

		TEST_F(VideoRenderQualityTrackerTest, when25HzTo60Hz_hasJudder) {
		Configuration c;
		Helper h(40, c);
		h.render({33.33, 33.33, 50.0});
		h.render({33.33, 33.33, 50.0});
		h.render({33.33, 33.33, 50.0});
		h.render({33.33, 33.33, 50.0});
		h.render({33.33, 33.33, 50.0});
		h.render({33.33, 33.33, 50.0});
		EXPECT_GT(h.getMetrics().judderScoreHistogram.getCount(), 0);
		}

		TEST_F(VideoRenderQualityTrackerTest, when50HzTo60Hz_hasJudder) {
		Configuration c;
		Helper h(20, c);
		h.render({16.66, 16.66, 16.66, 33.33});
		h.render({16.66, 16.66, 16.66, 33.33});
		h.render({16.66, 16.66, 16.66, 33.33});
		h.render({16.66, 16.66, 16.66, 33.33});
		h.render({16.66, 16.66, 16.66, 33.33});
		h.render({16.66, 16.66, 16.66, 33.33});
		EXPECT_GT(h.getMetrics().judderScoreHistogram.getCount(), 0);
		}

		TEST_F(VideoRenderQualityTrackerTest, when30HzTo50Hz_hasJudder) {
		Configuration c;
		Helper h(33.33, c);
		h.render({40.0, 40.0, 40.0, 60.0});
		h.render({40.0, 40.0, 40.0, 60.0});
		h.render({40.0, 40.0, 40.0, 60.0});
		h.render({40.0, 40.0, 40.0, 60.0});
		h.render({40.0, 40.0, 40.0, 60.0});
		EXPECT_GT(h.getMetrics().judderScoreHistogram.getCount(), 0);
		}

		TEST_F(VideoRenderQualityTrackerTest, whenSmallVariancePulldown24HzTo60Hz_hasNoJudder) {
		Configuration c;
		Helper h(41.66, c);
		h.render({52.0, 31.33, 52.0, 31.33, 52.0, 31.33});
		EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 0);
		}

		TEST_F(VideoRenderQualityTrackerTest, whenBad24HzTo60Hz_hasJudder) {
		Configuration c;
		Helper h(41.66, c);
		h.render({50.0, 33.33, 50.0, 33.33, /* no 50 between 33s */ 33.33, 50.0, 33.33});
		EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 1);
		}

		TEST_F(VideoRenderQualityTrackerTest, capturesJudderScoreHistogram) {
		Configuration c;
		c.judderErrorToleranceUs = 2000;
		c.judderScoreHistogramBuckets = {1, 5, 8};
		Helper h(16, c);
		h.render({16, 16, 23, 16, 16, 10, 16, 4, 16, 20, 16, 16});
		EXPECT_EQ(h.getMetrics().judderScoreHistogram.emit(), "0{1,2}1");
		EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 4);
		EXPECT_EQ(h.getMetrics().judderScoreHistogram.getMin(), 4);
		EXPECT_EQ(h.getMetrics().judderScoreHistogram.getMax(), 12);
		EXPECT_EQ(h.getMetrics().judderScoreHistogram.getAvg(), (7 + 6 + 12 + 4) / 4);
		}

		TEST_F(VideoRenderQualityTrackerTest, ranksJudderScoresInOrder) {
		// Each rendering is ranked from best to worst from a user experience
		Configuration c;
		c.judderErrorToleranceUs = 2000;
		c.judderScoreHistogramBuckets = {0, 1000};
		int64_t previousScore = 0;

		// 30fps poorly displayed at 60Hz
		{
		Helper h(33.33, c);
		h.render({33.33, 33.33, 16.66, 50.0, 33.33, 33.33});
		int64_t scoreBad30fpsTo60Hz = h.getMetrics().judderScoreHistogram.getMax();
		EXPECT_GT(scoreBad30fpsTo60Hz, previousScore);
		previousScore = scoreBad30fpsTo60Hz;
		}

		// 25fps displayed at 60hz
		{
		Helper h(40, c);
		h.render({33.33, 33.33, 50.0});
		h.render({33.33, 33.33, 50.0});
		h.render({33.33, 33.33, 50.0});
		h.render({33.33, 33.33, 50.0});
		h.render({33.33, 33.33, 50.0});
		h.render({33.33, 33.33, 50.0});
		int64_t score25fpsTo60hz = h.getMetrics().judderScoreHistogram.getMax();
		EXPECT_GT(score25fpsTo60hz, previousScore);
		previousScore = score25fpsTo60hz;
		}

		// 50fps displayed at 60hz
		{
		Helper h(20, c);
		h.render({16.66, 16.66, 16.66, 33.33});
		h.render({16.66, 16.66, 16.66, 33.33});
		h.render({16.66, 16.66, 16.66, 33.33});
		h.render({16.66, 16.66, 16.66, 33.33});
		h.render({16.66, 16.66, 16.66, 33.33});
		h.render({16.66, 16.66, 16.66, 33.33});
		int64_t score50fpsTo60hz = h.getMetrics().judderScoreHistogram.getMax();
		EXPECT_GT(score50fpsTo60hz, previousScore);
		previousScore = score50fpsTo60hz;
		}

		// 24fps poorly displayed at 60Hz
		{
		Helper h(41.66, c);
		h.render({50.0, 33.33, 50.0, 33.33, 33.33, 50.0, 33.33});
		int64_t scoreBad24HzTo60Hz = h.getMetrics().judderScoreHistogram.getMax();
		EXPECT_GT(scoreBad24HzTo60Hz, previousScore);
		previousScore = scoreBad24HzTo60Hz;
		}

		// 30fps displayed at 50hz
		{
		Helper h(33.33, c);
		h.render({40.0, 40.0, 40.0, 60.0});
		h.render({40.0, 40.0, 40.0, 60.0});
		h.render({40.0, 40.0, 40.0, 60.0});
		h.render({40.0, 40.0, 40.0, 60.0});
		h.render({40.0, 40.0, 40.0, 60.0});
		int64_t score30fpsTo50hz = h.getMetrics().judderScoreHistogram.getMax();
		EXPECT_GT(score30fpsTo50hz, previousScore);
		previousScore = score30fpsTo50hz;
		}

		// 24fps displayed at 50Hz
		{
		Helper h(41.66, c);
		h.render(40.0, 11);
		h.render(60.0, 1);
		h.render(40.0, 11);
		h.render(60.0, 1);
		h.render(40.0, 11);
		int64_t score24HzTo50Hz = h.getMetrics().judderScoreHistogram.getMax();
		EXPECT_GT(score24HzTo50Hz, previousScore);
		previousScore = score24HzTo50Hz;
		}
		}

		} // android