Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 3b5d4b2e authored by Brian Lindahl's avatar Brian Lindahl
Browse files

Add histogram metrics for video playback judder 

Bug: 234833109
Test: atest VideoRenderQualityTracker_test
Change-Id: I3a7f36384ab5e79f5e0e706a8f62a1e4854587a0
parent c4e9deef

media/libstagefright/MediaCodec.cpp

+12 −0
Original line number Diff line number Diff line
@@ -215,6 +215,11 @@ static const char *kCodecFreezeDistanceAverage = "android.media.mediacodec.freez 
static const char *kCodecFreezeDistanceHistogram =

        "android.media.mediacodec.freeze.distance.histogram";



static const char *kCodecJudderCount = "android.media.mediacodec.judder.count";

static const char *kCodecJudderScoreAverage = "android.media.mediacodec.judder.average";

static const char *kCodecJudderScoreMax = "android.media.mediacodec.judder.max";

static const char *kCodecJudderScoreHistogram = "android.media.mediacodec.judder.histogram";



/* -1: shaper disabled

   >=0: number of fields changed */

static const char *kCodecShapingEnhanced = "android.media.mediacodec.shaped";
@@ -1135,6 +1140,13 @@ void MediaCodec::updateMediametrics() { 
            mediametrics_setInt64(mMetricsHandle, kCodecFreezeDistanceAverage, histogram.getAvg());

            mediametrics_setString(mMetricsHandle, kCodecFreezeDistanceHistogram, histogram.emit());

        }

        if (m.judderScoreHistogram.getCount() >= 1) {

            const MediaHistogram &histogram = m.judderScoreHistogram;

            mediametrics_setInt64(mMetricsHandle, kCodecJudderCount, histogram.getCount());

            mediametrics_setInt64(mMetricsHandle, kCodecJudderScoreAverage, histogram.getAvg());

            mediametrics_setInt64(mMetricsHandle, kCodecJudderScoreMax, histogram.getMax());

            mediametrics_setString(mMetricsHandle, kCodecJudderScoreHistogram, histogram.emit());

        }

    }



    if (mLatencyHist.getCount() != 0 ) {

media/libstagefright/VideoRenderQualityTracker.cpp

+55 −0
Original line number Diff line number Diff line
@@ -55,6 +55,10 @@ VideoRenderQualityTracker::Configuration::Configuration() { 
    freezeDurationMsHistogramBuckets = {1, 20, 40, 60, 80, 100, 120, 150, 175, 225, 300, 400, 500};

    freezeDistanceMsHistogramBuckets = {0, 20, 100, 400, 1000, 2000, 3000, 4000, 8000, 15000, 30000,

                                        60000};



    // Judder configuration

    judderErrorToleranceUs = 2000;

    judderScoreHistogramBuckets = {1, 4, 5, 9, 11, 20, 30, 40, 50, 60, 70, 80};

}



VideoRenderQualityTracker::VideoRenderQualityTracker() : mConfiguration(Configuration()) {
@@ -239,6 +243,14 @@ void VideoRenderQualityTracker::processMetricsForRenderedFrame(int64_t contentTi 
        processFreeze(actualRenderTimeUs, mLastRenderTimeUs, mLastFreezeEndTimeUs, mMetrics);

        mLastFreezeEndTimeUs = actualRenderTimeUs;

    }



    // Judder is computed on the prior video frame, not the current video frame

    int64_t judderScore = computePreviousJudderScore(mActualFrameDurationUs,

                                                     mContentFrameDurationUs,

                                                     mConfiguration);

    if (judderScore != 0) {

        mMetrics.judderScoreHistogram.insert(judderScore);

    }

}



void VideoRenderQualityTracker::processFreeze(int64_t actualRenderTimeUs, int64_t lastRenderTimeUs,
@@ -252,10 +264,53 @@ void VideoRenderQualityTracker::processFreeze(int64_t actualRenderTimeUs, int64_ 
    }

}



int64_t VideoRenderQualityTracker::computePreviousJudderScore(

        const FrameDurationUs &actualFrameDurationUs,

        const FrameDurationUs &contentFrameDurationUs,

        const Configuration &c) {

    // If the frame before or after was dropped, then don't generate a judder score, since any

    // problems with frame drops are scored as a freeze instead.

    if (actualFrameDurationUs[0] == -1 || actualFrameDurationUs[1] == -1 ||

        actualFrameDurationUs[2] == -1) {

        return 0;

    }



    // Don't score judder for when playback is paused or rebuffering (long frame duration), or if

    // the player is intentionally playing each frame at a slow rate (e.g. half-rate). If the long

    // frame duration was unintentional, it is assumed that this will be coupled with a later frame

    // drop, and be scored as a freeze instead of judder.

    if (actualFrameDurationUs[1] >= 2 * contentFrameDurationUs[1]) {

        return 0;

    }



    // The judder score is based on the error of this frame

    int64_t errorUs = actualFrameDurationUs[1] - contentFrameDurationUs[1];

    // Don't score judder if the previous frame has high error, but this frame has low error

    if (abs(errorUs) < c.judderErrorToleranceUs) {

        return 0;

    }



    // Add a penalty if this frame has judder that amplifies the problem introduced by previous

    // judder, instead of catching up for the previous judder (50, 16, 16, 50) vs (50, 16, 50, 16)

    int64_t previousErrorUs = actualFrameDurationUs[2] - contentFrameDurationUs[2];

    // Don't add the pentalty for errors from the previous frame if the previous frame has low error

    if (abs(previousErrorUs) >= c.judderErrorToleranceUs) {

        errorUs = abs(errorUs) + abs(errorUs + previousErrorUs);

    }



    // Avoid scoring judder for 3:2 pulldown or other minimally-small frame duration errors

    if (abs(errorUs) < contentFrameDurationUs[1] / 4) {

        return 0;

    }



    return abs(errorUs) / 1000; // error in millis to keep numbers small

}



void VideoRenderQualityTracker::configureHistograms(VideoRenderQualityMetrics &m,

                                                    const Configuration &c) {

    m.freezeDurationMsHistogram.setup(c.freezeDurationMsHistogramBuckets);

    m.freezeDistanceMsHistogram.setup(c.freezeDistanceMsHistogramBuckets);

    m.judderScoreHistogram.setup(c.judderScoreHistogramBuckets);

}



int64_t VideoRenderQualityTracker::nowUs() {

media/libstagefright/include/media/stagefright/VideoRenderQualityTracker.h

+11 −0
Original line number Diff line number Diff line
@@ -63,6 +63,9 @@ struct VideoRenderQualityMetrics { 


    // A histogram of the durations between each freeze.

    MediaHistogram freezeDistanceMsHistogram;



    // A histogram of the judder scores.

    MediaHistogram judderScoreHistogram;

};



///////////////////////////////////////////////////////
@@ -113,6 +116,9 @@ public: 
        // Freeze configuration

        std::vector<int64_t> freezeDurationMsHistogramBuckets;

        std::vector<int64_t> freezeDistanceMsHistogramBuckets;



        int32_t judderErrorToleranceUs;

        std::vector<int64_t> judderScoreHistogramBuckets;

    };



    VideoRenderQualityTracker();
@@ -197,6 +203,11 @@ private: 
    static void processFreeze(int64_t actualRenderTimeUs, int64_t lastRenderTimeUs,

                              int64_t lastFreezeEndTimeUs, VideoRenderQualityMetrics &m);



    // Compute a judder score for the previously-rendered frame.

    static int64_t computePreviousJudderScore(const FrameDurationUs &actualRenderDurationUs,

                                              const FrameDurationUs &contentRenderDurationUs,

                                              const Configuration &c);



    // Check to see if a discontinuity has occurred by examining the content time and the

    // app-desired render time. If so, reset some internal state.

    bool resetIfDiscontinuity(int64_t contentTimeUs, int64_t desiredRenderTimeUs);

media/libstagefright/tests/VideoRenderQualityTracker_test.cpp

+206 −2
Original line number Diff line number Diff line
@@ -50,12 +50,13 @@ public: 
        }

    }



    void render(int numFrames) {

    void render(int numFrames, float durationMs = -1) {

        int64_t durationUs = durationMs < 0 ? mContentFrameDurationUs : durationMs * 1000;

        for (int i = 0; i < numFrames; ++i) {

            mVideoRenderQualityTracker.onFrameReleased(mMediaTimeUs);

            mVideoRenderQualityTracker.onFrameRendered(mMediaTimeUs, mClockTimeNs);

            mMediaTimeUs += mContentFrameDurationUs;

            mClockTimeNs += mContentFrameDurationUs * 1000;

            mClockTimeNs += durationUs * 1000;

        }

    }
@@ -291,4 +292,207 @@ TEST_F(VideoRenderQualityTrackerTest, capturesFreezeDistanceHistogram) { 
                                                                  6 * 17) / 6);

}



TEST_F(VideoRenderQualityTrackerTest, when60hz_hasNoJudder) {

    Configuration c;

    Helper h(16.66, c); // ~24Hz

    h.render({16.66, 16.66, 16.66, 16.66, 16.66, 16.66, 16.66});

    EXPECT_LE(h.getMetrics().judderScoreHistogram.getMax(), 0);

    EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 0);

}



TEST_F(VideoRenderQualityTrackerTest, whenSmallVariance60hz_hasNoJudder) {

    Configuration c;

    Helper h(16.66, c); // ~24Hz

    h.render({14, 18, 14, 18, 14, 18, 14, 18});

    EXPECT_LE(h.getMetrics().judderScoreHistogram.getMax(), 0);

    EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 0);

}



TEST_F(VideoRenderQualityTrackerTest, whenBadSmallVariance60Hz_hasJudder) {

    Configuration c;

    Helper h(16.66, c); // ~24Hz

    h.render({14, 18, 14, /* no 18 between 14s */ 14, 18, 14, 18});

    EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 1);

}



TEST_F(VideoRenderQualityTrackerTest, when30Hz_hasNoJudder) {

    Configuration c;

    Helper h(33.33, c);

    h.render({33.33, 33.33, 33.33, 33.33, 33.33, 33.33});

    EXPECT_LE(h.getMetrics().judderScoreHistogram.getMax(), 0);

    EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 0);

}



TEST_F(VideoRenderQualityTrackerTest, whenSmallVariance30Hz_hasNoJudder) {

    Configuration c;

    Helper h(33.33, c);

    h.render({29.0, 35.0, 29.0, 35.0, 29.0, 35.0});

    EXPECT_LE(h.getMetrics().judderScoreHistogram.getMax(), 0);

    EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 0);

}



TEST_F(VideoRenderQualityTrackerTest, whenBadSmallVariance30Hz_hasJudder) {

    Configuration c;

    Helper h(33.33, c);

    h.render({29.0, 35.0, 29.0, /* no 35 between 29s */ 29.0, 35.0, 29.0, 35.0});

    EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 1);

}



TEST_F(VideoRenderQualityTrackerTest, whenBad30HzTo60Hz_hasJudder) {

    Configuration c;

    Helper h(33.33, c);

    h.render({33.33, 33.33, 50.0, /* frame stayed 1 vsync too long */ 16.66, 33.33, 33.33});

    EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 2); // note: 2 counts of judder

}



TEST_F(VideoRenderQualityTrackerTest, when24HzTo60Hz_hasNoJudder) {

    Configuration c;

    Helper h(41.66, c);

    h.render({50.0, 33.33, 50.0, 33.33, 50.0, 33.33});

    EXPECT_LE(h.getMetrics().judderScoreHistogram.getMax(), 0);

    EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 0);

}



TEST_F(VideoRenderQualityTrackerTest, when25HzTo60Hz_hasJudder) {

    Configuration c;

    Helper h(40, c);

    h.render({33.33, 33.33, 50.0});

    h.render({33.33, 33.33, 50.0});

    h.render({33.33, 33.33, 50.0});

    h.render({33.33, 33.33, 50.0});

    h.render({33.33, 33.33, 50.0});

    h.render({33.33, 33.33, 50.0});

    EXPECT_GT(h.getMetrics().judderScoreHistogram.getCount(), 0);

}



TEST_F(VideoRenderQualityTrackerTest, when50HzTo60Hz_hasJudder) {

    Configuration c;

    Helper h(20, c);

    h.render({16.66, 16.66, 16.66, 33.33});

    h.render({16.66, 16.66, 16.66, 33.33});

    h.render({16.66, 16.66, 16.66, 33.33});

    h.render({16.66, 16.66, 16.66, 33.33});

    h.render({16.66, 16.66, 16.66, 33.33});

    h.render({16.66, 16.66, 16.66, 33.33});

    EXPECT_GT(h.getMetrics().judderScoreHistogram.getCount(), 0);

}



TEST_F(VideoRenderQualityTrackerTest, when30HzTo50Hz_hasJudder) {

    Configuration c;

    Helper h(33.33, c);

    h.render({40.0, 40.0, 40.0, 60.0});

    h.render({40.0, 40.0, 40.0, 60.0});

    h.render({40.0, 40.0, 40.0, 60.0});

    h.render({40.0, 40.0, 40.0, 60.0});

    h.render({40.0, 40.0, 40.0, 60.0});

    EXPECT_GT(h.getMetrics().judderScoreHistogram.getCount(), 0);

}



TEST_F(VideoRenderQualityTrackerTest, whenSmallVariancePulldown24HzTo60Hz_hasNoJudder) {

    Configuration c;

    Helper h(41.66, c);

    h.render({52.0, 31.33, 52.0, 31.33, 52.0, 31.33});

    EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 0);

}



TEST_F(VideoRenderQualityTrackerTest, whenBad24HzTo60Hz_hasJudder) {

    Configuration c;

    Helper h(41.66, c);

    h.render({50.0, 33.33, 50.0, 33.33, /* no 50 between 33s */ 33.33, 50.0, 33.33});

    EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 1);

}



TEST_F(VideoRenderQualityTrackerTest, capturesJudderScoreHistogram) {

    Configuration c;

    c.judderErrorToleranceUs = 2000;

    c.judderScoreHistogramBuckets = {1, 5, 8};

    Helper h(16, c);

    h.render({16, 16, 23, 16, 16, 10, 16, 4, 16, 20, 16, 16});

    EXPECT_EQ(h.getMetrics().judderScoreHistogram.emit(), "0{1,2}1");

    EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 4);

    EXPECT_EQ(h.getMetrics().judderScoreHistogram.getMin(), 4);

    EXPECT_EQ(h.getMetrics().judderScoreHistogram.getMax(), 12);

    EXPECT_EQ(h.getMetrics().judderScoreHistogram.getAvg(), (7 + 6 + 12 + 4) / 4);

}



TEST_F(VideoRenderQualityTrackerTest, ranksJudderScoresInOrder) {

    // Each rendering is ranked from best to worst from a user experience

    Configuration c;

    c.judderErrorToleranceUs = 2000;

    c.judderScoreHistogramBuckets = {0, 1000};

    int64_t previousScore = 0;



    // 30fps poorly displayed at 60Hz

    {

        Helper h(33.33, c);

        h.render({33.33, 33.33, 16.66, 50.0, 33.33, 33.33});

        int64_t scoreBad30fpsTo60Hz = h.getMetrics().judderScoreHistogram.getMax();

        EXPECT_GT(scoreBad30fpsTo60Hz, previousScore);

        previousScore = scoreBad30fpsTo60Hz;

    }



    // 25fps displayed at 60hz

    {

        Helper h(40, c);

        h.render({33.33, 33.33, 50.0});

        h.render({33.33, 33.33, 50.0});

        h.render({33.33, 33.33, 50.0});

        h.render({33.33, 33.33, 50.0});

        h.render({33.33, 33.33, 50.0});

        h.render({33.33, 33.33, 50.0});

        int64_t score25fpsTo60hz = h.getMetrics().judderScoreHistogram.getMax();

        EXPECT_GT(score25fpsTo60hz, previousScore);

        previousScore = score25fpsTo60hz;

    }



    // 50fps displayed at 60hz

    {

        Helper h(20, c);

        h.render({16.66, 16.66, 16.66, 33.33});

        h.render({16.66, 16.66, 16.66, 33.33});

        h.render({16.66, 16.66, 16.66, 33.33});

        h.render({16.66, 16.66, 16.66, 33.33});

        h.render({16.66, 16.66, 16.66, 33.33});

        h.render({16.66, 16.66, 16.66, 33.33});

        int64_t score50fpsTo60hz = h.getMetrics().judderScoreHistogram.getMax();

        EXPECT_GT(score50fpsTo60hz, previousScore);

        previousScore = score50fpsTo60hz;

    }



    // 24fps poorly displayed at 60Hz

    {

        Helper h(41.66, c);

        h.render({50.0, 33.33, 50.0, 33.33, 33.33, 50.0, 33.33});

        int64_t scoreBad24HzTo60Hz = h.getMetrics().judderScoreHistogram.getMax();

        EXPECT_GT(scoreBad24HzTo60Hz, previousScore);

        previousScore = scoreBad24HzTo60Hz;

    }



    // 30fps displayed at 50hz

    {

        Helper h(33.33, c);

        h.render({40.0, 40.0, 40.0, 60.0});

        h.render({40.0, 40.0, 40.0, 60.0});

        h.render({40.0, 40.0, 40.0, 60.0});

        h.render({40.0, 40.0, 40.0, 60.0});

        h.render({40.0, 40.0, 40.0, 60.0});

        int64_t score30fpsTo50hz = h.getMetrics().judderScoreHistogram.getMax();

        EXPECT_GT(score30fpsTo50hz, previousScore);

        previousScore = score30fpsTo50hz;

    }



    // 24fps displayed at 50Hz

    {

        Helper h(41.66, c);

        h.render(40.0, 11);

        h.render(60.0, 1);

        h.render(40.0, 11);

        h.render(60.0, 1);

        h.render(40.0, 11);

        int64_t score24HzTo50Hz = h.getMetrics().judderScoreHistogram.getMax();

        EXPECT_GT(score24HzTo50Hz, previousScore);

        previousScore = score24HzTo50Hz;

    }

}



} // android