Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 0866813c authored by Brian Lindahl's avatar Brian Lindahl Committed by Automerger Merge Worker
Browse files

Merge "Add histogram metrics for video playback judder" into udc-dev am: ceed8069 

Original change: https://googleplex-android-review.googlesource.com/c/platform/frameworks/av/+/23049641



Change-Id: Iac35be2bd5b48e720913ef09d12de1ee7218a534
Signed-off-by: default avatarAutomerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
parents c308a822 ceed8069

media/libstagefright/MediaCodec.cpp

+12 −0
Original line number Diff line number Diff line
@@ -215,6 +215,11 @@ static const char *kCodecFreezeDistanceAverage = "android.media.mediacodec.freez 
static const char *kCodecFreezeDistanceHistogram =

        "android.media.mediacodec.freeze.distance.histogram";



static const char *kCodecJudderCount = "android.media.mediacodec.judder.count";

static const char *kCodecJudderScoreAverage = "android.media.mediacodec.judder.average";

static const char *kCodecJudderScoreMax = "android.media.mediacodec.judder.max";

static const char *kCodecJudderScoreHistogram = "android.media.mediacodec.judder.histogram";



/* -1: shaper disabled

   >=0: number of fields changed */

static const char *kCodecShapingEnhanced = "android.media.mediacodec.shaped";
@@ -1135,6 +1140,13 @@ void MediaCodec::updateMediametrics() { 
            mediametrics_setInt64(mMetricsHandle, kCodecFreezeDistanceAverage, histogram.getAvg());

            mediametrics_setString(mMetricsHandle, kCodecFreezeDistanceHistogram, histogram.emit());

        }

        if (m.judderScoreHistogram.getCount() >= 1) {

            const MediaHistogram &histogram = m.judderScoreHistogram;

            mediametrics_setInt64(mMetricsHandle, kCodecJudderCount, histogram.getCount());

            mediametrics_setInt64(mMetricsHandle, kCodecJudderScoreAverage, histogram.getAvg());

            mediametrics_setInt64(mMetricsHandle, kCodecJudderScoreMax, histogram.getMax());

            mediametrics_setString(mMetricsHandle, kCodecJudderScoreHistogram, histogram.emit());

        }

    }



    if (mLatencyHist.getCount() != 0 ) {

media/libstagefright/VideoRenderQualityTracker.cpp

+55 −0
Original line number Diff line number Diff line
@@ -55,6 +55,10 @@ VideoRenderQualityTracker::Configuration::Configuration() { 
    freezeDurationMsHistogramBuckets = {1, 20, 40, 60, 80, 100, 120, 150, 175, 225, 300, 400, 500};

    freezeDistanceMsHistogramBuckets = {0, 20, 100, 400, 1000, 2000, 3000, 4000, 8000, 15000, 30000,

                                        60000};



    // Judder configuration

    judderErrorToleranceUs = 2000;

    judderScoreHistogramBuckets = {1, 4, 5, 9, 11, 20, 30, 40, 50, 60, 70, 80};

}



VideoRenderQualityTracker::VideoRenderQualityTracker() : mConfiguration(Configuration()) {
@@ -239,6 +243,14 @@ void VideoRenderQualityTracker::processMetricsForRenderedFrame(int64_t contentTi 
        processFreeze(actualRenderTimeUs, mLastRenderTimeUs, mLastFreezeEndTimeUs, mMetrics);

        mLastFreezeEndTimeUs = actualRenderTimeUs;

    }



    // Judder is computed on the prior video frame, not the current video frame

    int64_t judderScore = computePreviousJudderScore(mActualFrameDurationUs,

                                                     mContentFrameDurationUs,

                                                     mConfiguration);

    if (judderScore != 0) {

        mMetrics.judderScoreHistogram.insert(judderScore);

    }

}



void VideoRenderQualityTracker::processFreeze(int64_t actualRenderTimeUs, int64_t lastRenderTimeUs,
@@ -252,10 +264,53 @@ void VideoRenderQualityTracker::processFreeze(int64_t actualRenderTimeUs, int64_ 
    }

}



int64_t VideoRenderQualityTracker::computePreviousJudderScore(

        const FrameDurationUs &actualFrameDurationUs,

        const FrameDurationUs &contentFrameDurationUs,

        const Configuration &c) {

    // If the frame before or after was dropped, then don't generate a judder score, since any

    // problems with frame drops are scored as a freeze instead.

    if (actualFrameDurationUs[0] == -1 || actualFrameDurationUs[1] == -1 ||

        actualFrameDurationUs[2] == -1) {

        return 0;

    }



    // Don't score judder for when playback is paused or rebuffering (long frame duration), or if

    // the player is intentionally playing each frame at a slow rate (e.g. half-rate). If the long

    // frame duration was unintentional, it is assumed that this will be coupled with a later frame

    // drop, and be scored as a freeze instead of judder.

    if (actualFrameDurationUs[1] >= 2 * contentFrameDurationUs[1]) {

        return 0;

    }



    // The judder score is based on the error of this frame

    int64_t errorUs = actualFrameDurationUs[1] - contentFrameDurationUs[1];

    // Don't score judder if the previous frame has high error, but this frame has low error

    if (abs(errorUs) < c.judderErrorToleranceUs) {

        return 0;

    }



    // Add a penalty if this frame has judder that amplifies the problem introduced by previous

    // judder, instead of catching up for the previous judder (50, 16, 16, 50) vs (50, 16, 50, 16)

    int64_t previousErrorUs = actualFrameDurationUs[2] - contentFrameDurationUs[2];

    // Don't add the pentalty for errors from the previous frame if the previous frame has low error

    if (abs(previousErrorUs) >= c.judderErrorToleranceUs) {

        errorUs = abs(errorUs) + abs(errorUs + previousErrorUs);

    }



    // Avoid scoring judder for 3:2 pulldown or other minimally-small frame duration errors

    if (abs(errorUs) < contentFrameDurationUs[1] / 4) {

        return 0;

    }



    return abs(errorUs) / 1000; // error in millis to keep numbers small

}



void VideoRenderQualityTracker::configureHistograms(VideoRenderQualityMetrics &m,

                                                    const Configuration &c) {

    m.freezeDurationMsHistogram.setup(c.freezeDurationMsHistogramBuckets);

    m.freezeDistanceMsHistogram.setup(c.freezeDistanceMsHistogramBuckets);

    m.judderScoreHistogram.setup(c.judderScoreHistogramBuckets);

}



int64_t VideoRenderQualityTracker::nowUs() {

media/libstagefright/include/media/stagefright/VideoRenderQualityTracker.h

+11 −0
Original line number Diff line number Diff line
@@ -63,6 +63,9 @@ struct VideoRenderQualityMetrics { 


    // A histogram of the durations between each freeze.

    MediaHistogram freezeDistanceMsHistogram;



    // A histogram of the judder scores.

    MediaHistogram judderScoreHistogram;

};



///////////////////////////////////////////////////////
@@ -113,6 +116,9 @@ public: 
        // Freeze configuration

        std::vector<int64_t> freezeDurationMsHistogramBuckets;

        std::vector<int64_t> freezeDistanceMsHistogramBuckets;



        int32_t judderErrorToleranceUs;

        std::vector<int64_t> judderScoreHistogramBuckets;

    };



    VideoRenderQualityTracker();
@@ -197,6 +203,11 @@ private: 
    static void processFreeze(int64_t actualRenderTimeUs, int64_t lastRenderTimeUs,

                              int64_t lastFreezeEndTimeUs, VideoRenderQualityMetrics &m);



    // Compute a judder score for the previously-rendered frame.

    static int64_t computePreviousJudderScore(const FrameDurationUs &actualRenderDurationUs,

                                              const FrameDurationUs &contentRenderDurationUs,

                                              const Configuration &c);



    // Check to see if a discontinuity has occurred by examining the content time and the

    // app-desired render time. If so, reset some internal state.

    bool resetIfDiscontinuity(int64_t contentTimeUs, int64_t desiredRenderTimeUs);

media/libstagefright/tests/VideoRenderQualityTracker_test.cpp

+206 −2
Original line number Diff line number Diff line
@@ -50,12 +50,13 @@ public: 
        }

    }



    void render(int numFrames) {

    void render(int numFrames, float durationMs = -1) {

        int64_t durationUs = durationMs < 0 ? mContentFrameDurationUs : durationMs * 1000;

        for (int i = 0; i < numFrames; ++i) {

            mVideoRenderQualityTracker.onFrameReleased(mMediaTimeUs);

            mVideoRenderQualityTracker.onFrameRendered(mMediaTimeUs, mClockTimeNs);

            mMediaTimeUs += mContentFrameDurationUs;

            mClockTimeNs += mContentFrameDurationUs * 1000;

            mClockTimeNs += durationUs * 1000;

        }

    }
@@ -291,4 +292,207 @@ TEST_F(VideoRenderQualityTrackerTest, capturesFreezeDistanceHistogram) { 
                                                                  6 * 17) / 6);

}



TEST_F(VideoRenderQualityTrackerTest, when60hz_hasNoJudder) {

    Configuration c;

    Helper h(16.66, c); // ~24Hz

    h.render({16.66, 16.66, 16.66, 16.66, 16.66, 16.66, 16.66});

    EXPECT_LE(h.getMetrics().judderScoreHistogram.getMax(), 0);

    EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 0);

}



TEST_F(VideoRenderQualityTrackerTest, whenSmallVariance60hz_hasNoJudder) {

    Configuration c;

    Helper h(16.66, c); // ~24Hz

    h.render({14, 18, 14, 18, 14, 18, 14, 18});

    EXPECT_LE(h.getMetrics().judderScoreHistogram.getMax(), 0);

    EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 0);

}



TEST_F(VideoRenderQualityTrackerTest, whenBadSmallVariance60Hz_hasJudder) {

    Configuration c;

    Helper h(16.66, c); // ~24Hz

    h.render({14, 18, 14, /* no 18 between 14s */ 14, 18, 14, 18});

    EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 1);

}



TEST_F(VideoRenderQualityTrackerTest, when30Hz_hasNoJudder) {

    Configuration c;

    Helper h(33.33, c);

    h.render({33.33, 33.33, 33.33, 33.33, 33.33, 33.33});

    EXPECT_LE(h.getMetrics().judderScoreHistogram.getMax(), 0);

    EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 0);

}



TEST_F(VideoRenderQualityTrackerTest, whenSmallVariance30Hz_hasNoJudder) {

    Configuration c;

    Helper h(33.33, c);

    h.render({29.0, 35.0, 29.0, 35.0, 29.0, 35.0});

    EXPECT_LE(h.getMetrics().judderScoreHistogram.getMax(), 0);

    EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 0);

}



TEST_F(VideoRenderQualityTrackerTest, whenBadSmallVariance30Hz_hasJudder) {

    Configuration c;

    Helper h(33.33, c);

    h.render({29.0, 35.0, 29.0, /* no 35 between 29s */ 29.0, 35.0, 29.0, 35.0});

    EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 1);

}



TEST_F(VideoRenderQualityTrackerTest, whenBad30HzTo60Hz_hasJudder) {

    Configuration c;

    Helper h(33.33, c);

    h.render({33.33, 33.33, 50.0, /* frame stayed 1 vsync too long */ 16.66, 33.33, 33.33});

    EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 2); // note: 2 counts of judder

}



TEST_F(VideoRenderQualityTrackerTest, when24HzTo60Hz_hasNoJudder) {

    Configuration c;

    Helper h(41.66, c);

    h.render({50.0, 33.33, 50.0, 33.33, 50.0, 33.33});

    EXPECT_LE(h.getMetrics().judderScoreHistogram.getMax(), 0);

    EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 0);

}



TEST_F(VideoRenderQualityTrackerTest, when25HzTo60Hz_hasJudder) {

    Configuration c;

    Helper h(40, c);

    h.render({33.33, 33.33, 50.0});

    h.render({33.33, 33.33, 50.0});

    h.render({33.33, 33.33, 50.0});

    h.render({33.33, 33.33, 50.0});

    h.render({33.33, 33.33, 50.0});

    h.render({33.33, 33.33, 50.0});

    EXPECT_GT(h.getMetrics().judderScoreHistogram.getCount(), 0);

}



TEST_F(VideoRenderQualityTrackerTest, when50HzTo60Hz_hasJudder) {

    Configuration c;

    Helper h(20, c);

    h.render({16.66, 16.66, 16.66, 33.33});

    h.render({16.66, 16.66, 16.66, 33.33});

    h.render({16.66, 16.66, 16.66, 33.33});

    h.render({16.66, 16.66, 16.66, 33.33});

    h.render({16.66, 16.66, 16.66, 33.33});

    h.render({16.66, 16.66, 16.66, 33.33});

    EXPECT_GT(h.getMetrics().judderScoreHistogram.getCount(), 0);

}



TEST_F(VideoRenderQualityTrackerTest, when30HzTo50Hz_hasJudder) {

    Configuration c;

    Helper h(33.33, c);

    h.render({40.0, 40.0, 40.0, 60.0});

    h.render({40.0, 40.0, 40.0, 60.0});

    h.render({40.0, 40.0, 40.0, 60.0});

    h.render({40.0, 40.0, 40.0, 60.0});

    h.render({40.0, 40.0, 40.0, 60.0});

    EXPECT_GT(h.getMetrics().judderScoreHistogram.getCount(), 0);

}



TEST_F(VideoRenderQualityTrackerTest, whenSmallVariancePulldown24HzTo60Hz_hasNoJudder) {

    Configuration c;

    Helper h(41.66, c);

    h.render({52.0, 31.33, 52.0, 31.33, 52.0, 31.33});

    EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 0);

}



TEST_F(VideoRenderQualityTrackerTest, whenBad24HzTo60Hz_hasJudder) {

    Configuration c;

    Helper h(41.66, c);

    h.render({50.0, 33.33, 50.0, 33.33, /* no 50 between 33s */ 33.33, 50.0, 33.33});

    EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 1);

}



TEST_F(VideoRenderQualityTrackerTest, capturesJudderScoreHistogram) {

    Configuration c;

    c.judderErrorToleranceUs = 2000;

    c.judderScoreHistogramBuckets = {1, 5, 8};

    Helper h(16, c);

    h.render({16, 16, 23, 16, 16, 10, 16, 4, 16, 20, 16, 16});

    EXPECT_EQ(h.getMetrics().judderScoreHistogram.emit(), "0{1,2}1");

    EXPECT_EQ(h.getMetrics().judderScoreHistogram.getCount(), 4);

    EXPECT_EQ(h.getMetrics().judderScoreHistogram.getMin(), 4);

    EXPECT_EQ(h.getMetrics().judderScoreHistogram.getMax(), 12);

    EXPECT_EQ(h.getMetrics().judderScoreHistogram.getAvg(), (7 + 6 + 12 + 4) / 4);

}



TEST_F(VideoRenderQualityTrackerTest, ranksJudderScoresInOrder) {

    // Each rendering is ranked from best to worst from a user experience

    Configuration c;

    c.judderErrorToleranceUs = 2000;

    c.judderScoreHistogramBuckets = {0, 1000};

    int64_t previousScore = 0;



    // 30fps poorly displayed at 60Hz

    {

        Helper h(33.33, c);

        h.render({33.33, 33.33, 16.66, 50.0, 33.33, 33.33});

        int64_t scoreBad30fpsTo60Hz = h.getMetrics().judderScoreHistogram.getMax();

        EXPECT_GT(scoreBad30fpsTo60Hz, previousScore);

        previousScore = scoreBad30fpsTo60Hz;

    }



    // 25fps displayed at 60hz

    {

        Helper h(40, c);

        h.render({33.33, 33.33, 50.0});

        h.render({33.33, 33.33, 50.0});

        h.render({33.33, 33.33, 50.0});

        h.render({33.33, 33.33, 50.0});

        h.render({33.33, 33.33, 50.0});

        h.render({33.33, 33.33, 50.0});

        int64_t score25fpsTo60hz = h.getMetrics().judderScoreHistogram.getMax();

        EXPECT_GT(score25fpsTo60hz, previousScore);

        previousScore = score25fpsTo60hz;

    }



    // 50fps displayed at 60hz

    {

        Helper h(20, c);

        h.render({16.66, 16.66, 16.66, 33.33});

        h.render({16.66, 16.66, 16.66, 33.33});

        h.render({16.66, 16.66, 16.66, 33.33});

        h.render({16.66, 16.66, 16.66, 33.33});

        h.render({16.66, 16.66, 16.66, 33.33});

        h.render({16.66, 16.66, 16.66, 33.33});

        int64_t score50fpsTo60hz = h.getMetrics().judderScoreHistogram.getMax();

        EXPECT_GT(score50fpsTo60hz, previousScore);

        previousScore = score50fpsTo60hz;

    }



    // 24fps poorly displayed at 60Hz

    {

        Helper h(41.66, c);

        h.render({50.0, 33.33, 50.0, 33.33, 33.33, 50.0, 33.33});

        int64_t scoreBad24HzTo60Hz = h.getMetrics().judderScoreHistogram.getMax();

        EXPECT_GT(scoreBad24HzTo60Hz, previousScore);

        previousScore = scoreBad24HzTo60Hz;

    }



    // 30fps displayed at 50hz

    {

        Helper h(33.33, c);

        h.render({40.0, 40.0, 40.0, 60.0});

        h.render({40.0, 40.0, 40.0, 60.0});

        h.render({40.0, 40.0, 40.0, 60.0});

        h.render({40.0, 40.0, 40.0, 60.0});

        h.render({40.0, 40.0, 40.0, 60.0});

        int64_t score30fpsTo50hz = h.getMetrics().judderScoreHistogram.getMax();

        EXPECT_GT(score30fpsTo50hz, previousScore);

        previousScore = score30fpsTo50hz;

    }



    // 24fps displayed at 50Hz

    {

        Helper h(41.66, c);

        h.render(40.0, 11);

        h.render(60.0, 1);

        h.render(40.0, 11);

        h.render(60.0, 1);

        h.render(40.0, 11);

        int64_t score24HzTo50Hz = h.getMetrics().judderScoreHistogram.getMax();

        EXPECT_GT(score24HzTo50Hz, previousScore);

        previousScore = score24HzTo50Hz;

    }

}



} // android