Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c2c7a270 authored by Brian Lindahl's avatar Brian Lindahl Committed by Gerrit Code Review
Browse files

Merge "Don't count frame drops after resuming from a pause as a video freeze" into main

parents 28c2a42b 9bd8b2fe

media/libstagefright/VideoRenderQualityTracker.cpp

+46 −12
Original line number Diff line number Diff line
@@ -126,6 +126,7 @@ void VideoRenderQualityMetrics::clear() { 
    contentFrameRate = FRAME_RATE_UNDETERMINED;

    desiredFrameRate = FRAME_RATE_UNDETERMINED;

    actualFrameRate = FRAME_RATE_UNDETERMINED;

    maxContentDroppedAfterPauseMs = 0;

    freezeEventCount = 0;

    freezeDurationMsHistogram.clear();

    freezeDistanceMsHistogram.clear();
@@ -144,6 +145,7 @@ VideoRenderQualityTracker::Configuration 
    getFlag(maxExpectedContentFrameDurationUs, "max_expected_content_frame_duration_us");

    getFlag(frameRateDetectionToleranceUs, "frame_rate_detection_tolerance_us");

    getFlag(liveContentFrameDropToleranceUs, "live_content_frame_drop_tolerance_us");

    getFlag(pauseAudioLatencyUs, "pause_audio_latency_us");

    getFlag(freezeDurationMsHistogramBuckets, "freeze_duration_ms_histogram_buckets");

    getFlag(freezeDurationMsHistogramToScore, "freeze_duration_ms_histogram_to_score");

    getFlag(freezeDistanceMsHistogramBuckets, "freeze_distance_ms_histogram_buckets");
@@ -181,6 +183,9 @@ VideoRenderQualityTracker::Configuration::Configuration() { 
    // because of frame drops for live content, or because the user is seeking.

    liveContentFrameDropToleranceUs = 200 * 1000;



    // After a pause is initiated, audio should likely stop playback within 200ms.

    pauseAudioLatencyUs = 200 * 1000;



    // Freeze configuration

    freezeDurationMsHistogramBuckets = {1, 20, 40, 60, 80, 100, 120, 150, 175, 225, 300, 400, 500};

    freezeDurationMsHistogramToScore = {1,  1,  1,  1,  1,   1,   1,   1,   1,   1,   1,   1,   1};
@@ -397,13 +402,13 @@ void VideoRenderQualityTracker::resetForDiscontinuity() { 
    mLastRenderTimeUs = -1;

    mLastFreezeEndTimeUs = -1;

    mLastJudderEndTimeUs = -1;

    mWasPreviousFrameDropped = false;

    mDroppedContentDurationUs = 0;

    mFreezeEvent.valid = false;

    mJudderEvent.valid = false;



    // Don't worry about tracking frame rendering times from now up until playback catches up to the

    // discontinuity. While stuttering or freezing could be found in the next few frames, the impact

    // to the user is is minimal, so better to just keep things simple and don't bother.

    // Don't worry about tracking frame rendering times from now up until playback catches up to

    // the discontinuity. While stuttering or freezing could be found in the next few frames, the

    // impact to the user is is minimal, so better to just keep things simple and don't bother.

    mNextExpectedRenderedFrameQueue = {};

    mTunnelFrameQueuedContentTimeUs = -1;
@@ -472,7 +477,7 @@ void VideoRenderQualityTracker::processMetricsForSkippedFrame(int64_t contentTim 
    updateFrameDurations(mDesiredFrameDurationUs, -1);

    updateFrameDurations(mActualFrameDurationUs, -1);

    updateFrameRate(mMetrics.contentFrameRate, mContentFrameDurationUs, mConfiguration);

    mWasPreviousFrameDropped = false;

    mDroppedContentDurationUs = 0;

}



void VideoRenderQualityTracker::processMetricsForDroppedFrame(int64_t contentTimeUs,
@@ -483,7 +488,9 @@ void VideoRenderQualityTracker::processMetricsForDroppedFrame(int64_t contentTim 
    updateFrameDurations(mActualFrameDurationUs, -1);

    updateFrameRate(mMetrics.contentFrameRate, mContentFrameDurationUs, mConfiguration);

    updateFrameRate(mMetrics.desiredFrameRate, mDesiredFrameDurationUs, mConfiguration);

    mWasPreviousFrameDropped = true;

    if (mContentFrameDurationUs[0] != -1) {

        mDroppedContentDurationUs += mContentFrameDurationUs[0];

    }

}



void VideoRenderQualityTracker::processMetricsForRenderedFrame(int64_t contentTimeUs,
@@ -491,6 +498,8 @@ void VideoRenderQualityTracker::processMetricsForRenderedFrame(int64_t contentTi 
                                                               int64_t actualRenderTimeUs,

                                                               FreezeEvent *freezeEventOut,

                                                               JudderEvent *judderEventOut) {

    const Configuration& c = mConfiguration;



    // Capture the timestamp at which the first frame was rendered

    if (mMetrics.firstRenderTimeUs == 0) {

        mMetrics.firstRenderTimeUs = actualRenderTimeUs;
@@ -513,12 +522,37 @@ void VideoRenderQualityTracker::processMetricsForRenderedFrame(int64_t contentTi 
    updateFrameRate(mMetrics.desiredFrameRate, mDesiredFrameDurationUs, mConfiguration);

    updateFrameRate(mMetrics.actualFrameRate, mActualFrameDurationUs, mConfiguration);



    // If the previous frame was dropped, there was a freeze if we've already rendered a frame

    if (mWasPreviousFrameDropped && mLastRenderTimeUs != -1) {

    // A freeze occurs if frames were dropped NOT after a discontinuity

    if (mDroppedContentDurationUs != 0 && mLastRenderTimeUs != -1) {

        // When pausing, audio playback may continue for a brief period of time after video

        // pauses while the audio buffers drain. When resuming, a small number of video frames

        // might be dropped to catch up to the audio position. This is acceptable behacvior and

        // should not count as a freeze.

        bool isLikelyCatchingUpAfterPause = false;

        // A pause can be detected if a freeze occurs for a longer period of time than the

        // content duration of the dropped frames. This strategy works because, for freeze

        // events (no video pause), the content duration of the dropped frames will closely track

        // the wall clock time (freeze duration). When pausing, however, the wall clock time

        // (freeze duration) will be longer than the content duration of the dropped frames

        // required to catch up to the audio position.

        const int64_t wallClockDurationUs = actualRenderTimeUs - mLastRenderTimeUs;

        // 200ms is chosen because it is larger than what a hiccup in the display pipeline could

        // likely be, but shorter than the duration for which a user could pause for.

        static const int32_t MAX_PIPELINE_HICCUP_DURATION_US = 200 * 1000;

        if (wallClockDurationUs > mDroppedContentDurationUs + MAX_PIPELINE_HICCUP_DURATION_US) {

            // Capture the amount of content that is dropped after pause, so we can push apps to be

            // better about this behavior.

            if (mDroppedContentDurationUs / 1000 > mMetrics.maxContentDroppedAfterPauseMs) {

                mMetrics.maxContentDroppedAfterPauseMs = int32_t(mDroppedContentDurationUs / 1000);

            }

            isLikelyCatchingUpAfterPause = mDroppedContentDurationUs <= c.pauseAudioLatencyUs;

        }

        if (!isLikelyCatchingUpAfterPause) {

            processFreeze(actualRenderTimeUs, mLastRenderTimeUs, mLastFreezeEndTimeUs, mFreezeEvent,

                        mMetrics, mConfiguration);

            mLastFreezeEndTimeUs = actualRenderTimeUs;

        }

    }

    maybeCaptureFreezeEvent(actualRenderTimeUs, mLastFreezeEndTimeUs, mFreezeEvent, mMetrics,

                            mConfiguration, freezeEventOut);
@@ -536,7 +570,7 @@ void VideoRenderQualityTracker::processMetricsForRenderedFrame(int64_t contentTi 
    maybeCaptureJudderEvent(actualRenderTimeUs, mLastJudderEndTimeUs, mJudderEvent, mMetrics,

                            mConfiguration, judderEventOut);



    mWasPreviousFrameDropped = false;

    mDroppedContentDurationUs = 0;

}



void VideoRenderQualityTracker::processFreeze(int64_t actualRenderTimeUs, int64_t lastRenderTimeUs,

media/libstagefright/include/media/stagefright/VideoRenderQualityTracker.h

+12 −2
Original line number Diff line number Diff line
@@ -63,6 +63,11 @@ struct VideoRenderQualityMetrics { 
    // post-render.

    float actualFrameRate;



    // The amount of content duration skipped by the app after a pause when video was trying to

    // resume. This sometimes happen when catching up to the audio position which continued playing

    // after video pauses.

    int32_t maxContentDroppedAfterPauseMs;



    // A histogram of the durations of freezes due to dropped/skipped frames.

    MediaHistogram<int32_t> freezeDurationMsHistogram;

    // The computed overall freeze score using the above histogram and score conversion table. The
@@ -155,6 +160,11 @@ public: 
        // seeking forward.

        int32_t liveContentFrameDropToleranceUs;



        // The amount of time it takes for audio to stop playback after a pause is initiated. Used

        // for providing some allowance of dropped video frames to catch back up to the audio

        // position when resuming playback.

        int32_t pauseAudioLatencyUs;



        // Freeze configuration

        //

        // The values used to distribute freeze durations across a histogram.
@@ -441,8 +451,8 @@ private: 
    // The render duration of the playback.

    int64_t mRenderDurationMs;



    // True if the previous frame was dropped.

    bool mWasPreviousFrameDropped;

    // The duration of the content that was dropped.

    int64_t mDroppedContentDurationUs;



    // The freeze event that's currently being tracked.

    FreezeEvent mFreezeEvent;

media/libstagefright/tests/VideoRenderQualityTracker_test.cpp

+59 −0
Original line number Diff line number Diff line
@@ -140,6 +140,7 @@ TEST_F(VideoRenderQualityTrackerTest, getFromServerConfigurableFlags_withDefault 
    EXPECT_EQ(c.maxExpectedContentFrameDurationUs, d.maxExpectedContentFrameDurationUs);

    EXPECT_EQ(c.frameRateDetectionToleranceUs, d.frameRateDetectionToleranceUs);

    EXPECT_EQ(c.liveContentFrameDropToleranceUs, d.liveContentFrameDropToleranceUs);

    EXPECT_EQ(c.pauseAudioLatencyUs, d.pauseAudioLatencyUs);

    EXPECT_EQ(c.freezeDurationMsHistogramBuckets, d.freezeDurationMsHistogramBuckets);

    EXPECT_EQ(c.freezeDurationMsHistogramToScore, d.freezeDurationMsHistogramToScore);

    EXPECT_EQ(c.freezeDistanceMsHistogramBuckets, d.freezeDistanceMsHistogramBuckets);
@@ -171,6 +172,7 @@ TEST_F(VideoRenderQualityTrackerTest, getFromServerConfigurableFlags_withEmpty) 
    EXPECT_EQ(c.maxExpectedContentFrameDurationUs, d.maxExpectedContentFrameDurationUs);

    EXPECT_EQ(c.frameRateDetectionToleranceUs, d.frameRateDetectionToleranceUs);

    EXPECT_EQ(c.liveContentFrameDropToleranceUs, d.liveContentFrameDropToleranceUs);

    EXPECT_EQ(c.pauseAudioLatencyUs, d.pauseAudioLatencyUs);

    EXPECT_EQ(c.freezeDurationMsHistogramBuckets, d.freezeDurationMsHistogramBuckets);

    EXPECT_EQ(c.freezeDurationMsHistogramToScore, d.freezeDurationMsHistogramToScore);

    EXPECT_EQ(c.freezeDistanceMsHistogramBuckets, d.freezeDistanceMsHistogramBuckets);
@@ -202,6 +204,7 @@ TEST_F(VideoRenderQualityTrackerTest, getFromServerConfigurableFlags_withInvalid 
    EXPECT_EQ(c.maxExpectedContentFrameDurationUs, d.maxExpectedContentFrameDurationUs);

    EXPECT_EQ(c.frameRateDetectionToleranceUs, d.frameRateDetectionToleranceUs);

    EXPECT_EQ(c.liveContentFrameDropToleranceUs, d.liveContentFrameDropToleranceUs);

    EXPECT_EQ(c.pauseAudioLatencyUs, d.pauseAudioLatencyUs);

    EXPECT_EQ(c.freezeDurationMsHistogramBuckets, d.freezeDurationMsHistogramBuckets);

    EXPECT_EQ(c.freezeDurationMsHistogramToScore, d.freezeDurationMsHistogramToScore);

    EXPECT_EQ(c.freezeDistanceMsHistogramBuckets, d.freezeDistanceMsHistogramBuckets);
@@ -233,6 +236,8 @@ TEST_F(VideoRenderQualityTrackerTest, getFromServerConfigurableFlags_withAlmostV 
                return "10b0";

            } else if (flag == "render_metrics_live_content_frame_drop_tolerance_us") {

                return "c100";

            } else if (flag == "render_metrics_pause_audio_latency_us") {

                return "1ab0";

            } else if (flag == "render_metrics_freeze_duration_ms_histogram_buckets") {

                return "1,5300,3b400,123";

            } else if (flag == "render_metrics_freeze_duration_ms_histogram_to_score") {
@@ -276,6 +281,7 @@ TEST_F(VideoRenderQualityTrackerTest, getFromServerConfigurableFlags_withAlmostV 
    EXPECT_EQ(c.maxExpectedContentFrameDurationUs, d.maxExpectedContentFrameDurationUs);

    EXPECT_EQ(c.frameRateDetectionToleranceUs, d.frameRateDetectionToleranceUs);

    EXPECT_EQ(c.liveContentFrameDropToleranceUs, d.liveContentFrameDropToleranceUs);

    EXPECT_EQ(c.pauseAudioLatencyUs, d.pauseAudioLatencyUs);

    EXPECT_EQ(c.freezeDurationMsHistogramBuckets, d.freezeDurationMsHistogramBuckets);

    EXPECT_EQ(c.freezeDurationMsHistogramToScore, d.freezeDurationMsHistogramToScore);

    EXPECT_EQ(c.freezeDistanceMsHistogramBuckets, d.freezeDistanceMsHistogramBuckets);
@@ -307,6 +313,8 @@ TEST_F(VideoRenderQualityTrackerTest, getFromServerConfigurableFlags_withValid) 
                return "3000";

            } else if (flag == "render_metrics_live_content_frame_drop_tolerance_us") {

                return "4000";

            } else if (flag == "render_metrics_pause_audio_latency_us") {

                return "300000";

            } else if (flag == "render_metrics_freeze_duration_ms_histogram_buckets") {

                return "100,200,300,400";

            } else if (flag == "render_metrics_freeze_duration_ms_histogram_to_score") {
@@ -359,6 +367,8 @@ TEST_F(VideoRenderQualityTrackerTest, getFromServerConfigurableFlags_withValid) 
    EXPECT_NE(c.frameRateDetectionToleranceUs, d.frameRateDetectionToleranceUs);

    EXPECT_EQ(c.liveContentFrameDropToleranceUs, 4000);

    EXPECT_NE(c.liveContentFrameDropToleranceUs, d.liveContentFrameDropToleranceUs);

    EXPECT_EQ(c.pauseAudioLatencyUs, 300000);

    EXPECT_NE(c.pauseAudioLatencyUs, d.pauseAudioLatencyUs);

    {

        std::vector<int32_t> expected({100,200,300,400});

        EXPECT_EQ(c.freezeDurationMsHistogramBuckets, expected);
@@ -1181,4 +1191,53 @@ TEST_F(VideoRenderQualityTrackerTest, 


    EXPECT_EQ(h.getTraceTriggeredCount(), 0);

}



TEST_F(VideoRenderQualityTrackerTest, doesNotCountCatchUpAfterPauseAsFreeze) {

    Configuration c;

    c.enabled = true;

    c.pauseAudioLatencyUs = 200 * 1000; // allows for up to 10 frames to be dropped to catch up

                                        // to the audio position

    Helper h(20, c);

    // A few frames followed by a long pause

    h.render({20, 20, 1000});

    h.drop(10); // simulate catching up to audio

    h.render({20, 20, 1000});

    h.drop(11); // simulate catching up to audio but then also dropping frames

    h.render({20});



    // Only 1 freeze is counted because the first freeze (200ms) because it's equal to or below the

    // pause latency allowance, and the algorithm assumes a legitimate case of the video trying to

    // catch up to the audio position, which continued to play for a short period of time (less than

    // 200ms) after the pause was initiated

    EXPECT_EQ(h.getMetrics().freezeDurationMsHistogram.getCount(), 1);

}



TEST_F(VideoRenderQualityTrackerTest, capturesMaximumContentDroppedAfterPause) {

    Configuration c;

    c.enabled = true;

    c.pauseAudioLatencyUs = 200 * 1000; // allows for up to 10 frames to be dropped to catch up

                                        // to the audio position

    Helper h(20, c);



    // Freezes are below the pause latency are captured

    h.render({20, 20, 1000});

    h.drop(6);

    h.render({20, 20, 1000});

    h.drop(8);

    h.render({20, 20, 1000});

    h.drop(7);

    h.render({20});

    EXPECT_EQ(h.getMetrics().maxContentDroppedAfterPauseMs, 8 * 20);



    // Freezes are above the pause latency are also captured

    h.render({20, 20, 1000});

    h.drop(10);

    h.render({20, 20, 1000});

    h.drop(12);

    h.render({20, 20, 1000});

    h.drop(11);

    h.render({20});

    EXPECT_EQ(h.getMetrics().maxContentDroppedAfterPauseMs, 12 * 20);

}



} // android