Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit ff3068d6 authored by Brian Lindahl's avatar Brian Lindahl Committed by Automerger Merge Worker
Browse files

Merge "Add recording of detailed information for freezes and judders" into... 

Merge "Add recording of detailed information for freezes and judders" into udc-dev am: be950265 am: b24c91dd am: b763fd5e

Original change: https://googleplex-android-review.googlesource.com/c/platform/frameworks/av/+/23353964



Change-Id: Ia0472ffeb0e1fc15abff2986d3d3e4fd7112cb21
Signed-off-by: default avatarAutomerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
parents b0201743 b763fd5e

media/libstagefright/MediaCodec.cpp

+97 −5
Original line number Diff line number Diff line
@@ -19,12 +19,12 @@ 
#define LOG_TAG "MediaCodec"

#include <utils/Log.h>



#include <set>

#include <random>

#include <stdlib.h>

#include <dlfcn.h>

#include <inttypes.h>

#include <random>

#include <set>

#include <stdlib.h>

#include <dlfcn.h>

#include <string>



#include <C2Buffer.h>
@@ -90,6 +90,8 @@ using aidl::android::media::BnResourceManagerClient; 
using aidl::android::media::IResourceManagerClient;

using aidl::android::media::IResourceManagerService;

using aidl::android::media::ClientInfoParcel;

using FreezeEvent = VideoRenderQualityTracker::FreezeEvent;

using JudderEvent = VideoRenderQualityTracker::JudderEvent;



// key for media statistics

static const char *kCodecKeyName = "codec";
@@ -212,6 +214,7 @@ static const char *kCodecFramesSkipped = "android.media.mediacodec.frames-skippe 
static const char *kCodecFramerateContent = "android.media.mediacodec.framerate-content";

static const char *kCodecFramerateDesired = "android.media.mediacodec.framerate-desired";

static const char *kCodecFramerateActual = "android.media.mediacodec.framerate-actual";

// Freeze

static const char *kCodecFreezeCount = "android.media.mediacodec.freeze-count";

static const char *kCodecFreezeScore = "android.media.mediacodec.freeze-score";

static const char *kCodecFreezeRate = "android.media.mediacodec.freeze-rate";
@@ -226,6 +229,7 @@ static const char *kCodecFreezeDistanceMsHistogram = 
        "android.media.mediacodec.freeze-distance-ms-histogram";

static const char *kCodecFreezeDistanceMsHistogramBuckets =

        "android.media.mediacodec.freeze-distance-ms-histogram-buckets";

// Judder

static const char *kCodecJudderCount = "android.media.mediacodec.judder-count";

static const char *kCodecJudderScore = "android.media.mediacodec.judder-score";

static const char *kCodecJudderRate = "android.media.mediacodec.judder-rate";
@@ -234,6 +238,32 @@ static const char *kCodecJudderScoreMax = "android.media.mediacodec.judder-score 
static const char *kCodecJudderScoreHistogram = "android.media.mediacodec.judder-score-histogram";

static const char *kCodecJudderScoreHistogramBuckets =

        "android.media.mediacodec.judder-score-histogram-buckets";

// Freeze event

static const char *kCodecFreezeEventCount = "android.media.mediacodec.freeze-event-count";

static const char *kFreezeEventKeyName = "freeze";

static const char *kFreezeEventInitialTimeUs = "android.media.mediacodec.freeze.initial-time-us";

static const char *kFreezeEventDurationMs = "android.media.mediacodec.freeze.duration-ms";

static const char *kFreezeEventCount = "android.media.mediacodec.freeze.count";

static const char *kFreezeEventAvgDurationMs = "android.media.mediacodec.freeze.avg-duration-ms";

static const char *kFreezeEventAvgDistanceMs = "android.media.mediacodec.freeze.avg-distance-ms";

static const char *kFreezeEventDetailsDurationMs =

        "android.media.mediacodec.freeze.detail-duration-ms";

static const char *kFreezeEventDetailsDistanceMs =

        "android.media.mediacodec.freeze.detail-distance-ms";

// Judder event

static const char *kCodecJudderEventCount = "android.media.mediacodec.judder-event-count";

static const char *kJudderEventKeyName = "judder";

static const char *kJudderEventInitialTimeUs = "android.media.mediacodec.judder.initial-time-us";

static const char *kJudderEventDurationMs = "android.media.mediacodec.judder.duration-ms";

static const char *kJudderEventCount = "android.media.mediacodec.judder.count";

static const char *kJudderEventAvgScore = "android.media.mediacodec.judder.avg-score";

static const char *kJudderEventAvgDistanceMs = "android.media.mediacodec.judder.avg-distance-ms";

static const char *kJudderEventDetailsActualDurationUs =

        "android.media.mediacodec.judder.detail-actual-duration-us";

static const char *kJudderEventDetailsContentDurationUs =

        "android.media.mediacodec.judder.detail-content-duration-us";

static const char *kJudderEventDetailsDistanceMs =

        "android.media.mediacodec.judder.detail-distance-ms";



// XXX suppress until we get our representation right

static bool kEmitHistogram = false;
@@ -1169,6 +1199,12 @@ void MediaCodec::updateMediametrics() { 
            mediametrics_setString(mMetricsHandle, kCodecJudderScoreHistogramBuckets,

                                   h.emitBuckets());

        }

        if (m.freezeEventCount != 0) {

            mediametrics_setInt32(mMetricsHandle, kCodecFreezeEventCount, m.freezeEventCount);

        }

        if (m.judderEventCount != 0) {

            mediametrics_setInt32(mMetricsHandle, kCodecJudderEventCount, m.judderEventCount);

        }

    }



    if (mLatencyHist.getCount() != 0 ) {
@@ -1407,6 +1443,53 @@ void MediaCodec::updateEphemeralMediametrics(mediametrics_handle_t item) { 
    }

}



static std::string emitVector(std::vector<int32_t> vector) {

    std::ostringstream sstr;

    for (int i = 0; i < vector.size(); ++i) {

        if (i != 0) {

            sstr << ',';

        }

        sstr << vector[i];

    }

    return sstr.str();

}



static void reportToMediaMetricsIfValid(const FreezeEvent &e) {

    if (e.valid) {

        mediametrics_handle_t handle = mediametrics_create(kFreezeEventKeyName);

        mediametrics_setInt64(handle, kFreezeEventInitialTimeUs, e.initialTimeUs);

        mediametrics_setInt32(handle, kFreezeEventDurationMs, e.durationMs);

        mediametrics_setInt64(handle, kFreezeEventCount, e.count);

        mediametrics_setInt32(handle, kFreezeEventAvgDurationMs, e.sumDurationMs / e.count);

        mediametrics_setInt32(handle, kFreezeEventAvgDistanceMs, e.sumDistanceMs / e.count);

        mediametrics_setString(handle, kFreezeEventDetailsDurationMs,

                               emitVector(e.details.durationMs));

        mediametrics_setString(handle, kFreezeEventDetailsDistanceMs,

                               emitVector(e.details.distanceMs));

        mediametrics_selfRecord(handle);

        mediametrics_delete(handle);

    }

}



static void reportToMediaMetricsIfValid(const JudderEvent &e) {

    if (e.valid) {

        mediametrics_handle_t handle = mediametrics_create(kJudderEventKeyName);

        mediametrics_setInt64(handle, kJudderEventInitialTimeUs, e.initialTimeUs);

        mediametrics_setInt32(handle, kJudderEventDurationMs, e.durationMs);

        mediametrics_setInt64(handle, kJudderEventCount, e.count);

        mediametrics_setInt32(handle, kJudderEventAvgScore, e.sumScore / e.count);

        mediametrics_setInt32(handle, kJudderEventAvgDistanceMs, e.sumDistanceMs / e.count);

        mediametrics_setString(handle, kJudderEventDetailsActualDurationUs,

                               emitVector(e.details.actualRenderDurationUs));

        mediametrics_setString(handle, kJudderEventDetailsContentDurationUs,

                               emitVector(e.details.contentRenderDurationUs));

        mediametrics_setString(handle, kJudderEventDetailsDistanceMs,

                               emitVector(e.details.distanceMs));

        mediametrics_selfRecord(handle);

        mediametrics_delete(handle);

    }

}



void MediaCodec::flushMediametrics() {

    ALOGD("flushMediametrics");
@@ -1425,6 +1508,10 @@ void MediaCodec::flushMediametrics() { 
    }

    // we no longer have anything pending upload

    mMetricsToUpload = false;



    // Freeze and judder events are reported separately

    reportToMediaMetricsIfValid(mVideoRenderQualityTracker.getAndResetFreezeEvent());

    reportToMediaMetricsIfValid(mVideoRenderQualityTracker.getAndResetJudderEvent());

}



void MediaCodec::updateLowLatency(const sp<AMessage> &msg) {
@@ -1538,7 +1625,12 @@ void MediaCodec::processRenderedFrames(const sp<AMessage> &msg) { 
            // Tunneled frames use INT64_MAX to indicate end-of-stream, so don't report it as a

            // rendered frame.

            if (!mTunneled || mediaTimeUs != INT64_MAX) {

                mVideoRenderQualityTracker.onFrameRendered(mediaTimeUs, renderTimeNs);

                FreezeEvent freezeEvent;

                JudderEvent judderEvent;

                mVideoRenderQualityTracker.onFrameRendered(mediaTimeUs, renderTimeNs, &freezeEvent,

                                                           &judderEvent);

                reportToMediaMetricsIfValid(freezeEvent);

                reportToMediaMetricsIfValid(judderEvent);

            }

        }

    }

media/libstagefright/VideoRenderQualityTracker.cpp

+153 −11
Original line number Diff line number Diff line
@@ -42,8 +42,10 @@ void VideoRenderQualityMetrics::clear() { 
    contentFrameRate = FRAME_RATE_UNDETERMINED;

    desiredFrameRate = FRAME_RATE_UNDETERMINED;

    actualFrameRate = FRAME_RATE_UNDETERMINED;

    freezeEventCount = 0;

    freezeDurationMsHistogram.clear();

    freezeDistanceMsHistogram.clear();

    judderEventCount = 0;

    judderScoreHistogram.clear();

}
@@ -69,11 +71,17 @@ VideoRenderQualityTracker::Configuration::Configuration() { 
    freezeDurationMsHistogramToScore = {1,  1,  1,  1,  1,   1,   1,   1,   1,   1,   1,   1,   1};

    freezeDistanceMsHistogramBuckets = {0, 20, 100, 400, 1000, 2000, 3000, 4000, 8000, 15000, 30000,

                                        60000};

    freezeEventMax = 0; // enabled only when debugging

    freezeEventDetailsMax = 20;

    freezeEventDistanceToleranceMs = 60000; // lump freeze occurrences together when 60s or less



    // Judder configuration

    judderErrorToleranceUs = 2000;

    judderScoreHistogramBuckets = {1, 4, 5, 9, 11, 20, 30, 40, 50, 60, 70, 80};

    judderScoreHistogramToScore = {1, 1, 1, 1,  1,  1,  1,  1,  1,  1,  1,  1};

    judderEventMax = 0; // enabled only when debugging

    judderEventDetailsMax = 20;

    judderEventDistanceToleranceMs = 5000; // lump judder occurrences together when 5s or less

}



VideoRenderQualityTracker::VideoRenderQualityTracker() : mConfiguration(Configuration()) {
@@ -139,7 +147,9 @@ void VideoRenderQualityTracker::onFrameReleased(int64_t contentTimeUs, 
    mLastContentTimeUs = contentTimeUs;

}



void VideoRenderQualityTracker::onFrameRendered(int64_t contentTimeUs, int64_t actualRenderTimeNs) {

void VideoRenderQualityTracker::onFrameRendered(int64_t contentTimeUs, int64_t actualRenderTimeNs,

                                                FreezeEvent *freezeEventOut,

                                                JudderEvent *judderEventOut) {

    if (!mConfiguration.enabled) {

        return;

    }
@@ -183,10 +193,23 @@ void VideoRenderQualityTracker::onFrameRendered(int64_t contentTimeUs, int64_t a 
                                      nextExpectedFrame.desiredRenderTimeUs);

    }

    processMetricsForRenderedFrame(nextExpectedFrame.contentTimeUs,

                                   nextExpectedFrame.desiredRenderTimeUs, actualRenderTimeUs);

                                   nextExpectedFrame.desiredRenderTimeUs, actualRenderTimeUs,

                                   freezeEventOut, judderEventOut);

    mLastRenderTimeUs = actualRenderTimeUs;

}



VideoRenderQualityTracker::FreezeEvent VideoRenderQualityTracker::getAndResetFreezeEvent() {

    FreezeEvent event = std::move(mFreezeEvent);

    mFreezeEvent.valid = false;

    return event;

}



VideoRenderQualityTracker::JudderEvent VideoRenderQualityTracker::getAndResetJudderEvent() {

    JudderEvent event = std::move(mJudderEvent);

    mJudderEvent.valid = false;

    return event;

}



const VideoRenderQualityMetrics &VideoRenderQualityTracker::getMetrics() {

    if (!mConfiguration.enabled) {

        return mMetrics;
@@ -232,7 +255,10 @@ void VideoRenderQualityTracker::resetForDiscontinuity() { 
    mLastContentTimeUs = -1;

    mLastRenderTimeUs = -1;

    mLastFreezeEndTimeUs = -1;

    mLastJudderEndTimeUs = -1;

    mWasPreviousFrameDropped = false;

    mFreezeEvent.valid = false;

    mJudderEvent.valid = false;



    // Don't worry about tracking frame rendering times from now up until playback catches up to the

    // discontinuity. While stuttering or freezing could be found in the next few frames, the impact
@@ -315,7 +341,9 @@ void VideoRenderQualityTracker::processMetricsForDroppedFrame(int64_t contentTim 


void VideoRenderQualityTracker::processMetricsForRenderedFrame(int64_t contentTimeUs,

                                                               int64_t desiredRenderTimeUs,

                                                               int64_t actualRenderTimeUs) {

                                                               int64_t actualRenderTimeUs,

                                                               FreezeEvent *freezeEventOut,

                                                               JudderEvent *judderEventOut) {

    // Capture the timestamp at which the first frame was rendered

    if (mMetrics.firstRenderTimeUs == 0) {

        mMetrics.firstRenderTimeUs = actualRenderTimeUs;
@@ -338,30 +366,86 @@ void VideoRenderQualityTracker::processMetricsForRenderedFrame(int64_t contentTi 


    // If the previous frame was dropped, there was a freeze if we've already rendered a frame

    if (mWasPreviousFrameDropped && mLastRenderTimeUs != -1) {

        processFreeze(actualRenderTimeUs, mLastRenderTimeUs, mLastFreezeEndTimeUs, mMetrics);

        processFreeze(actualRenderTimeUs, mLastRenderTimeUs, mLastFreezeEndTimeUs, mFreezeEvent,

                      mMetrics, mConfiguration);

        mLastFreezeEndTimeUs = actualRenderTimeUs;

    }

    maybeCaptureFreezeEvent(actualRenderTimeUs, mLastFreezeEndTimeUs, mFreezeEvent, mMetrics,

                            mConfiguration, freezeEventOut);



    // Judder is computed on the prior video frame, not the current video frame

    int64_t judderScore = computePreviousJudderScore(mActualFrameDurationUs,

                                                     mContentFrameDurationUs,

                                                     mConfiguration);

    int64_t judderTimeUs = actualRenderTimeUs - mActualFrameDurationUs[0] -

                           mActualFrameDurationUs[1];

    if (judderScore != 0) {

        mMetrics.judderScoreHistogram.insert(judderScore);

        processJudder(judderScore, judderTimeUs, mLastJudderEndTimeUs, mActualFrameDurationUs,

                      mContentFrameDurationUs, mJudderEvent, mMetrics, mConfiguration);

        mLastJudderEndTimeUs = judderTimeUs + mActualFrameDurationUs[1];

    }

    maybeCaptureJudderEvent(actualRenderTimeUs, mLastJudderEndTimeUs, mJudderEvent, mMetrics,

                            mConfiguration, judderEventOut);



    mWasPreviousFrameDropped = false;

}



void VideoRenderQualityTracker::processFreeze(int64_t actualRenderTimeUs, int64_t lastRenderTimeUs,

                                              int64_t lastFreezeEndTimeUs,

                                              VideoRenderQualityMetrics &m) {

    int64_t freezeDurationMs = (actualRenderTimeUs - lastRenderTimeUs) / 1000;

    m.freezeDurationMsHistogram.insert(freezeDurationMs);

                                              int64_t lastFreezeEndTimeUs, FreezeEvent &e,

                                              VideoRenderQualityMetrics &m,

                                              const Configuration &c) {

    int32_t durationMs = int32_t((actualRenderTimeUs - lastRenderTimeUs) / 1000);

    m.freezeDurationMsHistogram.insert(durationMs);

    int32_t distanceMs = -1;

    if (lastFreezeEndTimeUs != -1) {

        int64_t distanceSinceLastFreezeMs = (lastRenderTimeUs - lastFreezeEndTimeUs) / 1000;

        m.freezeDistanceMsHistogram.insert(distanceSinceLastFreezeMs);

        // The distance to the last freeze is measured from the end of the last freze to the start

        // of this freeze.

        distanceMs = int32_t((lastRenderTimeUs - lastFreezeEndTimeUs) / 1000);

        m.freezeDistanceMsHistogram.insert(distanceMs);

    }

    if (c.freezeEventMax > 0) {

        if (e.valid == false) {

            m.freezeEventCount++;

            e.valid = true;

            e.initialTimeUs = lastRenderTimeUs;

            e.durationMs = 0;

            e.sumDurationMs = 0;

            e.sumDistanceMs = 0;

            e.count = 0;

            e.details.durationMs.clear();

            e.details.distanceMs.clear();

        } else if (distanceMs != -1) {

            e.durationMs += distanceMs;

            e.sumDistanceMs += distanceMs;

        }

        e.durationMs += durationMs;

        e.count++;

        e.sumDurationMs += durationMs;

        if (e.details.durationMs.size() < c.freezeEventDetailsMax) {

            e.details.durationMs.push_back(durationMs);

            e.details.distanceMs.push_back(distanceMs);

        }

    }

}



void VideoRenderQualityTracker::maybeCaptureFreezeEvent(int64_t actualRenderTimeUs,

                                                        int64_t lastFreezeEndTimeUs, FreezeEvent &e,

                                                        const VideoRenderQualityMetrics & m,

                                                        const Configuration &c,

                                                        FreezeEvent *freezeEventOut) {

    if (lastFreezeEndTimeUs == -1 || !e.valid) {

        return;

    }

    // Future freeze occurrences are still pulled into the current freeze event if under tolerance

    int64_t distanceMs = (actualRenderTimeUs - lastFreezeEndTimeUs) / 1000;

    if (distanceMs < c.freezeEventDistanceToleranceMs) {

        return;

    }

    if (freezeEventOut != nullptr && m.freezeEventCount <= c.freezeEventMax) {

        *freezeEventOut = std::move(e);

    }

    // start recording a new freeze event after pushing the current one back to the caller

    e.valid = false;

}



int64_t VideoRenderQualityTracker::computePreviousJudderScore(
@@ -406,6 +490,64 @@ int64_t VideoRenderQualityTracker::computePreviousJudderScore( 
    return abs(errorUs) / 1000; // error in millis to keep numbers small

}



void VideoRenderQualityTracker::processJudder(int32_t judderScore, int64_t judderTimeUs,

                                              int64_t lastJudderEndTime,

                                              const FrameDurationUs &actualDurationUs,

                                              const FrameDurationUs &contentDurationUs,

                                              JudderEvent &e, VideoRenderQualityMetrics &m,

                                              const Configuration &c) {

    int32_t distanceMs = -1;

    if (lastJudderEndTime != -1) {

        distanceMs = int32_t((judderTimeUs - lastJudderEndTime) / 1000);

    }

    m.judderScoreHistogram.insert(judderScore);

    if (c.judderEventMax > 0) {

        if (!e.valid) {

            m.judderEventCount++;

            e.valid = true;

            e.initialTimeUs = judderTimeUs;

            e.durationMs = 0;

            e.sumScore = 0;

            e.sumDistanceMs = 0;

            e.count = 0;

            e.details.contentRenderDurationUs.clear();

            e.details.actualRenderDurationUs.clear();

            e.details.distanceMs.clear();

        } else if (distanceMs != -1) {

            e.durationMs += distanceMs;

            e.sumDistanceMs += distanceMs;

        }

        e.durationMs += actualDurationUs[1] / 1000;

        e.count++;

        e.sumScore += judderScore;

        if (e.details.contentRenderDurationUs.size() < c.judderEventDetailsMax) {

            e.details.actualRenderDurationUs.push_back(actualDurationUs[1]);

            e.details.contentRenderDurationUs.push_back(contentDurationUs[1]);

            e.details.distanceMs.push_back(distanceMs);

        }

    }

}



void VideoRenderQualityTracker::maybeCaptureJudderEvent(int64_t actualRenderTimeUs,

                                                        int64_t lastJudderEndTimeUs, JudderEvent &e,

                                                        const VideoRenderQualityMetrics &m,

                                                        const Configuration &c,

                                                        JudderEvent *judderEventOut) {

    if (lastJudderEndTimeUs == -1 || !e.valid) {

        return;

    }

    // Future judder occurrences are still pulled into the current judder event if under tolerance

    int64_t distanceMs = (actualRenderTimeUs - lastJudderEndTimeUs) / 1000;

    if (distanceMs < c.judderEventDistanceToleranceMs) {

        return;

    }

    if (judderEventOut != nullptr && m.judderEventCount <= c.judderEventMax) {

        *judderEventOut = std::move(e);

    }

    // start recording a new judder event after pushing the current one back to the caller

    e.valid = false;

}



void VideoRenderQualityTracker::configureHistograms(VideoRenderQualityMetrics &m,

                                                    const Configuration &c) {

    m.freezeDurationMsHistogram.setup(c.freezeDurationMsHistogramBuckets);

media/libstagefright/include/media/stagefright/VideoRenderQualityTracker.h

+116 −5

File changed.

Preview size limit exceeded, changes collapsed.

media/libstagefright/tests/VideoRenderQualityTracker_test.cpp

+162 −2

File changed.

Preview size limit exceeded, changes collapsed.

services/mediametrics/MediaMetricsService.cpp

+2 −0
Original line number Diff line number Diff line
@@ -524,6 +524,8 @@ bool MediaMetricsService::isContentValid(const mediametrics::Item *item, bool is 
                                     "audiotrack",

                                     // other media

                                     "codec",

                                     "freeze",

                                     "judder",

                                     "extractor",

                                     "mediadrm",

                                     "mediaparser",