Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 1d72cc7f authored by Brian Lindahl's avatar Brian Lindahl Committed by Automerger Merge Worker
Browse files

Merge "Add recording of detailed information for freezes and judders" into udc-dev am: be950265

parents 5b0f43dc be950265
Loading
Loading
Loading
Loading
+97 −5
Original line number Diff line number Diff line
@@ -19,12 +19,12 @@
#define LOG_TAG "MediaCodec"
#include <utils/Log.h>

#include <set>
#include <random>
#include <stdlib.h>
#include <dlfcn.h>
#include <inttypes.h>
#include <random>
#include <set>
#include <stdlib.h>
#include <dlfcn.h>
#include <string>

#include <C2Buffer.h>

@@ -90,6 +90,8 @@ using aidl::android::media::BnResourceManagerClient;
using aidl::android::media::IResourceManagerClient;
using aidl::android::media::IResourceManagerService;
using aidl::android::media::ClientInfoParcel;
using FreezeEvent = VideoRenderQualityTracker::FreezeEvent;
using JudderEvent = VideoRenderQualityTracker::JudderEvent;

// key for media statistics
static const char *kCodecKeyName = "codec";
@@ -212,6 +214,7 @@ static const char *kCodecFramesSkipped = "android.media.mediacodec.frames-skippe
static const char *kCodecFramerateContent = "android.media.mediacodec.framerate-content";
static const char *kCodecFramerateDesired = "android.media.mediacodec.framerate-desired";
static const char *kCodecFramerateActual = "android.media.mediacodec.framerate-actual";
// Freeze
static const char *kCodecFreezeCount = "android.media.mediacodec.freeze-count";
static const char *kCodecFreezeScore = "android.media.mediacodec.freeze-score";
static const char *kCodecFreezeRate = "android.media.mediacodec.freeze-rate";
@@ -226,6 +229,7 @@ static const char *kCodecFreezeDistanceMsHistogram =
        "android.media.mediacodec.freeze-distance-ms-histogram";
static const char *kCodecFreezeDistanceMsHistogramBuckets =
        "android.media.mediacodec.freeze-distance-ms-histogram-buckets";
// Judder
static const char *kCodecJudderCount = "android.media.mediacodec.judder-count";
static const char *kCodecJudderScore = "android.media.mediacodec.judder-score";
static const char *kCodecJudderRate = "android.media.mediacodec.judder-rate";
@@ -234,6 +238,32 @@ static const char *kCodecJudderScoreMax = "android.media.mediacodec.judder-score
static const char *kCodecJudderScoreHistogram = "android.media.mediacodec.judder-score-histogram";
static const char *kCodecJudderScoreHistogramBuckets =
        "android.media.mediacodec.judder-score-histogram-buckets";
// Freeze event
static const char *kCodecFreezeEventCount = "android.media.mediacodec.freeze-event-count";
static const char *kFreezeEventKeyName = "freeze";
static const char *kFreezeEventInitialTimeUs = "android.media.mediacodec.freeze.initial-time-us";
static const char *kFreezeEventDurationMs = "android.media.mediacodec.freeze.duration-ms";
static const char *kFreezeEventCount = "android.media.mediacodec.freeze.count";
static const char *kFreezeEventAvgDurationMs = "android.media.mediacodec.freeze.avg-duration-ms";
static const char *kFreezeEventAvgDistanceMs = "android.media.mediacodec.freeze.avg-distance-ms";
static const char *kFreezeEventDetailsDurationMs =
        "android.media.mediacodec.freeze.detail-duration-ms";
static const char *kFreezeEventDetailsDistanceMs =
        "android.media.mediacodec.freeze.detail-distance-ms";
// Judder event
static const char *kCodecJudderEventCount = "android.media.mediacodec.judder-event-count";
static const char *kJudderEventKeyName = "judder";
static const char *kJudderEventInitialTimeUs = "android.media.mediacodec.judder.initial-time-us";
static const char *kJudderEventDurationMs = "android.media.mediacodec.judder.duration-ms";
static const char *kJudderEventCount = "android.media.mediacodec.judder.count";
static const char *kJudderEventAvgScore = "android.media.mediacodec.judder.avg-score";
static const char *kJudderEventAvgDistanceMs = "android.media.mediacodec.judder.avg-distance-ms";
static const char *kJudderEventDetailsActualDurationUs =
        "android.media.mediacodec.judder.detail-actual-duration-us";
static const char *kJudderEventDetailsContentDurationUs =
        "android.media.mediacodec.judder.detail-content-duration-us";
static const char *kJudderEventDetailsDistanceMs =
        "android.media.mediacodec.judder.detail-distance-ms";

// XXX suppress until we get our representation right
static bool kEmitHistogram = false;
@@ -1169,6 +1199,12 @@ void MediaCodec::updateMediametrics() {
            mediametrics_setString(mMetricsHandle, kCodecJudderScoreHistogramBuckets,
                                   h.emitBuckets());
        }
        if (m.freezeEventCount != 0) {
            mediametrics_setInt32(mMetricsHandle, kCodecFreezeEventCount, m.freezeEventCount);
        }
        if (m.judderEventCount != 0) {
            mediametrics_setInt32(mMetricsHandle, kCodecJudderEventCount, m.judderEventCount);
        }
    }

    if (mLatencyHist.getCount() != 0 ) {
@@ -1407,6 +1443,53 @@ void MediaCodec::updateEphemeralMediametrics(mediametrics_handle_t item) {
    }
}

static std::string emitVector(std::vector<int32_t> vector) {
    std::ostringstream sstr;
    for (int i = 0; i < vector.size(); ++i) {
        if (i != 0) {
            sstr << ',';
        }
        sstr << vector[i];
    }
    return sstr.str();
}

static void reportToMediaMetricsIfValid(const FreezeEvent &e) {
    if (e.valid) {
        mediametrics_handle_t handle = mediametrics_create(kFreezeEventKeyName);
        mediametrics_setInt64(handle, kFreezeEventInitialTimeUs, e.initialTimeUs);
        mediametrics_setInt32(handle, kFreezeEventDurationMs, e.durationMs);
        mediametrics_setInt64(handle, kFreezeEventCount, e.count);
        mediametrics_setInt32(handle, kFreezeEventAvgDurationMs, e.sumDurationMs / e.count);
        mediametrics_setInt32(handle, kFreezeEventAvgDistanceMs, e.sumDistanceMs / e.count);
        mediametrics_setString(handle, kFreezeEventDetailsDurationMs,
                               emitVector(e.details.durationMs));
        mediametrics_setString(handle, kFreezeEventDetailsDistanceMs,
                               emitVector(e.details.distanceMs));
        mediametrics_selfRecord(handle);
        mediametrics_delete(handle);
    }
}

static void reportToMediaMetricsIfValid(const JudderEvent &e) {
    if (e.valid) {
        mediametrics_handle_t handle = mediametrics_create(kJudderEventKeyName);
        mediametrics_setInt64(handle, kJudderEventInitialTimeUs, e.initialTimeUs);
        mediametrics_setInt32(handle, kJudderEventDurationMs, e.durationMs);
        mediametrics_setInt64(handle, kJudderEventCount, e.count);
        mediametrics_setInt32(handle, kJudderEventAvgScore, e.sumScore / e.count);
        mediametrics_setInt32(handle, kJudderEventAvgDistanceMs, e.sumDistanceMs / e.count);
        mediametrics_setString(handle, kJudderEventDetailsActualDurationUs,
                               emitVector(e.details.actualRenderDurationUs));
        mediametrics_setString(handle, kJudderEventDetailsContentDurationUs,
                               emitVector(e.details.contentRenderDurationUs));
        mediametrics_setString(handle, kJudderEventDetailsDistanceMs,
                               emitVector(e.details.distanceMs));
        mediametrics_selfRecord(handle);
        mediametrics_delete(handle);
    }
}

void MediaCodec::flushMediametrics() {
    ALOGD("flushMediametrics");

@@ -1425,6 +1508,10 @@ void MediaCodec::flushMediametrics() {
    }
    // we no longer have anything pending upload
    mMetricsToUpload = false;

    // Freeze and judder events are reported separately
    reportToMediaMetricsIfValid(mVideoRenderQualityTracker.getAndResetFreezeEvent());
    reportToMediaMetricsIfValid(mVideoRenderQualityTracker.getAndResetJudderEvent());
}

void MediaCodec::updateLowLatency(const sp<AMessage> &msg) {
@@ -1538,7 +1625,12 @@ void MediaCodec::processRenderedFrames(const sp<AMessage> &msg) {
            // Tunneled frames use INT64_MAX to indicate end-of-stream, so don't report it as a
            // rendered frame.
            if (!mTunneled || mediaTimeUs != INT64_MAX) {
                mVideoRenderQualityTracker.onFrameRendered(mediaTimeUs, renderTimeNs);
                FreezeEvent freezeEvent;
                JudderEvent judderEvent;
                mVideoRenderQualityTracker.onFrameRendered(mediaTimeUs, renderTimeNs, &freezeEvent,
                                                           &judderEvent);
                reportToMediaMetricsIfValid(freezeEvent);
                reportToMediaMetricsIfValid(judderEvent);
            }
        }
    }
+153 −11
Original line number Diff line number Diff line
@@ -42,8 +42,10 @@ void VideoRenderQualityMetrics::clear() {
    contentFrameRate = FRAME_RATE_UNDETERMINED;
    desiredFrameRate = FRAME_RATE_UNDETERMINED;
    actualFrameRate = FRAME_RATE_UNDETERMINED;
    freezeEventCount = 0;
    freezeDurationMsHistogram.clear();
    freezeDistanceMsHistogram.clear();
    judderEventCount = 0;
    judderScoreHistogram.clear();
}

@@ -69,11 +71,17 @@ VideoRenderQualityTracker::Configuration::Configuration() {
    freezeDurationMsHistogramToScore = {1,  1,  1,  1,  1,   1,   1,   1,   1,   1,   1,   1,   1};
    freezeDistanceMsHistogramBuckets = {0, 20, 100, 400, 1000, 2000, 3000, 4000, 8000, 15000, 30000,
                                        60000};
    freezeEventMax = 0; // enabled only when debugging
    freezeEventDetailsMax = 20;
    freezeEventDistanceToleranceMs = 60000; // lump freeze occurrences together when 60s or less

    // Judder configuration
    judderErrorToleranceUs = 2000;
    judderScoreHistogramBuckets = {1, 4, 5, 9, 11, 20, 30, 40, 50, 60, 70, 80};
    judderScoreHistogramToScore = {1, 1, 1, 1,  1,  1,  1,  1,  1,  1,  1,  1};
    judderEventMax = 0; // enabled only when debugging
    judderEventDetailsMax = 20;
    judderEventDistanceToleranceMs = 5000; // lump judder occurrences together when 5s or less
}

VideoRenderQualityTracker::VideoRenderQualityTracker() : mConfiguration(Configuration()) {
@@ -139,7 +147,9 @@ void VideoRenderQualityTracker::onFrameReleased(int64_t contentTimeUs,
    mLastContentTimeUs = contentTimeUs;
}

void VideoRenderQualityTracker::onFrameRendered(int64_t contentTimeUs, int64_t actualRenderTimeNs) {
void VideoRenderQualityTracker::onFrameRendered(int64_t contentTimeUs, int64_t actualRenderTimeNs,
                                                FreezeEvent *freezeEventOut,
                                                JudderEvent *judderEventOut) {
    if (!mConfiguration.enabled) {
        return;
    }
@@ -183,10 +193,23 @@ void VideoRenderQualityTracker::onFrameRendered(int64_t contentTimeUs, int64_t a
                                      nextExpectedFrame.desiredRenderTimeUs);
    }
    processMetricsForRenderedFrame(nextExpectedFrame.contentTimeUs,
                                   nextExpectedFrame.desiredRenderTimeUs, actualRenderTimeUs);
                                   nextExpectedFrame.desiredRenderTimeUs, actualRenderTimeUs,
                                   freezeEventOut, judderEventOut);
    mLastRenderTimeUs = actualRenderTimeUs;
}

VideoRenderQualityTracker::FreezeEvent VideoRenderQualityTracker::getAndResetFreezeEvent() {
    FreezeEvent event = std::move(mFreezeEvent);
    mFreezeEvent.valid = false;
    return event;
}

VideoRenderQualityTracker::JudderEvent VideoRenderQualityTracker::getAndResetJudderEvent() {
    JudderEvent event = std::move(mJudderEvent);
    mJudderEvent.valid = false;
    return event;
}

const VideoRenderQualityMetrics &VideoRenderQualityTracker::getMetrics() {
    if (!mConfiguration.enabled) {
        return mMetrics;
@@ -232,7 +255,10 @@ void VideoRenderQualityTracker::resetForDiscontinuity() {
    mLastContentTimeUs = -1;
    mLastRenderTimeUs = -1;
    mLastFreezeEndTimeUs = -1;
    mLastJudderEndTimeUs = -1;
    mWasPreviousFrameDropped = false;
    mFreezeEvent.valid = false;
    mJudderEvent.valid = false;

    // Don't worry about tracking frame rendering times from now up until playback catches up to the
    // discontinuity. While stuttering or freezing could be found in the next few frames, the impact
@@ -315,7 +341,9 @@ void VideoRenderQualityTracker::processMetricsForDroppedFrame(int64_t contentTim

void VideoRenderQualityTracker::processMetricsForRenderedFrame(int64_t contentTimeUs,
                                                               int64_t desiredRenderTimeUs,
                                                               int64_t actualRenderTimeUs) {
                                                               int64_t actualRenderTimeUs,
                                                               FreezeEvent *freezeEventOut,
                                                               JudderEvent *judderEventOut) {
    // Capture the timestamp at which the first frame was rendered
    if (mMetrics.firstRenderTimeUs == 0) {
        mMetrics.firstRenderTimeUs = actualRenderTimeUs;
@@ -338,30 +366,86 @@ void VideoRenderQualityTracker::processMetricsForRenderedFrame(int64_t contentTi

    // If the previous frame was dropped, there was a freeze if we've already rendered a frame
    if (mWasPreviousFrameDropped && mLastRenderTimeUs != -1) {
        processFreeze(actualRenderTimeUs, mLastRenderTimeUs, mLastFreezeEndTimeUs, mMetrics);
        processFreeze(actualRenderTimeUs, mLastRenderTimeUs, mLastFreezeEndTimeUs, mFreezeEvent,
                      mMetrics, mConfiguration);
        mLastFreezeEndTimeUs = actualRenderTimeUs;
    }
    maybeCaptureFreezeEvent(actualRenderTimeUs, mLastFreezeEndTimeUs, mFreezeEvent, mMetrics,
                            mConfiguration, freezeEventOut);

    // Judder is computed on the prior video frame, not the current video frame
    int64_t judderScore = computePreviousJudderScore(mActualFrameDurationUs,
                                                     mContentFrameDurationUs,
                                                     mConfiguration);
    int64_t judderTimeUs = actualRenderTimeUs - mActualFrameDurationUs[0] -
                           mActualFrameDurationUs[1];
    if (judderScore != 0) {
        mMetrics.judderScoreHistogram.insert(judderScore);
        processJudder(judderScore, judderTimeUs, mLastJudderEndTimeUs, mActualFrameDurationUs,
                      mContentFrameDurationUs, mJudderEvent, mMetrics, mConfiguration);
        mLastJudderEndTimeUs = judderTimeUs + mActualFrameDurationUs[1];
    }
    maybeCaptureJudderEvent(actualRenderTimeUs, mLastJudderEndTimeUs, mJudderEvent, mMetrics,
                            mConfiguration, judderEventOut);

    mWasPreviousFrameDropped = false;
}

void VideoRenderQualityTracker::processFreeze(int64_t actualRenderTimeUs, int64_t lastRenderTimeUs,
                                              int64_t lastFreezeEndTimeUs,
                                              VideoRenderQualityMetrics &m) {
    int64_t freezeDurationMs = (actualRenderTimeUs - lastRenderTimeUs) / 1000;
    m.freezeDurationMsHistogram.insert(freezeDurationMs);
                                              int64_t lastFreezeEndTimeUs, FreezeEvent &e,
                                              VideoRenderQualityMetrics &m,
                                              const Configuration &c) {
    int32_t durationMs = int32_t((actualRenderTimeUs - lastRenderTimeUs) / 1000);
    m.freezeDurationMsHistogram.insert(durationMs);
    int32_t distanceMs = -1;
    if (lastFreezeEndTimeUs != -1) {
        int64_t distanceSinceLastFreezeMs = (lastRenderTimeUs - lastFreezeEndTimeUs) / 1000;
        m.freezeDistanceMsHistogram.insert(distanceSinceLastFreezeMs);
        // The distance to the last freeze is measured from the end of the last freze to the start
        // of this freeze.
        distanceMs = int32_t((lastRenderTimeUs - lastFreezeEndTimeUs) / 1000);
        m.freezeDistanceMsHistogram.insert(distanceMs);
    }
    if (c.freezeEventMax > 0) {
        if (e.valid == false) {
            m.freezeEventCount++;
            e.valid = true;
            e.initialTimeUs = lastRenderTimeUs;
            e.durationMs = 0;
            e.sumDurationMs = 0;
            e.sumDistanceMs = 0;
            e.count = 0;
            e.details.durationMs.clear();
            e.details.distanceMs.clear();
        } else if (distanceMs != -1) {
            e.durationMs += distanceMs;
            e.sumDistanceMs += distanceMs;
        }
        e.durationMs += durationMs;
        e.count++;
        e.sumDurationMs += durationMs;
        if (e.details.durationMs.size() < c.freezeEventDetailsMax) {
            e.details.durationMs.push_back(durationMs);
            e.details.distanceMs.push_back(distanceMs);
        }
    }
}

void VideoRenderQualityTracker::maybeCaptureFreezeEvent(int64_t actualRenderTimeUs,
                                                        int64_t lastFreezeEndTimeUs, FreezeEvent &e,
                                                        const VideoRenderQualityMetrics & m,
                                                        const Configuration &c,
                                                        FreezeEvent *freezeEventOut) {
    if (lastFreezeEndTimeUs == -1 || !e.valid) {
        return;
    }
    // Future freeze occurrences are still pulled into the current freeze event if under tolerance
    int64_t distanceMs = (actualRenderTimeUs - lastFreezeEndTimeUs) / 1000;
    if (distanceMs < c.freezeEventDistanceToleranceMs) {
        return;
    }
    if (freezeEventOut != nullptr && m.freezeEventCount <= c.freezeEventMax) {
        *freezeEventOut = std::move(e);
    }
    // start recording a new freeze event after pushing the current one back to the caller
    e.valid = false;
}

int64_t VideoRenderQualityTracker::computePreviousJudderScore(
@@ -406,6 +490,64 @@ int64_t VideoRenderQualityTracker::computePreviousJudderScore(
    return abs(errorUs) / 1000; // error in millis to keep numbers small
}

void VideoRenderQualityTracker::processJudder(int32_t judderScore, int64_t judderTimeUs,
                                              int64_t lastJudderEndTime,
                                              const FrameDurationUs &actualDurationUs,
                                              const FrameDurationUs &contentDurationUs,
                                              JudderEvent &e, VideoRenderQualityMetrics &m,
                                              const Configuration &c) {
    int32_t distanceMs = -1;
    if (lastJudderEndTime != -1) {
        distanceMs = int32_t((judderTimeUs - lastJudderEndTime) / 1000);
    }
    m.judderScoreHistogram.insert(judderScore);
    if (c.judderEventMax > 0) {
        if (!e.valid) {
            m.judderEventCount++;
            e.valid = true;
            e.initialTimeUs = judderTimeUs;
            e.durationMs = 0;
            e.sumScore = 0;
            e.sumDistanceMs = 0;
            e.count = 0;
            e.details.contentRenderDurationUs.clear();
            e.details.actualRenderDurationUs.clear();
            e.details.distanceMs.clear();
        } else if (distanceMs != -1) {
            e.durationMs += distanceMs;
            e.sumDistanceMs += distanceMs;
        }
        e.durationMs += actualDurationUs[1] / 1000;
        e.count++;
        e.sumScore += judderScore;
        if (e.details.contentRenderDurationUs.size() < c.judderEventDetailsMax) {
            e.details.actualRenderDurationUs.push_back(actualDurationUs[1]);
            e.details.contentRenderDurationUs.push_back(contentDurationUs[1]);
            e.details.distanceMs.push_back(distanceMs);
        }
    }
}

void VideoRenderQualityTracker::maybeCaptureJudderEvent(int64_t actualRenderTimeUs,
                                                        int64_t lastJudderEndTimeUs, JudderEvent &e,
                                                        const VideoRenderQualityMetrics &m,
                                                        const Configuration &c,
                                                        JudderEvent *judderEventOut) {
    if (lastJudderEndTimeUs == -1 || !e.valid) {
        return;
    }
    // Future judder occurrences are still pulled into the current judder event if under tolerance
    int64_t distanceMs = (actualRenderTimeUs - lastJudderEndTimeUs) / 1000;
    if (distanceMs < c.judderEventDistanceToleranceMs) {
        return;
    }
    if (judderEventOut != nullptr && m.judderEventCount <= c.judderEventMax) {
        *judderEventOut = std::move(e);
    }
    // start recording a new judder event after pushing the current one back to the caller
    e.valid = false;
}

void VideoRenderQualityTracker::configureHistograms(VideoRenderQualityMetrics &m,
                                                    const Configuration &c) {
    m.freezeDurationMsHistogram.setup(c.freezeDurationMsHistogramBuckets);
+116 −5

File changed.

Preview size limit exceeded, changes collapsed.

+162 −2

File changed.

Preview size limit exceeded, changes collapsed.

+2 −0
Original line number Diff line number Diff line
@@ -524,6 +524,8 @@ bool MediaMetricsService::isContentValid(const mediametrics::Item *item, bool is
                                     "audiotrack",
                                     // other media
                                     "codec",
                                     "freeze",
                                     "judder",
                                     "extractor",
                                     "mediadrm",
                                     "mediaparser",