Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 7d1066db authored by Brian Lindahl's avatar Brian Lindahl Committed by Android (Google) Code Review
Browse files

Merge changes I5e3a8b02,Ie3995fb9,Ie8b4d2c8 into udc-dev 

* changes:
  Add histogram metrics for video playback freezes
  Move Histogram out of MediaCodec so it can be used for render quality metrics
  Capture basic metrics for video frame rendering
parents 4cf4c1da c4e9deef

media/libmediametrics/MediaMetrics.cpp

+13 −0
Original line number Diff line number Diff line
@@ -86,6 +86,11 @@ void mediametrics_setRate(mediametrics_handle_t handle, attr_t attr, 
    if (item != NULL) item->setRate(attr, count, duration);

}



void mediametrics_setString(mediametrics_handle_t handle, attr_t attr,

                                 const std::string &string) {

    mediametrics_setCString(handle, attr, string.c_str());

}



void mediametrics_setCString(mediametrics_handle_t handle, attr_t attr,

                                 const char *value) {

    Item *item = (Item *) handle;
@@ -152,6 +157,14 @@ bool mediametrics_getRate(mediametrics_handle_t handle, attr_t attr, 
    return item->getRate(attr, count, duration, rate);

}



bool mediametrics_getString(mediametrics_handle_t handle, attr_t attr,

                                 std::string *string) {

    Item *item = (Item *) handle;

    if (item == NULL) return false;



    return item->getString(attr, string);

}



// NB: caller owns the string that comes back, is responsible for freeing it

bool mediametrics_getCString(mediametrics_handle_t handle, attr_t attr,

                                 char **value) {

media/libmediametrics/include/media/MediaMetrics.h

+8 −1
Original line number Diff line number Diff line
@@ -95,4 +95,11 @@ bool mediametrics_getAttributes(mediametrics_handle_t handle, char **buffer, siz 


__END_DECLS



#ifdef __cplusplus

#include <string>

void mediametrics_setString(mediametrics_handle_t handle, attr_t attr,

                            const std::string &value);

bool mediametrics_getString(mediametrics_handle_t handle, attr_t attr, std::string *value);

#endif // __cplusplus



#endif

media/libmediametrics/include/media/MediaMetricsItem.h

+3 −3
Original line number Diff line number Diff line
@@ -1048,6 +1048,9 @@ public: 
        }

        return true;

    }

    bool getString(const char *key, std::string *value) const {

        return get(key, value);

    }

    // Caller owns the returned string

    bool getCString(const char *key, char **value) const {

        std::string s;
@@ -1057,9 +1060,6 @@ public: 
        }

        return false;

    }

    bool getString(const char *key, std::string *value) const {

        return get(key, value);

    }



    const Prop::Elem* get(const char *key) const {

        const Prop *prop = findProp(key);

media/libstagefright/Android.bp

+2 −0
Original line number Diff line number Diff line
@@ -255,6 +255,7 @@ cc_library { 
        "MediaCodecSource.cpp",

        "MediaExtractor.cpp",

        "MediaExtractorFactory.cpp",

        "MediaHistogram.cpp",

        "MediaSource.cpp",

        "MediaSync.cpp",

        "MediaTrack.cpp",
@@ -270,6 +271,7 @@ cc_library { 
        "SurfaceUtils.cpp",

        "ThrottledSource.cpp",

        "Utils.cpp",

        "VideoRenderQualityTracker.cpp",

        "VideoFrameSchedulerBase.cpp",

        "VideoFrameScheduler.cpp",

    ],

media/libstagefright/MediaCodec.cpp

+85 −111
Original line number Diff line number Diff line
@@ -30,7 +30,6 @@ 
#include <C2Buffer.h>



#include "include/SoftwareRenderer.h"

#include "PlaybackDurationAccumulator.h"



#include <android/binder_manager.h>

#include <android/content/pm/IPackageManagerNative.h>
@@ -199,6 +198,23 @@ static const char *kCodecRecentLatencyHist = "android.media.mediacodec.recent.hi 
static const char *kCodecPlaybackDurationSec =

        "android.media.mediacodec.playback-duration-sec"; /* in sec */



static const char *kCodecFramesReleased = "android.media.mediacodec.frames.released";

static const char *kCodecFramesRendered = "android.media.mediacodec.frames.rendered";

static const char *kCodecFramesSkipped = "android.media.mediacodec.frames.skipped";

static const char *kCodecFramesDropped = "android.media.mediacodec.frames.dropped";

static const char *kCodecFramerateContent = "android.media.mediacodec.framerate.content";

static const char *kCodecFramerateDesired = "android.media.mediacodec.framerate.desired";

static const char *kCodecFramerateActual = "android.media.mediacodec.framerate.actual";



static const char *kCodecFreezeCount = "android.media.mediacodec.freeze.count";

static const char *kCodecFreezeDurationAverage = "android.media.mediacodec.freeze.duration.average";

static const char *kCodecFreezeDurationMax = "android.media.mediacodec.freeze.duration.max";

static const char *kCodecFreezeDurationHistogram =

        "android.media.mediacodec.freeze.duration.histogram";

static const char *kCodecFreezeDistanceAverage = "android.media.mediacodec.freeze.distance.average";

static const char *kCodecFreezeDistanceHistogram =

        "android.media.mediacodec.freeze.distance.histogram";



/* -1: shaper disabled

   >=0: number of fields changed */

static const char *kCodecShapingEnhanced = "android.media.mediacodec.shaped";
@@ -960,8 +976,7 @@ MediaCodec::MediaCodec( 
      mHaveInputSurface(false),

      mHavePendingInputBuffers(false),

      mCpuBoostRequested(false),

      mPlaybackDurationAccumulator(new PlaybackDurationAccumulator()),

      mIsSurfaceToScreen(false),

      mIsSurfaceToDisplay(false),

      mLatencyUnknown(0),

      mBytesEncoded(0),

      mEarliestEncodedPtsUs(INT64_MAX),
@@ -1096,6 +1111,32 @@ void MediaCodec::updateMediametrics() { 
    mediametrics_setInt32(mMetricsHandle, kCodecResolutionChangeCount,

            mReliabilityContextMetrics.resolutionChangeCount);



    // Video rendering quality metrics

    {

        const VideoRenderQualityMetrics& m = mVideoRenderQualityTracker.getMetrics();

        if (m.frameRenderedCount > 0) {

            mediametrics_setInt64(mMetricsHandle, kCodecFramesReleased, m.frameReleasedCount);

            mediametrics_setInt64(mMetricsHandle, kCodecFramesRendered, m.frameRenderedCount);

            mediametrics_setInt64(mMetricsHandle, kCodecFramesSkipped, m.frameSkippedCount);

            mediametrics_setInt64(mMetricsHandle, kCodecFramesDropped, m.frameDroppedCount);

            mediametrics_setDouble(mMetricsHandle, kCodecFramerateContent, m.contentFrameRate);

            mediametrics_setDouble(mMetricsHandle, kCodecFramerateDesired, m.desiredFrameRate);

            mediametrics_setDouble(mMetricsHandle, kCodecFramerateActual, m.actualFrameRate);

        }

        if (m.freezeDurationMsHistogram.getCount() >= 1) {

            const MediaHistogram &histogram = m.freezeDurationMsHistogram;

            mediametrics_setInt64(mMetricsHandle, kCodecFreezeCount, histogram.getCount());

            mediametrics_setInt64(mMetricsHandle, kCodecFreezeDurationAverage, histogram.getAvg());

            mediametrics_setInt64(mMetricsHandle, kCodecFreezeDurationMax, histogram.getMax());

            mediametrics_setString(mMetricsHandle, kCodecFreezeDurationHistogram, histogram.emit());

        }

        if (m.freezeDistanceMsHistogram.getCount() >= 1) {

            const MediaHistogram &histogram = m.freezeDistanceMsHistogram;

            mediametrics_setInt64(mMetricsHandle, kCodecFreezeDistanceAverage, histogram.getAvg());

            mediametrics_setString(mMetricsHandle, kCodecFreezeDistanceHistogram, histogram.emit());

        }

    }



    if (mLatencyHist.getCount() != 0 ) {

        mediametrics_setInt64(mMetricsHandle, kCodecLatencyMax, mLatencyHist.getMax());

        mediametrics_setInt64(mMetricsHandle, kCodecLatencyMin, mLatencyHist.getMin());
@@ -1111,7 +1152,7 @@ void MediaCodec::updateMediametrics() { 
    if (mLatencyUnknown > 0) {

        mediametrics_setInt64(mMetricsHandle, kCodecLatencyUnknown, mLatencyUnknown);

    }

    int64_t playbackDurationSec = mPlaybackDurationAccumulator->getDurationInSeconds();

    int64_t playbackDurationSec = mPlaybackDurationAccumulator.getDurationInSeconds();

    if (playbackDurationSec > 0) {

        mediametrics_setInt64(mMetricsHandle, kCodecPlaybackDurationSec, playbackDurationSec);

    }
@@ -1302,7 +1343,7 @@ void MediaCodec::updateEphemeralMediametrics(mediametrics_handle_t item) { 
        return;

    }



    Histogram recentHist;

    MediaHistogram recentHist;



    // build an empty histogram

    recentHist.setup(kLatencyHistBuckets, kLatencyHistWidth, kLatencyHistFloor);
@@ -1436,116 +1477,34 @@ void MediaCodec::updateTunnelPeek(const sp<AMessage> &msg) { 
    ALOGV("TunnelPeekState: %s -> %s", asString(previousState), asString(mTunnelPeekState));

}



void MediaCodec::updatePlaybackDuration(const sp<AMessage> &msg) {

void MediaCodec::processRenderedFrames(const sp<AMessage> &msg) {

    int what = 0;

    msg->findInt32("what", &what);

    if (msg->what() != kWhatCodecNotify && what != kWhatOutputFramesRendered) {

        static bool logged = false;

        if (!logged) {

            logged = true;

            ALOGE("updatePlaybackDuration: expected kWhatOuputFramesRendered (%d)", msg->what());

        }

        return;

            ALOGE("processRenderedFrames: expected kWhatOutputFramesRendered (%d)", msg->what());

        }

    // Playback duration only counts if the buffers are going to the screen.

    if (!mIsSurfaceToScreen) {

        return;

    }

    // Rendered frames only matter if they're being sent to the display

    if (mIsSurfaceToDisplay) {

        int64_t renderTimeNs;

    size_t index = 0;

    while (msg->findInt64(AStringPrintf("%zu-system-nano", index++).c_str(), &renderTimeNs)) {

        mPlaybackDurationAccumulator->processRenderTime(renderTimeNs);

    }

}



bool MediaCodec::Histogram::setup(int nbuckets, int64_t width, int64_t floor)

{

    if (nbuckets <= 0 || width <= 0) {

        return false;

    }



    // get histogram buckets

    if (nbuckets == mBucketCount && mBuckets != NULL) {

        // reuse our existing buffer

        memset(mBuckets, 0, sizeof(*mBuckets) * mBucketCount);

    } else {

        // get a new pre-zeroed buffer

        int64_t *newbuckets = (int64_t *)calloc(nbuckets, sizeof (*mBuckets));

        if (newbuckets == NULL) {

            goto bad;

        }

        if (mBuckets != NULL)

            free(mBuckets);

        mBuckets = newbuckets;

    }



    mWidth = width;

    mFloor = floor;

    mCeiling = floor + nbuckets * width;

    mBucketCount = nbuckets;



    mMin = INT64_MAX;

    mMax = INT64_MIN;

    mSum = 0;

    mCount = 0;

    mBelow = mAbove = 0;



    return true;



  bad:

    if (mBuckets != NULL) {

        free(mBuckets);

        mBuckets = NULL;

    }



    return false;

}



void MediaCodec::Histogram::insert(int64_t sample)

{

    // histogram is not set up

    if (mBuckets == NULL) {

        return;

    }



    mCount++;

    mSum += sample;

    if (mMin > sample) mMin = sample;

    if (mMax < sample) mMax = sample;



    if (sample < mFloor) {

        mBelow++;

    } else if (sample >= mCeiling) {

        mAbove++;

    } else {

        int64_t slot = (sample - mFloor) / mWidth;

        CHECK(slot < mBucketCount);

        mBuckets[slot]++;

    }

    return;

        for (size_t index = 0;

            msg->findInt64(AStringPrintf("%zu-system-nano", index).c_str(), &renderTimeNs);

            index++) {

            // Capture metrics for playback duration

            mPlaybackDurationAccumulator.onFrameRendered(renderTimeNs);

            // Capture metrics for quality

            int64_t mediaTimeUs = 0;

            if (!msg->findInt64(AStringPrintf("%zu-media-time-us", index).c_str(), &mediaTimeUs)) {

                ALOGE("processRenderedFrames: no media time found");

                continue;

            }



std::string MediaCodec::Histogram::emit()

{

    std::string value;

    char buffer[64];



    // emits:  width,Below{bucket0,bucket1,...., bucketN}above

    // unconfigured will emit: 0,0{}0

    // XXX: is this best representation?

    snprintf(buffer, sizeof(buffer), "%" PRId64 ",%" PRId64 ",%" PRId64 "{",

             mFloor, mWidth, mBelow);

    value = buffer;

    for (int i = 0; i < mBucketCount; i++) {

        if (i != 0) {

            value = value + ",";

            mVideoRenderQualityTracker.onFrameRendered(mediaTimeUs, renderTimeNs);

        }

        snprintf(buffer, sizeof(buffer), "%" PRId64, mBuckets[i]);

        value = value + buffer;

    }

    snprintf(buffer, sizeof(buffer), "}%" PRId64 , mAbove);

    value = value + buffer;

    return value;

}



// when we send a buffer to the codec;
@@ -3964,7 +3923,7 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) { 
                                asString(previousState),

                                asString(TunnelPeekState::kBufferRendered));

                    }

                    updatePlaybackDuration(msg);

                    processRenderedFrames(msg);

                    // check that we have a notification set

                    if (mOnFrameRenderedNotification != NULL) {

                        sp<AMessage> notify = mOnFrameRenderedNotification->dup();
@@ -4158,6 +4117,10 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) { 
                              mState, stateString(mState).c_str());

                        break;

                    }



                    if (mIsSurfaceToDisplay) {

                        mVideoRenderQualityTracker.resetForDiscontinuity();

                    }

                    // Notify the RM that the codec has been stopped.

                    ClientConfigParcel clientConfig;

                    initClientConfigParcel(clientConfig);
@@ -4213,6 +4176,10 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) { 
                        break;

                    }



                    if (mIsSurfaceToDisplay) {

                        mVideoRenderQualityTracker.resetForDiscontinuity();

                    }



                    if (mFlags & kFlagIsAsync) {

                        setState(FLUSHED);

                    } else {
@@ -5927,7 +5894,9 @@ status_t MediaCodec::onReleaseOutputBuffer(const sp<AMessage> &msg) { 


        // If rendering to the screen, then schedule a time in the future to poll to see if this

        // frame was ever rendered to seed onFrameRendered callbacks.

        if (mIsSurfaceToScreen) {

        if (mIsSurfaceToDisplay) {

            noRenderTime ? mVideoRenderQualityTracker.onFrameReleased(mediaTimeUs)

                         : mVideoRenderQualityTracker.onFrameReleased(mediaTimeUs, renderTimeNs);

            // can't initialize this in the constructor because the Looper parent class needs to be

            // initialized first

            if (mMsgPollForRenderedBuffers == nullptr) {
@@ -5957,6 +5926,11 @@ status_t MediaCodec::onReleaseOutputBuffer(const sp<AMessage> &msg) { 
            ALOGI("rendring output error %d", err);

        }

    } else {

        if (mIsSurfaceToDisplay) {

            int64_t mediaTimeUs = -1;

            buffer->meta()->findInt64("timeUs", &mediaTimeUs);

            mVideoRenderQualityTracker.onFrameSkipped(mediaTimeUs);

        }

        mBufferChannel->discardBuffer(buffer);

    }
@@ -6023,7 +5997,7 @@ status_t MediaCodec::connectToSurface(const sp<Surface> &surface) { 


        // in case we don't connect, ensure that we don't signal the surface is

        // connected to the screen

        mIsSurfaceToScreen = false;

        mIsSurfaceToDisplay = false;



        err = nativeWindowConnect(surface.get(), "connectToSurface");

        if (err == OK) {
@@ -6053,7 +6027,7 @@ status_t MediaCodec::connectToSurface(const sp<Surface> &surface) { 
            // keep track whether or not the buffers of the connected surface go to the screen

            int result = 0;

            surface->query(NATIVE_WINDOW_QUEUES_TO_WINDOW_COMPOSER, &result);

            mIsSurfaceToScreen = result != 0;

            mIsSurfaceToDisplay = result != 0;

        }

    }

    // do not return ALREADY_EXISTS unless surfaces are the same
@@ -6071,7 +6045,7 @@ status_t MediaCodec::disconnectFromSurface() { 
        }

        // assume disconnected even on error

        mSurface.clear();

        mIsSurfaceToScreen = false;

        mIsSurfaceToDisplay = false;

    }

    return err;

}