Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit ff74e9d7 authored by Brian Lindahl's avatar Brian Lindahl
Browse files

Support video render metrics for OMX

Currently, no devices on Android 14 use OMX codecs, however it is
expected that TV devices using OMX will upgrade to Android 14
eventually. Add support for video render metrics for these device
upgrades so that we have universal video render metrics on all Android
14 devices.

Bug: 286919416
Bug: 234833109
Test: atest DecoderRenderTest on ADT-3 on Android 12
Test: Play youtube videos on ADT-3 on Android 12
Change-Id: Ia22d976bc2ba423fd8cd71110b64b4715326fe55
parent 050693eb
Loading
Loading
Loading
Loading
+2 −2
Original line number Diff line number Diff line
@@ -46,6 +46,7 @@
#include <media/stagefright/BufferProducerWrapper.h>
#include <media/stagefright/MediaCodecConstants.h>
#include <media/stagefright/PersistentSurface.h>
#include <media/stagefright/RenderedFrameInfo.h>
#include <utils/NativeHandle.h>

#include "C2OMXNode.h"
@@ -672,8 +673,7 @@ public:
    }

    void onOutputFramesRendered(int64_t mediaTimeUs, nsecs_t renderTimeNs) override {
        mCodec->mCallback->onOutputFramesRendered(
                {RenderedFrameInfo(mediaTimeUs, renderTimeNs)});
        mCodec->mCallback->onOutputFramesRendered({RenderedFrameInfo(mediaTimeUs, renderTimeNs)});
    }

    void onOutputBuffersChanged() override {
+115 −92
Original line number Diff line number Diff line
@@ -43,6 +43,7 @@
#include <media/stagefright/MediaDefs.h>
#include <media/stagefright/OMXClient.h>
#include <media/stagefright/PersistentSurface.h>
#include <media/stagefright/RenderedFrameInfo.h>
#include <media/stagefright/SurfaceUtils.h>
#include <media/hardware/HardwareAPI.h>
#include <media/MediaBufferHolder.h>
@@ -632,7 +633,8 @@ std::shared_ptr<BufferChannelBase> ACodec::getBufferChannel() {
    if (!mBufferChannel) {
        mBufferChannel = std::make_shared<ACodecBufferChannel>(
                new AMessage(kWhatInputBufferFilled, this),
                new AMessage(kWhatOutputBufferDrained, this));
                new AMessage(kWhatOutputBufferDrained, this),
                new AMessage(kWhatPollForRenderedBuffers, this));
    }
    return mBufferChannel;
}
@@ -742,6 +744,7 @@ status_t ACodec::handleSetSurface(const sp<Surface> &surface) {
    // if we have not yet started the codec, we can simply set the native window
    if (mBuffers[kPortIndexInput].size() == 0) {
        mNativeWindow = surface;
        initializeFrameTracking();
        return OK;
    }

@@ -850,6 +853,7 @@ status_t ACodec::handleSetSurface(const sp<Surface> &surface) {

    mNativeWindow = nativeWindow;
    mNativeWindowUsageBits = usageBits;
    initializeFrameTracking();
    return OK;
}

@@ -960,7 +964,6 @@ status_t ACodec::allocateBuffersOnPort(OMX_U32 portIndex) {
                BufferInfo info;
                info.mStatus = BufferInfo::OWNED_BY_US;
                info.mFenceFd = -1;
                info.mRenderInfo = NULL;
                info.mGraphicBuffer = NULL;
                info.mNewGraphicBuffer = false;

@@ -1228,6 +1231,7 @@ status_t ACodec::configureOutputBuffersFromNativeWindow(

    *bufferCount = def.nBufferCountActual;
    *bufferSize =  def.nBufferSize;
    initializeFrameTracking();
    return err;
}

@@ -1266,7 +1270,6 @@ status_t ACodec::allocateOutputBuffersFromNativeWindow() {
        info.mStatus = BufferInfo::OWNED_BY_US;
        info.mFenceFd = fenceFd;
        info.mIsReadFence = false;
        info.mRenderInfo = NULL;
        info.mGraphicBuffer = graphicBuffer;
        info.mNewGraphicBuffer = false;
        info.mDequeuedAt = mDequeueCounter;
@@ -1343,7 +1346,6 @@ status_t ACodec::allocateOutputMetadataBuffers() {
        BufferInfo info;
        info.mStatus = BufferInfo::OWNED_BY_NATIVE_WINDOW;
        info.mFenceFd = -1;
        info.mRenderInfo = NULL;
        info.mGraphicBuffer = NULL;
        info.mNewGraphicBuffer = false;
        info.mDequeuedAt = mDequeueCounter;
@@ -1439,42 +1441,6 @@ status_t ACodec::cancelBufferToNativeWindow(BufferInfo *info) {
    return err;
}

void ACodec::updateRenderInfoForDequeuedBuffer(
        ANativeWindowBuffer *buf, int fenceFd, BufferInfo *info) {

    info->mRenderInfo =
        mRenderTracker.updateInfoForDequeuedBuffer(
                buf, fenceFd, info - &mBuffers[kPortIndexOutput][0]);

    // check for any fences already signaled
    notifyOfRenderedFrames(false /* dropIncomplete */, info->mRenderInfo);
}

void ACodec::onFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano) {
    if (mRenderTracker.onFrameRendered(mediaTimeUs, systemNano) != OK) {
        mRenderTracker.dumpRenderQueue();
    }
}

void ACodec::notifyOfRenderedFrames(bool dropIncomplete, FrameRenderTracker::Info *until) {
    std::list<FrameRenderTracker::Info> done =
        mRenderTracker.checkFencesAndGetRenderedFrames(until, dropIncomplete);

    // unlink untracked frames
    for (std::list<FrameRenderTracker::Info>::const_iterator it = done.cbegin();
            it != done.cend(); ++it) {
        ssize_t index = it->getIndex();
        if (index >= 0 && (size_t)index < mBuffers[kPortIndexOutput].size()) {
            mBuffers[kPortIndexOutput][index].mRenderInfo = NULL;
        } else if (index >= 0) {
            // THIS SHOULD NEVER HAPPEN
            ALOGE("invalid index %zd in %zu", index, mBuffers[kPortIndexOutput].size());
        }
    }

    mCallback->onOutputFramesRendered(done);
}

void ACodec::onFirstTunnelFrameReady() {
    mCallback->onFirstTunnelFrameReady();
}
@@ -1529,7 +1495,6 @@ ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() {

                info->mStatus = BufferInfo::OWNED_BY_US;
                info->setWriteFence(fenceFd, "dequeueBufferFromNativeWindow");
                updateRenderInfoForDequeuedBuffer(buf, fenceFd, info);
                return info;
            }
        }
@@ -1574,18 +1539,96 @@ ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() {
    oldest->mNewGraphicBuffer = true;
    oldest->mStatus = BufferInfo::OWNED_BY_US;
    oldest->setWriteFence(fenceFd, "dequeueBufferFromNativeWindow for oldest");
    mRenderTracker.untrackFrame(oldest->mRenderInfo);
    oldest->mRenderInfo = NULL;

    ALOGV("replaced oldest buffer #%u with age %u, graphicBuffer %p",
            (unsigned)(oldest - &mBuffers[kPortIndexOutput][0]),
            mDequeueCounter - oldest->mDequeuedAt,
            oldest->mGraphicBuffer->handle);

    updateRenderInfoForDequeuedBuffer(buf, fenceFd, oldest);
    return oldest;
}

void ACodec::initializeFrameTracking() {
    mTrackedFrames.clear();

    int isWindowToDisplay = 0;
    mNativeWindow->query(mNativeWindow.get(), NATIVE_WINDOW_QUEUES_TO_WINDOW_COMPOSER,
            &isWindowToDisplay);
    mIsWindowToDisplay = isWindowToDisplay == 1;
    // No frame tracking is needed if we're not sending frames to the display
    if (!mIsWindowToDisplay) {
        // Return early so we don't call into SurfaceFlinger (requiring permissions)
        return;
    }

    int hasPresentFenceTimes = 0;
    mNativeWindow->query(mNativeWindow.get(), NATIVE_WINDOW_FRAME_TIMESTAMPS_SUPPORTS_PRESENT,
            &hasPresentFenceTimes);
    mHasPresentFenceTimes = hasPresentFenceTimes == 1;
    if (!mHasPresentFenceTimes) {
        ALOGI("Using latch times for frame rendered signals - present fences not supported");
    }

    status_t err = native_window_enable_frame_timestamps(mNativeWindow.get(), true);
    if (err) {
        ALOGE("Failed to enable frame timestamps (%d)", err);
    }
}

void ACodec::trackReleasedFrame(int64_t frameId, int64_t mediaTimeUs, int64_t desiredRenderTimeNs) {
    // If the render time is earlier than now, then we're suggesting it should be rendered ASAP,
    // so track the frame as if the desired render time is now.
    int64_t nowNs = systemTime(SYSTEM_TIME_MONOTONIC);
    if (desiredRenderTimeNs < nowNs) {
        desiredRenderTimeNs = nowNs;
    }
    // We've just queued a frame to the surface, so keep track of it and later check to see if it is
    // actually rendered.
    TrackedFrame frame;
    frame.id = frameId;
    frame.mediaTimeUs = mediaTimeUs;
    frame.desiredRenderTimeNs = desiredRenderTimeNs;
    mTrackedFrames.push_back(frame);
}

void ACodec::pollForRenderedFrames() {
    std::list<RenderedFrameInfo> renderedFrameInfos;
    // Scan all frames and check to see if the frames that SHOULD have been rendered by now, have,
    // in fact, been rendered.
    int64_t nowNs = systemTime(SYSTEM_TIME_MONOTONIC);
    while (!mTrackedFrames.empty()) {
        TrackedFrame & frame = mTrackedFrames.front();
        // Frames that should have been rendered at least 100ms in the past are checked
        if (frame.desiredRenderTimeNs > nowNs - 100*1000*1000LL) {
            break;
        }

        status_t err;
        nsecs_t latchOrPresentTimeNs = NATIVE_WINDOW_TIMESTAMP_INVALID;
        err = native_window_get_frame_timestamps(mNativeWindow.get(), frame.id,
                /* outRequestedPresentTime */ nullptr, /* outAcquireTime */ nullptr,
                mHasPresentFenceTimes ? nullptr : &latchOrPresentTimeNs, // latch time
                /* outFirstRefreshStartTime */ nullptr, /* outLastRefreshStartTime */ nullptr,
                /* outGpuCompositionDoneTime */ nullptr,
                mHasPresentFenceTimes ? &latchOrPresentTimeNs : nullptr, // display present time,
                /* outDequeueReadyTime */ nullptr, /* outReleaseTime */ nullptr);
        if (err) {
            ALOGE("Failed to get frame timestamps for %lld: %d", (long long) frame.id, err);
        }
        // If we don't have a render time by now, then consider the frame as dropped
        if (latchOrPresentTimeNs != NATIVE_WINDOW_TIMESTAMP_PENDING &&
            latchOrPresentTimeNs != NATIVE_WINDOW_TIMESTAMP_INVALID) {
            renderedFrameInfos.push_back(RenderedFrameInfo(frame.mediaTimeUs,
                                                           latchOrPresentTimeNs));
        }

        mTrackedFrames.pop_front();
    }

    if (!renderedFrameInfos.empty()) {
        mCallback->onOutputFramesRendered(renderedFrameInfos);
    }
}

status_t ACodec::freeBuffersOnPort(OMX_U32 portIndex) {
    if (portIndex == kPortIndexInput) {
        mBufferChannel->setInputBufferArray({});
@@ -1661,11 +1704,6 @@ status_t ACodec::freeBuffer(OMX_U32 portIndex, size_t i) {
        ::close(info->mFenceFd);
    }

    if (portIndex == kPortIndexOutput) {
        mRenderTracker.untrackFrame(info->mRenderInfo, i);
        info->mRenderInfo = NULL;
    }

    // remove buffer even if mOMXNode->freeBuffer fails
    mBuffers[portIndex].erase(mBuffers[portIndex].begin() + i);
    return err;
@@ -6030,22 +6068,10 @@ bool ACodec::BaseState::onOMXMessageList(const sp<AMessage> &msg) {
    sp<RefBase> obj;
    CHECK(msg->findObject("messages", &obj));
    sp<MessageList> msgList = static_cast<MessageList *>(obj.get());

    bool receivedRenderedEvents = false;
    for (std::list<sp<AMessage>>::const_iterator it = msgList->getList().cbegin();
          it != msgList->getList().cend(); ++it) {
        (*it)->setWhat(ACodec::kWhatOMXMessageItem);
        mCodec->handleMessage(*it);
        int32_t type;
        CHECK((*it)->findInt32("type", &type));
        if (type == omx_message::FRAME_RENDERED) {
            receivedRenderedEvents = true;
        }
    }

    if (receivedRenderedEvents) {
        // NOTE: all buffers are rendered in this case
        mCodec->notifyOfRenderedFrames();
    }
    return true;
}
@@ -6554,15 +6580,6 @@ bool ACodec::BaseState::onOMXFillBufferDone(
    info->mDequeuedAt = ++mCodec->mDequeueCounter;
    info->mStatus = BufferInfo::OWNED_BY_US;

    if (info->mRenderInfo != NULL) {
        // The fence for an emptied buffer must have signaled, but there still could be queued
        // or out-of-order dequeued buffers in the render queue prior to this buffer. Drop these,
        // as we will soon requeue this buffer to the surface. While in theory we could still keep
        // track of buffers that are requeued to the surface, it is better to add support to the
        // buffer-queue to notify us of released buffers and their fences (in the future).
        mCodec->notifyOfRenderedFrames(true /* dropIncomplete */);
    }

    // byte buffers cannot take fences, so wait for any fence now
    if (mCodec->mNativeWindow == NULL) {
        (void)mCodec->waitForFence(fenceFd, "onOMXFillBufferDone");
@@ -6769,14 +6786,6 @@ void ACodec::BaseState::onOutputBufferDrained(const sp<AMessage> &msg) {
            mCodec->mLastHdr10PlusBuffer = hdr10PlusInfo;
        }

        // save buffers sent to the surface so we can get render time when they return
        int64_t mediaTimeUs = -1;
        buffer->meta()->findInt64("timeUs", &mediaTimeUs);
        if (mediaTimeUs >= 0) {
            mCodec->mRenderTracker.onFrameQueued(
                    mediaTimeUs, info->mGraphicBuffer, new Fence(::dup(info->mFenceFd)));
        }

        int64_t timestampNs = 0;
        if (!msg->findInt64("timestampNs", &timestampNs)) {
            // use media timestamp if client did not request a specific render timestamp
@@ -6790,11 +6799,25 @@ void ACodec::BaseState::onOutputBufferDrained(const sp<AMessage> &msg) {
        err = native_window_set_buffers_timestamp(mCodec->mNativeWindow.get(), timestampNs);
        ALOGW_IF(err != NO_ERROR, "failed to set buffer timestamp: %d", err);

        uint64_t frameId;
        err = native_window_get_next_frame_id(mCodec->mNativeWindow.get(), &frameId);

        info->checkReadFence("onOutputBufferDrained before queueBuffer");
        err = mCodec->mNativeWindow->queueBuffer(
                    mCodec->mNativeWindow.get(), info->mGraphicBuffer.get(), info->mFenceFd);
        // TODO(b/266211548): Poll the native window for rendered buffers, since when queueing
        // buffers, the frame event history delta is retrieved.

        int64_t mediaTimeUs = -1;
        buffer->meta()->findInt64("timeUs", &mediaTimeUs);
        if (mCodec->mIsWindowToDisplay) {
            mCodec->trackReleasedFrame(frameId, mediaTimeUs, timestampNs);
            mCodec->pollForRenderedFrames();
        } else {
            // When the surface is an intermediate surface, onFrameRendered is triggered immediately
            // when the frame is queued to the non-display surface
            mCodec->mCallback->onOutputFramesRendered({RenderedFrameInfo(mediaTimeUs,
                                                                         timestampNs)});
        }

        info->mFenceFd = -1;
        if (err == OK) {
            info->mStatus = BufferInfo::OWNED_BY_NATIVE_WINDOW;
@@ -7021,7 +7044,6 @@ bool ACodec::UninitializedState::onAllocateComponent(const sp<AMessage> &msg) {
    ++mCodec->mNodeGeneration;

    mCodec->mComponentName = componentName;
    mCodec->mRenderTracker.setComponentName(componentName);
    mCodec->mFlags = 0;

    if (componentName.endsWith(".secure")) {
@@ -7644,7 +7666,6 @@ void ACodec::ExecutingState::resume() {

void ACodec::ExecutingState::stateEntered() {
    ALOGV("[%s] Now Executing", mCodec->mComponentName.c_str());
    mCodec->mRenderTracker.clear(systemTime(CLOCK_MONOTONIC));
    mCodec->processDeferredMessages();
}

@@ -7755,7 +7776,15 @@ bool ACodec::ExecutingState::onMessageReceived(const sp<AMessage> &msg) {
                    mCodec->signalSubmitOutputMetadataBufferIfEOS_workaround();
                }
            }
            return true;
            handled = true;
            break;
        }

        case kWhatPollForRenderedBuffers:
        {
            mCodec->pollForRenderedFrames();
            handled = true;
            break;
        }

        default:
@@ -8439,7 +8468,7 @@ void ACodec::forceStateTransition(int generation) {
}

bool ACodec::ExecutingState::onOMXFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano) {
    mCodec->onFrameRendered(mediaTimeUs, systemNano);
    mCodec->mCallback->onOutputFramesRendered({RenderedFrameInfo(mediaTimeUs, systemNano)});
    return true;
}

@@ -8613,7 +8642,7 @@ void ACodec::OutputPortSettingsChangedState::stateEntered() {

bool ACodec::OutputPortSettingsChangedState::onOMXFrameRendered(
        int64_t mediaTimeUs, nsecs_t systemNano) {
    mCodec->onFrameRendered(mediaTimeUs, systemNano);
    mCodec->mCallback->onOutputFramesRendered({RenderedFrameInfo(mediaTimeUs, systemNano)});
    return true;
}

@@ -8644,10 +8673,6 @@ bool ACodec::OutputPortSettingsChangedState::onOMXEvent(
                            OMX_CommandPortEnable, kPortIndexOutput);
                }

                // Clear the RenderQueue in which queued GraphicBuffers hold the
                // actual buffer references in order to free them early.
                mCodec->mRenderTracker.clear(systemTime(CLOCK_MONOTONIC));

                if (err == OK) {
                    err = mCodec->allocateBuffersOnPort(kPortIndexOutput);
                    ALOGE_IF(err != OK, "Failed to allocate output port buffers after port "
@@ -9031,8 +9056,6 @@ void ACodec::FlushingState::changeStateIfWeOwnAllBuffers() {
        // the native window for rendering. Let's get those back as well.
        mCodec->waitUntilAllPossibleNativeWindowBuffersAreReturnedToUs();

        mCodec->mRenderTracker.clear(systemTime(CLOCK_MONOTONIC));

        mCodec->mCallback->onFlushCompleted();

        mCodec->mPortEOS[kPortIndexInput] =
+5 −2
Original line number Diff line number Diff line
@@ -32,6 +32,7 @@
#include <media/stagefright/foundation/ABuffer.h>
#include <media/stagefright/foundation/AMessage.h>
#include <media/stagefright/foundation/AUtils.h>
#include <media/stagefright/ACodec.h>
#include <media/stagefright/MediaCodec.h>
#include <media/MediaCodecBuffer.h>
#include <system/window.h>
@@ -87,9 +88,11 @@ ACodecBufferChannel::BufferInfo::BufferInfo(
}

ACodecBufferChannel::ACodecBufferChannel(
        const sp<AMessage> &inputBufferFilled, const sp<AMessage> &outputBufferDrained)
        const sp<AMessage> &inputBufferFilled, const sp<AMessage> &outputBufferDrained,
        const sp<AMessage> &pollForRenderedBuffers)
    : mInputBufferFilled(inputBufferFilled),
      mOutputBufferDrained(outputBufferDrained),
      mPollForRenderedBuffers(pollForRenderedBuffers),
      mHeapSeqNum(-1) {
}

@@ -488,7 +491,7 @@ status_t ACodecBufferChannel::renderOutputBuffer(
}

void ACodecBufferChannel::pollForRenderedBuffers() {
    // TODO(b/266211548): Poll the native window for rendered buffers.
    mPollForRenderedBuffers->post();
}

status_t ACodecBufferChannel::discardBuffer(const sp<MediaCodecBuffer> &buffer) {
+18 −7
Original line number Diff line number Diff line
@@ -79,6 +79,7 @@
#include <media/stagefright/MediaErrors.h>
#include <media/stagefright/OMXClient.h>
#include <media/stagefright/PersistentSurface.h>
#include <media/stagefright/RenderedFrameInfo.h>
#include <media/stagefright/SurfaceUtils.h>
#include <nativeloader/dlext_namespaces.h>
#include <private/android_filesystem_config.h>
@@ -750,7 +751,7 @@ public:
            const sp<AMessage> &outputFormat) override;
    virtual void onInputSurfaceDeclined(status_t err) override;
    virtual void onSignaledInputEOS(status_t err) override;
    virtual void onOutputFramesRendered(const std::list<FrameRenderTracker::Info> &done) override;
    virtual void onOutputFramesRendered(const std::list<RenderedFrameInfo> &done) override;
    virtual void onOutputBuffersChanged() override;
    virtual void onFirstTunnelFrameReady() override;
private:
@@ -859,7 +860,7 @@ void CodecCallback::onSignaledInputEOS(status_t err) {
    notify->post();
}

void CodecCallback::onOutputFramesRendered(const std::list<FrameRenderTracker::Info> &done) {
void CodecCallback::onOutputFramesRendered(const std::list<RenderedFrameInfo> &done) {
    sp<AMessage> notify(mNotify->dup());
    notify->setInt32("what", kWhatOutputFramesRendered);
    if (MediaCodec::CreateFramesRenderedMessage(done, notify)) {
@@ -5953,12 +5954,10 @@ status_t MediaCodec::handleLeftover(size_t index) {
    return onQueueInputBuffer(msg);
}

//static
size_t MediaCodec::CreateFramesRenderedMessage(
        const std::list<FrameRenderTracker::Info> &done, sp<AMessage> &msg) {
template<typename T>
static size_t CreateFramesRenderedMessageInternal(const std::list<T> &done, sp<AMessage> &msg) {
    size_t index = 0;
    for (std::list<FrameRenderTracker::Info>::const_iterator it = done.cbegin();
            it != done.cend(); ++it) {
    for (typename std::list<T>::const_iterator it = done.cbegin(); it != done.cend(); ++it) {
        if (it->getRenderTimeNs() < 0) {
            continue; // dropped frame from tracking
        }
@@ -5969,6 +5968,18 @@ size_t MediaCodec::CreateFramesRenderedMessage(
    return index;
}

//static
size_t MediaCodec::CreateFramesRenderedMessage(
        const std::list<RenderedFrameInfo> &done, sp<AMessage> &msg) {
    return CreateFramesRenderedMessageInternal(done, msg);
}

//static
size_t MediaCodec::CreateFramesRenderedMessage(
        const std::list<FrameRenderTracker::Info> &done, sp<AMessage> &msg) {
    return CreateFramesRenderedMessageInternal(done, msg);
}

status_t MediaCodec::onReleaseOutputBuffer(const sp<AMessage> &msg) {
    size_t index;
    CHECK(msg->findSize("index", &index));
+4 −1
Original line number Diff line number Diff line
@@ -29,6 +29,7 @@
#include <media/IOMX.h>

namespace android {
 struct ACodec;
namespace hardware {
class HidlMemory;
};
@@ -63,7 +64,8 @@ public:
    };

    ACodecBufferChannel(
            const sp<AMessage> &inputBufferFilled, const sp<AMessage> &outputBufferDrained);
            const sp<AMessage> &inputBufferFilled, const sp<AMessage> &outputBufferDrained,
            const sp<AMessage> &pollForRenderedBuffers);
    virtual ~ACodecBufferChannel();

    // BufferChannelBase interface
@@ -138,6 +140,7 @@ private:

    const sp<AMessage> mInputBufferFilled;
    const sp<AMessage> mOutputBufferDrained;
    const sp<AMessage> mPollForRenderedBuffers;

    sp<MemoryDealer> mDealer;
    sp<IMemory> mDecryptDestination;
Loading