Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit ac2d80f9 authored by Brian Lindahl's avatar Brian Lindahl Committed by Android (Google) Code Review
Browse files

Merge "Support video render metrics for OMX" into udc-qpr-dev

parents a3131f2a ff74e9d7

media/codec2/sfplugin/CCodec.cpp

+2 −2
Original line number Diff line number Diff line
@@ -46,6 +46,7 @@ 
#include <media/stagefright/BufferProducerWrapper.h>

#include <media/stagefright/MediaCodecConstants.h>

#include <media/stagefright/PersistentSurface.h>

#include <media/stagefright/RenderedFrameInfo.h>

#include <utils/NativeHandle.h>



#include "C2OMXNode.h"
@@ -672,8 +673,7 @@ public: 
    }



    void onOutputFramesRendered(int64_t mediaTimeUs, nsecs_t renderTimeNs) override {

        mCodec->mCallback->onOutputFramesRendered(

                {RenderedFrameInfo(mediaTimeUs, renderTimeNs)});

        mCodec->mCallback->onOutputFramesRendered({RenderedFrameInfo(mediaTimeUs, renderTimeNs)});

    }



    void onOutputBuffersChanged() override {

media/libstagefright/ACodec.cpp

+115 −92
Original line number Diff line number Diff line
@@ -43,6 +43,7 @@ 
#include <media/stagefright/MediaDefs.h>

#include <media/stagefright/OMXClient.h>

#include <media/stagefright/PersistentSurface.h>

#include <media/stagefright/RenderedFrameInfo.h>

#include <media/stagefright/SurfaceUtils.h>

#include <media/hardware/HardwareAPI.h>

#include <media/MediaBufferHolder.h>
@@ -632,7 +633,8 @@ std::shared_ptr<BufferChannelBase> ACodec::getBufferChannel() { 
    if (!mBufferChannel) {

        mBufferChannel = std::make_shared<ACodecBufferChannel>(

                new AMessage(kWhatInputBufferFilled, this),

                new AMessage(kWhatOutputBufferDrained, this));

                new AMessage(kWhatOutputBufferDrained, this),

                new AMessage(kWhatPollForRenderedBuffers, this));

    }

    return mBufferChannel;

}
@@ -742,6 +744,7 @@ status_t ACodec::handleSetSurface(const sp<Surface> &surface) { 
    // if we have not yet started the codec, we can simply set the native window

    if (mBuffers[kPortIndexInput].size() == 0) {

        mNativeWindow = surface;

        initializeFrameTracking();

        return OK;

    }
@@ -850,6 +853,7 @@ status_t ACodec::handleSetSurface(const sp<Surface> &surface) { 


    mNativeWindow = nativeWindow;

    mNativeWindowUsageBits = usageBits;

    initializeFrameTracking();

    return OK;

}
@@ -960,7 +964,6 @@ status_t ACodec::allocateBuffersOnPort(OMX_U32 portIndex) { 
                BufferInfo info;

                info.mStatus = BufferInfo::OWNED_BY_US;

                info.mFenceFd = -1;

                info.mRenderInfo = NULL;

                info.mGraphicBuffer = NULL;

                info.mNewGraphicBuffer = false;
@@ -1228,6 +1231,7 @@ status_t ACodec::configureOutputBuffersFromNativeWindow( 


    *bufferCount = def.nBufferCountActual;

    *bufferSize =  def.nBufferSize;

    initializeFrameTracking();

    return err;

}
@@ -1266,7 +1270,6 @@ status_t ACodec::allocateOutputBuffersFromNativeWindow() { 
        info.mStatus = BufferInfo::OWNED_BY_US;

        info.mFenceFd = fenceFd;

        info.mIsReadFence = false;

        info.mRenderInfo = NULL;

        info.mGraphicBuffer = graphicBuffer;

        info.mNewGraphicBuffer = false;

        info.mDequeuedAt = mDequeueCounter;
@@ -1343,7 +1346,6 @@ status_t ACodec::allocateOutputMetadataBuffers() { 
        BufferInfo info;

        info.mStatus = BufferInfo::OWNED_BY_NATIVE_WINDOW;

        info.mFenceFd = -1;

        info.mRenderInfo = NULL;

        info.mGraphicBuffer = NULL;

        info.mNewGraphicBuffer = false;

        info.mDequeuedAt = mDequeueCounter;
@@ -1439,42 +1441,6 @@ status_t ACodec::cancelBufferToNativeWindow(BufferInfo *info) { 
    return err;

}



void ACodec::updateRenderInfoForDequeuedBuffer(

        ANativeWindowBuffer *buf, int fenceFd, BufferInfo *info) {



    info->mRenderInfo =

        mRenderTracker.updateInfoForDequeuedBuffer(

                buf, fenceFd, info - &mBuffers[kPortIndexOutput][0]);



    // check for any fences already signaled

    notifyOfRenderedFrames(false /* dropIncomplete */, info->mRenderInfo);

}



void ACodec::onFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano) {

    if (mRenderTracker.onFrameRendered(mediaTimeUs, systemNano) != OK) {

        mRenderTracker.dumpRenderQueue();

    }

}



void ACodec::notifyOfRenderedFrames(bool dropIncomplete, FrameRenderTracker::Info *until) {

    std::list<FrameRenderTracker::Info> done =

        mRenderTracker.checkFencesAndGetRenderedFrames(until, dropIncomplete);



    // unlink untracked frames

    for (std::list<FrameRenderTracker::Info>::const_iterator it = done.cbegin();

            it != done.cend(); ++it) {

        ssize_t index = it->getIndex();

        if (index >= 0 && (size_t)index < mBuffers[kPortIndexOutput].size()) {

            mBuffers[kPortIndexOutput][index].mRenderInfo = NULL;

        } else if (index >= 0) {

            // THIS SHOULD NEVER HAPPEN

            ALOGE("invalid index %zd in %zu", index, mBuffers[kPortIndexOutput].size());

        }

    }



    mCallback->onOutputFramesRendered(done);

}



void ACodec::onFirstTunnelFrameReady() {

    mCallback->onFirstTunnelFrameReady();

}
@@ -1529,7 +1495,6 @@ ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() { 


                info->mStatus = BufferInfo::OWNED_BY_US;

                info->setWriteFence(fenceFd, "dequeueBufferFromNativeWindow");

                updateRenderInfoForDequeuedBuffer(buf, fenceFd, info);

                return info;

            }

        }
@@ -1574,18 +1539,96 @@ ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() { 
    oldest->mNewGraphicBuffer = true;

    oldest->mStatus = BufferInfo::OWNED_BY_US;

    oldest->setWriteFence(fenceFd, "dequeueBufferFromNativeWindow for oldest");

    mRenderTracker.untrackFrame(oldest->mRenderInfo);

    oldest->mRenderInfo = NULL;



    ALOGV("replaced oldest buffer #%u with age %u, graphicBuffer %p",

            (unsigned)(oldest - &mBuffers[kPortIndexOutput][0]),

            mDequeueCounter - oldest->mDequeuedAt,

            oldest->mGraphicBuffer->handle);



    updateRenderInfoForDequeuedBuffer(buf, fenceFd, oldest);

    return oldest;

}



void ACodec::initializeFrameTracking() {

    mTrackedFrames.clear();



    int isWindowToDisplay = 0;

    mNativeWindow->query(mNativeWindow.get(), NATIVE_WINDOW_QUEUES_TO_WINDOW_COMPOSER,

            &isWindowToDisplay);

    mIsWindowToDisplay = isWindowToDisplay == 1;

    // No frame tracking is needed if we're not sending frames to the display

    if (!mIsWindowToDisplay) {

        // Return early so we don't call into SurfaceFlinger (requiring permissions)

        return;

    }



    int hasPresentFenceTimes = 0;

    mNativeWindow->query(mNativeWindow.get(), NATIVE_WINDOW_FRAME_TIMESTAMPS_SUPPORTS_PRESENT,

            &hasPresentFenceTimes);

    mHasPresentFenceTimes = hasPresentFenceTimes == 1;

    if (!mHasPresentFenceTimes) {

        ALOGI("Using latch times for frame rendered signals - present fences not supported");

    }



    status_t err = native_window_enable_frame_timestamps(mNativeWindow.get(), true);

    if (err) {

        ALOGE("Failed to enable frame timestamps (%d)", err);

    }

}



void ACodec::trackReleasedFrame(int64_t frameId, int64_t mediaTimeUs, int64_t desiredRenderTimeNs) {

    // If the render time is earlier than now, then we're suggesting it should be rendered ASAP,

    // so track the frame as if the desired render time is now.

    int64_t nowNs = systemTime(SYSTEM_TIME_MONOTONIC);

    if (desiredRenderTimeNs < nowNs) {

        desiredRenderTimeNs = nowNs;

    }

    // We've just queued a frame to the surface, so keep track of it and later check to see if it is

    // actually rendered.

    TrackedFrame frame;

    frame.id = frameId;

    frame.mediaTimeUs = mediaTimeUs;

    frame.desiredRenderTimeNs = desiredRenderTimeNs;

    mTrackedFrames.push_back(frame);

}



void ACodec::pollForRenderedFrames() {

    std::list<RenderedFrameInfo> renderedFrameInfos;

    // Scan all frames and check to see if the frames that SHOULD have been rendered by now, have,

    // in fact, been rendered.

    int64_t nowNs = systemTime(SYSTEM_TIME_MONOTONIC);

    while (!mTrackedFrames.empty()) {

        TrackedFrame & frame = mTrackedFrames.front();

        // Frames that should have been rendered at least 100ms in the past are checked

        if (frame.desiredRenderTimeNs > nowNs - 100*1000*1000LL) {

            break;

        }



        status_t err;

        nsecs_t latchOrPresentTimeNs = NATIVE_WINDOW_TIMESTAMP_INVALID;

        err = native_window_get_frame_timestamps(mNativeWindow.get(), frame.id,

                /* outRequestedPresentTime */ nullptr, /* outAcquireTime */ nullptr,

                mHasPresentFenceTimes ? nullptr : &latchOrPresentTimeNs, // latch time

                /* outFirstRefreshStartTime */ nullptr, /* outLastRefreshStartTime */ nullptr,

                /* outGpuCompositionDoneTime */ nullptr,

                mHasPresentFenceTimes ? &latchOrPresentTimeNs : nullptr, // display present time,

                /* outDequeueReadyTime */ nullptr, /* outReleaseTime */ nullptr);

        if (err) {

            ALOGE("Failed to get frame timestamps for %lld: %d", (long long) frame.id, err);

        }

        // If we don't have a render time by now, then consider the frame as dropped

        if (latchOrPresentTimeNs != NATIVE_WINDOW_TIMESTAMP_PENDING &&

            latchOrPresentTimeNs != NATIVE_WINDOW_TIMESTAMP_INVALID) {

            renderedFrameInfos.push_back(RenderedFrameInfo(frame.mediaTimeUs,

                                                           latchOrPresentTimeNs));

        }



        mTrackedFrames.pop_front();

    }



    if (!renderedFrameInfos.empty()) {

        mCallback->onOutputFramesRendered(renderedFrameInfos);

    }

}



status_t ACodec::freeBuffersOnPort(OMX_U32 portIndex) {

    if (portIndex == kPortIndexInput) {

        mBufferChannel->setInputBufferArray({});
@@ -1661,11 +1704,6 @@ status_t ACodec::freeBuffer(OMX_U32 portIndex, size_t i) { 
        ::close(info->mFenceFd);

    }



    if (portIndex == kPortIndexOutput) {

        mRenderTracker.untrackFrame(info->mRenderInfo, i);

        info->mRenderInfo = NULL;

    }



    // remove buffer even if mOMXNode->freeBuffer fails

    mBuffers[portIndex].erase(mBuffers[portIndex].begin() + i);

    return err;
@@ -6030,22 +6068,10 @@ bool ACodec::BaseState::onOMXMessageList(const sp<AMessage> &msg) { 
    sp<RefBase> obj;

    CHECK(msg->findObject("messages", &obj));

    sp<MessageList> msgList = static_cast<MessageList *>(obj.get());



    bool receivedRenderedEvents = false;

    for (std::list<sp<AMessage>>::const_iterator it = msgList->getList().cbegin();

          it != msgList->getList().cend(); ++it) {

        (*it)->setWhat(ACodec::kWhatOMXMessageItem);

        mCodec->handleMessage(*it);

        int32_t type;

        CHECK((*it)->findInt32("type", &type));

        if (type == omx_message::FRAME_RENDERED) {

            receivedRenderedEvents = true;

        }

    }



    if (receivedRenderedEvents) {

        // NOTE: all buffers are rendered in this case

        mCodec->notifyOfRenderedFrames();

    }

    return true;

}
@@ -6554,15 +6580,6 @@ bool ACodec::BaseState::onOMXFillBufferDone( 
    info->mDequeuedAt = ++mCodec->mDequeueCounter;

    info->mStatus = BufferInfo::OWNED_BY_US;



    if (info->mRenderInfo != NULL) {

        // The fence for an emptied buffer must have signaled, but there still could be queued

        // or out-of-order dequeued buffers in the render queue prior to this buffer. Drop these,

        // as we will soon requeue this buffer to the surface. While in theory we could still keep

        // track of buffers that are requeued to the surface, it is better to add support to the

        // buffer-queue to notify us of released buffers and their fences (in the future).

        mCodec->notifyOfRenderedFrames(true /* dropIncomplete */);

    }



    // byte buffers cannot take fences, so wait for any fence now

    if (mCodec->mNativeWindow == NULL) {

        (void)mCodec->waitForFence(fenceFd, "onOMXFillBufferDone");
@@ -6769,14 +6786,6 @@ void ACodec::BaseState::onOutputBufferDrained(const sp<AMessage> &msg) { 
            mCodec->mLastHdr10PlusBuffer = hdr10PlusInfo;

        }



        // save buffers sent to the surface so we can get render time when they return

        int64_t mediaTimeUs = -1;

        buffer->meta()->findInt64("timeUs", &mediaTimeUs);

        if (mediaTimeUs >= 0) {

            mCodec->mRenderTracker.onFrameQueued(

                    mediaTimeUs, info->mGraphicBuffer, new Fence(::dup(info->mFenceFd)));

        }



        int64_t timestampNs = 0;

        if (!msg->findInt64("timestampNs", &timestampNs)) {

            // use media timestamp if client did not request a specific render timestamp
@@ -6790,11 +6799,25 @@ void ACodec::BaseState::onOutputBufferDrained(const sp<AMessage> &msg) { 
        err = native_window_set_buffers_timestamp(mCodec->mNativeWindow.get(), timestampNs);

        ALOGW_IF(err != NO_ERROR, "failed to set buffer timestamp: %d", err);



        uint64_t frameId;

        err = native_window_get_next_frame_id(mCodec->mNativeWindow.get(), &frameId);



        info->checkReadFence("onOutputBufferDrained before queueBuffer");

        err = mCodec->mNativeWindow->queueBuffer(

                    mCodec->mNativeWindow.get(), info->mGraphicBuffer.get(), info->mFenceFd);

        // TODO(b/266211548): Poll the native window for rendered buffers, since when queueing

        // buffers, the frame event history delta is retrieved.



        int64_t mediaTimeUs = -1;

        buffer->meta()->findInt64("timeUs", &mediaTimeUs);

        if (mCodec->mIsWindowToDisplay) {

            mCodec->trackReleasedFrame(frameId, mediaTimeUs, timestampNs);

            mCodec->pollForRenderedFrames();

        } else {

            // When the surface is an intermediate surface, onFrameRendered is triggered immediately

            // when the frame is queued to the non-display surface

            mCodec->mCallback->onOutputFramesRendered({RenderedFrameInfo(mediaTimeUs,

                                                                         timestampNs)});

        }



        info->mFenceFd = -1;

        if (err == OK) {

            info->mStatus = BufferInfo::OWNED_BY_NATIVE_WINDOW;
@@ -7021,7 +7044,6 @@ bool ACodec::UninitializedState::onAllocateComponent(const sp<AMessage> &msg) { 
    ++mCodec->mNodeGeneration;



    mCodec->mComponentName = componentName;

    mCodec->mRenderTracker.setComponentName(componentName);

    mCodec->mFlags = 0;



    if (componentName.endsWith(".secure")) {
@@ -7644,7 +7666,6 @@ void ACodec::ExecutingState::resume() { 


void ACodec::ExecutingState::stateEntered() {

    ALOGV("[%s] Now Executing", mCodec->mComponentName.c_str());

    mCodec->mRenderTracker.clear(systemTime(CLOCK_MONOTONIC));

    mCodec->processDeferredMessages();

}
@@ -7755,7 +7776,15 @@ bool ACodec::ExecutingState::onMessageReceived(const sp<AMessage> &msg) { 
                    mCodec->signalSubmitOutputMetadataBufferIfEOS_workaround();

                }

            }

            return true;

            handled = true;

            break;

        }



        case kWhatPollForRenderedBuffers:

        {

            mCodec->pollForRenderedFrames();

            handled = true;

            break;

        }



        default:
@@ -8439,7 +8468,7 @@ void ACodec::forceStateTransition(int generation) { 
}



bool ACodec::ExecutingState::onOMXFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano) {

    mCodec->onFrameRendered(mediaTimeUs, systemNano);

    mCodec->mCallback->onOutputFramesRendered({RenderedFrameInfo(mediaTimeUs, systemNano)});

    return true;

}
@@ -8613,7 +8642,7 @@ void ACodec::OutputPortSettingsChangedState::stateEntered() { 


bool ACodec::OutputPortSettingsChangedState::onOMXFrameRendered(

        int64_t mediaTimeUs, nsecs_t systemNano) {

    mCodec->onFrameRendered(mediaTimeUs, systemNano);

    mCodec->mCallback->onOutputFramesRendered({RenderedFrameInfo(mediaTimeUs, systemNano)});

    return true;

}
@@ -8644,10 +8673,6 @@ bool ACodec::OutputPortSettingsChangedState::onOMXEvent( 
                            OMX_CommandPortEnable, kPortIndexOutput);

                }



                // Clear the RenderQueue in which queued GraphicBuffers hold the

                // actual buffer references in order to free them early.

                mCodec->mRenderTracker.clear(systemTime(CLOCK_MONOTONIC));



                if (err == OK) {

                    err = mCodec->allocateBuffersOnPort(kPortIndexOutput);

                    ALOGE_IF(err != OK, "Failed to allocate output port buffers after port "
@@ -9031,8 +9056,6 @@ void ACodec::FlushingState::changeStateIfWeOwnAllBuffers() { 
        // the native window for rendering. Let's get those back as well.

        mCodec->waitUntilAllPossibleNativeWindowBuffersAreReturnedToUs();



        mCodec->mRenderTracker.clear(systemTime(CLOCK_MONOTONIC));



        mCodec->mCallback->onFlushCompleted();



        mCodec->mPortEOS[kPortIndexInput] =

media/libstagefright/ACodecBufferChannel.cpp

+5 −2
Original line number Diff line number Diff line
@@ -32,6 +32,7 @@ 
#include <media/stagefright/foundation/ABuffer.h>

#include <media/stagefright/foundation/AMessage.h>

#include <media/stagefright/foundation/AUtils.h>

#include <media/stagefright/ACodec.h>

#include <media/stagefright/MediaCodec.h>

#include <media/MediaCodecBuffer.h>

#include <system/window.h>
@@ -87,9 +88,11 @@ ACodecBufferChannel::BufferInfo::BufferInfo( 
}



ACodecBufferChannel::ACodecBufferChannel(

        const sp<AMessage> &inputBufferFilled, const sp<AMessage> &outputBufferDrained)

        const sp<AMessage> &inputBufferFilled, const sp<AMessage> &outputBufferDrained,

        const sp<AMessage> &pollForRenderedBuffers)

    : mInputBufferFilled(inputBufferFilled),

      mOutputBufferDrained(outputBufferDrained),

      mPollForRenderedBuffers(pollForRenderedBuffers),

      mHeapSeqNum(-1) {

}
@@ -488,7 +491,7 @@ status_t ACodecBufferChannel::renderOutputBuffer( 
}



void ACodecBufferChannel::pollForRenderedBuffers() {

    // TODO(b/266211548): Poll the native window for rendered buffers.

    mPollForRenderedBuffers->post();

}



status_t ACodecBufferChannel::discardBuffer(const sp<MediaCodecBuffer> &buffer) {

media/libstagefright/MediaCodec.cpp

+18 −7
Original line number Diff line number Diff line
@@ -79,6 +79,7 @@ 
#include <media/stagefright/MediaErrors.h>

#include <media/stagefright/OMXClient.h>

#include <media/stagefright/PersistentSurface.h>

#include <media/stagefright/RenderedFrameInfo.h>

#include <media/stagefright/SurfaceUtils.h>

#include <nativeloader/dlext_namespaces.h>

#include <private/android_filesystem_config.h>
@@ -750,7 +751,7 @@ public: 
            const sp<AMessage> &outputFormat) override;

    virtual void onInputSurfaceDeclined(status_t err) override;

    virtual void onSignaledInputEOS(status_t err) override;

    virtual void onOutputFramesRendered(const std::list<FrameRenderTracker::Info> &done) override;

    virtual void onOutputFramesRendered(const std::list<RenderedFrameInfo> &done) override;

    virtual void onOutputBuffersChanged() override;

    virtual void onFirstTunnelFrameReady() override;

private:
@@ -859,7 +860,7 @@ void CodecCallback::onSignaledInputEOS(status_t err) { 
    notify->post();

}



void CodecCallback::onOutputFramesRendered(const std::list<FrameRenderTracker::Info> &done) {

void CodecCallback::onOutputFramesRendered(const std::list<RenderedFrameInfo> &done) {

    sp<AMessage> notify(mNotify->dup());

    notify->setInt32("what", kWhatOutputFramesRendered);

    if (MediaCodec::CreateFramesRenderedMessage(done, notify)) {
@@ -5953,12 +5954,10 @@ status_t MediaCodec::handleLeftover(size_t index) { 
    return onQueueInputBuffer(msg);

}



//static

size_t MediaCodec::CreateFramesRenderedMessage(

        const std::list<FrameRenderTracker::Info> &done, sp<AMessage> &msg) {

template<typename T>

static size_t CreateFramesRenderedMessageInternal(const std::list<T> &done, sp<AMessage> &msg) {

    size_t index = 0;

    for (std::list<FrameRenderTracker::Info>::const_iterator it = done.cbegin();

            it != done.cend(); ++it) {

    for (typename std::list<T>::const_iterator it = done.cbegin(); it != done.cend(); ++it) {

        if (it->getRenderTimeNs() < 0) {

            continue; // dropped frame from tracking

        }
@@ -5969,6 +5968,18 @@ size_t MediaCodec::CreateFramesRenderedMessage( 
    return index;

}



//static

size_t MediaCodec::CreateFramesRenderedMessage(

        const std::list<RenderedFrameInfo> &done, sp<AMessage> &msg) {

    return CreateFramesRenderedMessageInternal(done, msg);

}



//static

size_t MediaCodec::CreateFramesRenderedMessage(

        const std::list<FrameRenderTracker::Info> &done, sp<AMessage> &msg) {

    return CreateFramesRenderedMessageInternal(done, msg);

}



status_t MediaCodec::onReleaseOutputBuffer(const sp<AMessage> &msg) {

    size_t index;

    CHECK(msg->findSize("index", &index));

media/libstagefright/include/ACodecBufferChannel.h

+4 −1
Original line number Diff line number Diff line
@@ -29,6 +29,7 @@ 
#include <media/IOMX.h>



namespace android {

 struct ACodec;

namespace hardware {

class HidlMemory;

};
@@ -63,7 +64,8 @@ public: 
    };



    ACodecBufferChannel(

            const sp<AMessage> &inputBufferFilled, const sp<AMessage> &outputBufferDrained);

            const sp<AMessage> &inputBufferFilled, const sp<AMessage> &outputBufferDrained,

            const sp<AMessage> &pollForRenderedBuffers);

    virtual ~ACodecBufferChannel();



    // BufferChannelBase interface
@@ -138,6 +140,7 @@ private: 


    const sp<AMessage> mInputBufferFilled;

    const sp<AMessage> mOutputBufferDrained;

    const sp<AMessage> mPollForRenderedBuffers;



    sp<MemoryDealer> mDealer;

    sp<IMemory> mDecryptDestination;