Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 8bd4d16a authored by Chong Zhang's avatar Chong Zhang
Browse files

Cap pts gap between adjacent frames to specified value 

- In the scenario of cast mirroring, encoding could be suspended
  for prolonged periods. Limiting the pts gap to workaround the
  problem where encoder's rate control logic produces huge frames
  after a long period of suspension.

- Repeat last frame a couple more times to get better quality
  on static scenes.

- Fix the timestamp on repeat frames (it was not set)

Bug: 11971963
Change-Id: I1d68ab3d269874bf3921aa429a985c5f63e428c7
(cherry picked from commit 94ee4b70)
parent bc69c8ba

include/media/IOMX.h

+1 −0
Original line number Diff line number Diff line
@@ -142,6 +142,7 @@ public: 
    enum InternalOptionType {

        INTERNAL_OPTION_SUSPEND,  // data is a bool

        INTERNAL_OPTION_REPEAT_PREVIOUS_FRAME_DELAY,  // data is an int64_t

        INTERNAL_OPTION_MAX_TIMESTAMP_GAP, // data is int64_t

    };

    virtual status_t setInternalOption(

            node_id node,

include/media/stagefright/ACodec.h

+1 −0
Original line number Diff line number Diff line
@@ -205,6 +205,7 @@ private: 
    int32_t mMetaDataBuffersToSubmit;



    int64_t mRepeatFrameDelayUs;

    int64_t mMaxPtsGapUs;



    status_t setCyclicIntraMacroblockRefresh(const sp<AMessage> &msg, int32_t mode);

    status_t allocateBuffersOnPort(OMX_U32 portIndex);

media/libstagefright/ACodec.cpp

+21 −1
Original line number Diff line number Diff line
@@ -371,7 +371,8 @@ ACodec::ACodec() 
      mDequeueCounter(0),

      mStoreMetaDataInOutputBuffers(false),

      mMetaDataBuffersToSubmit(0),

      mRepeatFrameDelayUs(-1ll) {

      mRepeatFrameDelayUs(-1ll),

      mMaxPtsGapUs(-1l) {

    mUninitializedState = new UninitializedState(this);

    mLoadedState = new LoadedState(this);

    mLoadedToIdleState = new LoadedToIdleState(this);
@@ -1109,6 +1110,10 @@ status_t ACodec::configureCodec( 
                    &mRepeatFrameDelayUs)) {

            mRepeatFrameDelayUs = -1ll;

        }



        if (!msg->findInt64("max-pts-gap-to-encoder", &mMaxPtsGapUs)) {

            mMaxPtsGapUs = -1l;

        }

    }



    // Always try to enable dynamic output buffers on native surface
@@ -3855,6 +3860,21 @@ void ACodec::LoadedState::onCreateInputSurface( 
        }

    }



    if (err == OK && mCodec->mMaxPtsGapUs > 0l) {

        err = mCodec->mOMX->setInternalOption(

                mCodec->mNode,

                kPortIndexInput,

                IOMX::INTERNAL_OPTION_MAX_TIMESTAMP_GAP,

                &mCodec->mMaxPtsGapUs,

                sizeof(mCodec->mMaxPtsGapUs));



        if (err != OK) {

            ALOGE("[%s] Unable to configure max timestamp gap (err %d)",

                  mCodec->mComponentName.c_str(),

                  err);

        }

    }



    if (err == OK) {

        notify->setObject("input-surface",

                new BufferProducerWrapper(bufferProducer));

media/libstagefright/omx/GraphicBufferSource.cpp

+99 −1
Original line number Diff line number Diff line
@@ -42,7 +42,11 @@ GraphicBufferSource::GraphicBufferSource(OMXNodeInstance* nodeInstance, 
    mEndOfStream(false),

    mEndOfStreamSent(false),

    mRepeatAfterUs(-1ll),

    mMaxTimestampGapUs(-1ll),

    mPrevOriginalTimeUs(-1ll),

    mPrevModifiedTimeUs(-1ll),

    mRepeatLastFrameGeneration(0),

    mRepeatLastFrameTimestamp(-1ll),

    mLatestSubmittedBufferId(-1),

    mLatestSubmittedBufferFrameNum(0),

    mLatestSubmittedBufferUseCount(0),
@@ -299,6 +303,32 @@ void GraphicBufferSource::codecBufferEmptied(OMX_BUFFERHEADERTYPE* header) { 
    return;

}



void GraphicBufferSource::codecBufferFilled(OMX_BUFFERHEADERTYPE* header) {

    Mutex::Autolock autoLock(mMutex);



    if (mMaxTimestampGapUs > 0ll

            && !(header->nFlags & OMX_BUFFERFLAG_CODECCONFIG)) {

        ssize_t index = mOriginalTimeUs.indexOfKey(header->nTimeStamp);

        if (index >= 0) {

            ALOGV("OUT timestamp: %lld -> %lld",

                    header->nTimeStamp, mOriginalTimeUs[index]);

            header->nTimeStamp = mOriginalTimeUs[index];

            mOriginalTimeUs.removeItemsAt(index);

        } else {

            // giving up the effort as encoder doesn't appear to preserve pts

            ALOGW("giving up limiting timestamp gap (pts = %lld)",

                    header->nTimeStamp);

            mMaxTimestampGapUs = -1ll;

        }

        if (mOriginalTimeUs.size() > BufferQueue::NUM_BUFFER_SLOTS) {

            // something terribly wrong must have happened, giving up...

            ALOGE("mOriginalTimeUs has too many entries (%d)",

                    mOriginalTimeUs.size());

            mMaxTimestampGapUs = -1ll;

        }

    }

}



void GraphicBufferSource::suspend(bool suspend) {

    Mutex::Autolock autoLock(mMutex);
@@ -431,6 +461,7 @@ bool GraphicBufferSource::repeatLatestSubmittedBuffer_l() { 
    BufferQueue::BufferItem item;

    item.mBuf = mLatestSubmittedBufferId;

    item.mFrameNumber = mLatestSubmittedBufferFrameNum;

    item.mTimestamp = mRepeatLastFrameTimestamp;



    status_t err = submitBuffer_l(item, cbi);
@@ -440,6 +471,20 @@ bool GraphicBufferSource::repeatLatestSubmittedBuffer_l() { 


    ++mLatestSubmittedBufferUseCount;



    /* repeat last frame up to kRepeatLastFrameCount times.

     * in case of static scene, a single repeat might not get rid of encoder

     * ghosting completely, refresh a couple more times to get better quality

     */

    if (--mRepeatLastFrameCount > 0) {

        mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000;



        if (mReflector != NULL) {

            sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector->id());

            msg->setInt32("generation", ++mRepeatLastFrameGeneration);

            msg->post(mRepeatAfterUs);

        }

    }



    return true;

}
@@ -460,8 +505,11 @@ void GraphicBufferSource::setLatestSubmittedBuffer_l( 


    mLatestSubmittedBufferId = item.mBuf;

    mLatestSubmittedBufferFrameNum = item.mFrameNumber;

    mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000;



    mLatestSubmittedBufferUseCount = 1;

    mRepeatBufferDeferred = false;

    mRepeatLastFrameCount = kRepeatLastFrameCount;



    if (mReflector != NULL) {

        sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector->id());
@@ -497,6 +545,39 @@ status_t GraphicBufferSource::signalEndOfInputStream() { 
    return OK;

}



int64_t GraphicBufferSource::getTimestamp(const BufferQueue::BufferItem &item) {

    int64_t timeUs = item.mTimestamp / 1000;



    if (mMaxTimestampGapUs > 0ll) {

        /* Cap timestamp gap between adjacent frames to specified max

         *

         * In the scenario of cast mirroring, encoding could be suspended for

         * prolonged periods. Limiting the pts gap to workaround the problem

         * where encoder's rate control logic produces huge frames after a

         * long period of suspension.

         */



        int64_t originalTimeUs = timeUs;

        if (mPrevOriginalTimeUs >= 0ll) {

            if (originalTimeUs < mPrevOriginalTimeUs) {

                // Drop the frame if it's going backward in time. Bad timestamp

                // could disrupt encoder's rate control completely.

                ALOGV("Dropping frame that's going backward in time");

                return -1;

            }

            int64_t timestampGapUs = originalTimeUs - mPrevOriginalTimeUs;

            timeUs = (timestampGapUs < mMaxTimestampGapUs ?

                    timestampGapUs : mMaxTimestampGapUs) + mPrevModifiedTimeUs;

        }

        mPrevOriginalTimeUs = originalTimeUs;

        mPrevModifiedTimeUs = timeUs;

        mOriginalTimeUs.add(timeUs, originalTimeUs);

        ALOGV("IN  timestamp: %lld -> %lld", originalTimeUs, timeUs);

    }



    return timeUs;

}



status_t GraphicBufferSource::submitBuffer_l(

        const BufferQueue::BufferItem &item, int cbi) {

    ALOGV("submitBuffer_l cbi=%d", cbi);
@@ -513,9 +594,15 @@ status_t GraphicBufferSource::submitBuffer_l( 
    memcpy(data, &type, 4);

    memcpy(data + 4, &handle, sizeof(buffer_handle_t));



    int64_t timeUs = getTimestamp(item);

    if (timeUs < 0ll) {

        ALOGE("Dropping frame with bad timestamp");

        return UNKNOWN_ERROR;

    }



    status_t err = mNodeInstance->emptyDirectBuffer(header, 0,

            4 + sizeof(buffer_handle_t), OMX_BUFFERFLAG_ENDOFFRAME,

            item.mTimestamp / 1000);

            timeUs);

    if (err != OK) {

        ALOGW("WARNING: emptyDirectBuffer failed: 0x%x", err);

        codecBuffer.mGraphicBuffer = NULL;
@@ -658,6 +745,17 @@ status_t GraphicBufferSource::setRepeatPreviousFrameDelayUs( 
    return OK;

}



status_t GraphicBufferSource::setMaxTimestampGapUs(int64_t maxGapUs) {

    Mutex::Autolock autoLock(mMutex);



    if (mExecuting || maxGapUs <= 0ll) {

        return INVALID_OPERATION;

    }



    mMaxTimestampGapUs = maxGapUs;



    return OK;

}

void GraphicBufferSource::onMessageReceived(const sp<AMessage> &msg) {

    switch (msg->what()) {

        case kWhatRepeatLastFrame:

media/libstagefright/omx/GraphicBufferSource.h

+24 −1
Original line number Diff line number Diff line
@@ -87,6 +87,10 @@ public: 
    // fill it with a new frame of data; otherwise, just mark it as available.

    void codecBufferEmptied(OMX_BUFFERHEADERTYPE* header);



    // Called when omx_message::FILL_BUFFER_DONE is received. (Currently the

    // buffer source will fix timestamp in the header if needed.)

    void codecBufferFilled(OMX_BUFFERHEADERTYPE* header);



    // This is called after the last input frame has been submitted.  We

    // need to submit an empty buffer with the EOS flag set.  If we don't

    // have a codec buffer ready, we just set the mEndOfStream flag.
@@ -105,6 +109,15 @@ public: 
    // state and once this behaviour is specified it cannot be reset.

    status_t setRepeatPreviousFrameDelayUs(int64_t repeatAfterUs);



    // When set, the timestamp fed to the encoder will be modified such that

    // the gap between two adjacent frames is capped at maxGapUs. Timestamp

    // will be restored to the original when the encoded frame is returned to

    // the client.

    // This is to solve a problem in certain real-time streaming case, where

    // encoder's rate control logic produces huge frames after a long period

    // of suspension on input.

    status_t setMaxTimestampGapUs(int64_t maxGapUs);



protected:

    // BufferQueue::ConsumerListener interface, called when a new frame of

    // data is available.  If we're executing and a codec buffer is
@@ -165,6 +178,7 @@ private: 


    void setLatestSubmittedBuffer_l(const BufferQueue::BufferItem &item);

    bool repeatLatestSubmittedBuffer_l();

    int64_t getTimestamp(const BufferQueue::BufferItem &item);



    // Lock, covers all member variables.

    mutable Mutex mMutex;
@@ -206,13 +220,22 @@ private: 
    enum {

        kWhatRepeatLastFrame,

    };



    enum {

        kRepeatLastFrameCount = 10,

    };

    int64_t mRepeatAfterUs;

    int64_t mMaxTimestampGapUs;



    KeyedVector<int64_t, int64_t> mOriginalTimeUs;

    int64_t mPrevOriginalTimeUs;

    int64_t mPrevModifiedTimeUs;



    sp<ALooper> mLooper;

    sp<AHandlerReflector<GraphicBufferSource> > mReflector;



    int32_t mRepeatLastFrameGeneration;

    int64_t mRepeatLastFrameTimestamp;

    int32_t mRepeatLastFrameCount;



    int mLatestSubmittedBufferId;

    uint64_t mLatestSubmittedBufferFrameNum;