Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c0a4d1a5 authored by Byeongjo Park's avatar Byeongjo Park Committed by Lajos Molnar
Browse files

VT: Enhancements on RTP depacketizer (2) 



  - Send FIR if there was no I-frame within a sec in packetLost
  - Fixed an issue dropped frame even it is completed. caused by
      ABuffer marked as 'damaged'.
  - Provides completed I-Frame only to prevent codec stuck.
  - Change time mapping as realtime since rtp time is already covered
      by jitter buffer.
  - Removed periodic RTCP:FIR requests if video call. Cause i-frame
      insertion should happened only if video stream is broken.
  - Give default value for uninitialize TrackInfo Value because
    video stucked at the first frame all the times by Uninitialized
    trash TimeScale & NormalPlaytime values make AccessUnit's ntp time
    wrong.

Bug: 165061754
Merged-in: Ied41ea57d4fa96d4c05d7f8eba3d96bcb8f3fc69
Change-Id: Ied41ea57d4fa96d4c05d7f8eba3d96bcb8f3fc69
Signed-off-by: default avatarKim Sungyeon <sy85.kim@samsung.com>
Signed-off-by: default avatarByeongjo Park <bjo.park@samsung.com>
parent 8d05479a

media/libmediaplayerservice/nuplayer/RTPSource.cpp

+12 −2
Original line number Diff line number Diff line
@@ -135,11 +135,16 @@ void NuPlayer::RTPSource::prepareAsync() { 


        if (info->mIsAudio) {

            mAudioTrack = source;

            info->mTimeScale = 16000;

        } else {

            mVideoTrack = source;

            info->mTimeScale = 90000;

        }



        info->mSource = source;

        info->mRTPTime = 0;

        info->mNormalPlaytimeUs = 0;

        info->mNPTMappingValid = false;

    }



    if (mInPreparationPhase) {
@@ -346,6 +351,11 @@ void NuPlayer::RTPSource::onPollBuffering() { 
    schedulePollBuffering();

}



bool NuPlayer::RTPSource::isRealTime() const {

    ALOGD("RTPSource::isRealTime=%d", true);

    return true;

}



void NuPlayer::RTPSource::onMessageReceived(const sp<AMessage> &msg) {

    ALOGV("onMessageReceived =%d", msg->what());
@@ -428,7 +438,6 @@ void NuPlayer::RTPSource::onMessageReceived(const sp<AMessage> &msg) { 
                    source->queueAccessUnit(accessUnit);

                    break;

                }

                */



                int64_t nptUs =

                    ((double)rtpTime - (double)info->mRTPTime)
@@ -436,7 +445,8 @@ void NuPlayer::RTPSource::onMessageReceived(const sp<AMessage> &msg) { 
                        * 1000000ll

                        + info->mNormalPlaytimeUs;



                accessUnit->meta()->setInt64("timeUs", nptUs);

                */

                accessUnit->meta()->setInt64("timeUs", ALooper::GetNowUs());



                source->queueAccessUnit(accessUnit);

            }

media/libmediaplayerservice/nuplayer/RTPSource.h

+2 −0
Original line number Diff line number Diff line
@@ -80,6 +80,8 @@ struct NuPlayer::RTPSource : public NuPlayer::Source { 
            int64_t seekTimeUs,

            MediaPlayerSeekMode mode = MediaPlayerSeekMode::SEEK_PREVIOUS_SYNC) override;



    virtual bool isRealTime() const;



    void onMessageReceived(const sp<AMessage> &msg);



    virtual void setTargetBitrate(int32_t bitrate) override;

media/libstagefright/rtsp/AAVCAssembler.cpp

+45 −16
Original line number Diff line number Diff line
@@ -37,7 +37,9 @@ AAVCAssembler::AAVCAssembler(const sp<AMessage> &notify) 
      mAccessUnitRTPTime(0),

      mNextExpectedSeqNoValid(false),

      mNextExpectedSeqNo(0),

      mAccessUnitDamaged(false) {

      mAccessUnitDamaged(false),

      mFirstIFrameProvided(false),

      mLastIFrameProvidedAt(0) {

}



AAVCAssembler::~AAVCAssembler() {
@@ -218,12 +220,27 @@ ARTPAssembler::AssemblyStatus AAVCAssembler::addNALUnit( 
    }

}



void AAVCAssembler::checkIFrameProvided(const sp<ABuffer> &buffer) {

    const uint8_t *data = buffer->data();

    unsigned nalType = data[0] & 0x1f;

    if (nalType == 0x5) {

        mFirstIFrameProvided = true;

        mLastIFrameProvidedAt = ALooper::GetNowUs() / 1000;



        uint32_t rtpTime;

        CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));

        ALOGD("got First I-frame to be decoded. rtpTime=%d, size=%zu", rtpTime, buffer->size());

    }

}



void AAVCAssembler::addSingleNALUnit(const sp<ABuffer> &buffer) {

    ALOGV("addSingleNALUnit of size %zu", buffer->size());

#if !LOG_NDEBUG

    hexdump(buffer->data(), buffer->size());

#endif



    checkIFrameProvided(buffer);



    uint32_t rtpTime;

    CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));
@@ -324,6 +341,8 @@ ARTPAssembler::AssemblyStatus AAVCAssembler::addFragmentedNALUnit( 
        complete = true;

    } else {

        List<sp<ABuffer> >::iterator it = ++queue->begin();

        int32_t connected = 1;

        bool snapped = false;

        while (it != queue->end()) {

            ALOGV("sequence length %zu", totalCount);
@@ -333,13 +352,17 @@ ARTPAssembler::AssemblyStatus AAVCAssembler::addFragmentedNALUnit( 
            size_t size = buffer->size();



            if ((uint32_t)buffer->int32Data() != expectedSeqNo) {

                ALOGV("sequence not complete, expected seqNo %d, got %d, pFrame %d",

                     expectedSeqNo, (uint32_t)buffer->int32Data(), pFrame);

                ALOGV("sequence not complete, expected seqNo %d, got %d, nalType %d",

                     expectedSeqNo, (uint32_t)buffer->int32Data(), nalType);

                snapped = true;



                if (!pFrame)

                    return WRONG_SEQUENCE_NUMBER;

            }



            if (!snapped)

                connected++;



            uint32_t rtpTime;

            CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));

            if (size < 2
@@ -363,12 +386,14 @@ ARTPAssembler::AssemblyStatus AAVCAssembler::addFragmentedNALUnit( 
            expectedSeqNo = buffer->int32Data() + 1;



            if (data[1] & 0x40) {

                if (pFrame && !recycleUnit(startSeqNo, expectedSeqNo, totalCount, 0.5f)) {

                if (pFrame && !recycleUnit(startSeqNo, expectedSeqNo,

                            connected, totalCount, 0.5f)) {

                    mNextExpectedSeqNo = expectedSeqNo;

                    deleteUnitUnderSeq(queue, mNextExpectedSeqNo);



                    return MALFORMED_PACKET;

                }



                // This is the last fragment.

                complete = true;

                break;
@@ -493,13 +518,15 @@ int32_t AAVCAssembler::pickProperSeq(const Q *q, uint32_t jit, int64_t play) { 
    return nextSeqNo;

}



bool AAVCAssembler::recycleUnit(uint32_t start, uint32_t end, size_t avail, float goodRatio) {

bool AAVCAssembler::recycleUnit(uint32_t start, uint32_t end, uint32_t connected,

        size_t avail, float goodRatio) {

    float total = end - start;

    float valid = connected;

    float exist = avail;

    bool isRecycle = (exist / total) >= goodRatio;

    bool isRecycle = (valid / total) >= goodRatio;



    ALOGV("checking p-frame losses.. recvBufs %f diff %f recycle? %d",

            exist, total, isRecycle);

    ALOGV("checking p-frame losses.. recvBufs %f valid %f diff %f recycle? %d",

            exist, valid, total, isRecycle);



    return isRecycle;

}
@@ -531,18 +558,20 @@ ARTPAssembler::AssemblyStatus AAVCAssembler::assembleMore( 
        const sp<ARTPSource> &source) {

    AssemblyStatus status = addNALUnit(source);

    if (status == MALFORMED_PACKET) {

        mAccessUnitDamaged = true;

        uint64_t timeAfterLastIFrame = (ALooper::GetNowUs() / 1000) - mLastIFrameProvidedAt;

        if (timeAfterLastIFrame > 1000) {

            ALOGV("request FIR to get a new I-Frame, time after "

                    "last I-Frame in miils %llu", (unsigned long long)timeAfterLastIFrame);

            source->onIssueFIRByAssembler();

        }

    }

    return status;

}



void AAVCAssembler::packetLost() {

    CHECK(mNextExpectedSeqNoValid);

    ALOGV("packetLost (expected %d)", mNextExpectedSeqNo);



    ALOGD("packetLost (expected %d)", mNextExpectedSeqNo);

    ++mNextExpectedSeqNo;



    mAccessUnitDamaged = true;

}



void AAVCAssembler::onByeReceived() {

media/libstagefright/rtsp/AAVCAssembler.h

+5 −1
Original line number Diff line number Diff line
@@ -47,9 +47,12 @@ private: 
    bool mNextExpectedSeqNoValid;

    uint32_t mNextExpectedSeqNo;

    bool mAccessUnitDamaged;

    bool mFirstIFrameProvided;

    uint64_t mLastIFrameProvidedAt;

    List<sp<ABuffer> > mNALUnits;



    int32_t addNack(const sp<ARTPSource> &source);

    void checkIFrameProvided(const sp<ABuffer> &buffer);

    AssemblyStatus addNALUnit(const sp<ARTPSource> &source);

    void addSingleNALUnit(const sp<ABuffer> &buffer);

    AssemblyStatus addFragmentedNALUnit(List<sp<ABuffer> > *queue);
@@ -58,7 +61,8 @@ private: 
    void submitAccessUnit();



    int32_t pickProperSeq(const Q *q, uint32_t jit, int64_t play);

    bool recycleUnit(uint32_t start, uint32_t end, size_t avail, float goodRatio);

    bool recycleUnit(uint32_t start, uint32_t end, uint32_t conneceted,

            size_t avail, float goodRatio);

    int32_t deleteUnitUnderSeq(Q *q, uint32_t seq);

    void printNowTimeUs(int64_t start, int64_t now, int64_t play);

    void printRTPTime(uint32_t rtp, int64_t play, uint32_t exp, bool isExp);

media/libstagefright/rtsp/ARTPConnection.cpp

+11 −1
Original line number Diff line number Diff line
@@ -448,7 +448,7 @@ void ARTPConnection::onPollStreams() { 
                continue;

            }



            // addNACK

            // add NACK and FIR that needs to be sent immediately.

            sp<ABuffer> buffer = new ABuffer(kMaxUDPSize);

            for (size_t i = 0; i < it->mSources.size(); ++i) {

                buffer->setRange(0, 0);
@@ -457,6 +457,13 @@ void ARTPConnection::onPollStreams() { 
                    ALOGV("Send NACK for lost %d Packets", cnt);

                    send(&*it, buffer);

                }



                buffer->setRange(0, 0);

                it->mSources.valueAt(i)->addFIR(buffer);

                if (buffer->size() > 0) {

                    ALOGD("Send FIR immediately for lost Packets");

                    send(&*it, buffer);

                }

            }



            ++it;
@@ -1032,6 +1039,9 @@ sp<ARTPSource> ARTPConnection::findSource(StreamInfo *info, uint32_t srcId) { 
        source = new ARTPSource(

                srcId, info->mSessionDesc, info->mIndex, info->mNotifyMsg);



        if (mFlags & kViLTEConnection)

            source->setPeriodicFIR(false);



        source->setSelfID(mSelfID);

        source->setJbTime(mJbTime > 0 ? mJbTime : 300);

        info->mSources.add(srcId, source);