Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit ce872bde authored by Kim Sungyeon's avatar Kim Sungyeon
Browse files

VT: Introduce dynamic jitter buffer for RTP source 



A jitter calculator decides how much time is proper jitter time.
This helps to have shorter buffering time as per network status.

The formula of the calculation is defined in 6.4.1 of RFC3550.

* Added unit postfix 'Ms' for all variables related to the jitter time.
** Unified all default jitter value to kStaticJitterTimeMs.
*** Now a RR includes interarrival jitter time information.

Bug: 183578712

Change-Id: I9c442a801029680fc6c5227a14c243f13c89e3fa
Signed-off-by: default avatarKim Sungyeon <sy85.kim@samsung.com>
parent 274e3534

media/libmediaplayerservice/nuplayer/RTPSource.cpp

+13 −6
Original line number Original line Diff line number Diff line
@@ -124,8 +124,16 @@ void NuPlayer::RTPSource::prepareAsync() { 
        // index(i) should be started from 1. 0 is reserved for [root]

        // index(i) should be started from 1. 0 is reserved for [root]

        mRTPConn->addStream(sockRtp, sockRtcp, desc, i + 1, notify, false);

        mRTPConn->addStream(sockRtp, sockRtcp, desc, i + 1, notify, false);

        mRTPConn->setSelfID(info->mSelfID);

        mRTPConn->setSelfID(info->mSelfID);

        mRTPConn->setJbTime(

        mRTPConn->setStaticJitterTimeMs(info->mJbTimeMs);

                (info->mJbTimeMs <= 3000 && info->mJbTimeMs >= 40) ? info->mJbTimeMs : 300);



        unsigned long PT;

        AString formatDesc, formatParams;

        // index(i) should be started from 1. 0 is reserved for [root]

        desc->getFormatType(i + 1, &PT, &formatDesc, &formatParams);



        int32_t clockRate, numChannels;

        ASessionDescription::ParseFormatDesc(formatDesc.c_str(), &clockRate, &numChannels);

        info->mTimeScale = clockRate;





        info->mRTPSocket = sockRtp;

        info->mRTPSocket = sockRtp;

        info->mRTCPSocket = sockRtcp;

        info->mRTCPSocket = sockRtcp;
@@ -146,10 +154,8 @@ void NuPlayer::RTPSource::prepareAsync() { 




        if (info->mIsAudio) {

        if (info->mIsAudio) {

            mAudioTrack = source;

            mAudioTrack = source;

            info->mTimeScale = 16000;

        } else {

        } else {

            mVideoTrack = source;

            mVideoTrack = source;

            info->mTimeScale = 90000;

        }

        }





        info->mSource = source;

        info->mSource = source;
@@ -680,7 +686,7 @@ status_t NuPlayer::RTPSource::setParameter(const String8 &key, const String8 &va 
        newTrackInfo.mIsAudio = isAudioKey;

        newTrackInfo.mIsAudio = isAudioKey;

        mTracks.push(newTrackInfo);

        mTracks.push(newTrackInfo);

        info = &mTracks.editTop();

        info = &mTracks.editTop();

        info->mJbTimeMs = 300;

        info->mJbTimeMs = kStaticJitterTimeMs;

    }

    }





    if (key == "rtp-param-mime-type") {

    if (key == "rtp-param-mime-type") {
@@ -724,7 +730,8 @@ status_t NuPlayer::RTPSource::setParameter(const String8 &key, const String8 &va 
        int64_t networkHandle = atoll(value);

        int64_t networkHandle = atoll(value);

        setSocketNetwork(networkHandle);

        setSocketNetwork(networkHandle);

    } else if (key == "rtp-param-jitter-buffer-time") {

    } else if (key == "rtp-param-jitter-buffer-time") {

        info->mJbTimeMs = atoi(value);

        // clamping min at 40, max at 3000

        info->mJbTimeMs = std::min(std::max(40, atoi(value)), 3000);

    }

    }





    return OK;

    return OK;

media/libstagefright/rtsp/AAVCAssembler.cpp

+59 −45
Original line number Original line Diff line number Diff line
@@ -34,6 +34,8 @@ 




namespace android {

namespace android {





const double JITTER_MULTIPLE = 1.5f;



// static

// static

AAVCAssembler::AAVCAssembler(const sp<AMessage> &notify)

AAVCAssembler::AAVCAssembler(const sp<AMessage> &notify)

    : mNotifyMsg(notify),

    : mNotifyMsg(notify),
@@ -123,22 +125,48 @@ ARTPAssembler::AssemblyStatus AAVCAssembler::addNALUnit( 




    int64_t rtpTime = findRTPTime(firstRTPTime, buffer);

    int64_t rtpTime = findRTPTime(firstRTPTime, buffer);





    int64_t startTime = source->mFirstSysTime / 1000;

    const int64_t startTimeMs = source->mFirstSysTime / 1000;

    int64_t nowTime = ALooper::GetNowUs() / 1000;

    const int64_t nowTimeMs = ALooper::GetNowUs() / 1000;

    int64_t playedTime = nowTime - startTime;

    const int64_t staticJbTimeMs = source->getStaticJitterTimeMs();

    const int64_t dynamicJbTimeMs = source->getDynamicJitterTimeMs();

    const int64_t clockRate = source->mClockRate;



    int64_t playedTimeMs = nowTimeMs - startTimeMs;

    int64_t playedTimeRtp = source->mFirstRtpTime + MsToRtp(playedTimeMs, clockRate);



    /**

     * Based on experience in real commercial network services,

     * 300 ms is a maximum heuristic jitter buffer time for video RTP service.

     */



    /**

     * The static(base) jitter is a kind of expected propagation time that we desire.

     * We can drop packets if it doesn't meet our standards.

     * If it gets shorter we can get faster response but can lose packets.

     * Expecting range : 50ms ~ 1000ms (But 300 ms would be practical upper bound)

     */

    const int64_t baseJbTimeRtp = MsToRtp(staticJbTimeMs, clockRate);

    /**

     * Dynamic jitter is a variance of interarrival time as defined in the 6.4.1 of RFC 3550.

     * We can regard this as a tolerance of every moments.

     * Expecting range : 0ms ~ 150ms (Not to over 300 ms practically)

     */

    const int64_t dynamicJbTimeRtp =                        // Max 150

            std::min(MsToRtp(dynamicJbTimeMs, clockRate), MsToRtp(150, clockRate));

    const int64_t jitterTimeRtp = baseJbTimeRtp + dynamicJbTimeRtp; // Total jitter time





    int64_t playedTimeRtp = source->mFirstRtpTime + playedTime * (int64_t)source->mClockRate / 1000;

    int64_t expiredTimeRtp = rtpTime + jitterTimeRtp;       // When does this buffer expire ? (T)

    const int64_t jitterTime = source->mJbTimeMs * (int64_t)source->mClockRate / 1000;

    int64_t diffTimeRtp = playedTimeRtp - expiredTimeRtp;

    bool isExpired = (diffTimeRtp >= 0);                    // It's expired if T is passed away

    bool isFirstLineBroken = (diffTimeRtp > jitterTimeRtp); // (T + jitter) is a standard tolerance





    int64_t expiredTimeInJb = rtpTime + jitterTime;

    int64_t finalMargin = dynamicJbTimeRtp * JITTER_MULTIPLE;

    bool isExpired = expiredTimeInJb <= (playedTimeRtp);

    bool isSecondLineBroken = (diffTimeRtp > jitterTimeRtp + finalMargin); // The Maginot line

    bool isTooLate200 = expiredTimeInJb < (playedTimeRtp - jitterTime);

    bool isTooLate300 = expiredTimeInJb < (playedTimeRtp - (jitterTime * 3 / 2));





    if (mShowQueue && mShowQueueCnt < 20) {

    if (mShowQueue && mShowQueueCnt < 20) {

        showCurrentQueue(queue);

        showCurrentQueue(queue);

        printNowTimeUs(startTime, nowTime, playedTime);

        printNowTimeMs(startTimeMs, nowTimeMs, playedTimeMs);

        printRTPTime(rtpTime, playedTimeRtp, expiredTimeInJb, isExpired);

        printRTPTime(rtpTime, playedTimeRtp, expiredTimeRtp, isExpired);

        mShowQueueCnt++;

        mShowQueueCnt++;

    }

    }
@@ -149,17 +177,23 @@ ARTPAssembler::AssemblyStatus AAVCAssembler::addNALUnit( 
        return NOT_ENOUGH_DATA;

        return NOT_ENOUGH_DATA;

    }

    }





    if (isTooLate200) {

    if (isFirstLineBroken) {

        ALOGW("=== WARNING === buffer arrived 200ms late. === WARNING === ");

        if (isSecondLineBroken) {

    }

            ALOGW("buffer too late ... \t Diff in Jb=%lld \t "

                    "Seq# %d \t ExpSeq# %d \t"

                    "JitterMs %lld + (%lld * %.3f)",

                    (long long)(diffTimeRtp),

                    buffer->int32Data(), mNextExpectedSeqNo,

                    (long long)staticJbTimeMs, (long long)dynamicJbTimeMs, JITTER_MULTIPLE + 1);

            printNowTimeMs(startTimeMs, nowTimeMs, playedTimeMs);

            printRTPTime(rtpTime, playedTimeRtp, expiredTimeRtp, isExpired);





    if (isTooLate300) {

            mNextExpectedSeqNo = pickProperSeq(queue, firstRTPTime, playedTimeRtp, jitterTimeRtp);

        ALOGW("buffer arrived after 300ms ... \t Diff in Jb=%lld \t Seq# %d",

        }  else {

                (long long)(playedTimeRtp - expiredTimeInJb), buffer->int32Data());

            ALOGW("=== WARNING === buffer arrived after %lld + %lld = %lld ms === WARNING === ",

        printNowTimeUs(startTime, nowTime, playedTime);

                    (long long)staticJbTimeMs, (long long)dynamicJbTimeMs,

        printRTPTime(rtpTime, playedTimeRtp, expiredTimeInJb, isExpired);

                    (long long)RtpToMs(jitterTimeRtp, clockRate));



        }

        mNextExpectedSeqNo = pickProperSeq(queue, firstRTPTime, playedTimeRtp, jitterTime);

    }

    }





    if (mNextExpectedSeqNoValid) {

    if (mNextExpectedSeqNoValid) {
@@ -170,6 +204,7 @@ ARTPAssembler::AssemblyStatus AAVCAssembler::addNALUnit( 
            source->noticeAbandonBuffer(cntRemove);

            source->noticeAbandonBuffer(cntRemove);

            ALOGW("delete %d of %d buffers", cntRemove, size);

            ALOGW("delete %d of %d buffers", cntRemove, size);

        }

        }



        if (queue->empty()) {

        if (queue->empty()) {

            return NOT_ENOUGH_DATA;

            return NOT_ENOUGH_DATA;

        }

        }
@@ -565,17 +600,6 @@ void AAVCAssembler::submitAccessUnit() { 
    msg->post();

    msg->post();

}

}





inline int64_t AAVCAssembler::findRTPTime(

        const uint32_t& firstRTPTime, const sp<ABuffer>& buffer) {

    /* If you want to +, -, * rtpTime, recommend to declare rtpTime as int64_t.

       Because rtpTime can be near UINT32_MAX. Beware the overflow. */

    int64_t rtpTime = 0;

    CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));

    // If the first overs 2^31 and rtp unders 2^31, the rtp value is overflowed one.

    int64_t overflowMask = (firstRTPTime & 0x80000000 & ~rtpTime) << 1;

    return rtpTime | overflowMask;

}



int32_t AAVCAssembler::pickProperSeq(const Queue *queue,

int32_t AAVCAssembler::pickProperSeq(const Queue *queue,

        uint32_t first, int64_t play, int64_t jit) {

        uint32_t first, int64_t play, int64_t jit) {

    sp<ABuffer> buffer = *(queue->begin());

    sp<ABuffer> buffer = *(queue->begin());
@@ -620,16 +644,6 @@ int32_t AAVCAssembler::deleteUnitUnderSeq(Queue *queue, uint32_t seq) { 
    return initSize - queue->size();

    return initSize - queue->size();

}

}





inline void AAVCAssembler::printNowTimeUs(int64_t start, int64_t now, int64_t play) {

    ALOGD("start=%lld, now=%lld, played=%lld",

            (long long)start, (long long)now, (long long)play);

}



inline void AAVCAssembler::printRTPTime(int64_t rtp, int64_t play, int64_t exp, bool isExp) {

    ALOGD("rtp-time(JB)=%lld, played-rtp-time(JB)=%lld, expired-rtp-time(JB)=%lld expired=%d",

            (long long)rtp, (long long)play, (long long)exp, isExp);

}



ARTPAssembler::AssemblyStatus AAVCAssembler::assembleMore(

ARTPAssembler::AssemblyStatus AAVCAssembler::assembleMore(

        const sp<ARTPSource> &source) {

        const sp<ARTPSource> &source) {

    AssemblyStatus status = addNALUnit(source);

    AssemblyStatus status = addNALUnit(source);

media/libstagefright/rtsp/AAVCAssembler.h

+0 −3
Original line number Original line Diff line number Diff line
@@ -63,13 +63,10 @@ private: 




    void submitAccessUnit();

    void submitAccessUnit();





    inline int64_t findRTPTime(const uint32_t& firstRTPTime, const sp<ABuffer>& buffer);

    int32_t pickProperSeq(const Queue *q, uint32_t first, int64_t play, int64_t jit);

    int32_t pickProperSeq(const Queue *q, uint32_t first, int64_t play, int64_t jit);

    bool recycleUnit(uint32_t start, uint32_t end, uint32_t connected,

    bool recycleUnit(uint32_t start, uint32_t end, uint32_t connected,

            size_t avail, float goodRatio);

            size_t avail, float goodRatio);

    int32_t deleteUnitUnderSeq(Queue *q, uint32_t seq);

    int32_t deleteUnitUnderSeq(Queue *q, uint32_t seq);

    void printNowTimeUs(int64_t start, int64_t now, int64_t play);

    void printRTPTime(int64_t rtp, int64_t play, int64_t exp, bool isExp);





    DISALLOW_EVIL_CONSTRUCTORS(AAVCAssembler);

    DISALLOW_EVIL_CONSTRUCTORS(AAVCAssembler);

};

};

media/libstagefright/rtsp/AHEVCAssembler.cpp

+58 −43
Original line number Original line Diff line number Diff line
@@ -41,6 +41,8 @@ 




namespace android {

namespace android {





const double JITTER_MULTIPLE = 1.5f;



// static

// static

AHEVCAssembler::AHEVCAssembler(const sp<AMessage> &notify)

AHEVCAssembler::AHEVCAssembler(const sp<AMessage> &notify)

    : mNotifyMsg(notify),

    : mNotifyMsg(notify),
@@ -130,23 +132,51 @@ ARTPAssembler::AssemblyStatus AHEVCAssembler::addNALUnit( 




    sp<ABuffer> buffer = *queue->begin();

    sp<ABuffer> buffer = *queue->begin();

    buffer->meta()->setObject("source", source);

    buffer->meta()->setObject("source", source);



    int64_t rtpTime = findRTPTime(firstRTPTime, buffer);

    int64_t rtpTime = findRTPTime(firstRTPTime, buffer);





    int64_t startTime = source->mFirstSysTime / 1000;

    const int64_t startTimeMs = source->mFirstSysTime / 1000;

    int64_t nowTime = ALooper::GetNowUs() / 1000;

    const int64_t nowTimeMs = ALooper::GetNowUs() / 1000;

    int64_t playedTime = nowTime - startTime;

    const int64_t staticJbTimeMs = source->getStaticJitterTimeMs();

    int64_t playedTimeRtp = source->mFirstRtpTime + playedTime * (int64_t)source->mClockRate / 1000;

    const int64_t dynamicJbTimeMs = source->getDynamicJitterTimeMs();

    const int64_t jitterTime = source->mJbTimeMs * (int64_t)source->mClockRate / 1000;

    const int64_t clockRate = source->mClockRate;



    int64_t playedTimeMs = nowTimeMs - startTimeMs;

    int64_t playedTimeRtp = source->mFirstRtpTime + MsToRtp(playedTimeMs, clockRate);



    /**

     * Based on experience in real commercial network services,

     * 300 ms is a maximum heuristic jitter buffer time for video RTP service.

     */



    /**

     * The static(base) jitter is a kind of expected propagation time that we desire.

     * We can drop packets if it doesn't meet our standards.

     * If it gets shorter we can get faster response but can lose packets.

     * Expecting range : 50ms ~ 1000ms (But 300 ms would be practical upper bound)

     */

    const int64_t baseJbTimeRtp = MsToRtp(staticJbTimeMs, clockRate);

    /**

     * Dynamic jitter is a variance of interarrival time as defined in the 6.4.1 of RFC 3550.

     * We can regard this as a tolerance of every moments.

     * Expecting range : 0ms ~ 150ms (Not to over 300 ms practically)

     */

    const int64_t dynamicJbTimeRtp =                        // Max 150

            std::min(MsToRtp(dynamicJbTimeMs, clockRate), MsToRtp(150, clockRate));

    const int64_t jitterTimeRtp = baseJbTimeRtp + dynamicJbTimeRtp; // Total jitter time





    int64_t expiredTimeInJb = rtpTime + jitterTime;

    int64_t expiredTimeRtp = rtpTime + jitterTimeRtp;       // When does this buffer expire ? (T)

    bool isExpired = expiredTimeInJb <= (playedTimeRtp);

    int64_t diffTimeRtp = playedTimeRtp - expiredTimeRtp;

    bool isTooLate200 = expiredTimeInJb < (playedTimeRtp - jitterTime);

    bool isExpired = (diffTimeRtp >= 0);                    // It's expired if T is passed away

    bool isTooLate300 = expiredTimeInJb < (playedTimeRtp - (jitterTime * 3 / 2));

    bool isFirstLineBroken = (diffTimeRtp > jitterTimeRtp); // (T + jitter) is a standard tolerance



    int64_t finalMargin = dynamicJbTimeRtp * JITTER_MULTIPLE;

    bool isSecondLineBroken = (diffTimeRtp > jitterTimeRtp + finalMargin); // The Maginot line





    if (mShowQueueCnt < 20) {

    if (mShowQueueCnt < 20) {

        showCurrentQueue(queue);

        showCurrentQueue(queue);

        printNowTimeUs(startTime, nowTime, playedTime);

        printNowTimeMs(startTimeMs, nowTimeMs, playedTimeMs);

        printRTPTime(rtpTime, playedTimeRtp, expiredTimeInJb, isExpired);

        printRTPTime(rtpTime, playedTimeRtp, expiredTimeRtp, isExpired);

        mShowQueueCnt++;

        mShowQueueCnt++;

    }

    }
@@ -157,17 +187,23 @@ ARTPAssembler::AssemblyStatus AHEVCAssembler::addNALUnit( 
        return NOT_ENOUGH_DATA;

        return NOT_ENOUGH_DATA;

    }

    }





    if (isTooLate200) {

    if (isFirstLineBroken) {

        ALOGW("=== WARNING === buffer arrived 200ms late. === WARNING === ");

        if (isSecondLineBroken) {

    }

            ALOGW("buffer too late ... \t Diff in Jb=%lld \t "



                    "Seq# %d \t ExpSeq# %d \t"

    if (isTooLate300) {

                    "JitterMs %lld + (%lld * %.3f)",

        ALOGW("buffer arrived after 300ms ... \t Diff in Jb=%lld \t Seq# %d",

                    (long long)(diffTimeRtp),

                (long long)(playedTimeRtp - expiredTimeInJb), buffer->int32Data());

                    buffer->int32Data(), mNextExpectedSeqNo,

        printNowTimeUs(startTime, nowTime, playedTime);

                    (long long)staticJbTimeMs, (long long)dynamicJbTimeMs, JITTER_MULTIPLE + 1);

        printRTPTime(rtpTime, playedTimeRtp, expiredTimeInJb, isExpired);

            printNowTimeMs(startTimeMs, nowTimeMs, playedTimeMs);

            printRTPTime(rtpTime, playedTimeRtp, expiredTimeRtp, isExpired);





        mNextExpectedSeqNo = pickProperSeq(queue, firstRTPTime, playedTimeRtp, jitterTime);

            mNextExpectedSeqNo = pickProperSeq(queue, firstRTPTime, playedTimeRtp, jitterTimeRtp);

        }  else {

            ALOGW("=== WARNING === buffer arrived after %lld + %lld = %lld ms === WARNING === ",

                    (long long)staticJbTimeMs, (long long)dynamicJbTimeMs,

                    (long long)RtpToMs(jitterTimeRtp, clockRate));

        }

    }

    }





    if (mNextExpectedSeqNoValid) {

    if (mNextExpectedSeqNoValid) {
@@ -578,17 +614,6 @@ void AHEVCAssembler::submitAccessUnit() { 
    msg->post();

    msg->post();

}

}





inline int64_t AHEVCAssembler::findRTPTime(

        const uint32_t& firstRTPTime, const sp<ABuffer>& buffer) {

    /* If you want to +, -, * rtpTime, recommend to declare rtpTime as int64_t.

       Because rtpTime can be near UINT32_MAX. Beware the overflow. */

    int64_t rtpTime = 0;

    CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));

    // If the first overs 2^31 and rtp unders 2^31, the rtp value is overflowed one.

    int64_t overflowMask = (firstRTPTime & 0x80000000 & ~rtpTime) << 1;

    return rtpTime | overflowMask;

}



int32_t AHEVCAssembler::pickProperSeq(const Queue *queue,

int32_t AHEVCAssembler::pickProperSeq(const Queue *queue,

        uint32_t first, int64_t play, int64_t jit) {

        uint32_t first, int64_t play, int64_t jit) {

    sp<ABuffer> buffer = *(queue->begin());

    sp<ABuffer> buffer = *(queue->begin());
@@ -633,16 +658,6 @@ int32_t AHEVCAssembler::deleteUnitUnderSeq(Queue *queue, uint32_t seq) { 
    return initSize - queue->size();

    return initSize - queue->size();

}

}





inline void AHEVCAssembler::printNowTimeUs(int64_t start, int64_t now, int64_t play) {

    ALOGD("start=%lld, now=%lld, played=%lld",

            (long long)start, (long long)now, (long long)play);

}



inline void AHEVCAssembler::printRTPTime(int64_t rtp, int64_t play, int64_t exp, bool isExp) {

    ALOGD("rtp-time(JB)=%lld, played-rtp-time(JB)=%lld, expired-rtp-time(JB)=%lld expired=%d",

            (long long)rtp, (long long)play, (long long)exp, isExp);

}



ARTPAssembler::AssemblyStatus AHEVCAssembler::assembleMore(

ARTPAssembler::AssemblyStatus AHEVCAssembler::assembleMore(

        const sp<ARTPSource> &source) {

        const sp<ARTPSource> &source) {

    AssemblyStatus status = addNALUnit(source);

    AssemblyStatus status = addNALUnit(source);

media/libstagefright/rtsp/AHEVCAssembler.h

+0 −3
Original line number Original line Diff line number Diff line
@@ -64,13 +64,10 @@ private: 




    void submitAccessUnit();

    void submitAccessUnit();





    inline int64_t findRTPTime(const uint32_t& firstRTPTime, const sp<ABuffer>& buffer);

    int32_t pickProperSeq(const Queue *q, uint32_t first, int64_t play, int64_t jit);

    int32_t pickProperSeq(const Queue *q, uint32_t first, int64_t play, int64_t jit);

    bool recycleUnit(uint32_t start, uint32_t end, uint32_t connected,

    bool recycleUnit(uint32_t start, uint32_t end, uint32_t connected,

             size_t avail, float goodRatio);

             size_t avail, float goodRatio);

    int32_t deleteUnitUnderSeq(Queue *queue, uint32_t seq);

    int32_t deleteUnitUnderSeq(Queue *queue, uint32_t seq);

    void printNowTimeUs(int64_t start, int64_t now, int64_t play);

    void printRTPTime(int64_t rtp, int64_t play, int64_t exp, bool isExp);





    DISALLOW_EVIL_CONSTRUCTORS(AHEVCAssembler);

    DISALLOW_EVIL_CONSTRUCTORS(AHEVCAssembler);

};

};