Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit bc978501 authored by Treehugger Robot's avatar Treehugger Robot Committed by Automerger Merge Worker
Browse files

Merge "VT: Introduce dynamic jitter buffer for RTP source" am: d71f5042 

Original change: https://android-review.googlesource.com/c/platform/frameworks/av/+/1652258

Change-Id: Ib9c63a4067c13bc9bfa9d002e80f3b0e7faa81ac
parents 38c9802d d71f5042

media/libmediaplayerservice/nuplayer/RTPSource.cpp

+13 −6
Original line number Diff line number Diff line
@@ -124,8 +124,16 @@ void NuPlayer::RTPSource::prepareAsync() { 
        // index(i) should be started from 1. 0 is reserved for [root]

        mRTPConn->addStream(sockRtp, sockRtcp, desc, i + 1, notify, false);

        mRTPConn->setSelfID(info->mSelfID);

        mRTPConn->setJbTime(

                (info->mJbTimeMs <= 3000 && info->mJbTimeMs >= 40) ? info->mJbTimeMs : 300);

        mRTPConn->setStaticJitterTimeMs(info->mJbTimeMs);



        unsigned long PT;

        AString formatDesc, formatParams;

        // index(i) should be started from 1. 0 is reserved for [root]

        desc->getFormatType(i + 1, &PT, &formatDesc, &formatParams);



        int32_t clockRate, numChannels;

        ASessionDescription::ParseFormatDesc(formatDesc.c_str(), &clockRate, &numChannels);

        info->mTimeScale = clockRate;



        info->mRTPSocket = sockRtp;

        info->mRTCPSocket = sockRtcp;
@@ -146,10 +154,8 @@ void NuPlayer::RTPSource::prepareAsync() { 


        if (info->mIsAudio) {

            mAudioTrack = source;

            info->mTimeScale = 16000;

        } else {

            mVideoTrack = source;

            info->mTimeScale = 90000;

        }



        info->mSource = source;
@@ -680,7 +686,7 @@ status_t NuPlayer::RTPSource::setParameter(const String8 &key, const String8 &va 
        newTrackInfo.mIsAudio = isAudioKey;

        mTracks.push(newTrackInfo);

        info = &mTracks.editTop();

        info->mJbTimeMs = 300;

        info->mJbTimeMs = kStaticJitterTimeMs;

    }



    if (key == "rtp-param-mime-type") {
@@ -724,7 +730,8 @@ status_t NuPlayer::RTPSource::setParameter(const String8 &key, const String8 &va 
        int64_t networkHandle = atoll(value);

        setSocketNetwork(networkHandle);

    } else if (key == "rtp-param-jitter-buffer-time") {

        info->mJbTimeMs = atoi(value);

        // clamping min at 40, max at 3000

        info->mJbTimeMs = std::min(std::max(40, atoi(value)), 3000);

    }



    return OK;

media/libstagefright/rtsp/AAVCAssembler.cpp

+59 −45
Original line number Diff line number Diff line
@@ -34,6 +34,8 @@ 


namespace android {



const double JITTER_MULTIPLE = 1.5f;



// static

AAVCAssembler::AAVCAssembler(const sp<AMessage> &notify)

    : mNotifyMsg(notify),
@@ -123,22 +125,48 @@ ARTPAssembler::AssemblyStatus AAVCAssembler::addNALUnit( 


    int64_t rtpTime = findRTPTime(firstRTPTime, buffer);



    int64_t startTime = source->mFirstSysTime / 1000;

    int64_t nowTime = ALooper::GetNowUs() / 1000;

    int64_t playedTime = nowTime - startTime;

    const int64_t startTimeMs = source->mFirstSysTime / 1000;

    const int64_t nowTimeMs = ALooper::GetNowUs() / 1000;

    const int64_t staticJbTimeMs = source->getStaticJitterTimeMs();

    const int64_t dynamicJbTimeMs = source->getDynamicJitterTimeMs();

    const int64_t clockRate = source->mClockRate;



    int64_t playedTimeMs = nowTimeMs - startTimeMs;

    int64_t playedTimeRtp = source->mFirstRtpTime + MsToRtp(playedTimeMs, clockRate);



    /**

     * Based on experience in real commercial network services,

     * 300 ms is a maximum heuristic jitter buffer time for video RTP service.

     */



    /**

     * The static(base) jitter is a kind of expected propagation time that we desire.

     * We can drop packets if it doesn't meet our standards.

     * If it gets shorter we can get faster response but can lose packets.

     * Expecting range : 50ms ~ 1000ms (But 300 ms would be practical upper bound)

     */

    const int64_t baseJbTimeRtp = MsToRtp(staticJbTimeMs, clockRate);

    /**

     * Dynamic jitter is a variance of interarrival time as defined in the 6.4.1 of RFC 3550.

     * We can regard this as a tolerance of every moments.

     * Expecting range : 0ms ~ 150ms (Not to over 300 ms practically)

     */

    const int64_t dynamicJbTimeRtp =                        // Max 150

            std::min(MsToRtp(dynamicJbTimeMs, clockRate), MsToRtp(150, clockRate));

    const int64_t jitterTimeRtp = baseJbTimeRtp + dynamicJbTimeRtp; // Total jitter time



    int64_t playedTimeRtp = source->mFirstRtpTime + playedTime * (int64_t)source->mClockRate / 1000;

    const int64_t jitterTime = source->mJbTimeMs * (int64_t)source->mClockRate / 1000;

    int64_t expiredTimeRtp = rtpTime + jitterTimeRtp;       // When does this buffer expire ? (T)

    int64_t diffTimeRtp = playedTimeRtp - expiredTimeRtp;

    bool isExpired = (diffTimeRtp >= 0);                    // It's expired if T is passed away

    bool isFirstLineBroken = (diffTimeRtp > jitterTimeRtp); // (T + jitter) is a standard tolerance



    int64_t expiredTimeInJb = rtpTime + jitterTime;

    bool isExpired = expiredTimeInJb <= (playedTimeRtp);

    bool isTooLate200 = expiredTimeInJb < (playedTimeRtp - jitterTime);

    bool isTooLate300 = expiredTimeInJb < (playedTimeRtp - (jitterTime * 3 / 2));

    int64_t finalMargin = dynamicJbTimeRtp * JITTER_MULTIPLE;

    bool isSecondLineBroken = (diffTimeRtp > jitterTimeRtp + finalMargin); // The Maginot line



    if (mShowQueue && mShowQueueCnt < 20) {

        showCurrentQueue(queue);

        printNowTimeUs(startTime, nowTime, playedTime);

        printRTPTime(rtpTime, playedTimeRtp, expiredTimeInJb, isExpired);

        printNowTimeMs(startTimeMs, nowTimeMs, playedTimeMs);

        printRTPTime(rtpTime, playedTimeRtp, expiredTimeRtp, isExpired);

        mShowQueueCnt++;

    }
@@ -149,17 +177,23 @@ ARTPAssembler::AssemblyStatus AAVCAssembler::addNALUnit( 
        return NOT_ENOUGH_DATA;

    }



    if (isTooLate200) {

        ALOGW("=== WARNING === buffer arrived 200ms late. === WARNING === ");

    }

    if (isFirstLineBroken) {

        if (isSecondLineBroken) {

            ALOGW("buffer too late ... \t Diff in Jb=%lld \t "

                    "Seq# %d \t ExpSeq# %d \t"

                    "JitterMs %lld + (%lld * %.3f)",

                    (long long)(diffTimeRtp),

                    buffer->int32Data(), mNextExpectedSeqNo,

                    (long long)staticJbTimeMs, (long long)dynamicJbTimeMs, JITTER_MULTIPLE + 1);

            printNowTimeMs(startTimeMs, nowTimeMs, playedTimeMs);

            printRTPTime(rtpTime, playedTimeRtp, expiredTimeRtp, isExpired);



    if (isTooLate300) {

        ALOGW("buffer arrived after 300ms ... \t Diff in Jb=%lld \t Seq# %d",

                (long long)(playedTimeRtp - expiredTimeInJb), buffer->int32Data());

        printNowTimeUs(startTime, nowTime, playedTime);

        printRTPTime(rtpTime, playedTimeRtp, expiredTimeInJb, isExpired);



        mNextExpectedSeqNo = pickProperSeq(queue, firstRTPTime, playedTimeRtp, jitterTime);

            mNextExpectedSeqNo = pickProperSeq(queue, firstRTPTime, playedTimeRtp, jitterTimeRtp);

        }  else {

            ALOGW("=== WARNING === buffer arrived after %lld + %lld = %lld ms === WARNING === ",

                    (long long)staticJbTimeMs, (long long)dynamicJbTimeMs,

                    (long long)RtpToMs(jitterTimeRtp, clockRate));

        }

    }



    if (mNextExpectedSeqNoValid) {
@@ -170,6 +204,7 @@ ARTPAssembler::AssemblyStatus AAVCAssembler::addNALUnit( 
            source->noticeAbandonBuffer(cntRemove);

            ALOGW("delete %d of %d buffers", cntRemove, size);

        }



        if (queue->empty()) {

            return NOT_ENOUGH_DATA;

        }
@@ -565,17 +600,6 @@ void AAVCAssembler::submitAccessUnit() { 
    msg->post();

}



inline int64_t AAVCAssembler::findRTPTime(

        const uint32_t& firstRTPTime, const sp<ABuffer>& buffer) {

    /* If you want to +, -, * rtpTime, recommend to declare rtpTime as int64_t.

       Because rtpTime can be near UINT32_MAX. Beware the overflow. */

    int64_t rtpTime = 0;

    CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));

    // If the first overs 2^31 and rtp unders 2^31, the rtp value is overflowed one.

    int64_t overflowMask = (firstRTPTime & 0x80000000 & ~rtpTime) << 1;

    return rtpTime | overflowMask;

}



int32_t AAVCAssembler::pickProperSeq(const Queue *queue,

        uint32_t first, int64_t play, int64_t jit) {

    sp<ABuffer> buffer = *(queue->begin());
@@ -620,16 +644,6 @@ int32_t AAVCAssembler::deleteUnitUnderSeq(Queue *queue, uint32_t seq) { 
    return initSize - queue->size();

}



inline void AAVCAssembler::printNowTimeUs(int64_t start, int64_t now, int64_t play) {

    ALOGD("start=%lld, now=%lld, played=%lld",

            (long long)start, (long long)now, (long long)play);

}



inline void AAVCAssembler::printRTPTime(int64_t rtp, int64_t play, int64_t exp, bool isExp) {

    ALOGD("rtp-time(JB)=%lld, played-rtp-time(JB)=%lld, expired-rtp-time(JB)=%lld expired=%d",

            (long long)rtp, (long long)play, (long long)exp, isExp);

}



ARTPAssembler::AssemblyStatus AAVCAssembler::assembleMore(

        const sp<ARTPSource> &source) {

    AssemblyStatus status = addNALUnit(source);

media/libstagefright/rtsp/AAVCAssembler.h

+0 −3
Original line number Diff line number Diff line
@@ -63,13 +63,10 @@ private: 


    void submitAccessUnit();



    inline int64_t findRTPTime(const uint32_t& firstRTPTime, const sp<ABuffer>& buffer);

    int32_t pickProperSeq(const Queue *q, uint32_t first, int64_t play, int64_t jit);

    bool recycleUnit(uint32_t start, uint32_t end, uint32_t connected,

            size_t avail, float goodRatio);

    int32_t deleteUnitUnderSeq(Queue *q, uint32_t seq);

    void printNowTimeUs(int64_t start, int64_t now, int64_t play);

    void printRTPTime(int64_t rtp, int64_t play, int64_t exp, bool isExp);



    DISALLOW_EVIL_CONSTRUCTORS(AAVCAssembler);

};

media/libstagefright/rtsp/AHEVCAssembler.cpp

+58 −43
Original line number Diff line number Diff line
@@ -41,6 +41,8 @@ 


namespace android {



const double JITTER_MULTIPLE = 1.5f;



// static

AHEVCAssembler::AHEVCAssembler(const sp<AMessage> &notify)

    : mNotifyMsg(notify),
@@ -130,23 +132,51 @@ ARTPAssembler::AssemblyStatus AHEVCAssembler::addNALUnit( 


    sp<ABuffer> buffer = *queue->begin();

    buffer->meta()->setObject("source", source);



    int64_t rtpTime = findRTPTime(firstRTPTime, buffer);



    int64_t startTime = source->mFirstSysTime / 1000;

    int64_t nowTime = ALooper::GetNowUs() / 1000;

    int64_t playedTime = nowTime - startTime;

    int64_t playedTimeRtp = source->mFirstRtpTime + playedTime * (int64_t)source->mClockRate / 1000;

    const int64_t jitterTime = source->mJbTimeMs * (int64_t)source->mClockRate / 1000;

    const int64_t startTimeMs = source->mFirstSysTime / 1000;

    const int64_t nowTimeMs = ALooper::GetNowUs() / 1000;

    const int64_t staticJbTimeMs = source->getStaticJitterTimeMs();

    const int64_t dynamicJbTimeMs = source->getDynamicJitterTimeMs();

    const int64_t clockRate = source->mClockRate;



    int64_t playedTimeMs = nowTimeMs - startTimeMs;

    int64_t playedTimeRtp = source->mFirstRtpTime + MsToRtp(playedTimeMs, clockRate);



    /**

     * Based on experience in real commercial network services,

     * 300 ms is a maximum heuristic jitter buffer time for video RTP service.

     */



    /**

     * The static(base) jitter is a kind of expected propagation time that we desire.

     * We can drop packets if it doesn't meet our standards.

     * If it gets shorter we can get faster response but can lose packets.

     * Expecting range : 50ms ~ 1000ms (But 300 ms would be practical upper bound)

     */

    const int64_t baseJbTimeRtp = MsToRtp(staticJbTimeMs, clockRate);

    /**

     * Dynamic jitter is a variance of interarrival time as defined in the 6.4.1 of RFC 3550.

     * We can regard this as a tolerance of every moments.

     * Expecting range : 0ms ~ 150ms (Not to over 300 ms practically)

     */

    const int64_t dynamicJbTimeRtp =                        // Max 150

            std::min(MsToRtp(dynamicJbTimeMs, clockRate), MsToRtp(150, clockRate));

    const int64_t jitterTimeRtp = baseJbTimeRtp + dynamicJbTimeRtp; // Total jitter time



    int64_t expiredTimeInJb = rtpTime + jitterTime;

    bool isExpired = expiredTimeInJb <= (playedTimeRtp);

    bool isTooLate200 = expiredTimeInJb < (playedTimeRtp - jitterTime);

    bool isTooLate300 = expiredTimeInJb < (playedTimeRtp - (jitterTime * 3 / 2));

    int64_t expiredTimeRtp = rtpTime + jitterTimeRtp;       // When does this buffer expire ? (T)

    int64_t diffTimeRtp = playedTimeRtp - expiredTimeRtp;

    bool isExpired = (diffTimeRtp >= 0);                    // It's expired if T is passed away

    bool isFirstLineBroken = (diffTimeRtp > jitterTimeRtp); // (T + jitter) is a standard tolerance



    int64_t finalMargin = dynamicJbTimeRtp * JITTER_MULTIPLE;

    bool isSecondLineBroken = (diffTimeRtp > jitterTimeRtp + finalMargin); // The Maginot line



    if (mShowQueueCnt < 20) {

        showCurrentQueue(queue);

        printNowTimeUs(startTime, nowTime, playedTime);

        printRTPTime(rtpTime, playedTimeRtp, expiredTimeInJb, isExpired);

        printNowTimeMs(startTimeMs, nowTimeMs, playedTimeMs);

        printRTPTime(rtpTime, playedTimeRtp, expiredTimeRtp, isExpired);

        mShowQueueCnt++;

    }
@@ -157,17 +187,23 @@ ARTPAssembler::AssemblyStatus AHEVCAssembler::addNALUnit( 
        return NOT_ENOUGH_DATA;

    }



    if (isTooLate200) {

        ALOGW("=== WARNING === buffer arrived 200ms late. === WARNING === ");

    }



    if (isTooLate300) {

        ALOGW("buffer arrived after 300ms ... \t Diff in Jb=%lld \t Seq# %d",

                (long long)(playedTimeRtp - expiredTimeInJb), buffer->int32Data());

        printNowTimeUs(startTime, nowTime, playedTime);

        printRTPTime(rtpTime, playedTimeRtp, expiredTimeInJb, isExpired);

    if (isFirstLineBroken) {

        if (isSecondLineBroken) {

            ALOGW("buffer too late ... \t Diff in Jb=%lld \t "

                    "Seq# %d \t ExpSeq# %d \t"

                    "JitterMs %lld + (%lld * %.3f)",

                    (long long)(diffTimeRtp),

                    buffer->int32Data(), mNextExpectedSeqNo,

                    (long long)staticJbTimeMs, (long long)dynamicJbTimeMs, JITTER_MULTIPLE + 1);

            printNowTimeMs(startTimeMs, nowTimeMs, playedTimeMs);

            printRTPTime(rtpTime, playedTimeRtp, expiredTimeRtp, isExpired);



        mNextExpectedSeqNo = pickProperSeq(queue, firstRTPTime, playedTimeRtp, jitterTime);

            mNextExpectedSeqNo = pickProperSeq(queue, firstRTPTime, playedTimeRtp, jitterTimeRtp);

        }  else {

            ALOGW("=== WARNING === buffer arrived after %lld + %lld = %lld ms === WARNING === ",

                    (long long)staticJbTimeMs, (long long)dynamicJbTimeMs,

                    (long long)RtpToMs(jitterTimeRtp, clockRate));

        }

    }



    if (mNextExpectedSeqNoValid) {
@@ -578,17 +614,6 @@ void AHEVCAssembler::submitAccessUnit() { 
    msg->post();

}



inline int64_t AHEVCAssembler::findRTPTime(

        const uint32_t& firstRTPTime, const sp<ABuffer>& buffer) {

    /* If you want to +, -, * rtpTime, recommend to declare rtpTime as int64_t.

       Because rtpTime can be near UINT32_MAX. Beware the overflow. */

    int64_t rtpTime = 0;

    CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));

    // If the first overs 2^31 and rtp unders 2^31, the rtp value is overflowed one.

    int64_t overflowMask = (firstRTPTime & 0x80000000 & ~rtpTime) << 1;

    return rtpTime | overflowMask;

}



int32_t AHEVCAssembler::pickProperSeq(const Queue *queue,

        uint32_t first, int64_t play, int64_t jit) {

    sp<ABuffer> buffer = *(queue->begin());
@@ -633,16 +658,6 @@ int32_t AHEVCAssembler::deleteUnitUnderSeq(Queue *queue, uint32_t seq) { 
    return initSize - queue->size();

}



inline void AHEVCAssembler::printNowTimeUs(int64_t start, int64_t now, int64_t play) {

    ALOGD("start=%lld, now=%lld, played=%lld",

            (long long)start, (long long)now, (long long)play);

}



inline void AHEVCAssembler::printRTPTime(int64_t rtp, int64_t play, int64_t exp, bool isExp) {

    ALOGD("rtp-time(JB)=%lld, played-rtp-time(JB)=%lld, expired-rtp-time(JB)=%lld expired=%d",

            (long long)rtp, (long long)play, (long long)exp, isExp);

}



ARTPAssembler::AssemblyStatus AHEVCAssembler::assembleMore(

        const sp<ARTPSource> &source) {

    AssemblyStatus status = addNALUnit(source);

media/libstagefright/rtsp/AHEVCAssembler.h

+0 −3
Original line number Diff line number Diff line
@@ -64,13 +64,10 @@ private: 


    void submitAccessUnit();



    inline int64_t findRTPTime(const uint32_t& firstRTPTime, const sp<ABuffer>& buffer);

    int32_t pickProperSeq(const Queue *q, uint32_t first, int64_t play, int64_t jit);

    bool recycleUnit(uint32_t start, uint32_t end, uint32_t connected,

             size_t avail, float goodRatio);

    int32_t deleteUnitUnderSeq(Queue *queue, uint32_t seq);

    void printNowTimeUs(int64_t start, int64_t now, int64_t play);

    void printRTPTime(int64_t rtp, int64_t play, int64_t exp, bool isExp);



    DISALLOW_EVIL_CONSTRUCTORS(AHEVCAssembler);

};