Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 4d5fc223 authored by Kim Sungyeon's avatar Kim Sungyeon Committed by Byeongjo Park
Browse files

VT: Send more RTP SR information to upper layer 



[Problem] Need some method to estimate frame process time of an opponent
[Cause] NTP & RTP time of SR was not proper.
        Becuase of this, SR have not been applicated usefully.
[Solution] Fix the time of SR to indicate its own timing.
           Send SR information from the opponent to upper layer.

Bug: 183578712

Signed-off-by: default avatarByeongjo Park <bjo.park@samsung.com>
Change-Id: I005c798759cc1609e04f766e8b0fe751ff5eede8
Signed-off-by: default avatarKim Sungyeon <sy85.kim@samsung.com>
parent 8f6d8f2a

media/libmediaplayerservice/nuplayer/NuPlayer.cpp

+44 −0
Original line number Diff line number Diff line
@@ -2854,10 +2854,43 @@ void NuPlayer::sendIMSRxNotice(const sp<AMessage> &msg) { 


    CHECK(msg->findInt32("payload-type", &payloadType));



    int32_t rtpSeq = 0, rtpTime = 0;

    int64_t ntpTime = 0, recvTimeUs = 0;



    Parcel in;

    in.writeInt32(payloadType);



    switch (payloadType) {

        case ARTPSource::RTP_FIRST_PACKET:

        {

            CHECK(msg->findInt32("rtp-time", &rtpTime));

            CHECK(msg->findInt32("rtp-seq-num", &rtpSeq));

            CHECK(msg->findInt64("recv-time-us", &recvTimeUs));

            in.writeInt32(rtpTime);

            in.writeInt32(rtpSeq);

            in.writeInt32(recvTimeUs >> 32);

            in.writeInt32(recvTimeUs & 0xFFFFFFFF);

            break;

        }

        case ARTPSource::RTCP_FIRST_PACKET:

        {

            CHECK(msg->findInt64("recv-time-us", &recvTimeUs));

            in.writeInt32(recvTimeUs >> 32);

            in.writeInt32(recvTimeUs & 0xFFFFFFFF);

            break;

        }

        case ARTPSource::RTCP_SR:

        {

            CHECK(msg->findInt32("rtp-time", &rtpTime));

            CHECK(msg->findInt64("ntp-time", &ntpTime));

            CHECK(msg->findInt64("recv-time-us", &recvTimeUs));

            in.writeInt32(rtpTime);

            in.writeInt32(ntpTime >> 32);

            in.writeInt32(ntpTime & 0xFFFFFFFF);

            in.writeInt32(recvTimeUs >> 32);

            in.writeInt32(recvTimeUs & 0xFFFFFFFF);

            break;

        }

        case ARTPSource::RTCP_TSFB:   // RTCP TSFB

        case ARTPSource::RTCP_PSFB:   // RTCP PSFB

        case ARTPSource::RTP_AUTODOWN:
@@ -2880,6 +2913,8 @@ void NuPlayer::sendIMSRxNotice(const sp<AMessage> &msg) { 
            int32_t feedbackType, bitrate;

            int32_t highestSeqNum, baseSeqNum, prevExpected;

            int32_t numBufRecv, prevNumBufRecv;

            int32_t latestRtpTime, jbTimeMs, rtpRtcpSrTimeGapMs;

            int64_t recvTimeUs;

            CHECK(msg->findInt32("feedback-type", &feedbackType));

            CHECK(msg->findInt32("bit-rate", &bitrate));

            CHECK(msg->findInt32("highest-seq-num", &highestSeqNum));
@@ -2887,6 +2922,10 @@ void NuPlayer::sendIMSRxNotice(const sp<AMessage> &msg) { 
            CHECK(msg->findInt32("prev-expected", &prevExpected));

            CHECK(msg->findInt32("num-buf-recv", &numBufRecv));

            CHECK(msg->findInt32("prev-num-buf-recv", &prevNumBufRecv));

            CHECK(msg->findInt32("latest-rtp-time", &latestRtpTime));

            CHECK(msg->findInt64("recv-time-us", &recvTimeUs));

            CHECK(msg->findInt32("rtp-jitter-time-ms", &jbTimeMs));

            CHECK(msg->findInt32("rtp-rtcpsr-time-gap-ms", &rtpRtcpSrTimeGapMs));

            in.writeInt32(feedbackType);

            in.writeInt32(bitrate);

            in.writeInt32(highestSeqNum);
@@ -2894,6 +2933,11 @@ void NuPlayer::sendIMSRxNotice(const sp<AMessage> &msg) { 
            in.writeInt32(prevExpected);

            in.writeInt32(numBufRecv);

            in.writeInt32(prevNumBufRecv);

            in.writeInt32(latestRtpTime);

            in.writeInt32(recvTimeUs >> 32);

            in.writeInt32(recvTimeUs & 0xFFFFFFFF);

            in.writeInt32(jbTimeMs);

            in.writeInt32(rtpRtcpSrTimeGapMs);

            break;

        }

        case ARTPSource::RTP_CVO:

media/libmediaplayerservice/nuplayer/RTPSource.cpp

+2 −12
Original line number Diff line number Diff line
@@ -395,23 +395,13 @@ void NuPlayer::RTPSource::onMessageReceived(const sp<AMessage> &msg) { 
                CHECK(msg->findInt64("ntp-time", (int64_t *)&ntpTime));



                onTimeUpdate(trackIndex, rtpTime, ntpTime);

                break;

            }



            int32_t firstRTCP;

            if (msg->findInt32("first-rtcp", &firstRTCP)) {

                // There won't be an access unit here, it's just a notification

                // that the data communication worked since we got the first

                // rtcp packet.

                ALOGV("first-rtcp");

                break;

            }



            int32_t IMSRxNotice;

            if (msg->findInt32("rtcp-event", &IMSRxNotice)) {

                int32_t payloadType, feedbackType;

                int32_t payloadType = 0, feedbackType = 0;

                CHECK(msg->findInt32("payload-type", &payloadType));

                CHECK(msg->findInt32("feedback-type", &feedbackType));

                msg->findInt32("feedback-type", &feedbackType);



                sp<AMessage> notify = dupNotify();

                notify->setInt32("what", kWhatIMSRxNotice);

media/libstagefright/rtsp/ARTPConnection.cpp

+22 −16
Original line number Diff line number Diff line
@@ -656,12 +656,6 @@ ssize_t ARTPConnection::send(const StreamInfo *info, const sp<ABuffer> buffer) { 
}



status_t ARTPConnection::parseRTP(StreamInfo *s, const sp<ABuffer> &buffer) {

    if (s->mNumRTPPacketsReceived++ == 0) {

        sp<AMessage> notify = s->mNotifyMsg->dup();

        notify->setInt32("first-rtp", true);

        notify->post();

    }



    size_t size = buffer->size();



    if (size < 12) {
@@ -743,9 +737,23 @@ status_t ARTPConnection::parseRTP(StreamInfo *s, const sp<ABuffer> &buffer) { 
        meta->setInt32("cvo", cvoDegrees);

    }



    buffer->setInt32Data(u16at(&data[2]));

    int32_t seq = u16at(&data[2]);

    buffer->setInt32Data(seq);

    buffer->setRange(payloadOffset, size - payloadOffset);



    if (s->mNumRTPPacketsReceived++ == 0) {

        sp<AMessage> notify = s->mNotifyMsg->dup();

        notify->setInt32("first-rtp", true);

        notify->setInt32("rtcp-event", 1);

        notify->setInt32("payload-type", ARTPSource::RTP_FIRST_PACKET);

        notify->setInt32("rtp-time", (int32_t)rtpTime);

        notify->setInt32("rtp-seq-num", seq);

        notify->setInt64("recv-time-us", ALooper::GetNowUs());

        notify->post();



        ALOGD("send first-rtp event to upper layer");

    }



    source->processRTPPacket(buffer);



    return OK;
@@ -802,14 +810,12 @@ status_t ARTPConnection::parseRTCP(StreamInfo *s, const sp<ABuffer> &buffer) { 
    if (s->mNumRTCPPacketsReceived++ == 0) {

        sp<AMessage> notify = s->mNotifyMsg->dup();

        notify->setInt32("first-rtcp", true);

        notify->setInt32("rtcp-event", 1);

        notify->setInt32("payload-type", ARTPSource::RTCP_FIRST_PACKET);

        notify->setInt64("recv-time-us", ALooper::GetNowUs());

        notify->post();



        ALOGI("send first-rtcp event to upper layer as ImsRxNotice");

        sp<AMessage> imsNotify = s->mNotifyMsg->dup();

        imsNotify->setInt32("rtcp-event", 1);

        imsNotify->setInt32("payload-type", 101);

        imsNotify->setInt32("feedback-type", 0);

        imsNotify->post();

        ALOGD("send first-rtcp event to upper layer");

    }



    const uint8_t *data = buffer->data();
@@ -906,7 +912,7 @@ status_t ARTPConnection::parseBYE( 
    int64_t nowUs = ALooper::GetNowUs();

    int32_t timeDiff = (nowUs - mLastBitrateReportTimeUs) / 1000000ll;

    int32_t bitrate = mCumulativeBytes * 8 / timeDiff;

    source->notifyPktInfo(bitrate, true /* isRegular */);

    source->notifyPktInfo(bitrate, nowUs, true /* isRegular */);



    source->byeReceived();
@@ -1140,7 +1146,7 @@ void ARTPConnection::checkRxBitrate(int64_t nowUs) { 
            for (size_t i = 0; i < s->mSources.size(); ++i) {

                sp<ARTPSource> source = s->mSources.valueAt(i);

                if (source->isNeedToEarlyNotify()) {

                    source->notifyPktInfo(bitrate, false /* isRegular */);

                    source->notifyPktInfo(bitrate, nowUs, false /* isRegular */);

                    mLastEarlyNotifyTimeUs = nowUs + (1000000ll * 3600 * 24); // after 1 day

                }

            }
@@ -1171,7 +1177,7 @@ void ARTPConnection::checkRxBitrate(int64_t nowUs) { 
            buffer->setRange(0, 0);

            for (size_t i = 0; i < s->mSources.size(); ++i) {

                sp<ARTPSource> source = s->mSources.valueAt(i);

                source->notifyPktInfo(bitrate, true /* isRegular */);

                source->notifyPktInfo(bitrate, nowUs, true /* isRegular */);

            }

            ++it;

        }

media/libstagefright/rtsp/ARTPSource.cpp

+68 −16
Original line number Diff line number Diff line
@@ -44,8 +44,7 @@ ARTPSource::ARTPSource( 
        uint32_t id,

        const sp<ASessionDescription> &sessionDesc, size_t index,

        const sp<AMessage> &notify)

    : mFirstSeqNumber(0),

      mFirstRtpTime(0),

    : mFirstRtpTime(0),

      mFirstSysTime(0),

      mClockRate(0),

      mFirstSsrc(0),
@@ -58,9 +57,13 @@ ARTPSource::ARTPSource( 
      mPrevNumBuffersReceived(0),

      mPrevExpectedForRR(0),

      mPrevNumBuffersReceivedForRR(0),

      mLatestRtpTime(0),

      mStaticJbTimeMs(kStaticJitterTimeMs),

      mLastNTPTime(0),

      mLastNTPTimeUpdateUs(0),

      mLastSrRtpTime(0),

      mLastSrNtpTime(0),

      mLastSrUpdateTimeUs(0),

      mIsFirstRtpRtcpGap(true),

      mAvgRtpRtcpGapMs(0),

      mIssueFIRRequests(false),

      mIssueFIRByAssembler(false),

      mLastFIRRequestUs(-1),
@@ -120,13 +123,17 @@ void ARTPSource::processRTPPacket(const sp<ABuffer> &buffer) { 
}



void ARTPSource::timeUpdate(uint32_t rtpTime, uint64_t ntpTime) {

    mLastNTPTime = ntpTime;

    mLastNTPTimeUpdateUs = ALooper::GetNowUs();

    mLastSrRtpTime = rtpTime;

    mLastSrNtpTime = ntpTime;

    mLastSrUpdateTimeUs = ALooper::GetNowUs();



    sp<AMessage> notify = mNotify->dup();

    notify->setInt32("time-update", true);

    notify->setInt32("rtp-time", rtpTime);

    notify->setInt64("ntp-time", ntpTime);

    notify->setInt32("rtcp-event", 1);

    notify->setInt32("payload-type", RTCP_SR);

    notify->setInt64("recv-time-us", mLastSrUpdateTimeUs);

    notify->post();

}
@@ -142,25 +149,65 @@ void ARTPSource::timeReset() { 
    mPrevNumBuffersReceived = 0;

    mPrevExpectedForRR = 0;

    mPrevNumBuffersReceivedForRR = 0;

    mLastNTPTime = 0;

    mLastNTPTimeUpdateUs = 0;

    mLatestRtpTime = 0;

    mLastSrRtpTime = 0;

    mLastSrNtpTime = 0;

    mLastSrUpdateTimeUs = 0;

    mIsFirstRtpRtcpGap = true;

    mAvgRtpRtcpGapMs = 0;

    mIssueFIRByAssembler = false;

    mLastFIRRequestUs = -1;

}



void ARTPSource::calcTimeGapRtpRtcp(const sp<ABuffer> &buffer) {

    if (mLastSrUpdateTimeUs == 0) {

        return;

    }



    int64_t elapsedMs = (ALooper::GetNowUs() - mLastSrUpdateTimeUs) / 1000;

    int64_t elapsedRtpTime = (elapsedMs * (mClockRate / 1000));

    uint32_t rtpTime;

    CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));



    int64_t anchorRtpTime = mLastSrRtpTime + elapsedRtpTime;

    int64_t rtpTimeGap = anchorRtpTime - rtpTime;

    // rtpTime can not be faster than it's anchor time.

    // because rtpTime(of rtp packet) represents it's a frame captured time and

    // anchorRtpTime(of rtcp:sr packet) represents it's a rtp packetized time.

    if (rtpTimeGap < 0 || rtpTimeGap > (mClockRate * 60)) {

        // ignore invalid delay gap such as negative delay or later than 1 min.

        return;

    }



    int64_t rtpTimeGapMs = (rtpTimeGap * 1000 / mClockRate);

    if (mIsFirstRtpRtcpGap) {

        mIsFirstRtpRtcpGap = false;

        mAvgRtpRtcpGapMs = rtpTimeGapMs;

    } else {

        // This is measuring avg rtp timestamp distance between rtp and rtcp:sr packet.

        // Rtp timestamp of rtp packet represents it's raw frame captured time.

        // Rtp timestamp of rtcp:sr packet represents it's packetization time.

        // So that, this value is showing how much time delayed to be a rtp packet

        // from a raw frame captured time.

        // This value maybe referred to know a/v sync and sender's own delay of this media stream.

        mAvgRtpRtcpGapMs = ((mAvgRtpRtcpGapMs * 15) + rtpTimeGapMs) / 16;

    }

}



bool ARTPSource::queuePacket(const sp<ABuffer> &buffer) {

    calcTimeGapRtpRtcp(buffer);

    uint32_t seqNum = (uint32_t)buffer->int32Data();



    int32_t ssrc = 0;

    int32_t ssrc = 0, rtpTime = 0;

    buffer->meta()->findInt32("ssrc", &ssrc);

    CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));

    mLatestRtpTime = rtpTime;



    if (mNumBuffersReceived++ == 0 && mFirstSysTime == 0) {

        uint32_t firstRtpTime;

        CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&firstRtpTime));

        mFirstSysTime = ALooper::GetNowUs();

        mHighestSeqNumber = seqNum;

        mBaseSeqNumber = seqNum;

        mFirstRtpTime = firstRtpTime;

        mFirstRtpTime = rtpTime;

        mFirstSsrc = ssrc;

        ALOGD("first-rtp arrived: first-rtp-time=%u, sys-time=%lld, seq-num=%u, ssrc=%d",

                mFirstRtpTime, (long long)mFirstSysTime, mHighestSeqNumber, mFirstSsrc);
@@ -363,11 +410,11 @@ void ARTPSource::addReceiverReport(const sp<ABuffer> &buffer) { 


    uint32_t LSR = 0;

    uint32_t DLSR = 0;

    if (mLastNTPTime != 0) {

        LSR = (mLastNTPTime >> 16) & 0xffffffff;

    if (mLastSrNtpTime != 0) {

        LSR = (mLastSrNtpTime >> 16) & 0xffffffff;



        DLSR = (uint32_t)

            ((ALooper::GetNowUs() - mLastNTPTimeUpdateUs) * 65536.0 / 1E6);

            ((ALooper::GetNowUs() - mLastSrUpdateTimeUs) * 65536.0 / 1E6);

    }



    data[24] = LSR >> 24;
@@ -576,7 +623,7 @@ bool ARTPSource::isNeedToEarlyNotify() { 
    return false;

}



void ARTPSource::notifyPktInfo(int32_t bitrate, bool isRegular) {

void ARTPSource::notifyPktInfo(int32_t bitrate, int64_t nowUs, bool isRegular) {

    int32_t payloadType = isRegular ? RTP_QUALITY : RTP_QUALITY_EMC;



    sp<AMessage> notify = mNotify->dup();
@@ -590,6 +637,11 @@ void ARTPSource::notifyPktInfo(int32_t bitrate, bool isRegular) { 
    notify->setInt32("prev-expected", mPrevExpected);

    notify->setInt32("num-buf-recv", mNumBuffersReceived);

    notify->setInt32("prev-num-buf-recv", mPrevNumBuffersReceived);

    notify->setInt32("latest-rtp-time", mLatestRtpTime);

    notify->setInt64("recv-time-us", nowUs);

    notify->setInt32("rtp-jitter-time-ms",

            std::max(getBaseJitterTimeMs(), getStaticJitterTimeMs()));

    notify->setInt32("rtp-rtcpsr-time-gap-ms", (int32_t)mAvgRtpRtcpGapMs);

    notify->post();



    if (isRegular) {

media/libstagefright/rtsp/ARTPSource.h

+12 −4
Original line number Diff line number Diff line
@@ -49,6 +49,8 @@ struct ARTPSource : public RefBase { 
        RTCP_FIRST_PACKET = 101,

        RTP_QUALITY = 102,

        RTP_QUALITY_EMC = 103,

        RTCP_SR = 200,

        RTCP_RR = 201,

        RTCP_TSFB = 205,

        RTCP_PSFB = 206,

        RTP_CVO = 300,
@@ -79,13 +81,12 @@ struct ARTPSource : public RefBase { 
    void putInterArrivalJitterData(uint32_t timeStamp, int64_t arrivalTime);



    bool isNeedToEarlyNotify();

    void notifyPktInfo(int32_t bitrate, bool isRegular);

    void notifyPktInfo(int32_t bitrate, int64_t nowUs, bool isRegular);

    // FIR needs to be sent by missing packet or broken video image.

    void onIssueFIRByAssembler();



    void noticeAbandonBuffer(int cnt=1);



    int32_t mFirstSeqNumber;

    uint32_t mFirstRtpTime;

    int64_t mFirstSysTime;

    int32_t mClockRate;
@@ -104,6 +105,8 @@ private: 
    uint32_t mPrevExpectedForRR;

    int32_t mPrevNumBuffersReceivedForRR;



    uint32_t mLatestRtpTime;



    List<sp<ABuffer> > mQueue;

    sp<ARTPAssembler> mAssembler;
@@ -121,8 +124,12 @@ private: 
    std::map<uint16_t, infoNACK> mNACKMap;

    int getSeqNumToNACK(List<int>& list, int size);



    uint64_t mLastNTPTime;

    int64_t mLastNTPTimeUpdateUs;

    uint32_t mLastSrRtpTime;

    uint64_t mLastSrNtpTime;

    int64_t mLastSrUpdateTimeUs;



    bool mIsFirstRtpRtcpGap;

    double mAvgRtpRtcpGapMs;



    bool mIssueFIRRequests;

    bool mIssueFIRByAssembler;
@@ -131,6 +138,7 @@ private: 


    sp<AMessage> mNotify;



    void calcTimeGapRtpRtcp(const sp<ABuffer> &buffer);

    bool queuePacket(const sp<ABuffer> &buffer);



    DISALLOW_EVIL_CONSTRUCTORS(ARTPSource);