Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 6ad0a2f1 authored by Jainishkumar Anghan's avatar Jainishkumar Anghan Committed by Lajos Molnar
Browse files

VT: Migration AAVCAssembler patches to AHEVCAssembler. 



 - CVO implementation for Rx side
 - Added an interface to set JitterBufferTime of AHEVCAssembler
 - Do not merge FU pkts if rtptimes are different
 - Implementation of a new frame drop rule for p-frame
 - Nack Implementation
 - Send FIR if there was no I-frame within a sec in packetLost

Bug: 165061754
Merged-in: Iad574dbfaa1644c2968128a0bf72a8ed323228c5
Change-Id: Iad574dbfaa1644c2968128a0bf72a8ed323228c5
Signed-off-by: default avatarJainishkumar Anghan <jai.anghan@samsung.com>
parent c0a4d1a5

media/libstagefright/HevcUtils.cpp

+48 −0
Original line number Diff line number Diff line
@@ -377,6 +377,54 @@ status_t HevcParameterSets::parseSps(const uint8_t* data, size_t size) { 
    return reader.overRead() ? ERROR_MALFORMED : OK;

}



void HevcParameterSets::FindHEVCDimensions(const sp<ABuffer> &SpsBuffer, int32_t *width, int32_t *height)

{

    ALOGD("FindHEVCDimensions");

    // See Rec. ITU-T H.265 v3 (04/2015) Chapter 7.3.2.2 for reference

    ABitReader reader(SpsBuffer->data() + 1, SpsBuffer->size() - 1);

    // Skip sps_video_parameter_set_id

    reader.skipBits(4);

    uint8_t maxSubLayersMinus1 = reader.getBitsWithFallback(3, 0);

    // Skip sps_temporal_id_nesting_flag;

    reader.skipBits(1);

    // Skip general profile

    reader.skipBits(96);

    if (maxSubLayersMinus1 > 0) {

        bool subLayerProfilePresentFlag[8];

        bool subLayerLevelPresentFlag[8];

        for (int i = 0; i < maxSubLayersMinus1; ++i) {

            subLayerProfilePresentFlag[i] = reader.getBitsWithFallback(1, 0);

            subLayerLevelPresentFlag[i] = reader.getBitsWithFallback(1, 0);

        }

        // Skip reserved

        reader.skipBits(2 * (8 - maxSubLayersMinus1));

        for (int i = 0; i < maxSubLayersMinus1; ++i) {

            if (subLayerProfilePresentFlag[i]) {

                // Skip profile

                reader.skipBits(88);

            }

            if (subLayerLevelPresentFlag[i]) {

                // Skip sub_layer_level_idc[i]

                reader.skipBits(8);

            }

        }

    }

    // Skip sps_seq_parameter_set_id

    skipUE(&reader);

    uint8_t chromaFormatIdc = parseUEWithFallback(&reader, 0);

    if (chromaFormatIdc == 3) {

        // Skip separate_colour_plane_flag

        reader.skipBits(1);

    }

    skipUE(&reader);

    skipUE(&reader);



    // pic_width_in_luma_samples

    *width = parseUEWithFallback(&reader, 0);

    // pic_height_in_luma_samples

    *height = parseUEWithFallback(&reader, 0);

}



status_t HevcParameterSets::parsePps(

        const uint8_t* data UNUSED_PARAM, size_t size UNUSED_PARAM) {

    return OK;

media/libstagefright/include/HevcUtils.h

+2 −0
Original line number Diff line number Diff line
@@ -90,6 +90,8 @@ public: 
    // Note that this method does not write the start code.

    bool write(size_t index, uint8_t* dest, size_t size);

    status_t makeHvcc(uint8_t *hvcc, size_t *hvccSize, size_t nalSizeLength);

    void FindHEVCDimensions(const sp<ABuffer> &SpsBuffer, int32_t *width,

                                                int32_t *height);



    Info getInfo() const { return mInfo; }

media/libstagefright/rtsp/AHEVCAssembler.cpp

+248 −26
Original line number Diff line number Diff line
@@ -25,6 +25,7 @@ 
#include <media/stagefright/foundation/ABuffer.h>

#include <media/stagefright/foundation/ADebug.h>

#include <media/stagefright/foundation/AMessage.h>

#include <include/HevcUtils.h>

#include <media/stagefright/foundation/hexdump.h>



#include <stdint.h>
@@ -46,7 +47,11 @@ AHEVCAssembler::AHEVCAssembler(const sp<AMessage> &notify) 
      mAccessUnitRTPTime(0),

      mNextExpectedSeqNoValid(false),

      mNextExpectedSeqNo(0),

      mAccessUnitDamaged(false) {

      mAccessUnitDamaged(false),

      mFirstIFrameProvided(false),

      mLastIFrameProvidedAt(0),

      mWidth(0),

      mHeight(0) {



      ALOGV("Constructor");

}
@@ -54,6 +59,63 @@ AHEVCAssembler::AHEVCAssembler(const sp<AMessage> &notify) 
AHEVCAssembler::~AHEVCAssembler() {

}



int32_t AHEVCAssembler::addNack(

        const sp<ARTPSource> &source) {

    List<sp<ABuffer>> *queue = source->queue();

    int32_t nackCount = 0;



    List<sp<ABuffer> >::iterator it = queue->begin();



    uint16_t queueHeadSeqNum;

    if (it != queue->end())

        queueHeadSeqNum = (*it)->int32Data();



    // move to the packet after that RTCP:NACK sent.

    while (it != queue->end()) {

        int32_t seqNum = (*it)->int32Data();

        if (seqNum < source->mHighestNackNumber)

            it++;

        else

            break;

    }



    int32_t nackStartAt = -1;



    while (it != queue->end()) {

        int32_t seqBeforeLast = (*it)->int32Data();

        // increase iterator.

        if ((++it) == queue->end())

            break;

        int32_t seqLast = (*it)->int32Data();



        if ((seqLast - seqBeforeLast) < 0) {

            ALOGD("addNack : found end of seqNum from(%d) to(%d)", seqBeforeLast, seqLast);

            source->mHighestNackNumber = 0;

        }



        // missed packet found

        if (seqLast > (seqBeforeLast + 1) &&

            // we didn't send RTCP:NACK for this packet yet.

            (seqLast - 1) > source->mHighestNackNumber) {

            source->mHighestNackNumber = seqLast -1;

            nackStartAt = seqBeforeLast + 1;

            break;

        }



    }



    if (nackStartAt != -1) {

        nackCount = source->mHighestNackNumber - nackStartAt + 1;

        ALOGD("addNack : nackCount=%d, nackFrom=%d, nackTo=%d", nackCount,

            nackStartAt, source->mHighestNackNumber);



        uint16_t mask = (uint16_t)(0xffff) >> (16-nackCount+1);

        source->setSeqNumToNACK(nackStartAt, mask, queueHeadSeqNum);

    }



    return nackCount;

}



ARTPAssembler::AssemblyStatus AHEVCAssembler::addNALUnit(

        const sp<ARTPSource> &source) {

    List<sp<ABuffer> > *queue = source->queue();
@@ -63,6 +125,7 @@ ARTPAssembler::AssemblyStatus AHEVCAssembler::addNALUnit( 
    }



    sp<ABuffer> buffer = *queue->begin();

    buffer->meta()->setObject("source", source);

    uint32_t rtpTime;

    CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));

    int64_t startTime = source->mFirstSysTime / 1000;
@@ -70,26 +133,45 @@ ARTPAssembler::AssemblyStatus AHEVCAssembler::addNALUnit( 
    int64_t playedTime = nowTime - startTime;

    int64_t playedTimeRtp = source->mFirstRtpTime +

        (((uint32_t)playedTime) * (source->mClockRate / 1000));

    const uint32_t expiredTimeInJb = rtpTime + (source->mClockRate / 5);

    const uint32_t jitterTime = (uint32_t)(source->mClockRate / ((float)1000 / (source->mJbTime)));

    uint32_t expiredTimeInJb = rtpTime + jitterTime;

    bool isExpired = expiredTimeInJb <= (playedTimeRtp);

    ALOGV("start=%lld, now=%lld, played=%lld", (long long)startTime,

            (long long)nowTime, (long long)playedTime);

    ALOGV("rtp-time(JB)=%u, played-rtp-time(JB)=%lld, expired-rtp-time(JB)=%u isExpired=%d",

            rtpTime, (long long)playedTimeRtp, expiredTimeInJb, isExpired);

    bool isTooLate200 = expiredTimeInJb < (playedTimeRtp - jitterTime);

    bool isTooLate300 = expiredTimeInJb < (playedTimeRtp - (jitterTime * 3 / 2));



    if (mShowQueueCnt < 20) {

        showCurrentQueue(queue);

        printNowTimeUs(startTime, nowTime, playedTime);

        printRTPTime(rtpTime, playedTimeRtp, expiredTimeInJb, isExpired);

        mShowQueueCnt++;

    }



    AHEVCAssembler::addNack(source);



    if (!isExpired) {

        ALOGV("buffering in jitter buffer.");

        return NOT_ENOUGH_DATA;

    }



    if (mNextExpectedSeqNoValid) {

        List<sp<ABuffer> >::iterator it = queue->begin();

        while (it != queue->end()) {

            if ((uint32_t)(*it)->int32Data() >= mNextExpectedSeqNo) {

                break;

    if (isTooLate200)

        ALOGW("=== WARNING === buffer arrived 200ms late. === WARNING === ");



    if (isTooLate300) {

        ALOGW("buffer arrived after 300ms ... \t Diff in Jb=%lld \t Seq# %d",

              ((long long)playedTimeRtp) - expiredTimeInJb, buffer->int32Data());

        printNowTimeUs(startTime, nowTime, playedTime);

        printRTPTime(rtpTime, playedTimeRtp, expiredTimeInJb, isExpired);



        mNextExpectedSeqNo = pickProperSeq(queue, jitterTime, playedTimeRtp);

    }



            it = queue->erase(it);

    if (mNextExpectedSeqNoValid) {

        int32_t size = queue->size();

        int32_t cntRemove = deleteUnitUnderSeq(queue, mNextExpectedSeqNo);



        if (cntRemove > 0) {

            source->noticeAbandonBuffer(cntRemove);

            ALOGW("delete %d of %d buffers", cntRemove, size);

        }



        if (queue->empty()) {
@@ -154,15 +236,68 @@ ARTPAssembler::AssemblyStatus AHEVCAssembler::addNALUnit( 
    }

}



void AHEVCAssembler::checkSpsUpdated(const sp<ABuffer> &buffer) {

    const uint8_t *data = buffer->data();

    HevcParameterSets paramSets;

    unsigned nalType = (data[0] >> 1) & H265_NALU_MASK;

    if (nalType == H265_NALU_SPS) {

        int32_t width = 0, height = 0;

        paramSets.FindHEVCDimensions(buffer, &width, &height);

        ALOGV("existing resolution (%u x %u)", mWidth, mHeight);

        if (width != mWidth || height != mHeight) {

            mFirstIFrameProvided = false;

            mWidth = width;

            mHeight = height;

            ALOGD("found a new resolution (%u x %u)", mWidth, mHeight);

        }

    }

}



void AHEVCAssembler::checkIFrameProvided(const sp<ABuffer> &buffer) {

    const uint8_t *data = buffer->data();

    unsigned nalType = (data[0] >> 1) & H265_NALU_MASK;

    if (nalType > 0x0F && nalType < 0x18) {

        mLastIFrameProvidedAt = ALooper::GetNowUs() / 1000;

        if (!mFirstIFrameProvided) {

            mFirstIFrameProvided = true;

            uint32_t rtpTime;

            CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));

            ALOGD("got First I-frame to be decoded. rtpTime=%d, size=%zu", rtpTime, buffer->size());

        }

    }

}



bool AHEVCAssembler::dropFramesUntilIframe(const sp<ABuffer> &buffer) {

    const uint8_t *data = buffer->data();

    unsigned nalType = (data[0] >> 1) & H265_NALU_MASK;

    if (!mFirstIFrameProvided && nalType < 0x10)

        return true;



    return false;

}



void AHEVCAssembler::addSingleNALUnit(const sp<ABuffer> &buffer) {

    ALOGV("addSingleNALUnit of size %zu", buffer->size());

#if !LOG_NDEBUG

    hexdump(buffer->data(), buffer->size());

#endif

    checkSpsUpdated(buffer);

    checkIFrameProvided(buffer);



    uint32_t rtpTime;

    CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));



    if (dropFramesUntilIframe(buffer)) {

        sp<ARTPSource> source = nullptr;

        buffer->meta()->findObject("source", (sp<android::RefBase>*)&source);

        if (source != nullptr) {

            ALOGD("Issued FIR to get the I-frame");

            source->onIssueFIRByAssembler();

        }

        ALOGD("drop P-frames till an I-frame provided. rtpTime %u", rtpTime);

        return;

    }



    if (!mNALUnits.empty() && rtpTime != mAccessUnitRTPTime) {

        submitAccessUnit();

    }
@@ -260,6 +395,11 @@ ARTPAssembler::AssemblyStatus AHEVCAssembler::addFragmentedNALUnit( 
    size_t totalCount = 1;

    bool complete = false;



    uint32_t rtpTimeStartAt;

    CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTimeStartAt));

    uint32_t startSeqNo = buffer->int32Data();

    bool pFrame = (nalType < 0x10);



    if (data[2] & 0x40) {

        // Huh? End bit also set on the first buffer.
@@ -268,6 +408,8 @@ ARTPAssembler::AssemblyStatus AHEVCAssembler::addFragmentedNALUnit( 
        complete = true;

    } else {

        List<sp<ABuffer> >::iterator it = ++queue->begin();

        int32_t connected = 1;

        bool snapped = false;

        while (it != queue->end()) {

            ALOGV("sequence length %zu", totalCount);
@@ -277,26 +419,30 @@ ARTPAssembler::AssemblyStatus AHEVCAssembler::addFragmentedNALUnit( 
            size_t size = buffer->size();



            if ((uint32_t)buffer->int32Data() != expectedSeqNo) {

                ALOGV("sequence not complete, expected seqNo %d, got %d",

                     expectedSeqNo, (uint32_t)buffer->int32Data());

                ALOGV("sequence not complete, expected seqNo %d, got %d, nalType %d",

                     expectedSeqNo, (uint32_t)buffer->int32Data(), nalType);

                snapped = true;



                if (!pFrame)

                    return WRONG_SEQUENCE_NUMBER;

            }



            if (!snapped)

                connected++;



            uint32_t rtpTime;

            CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));

            if (size < 3

                    || ((data[0] >> 1) & H265_NALU_MASK) != indicator

                    || (data[2] & H265_NALU_MASK) != nalType

                    || (data[2] & 0x80)) {

                    || (data[2] & 0x80)

                    || rtpTime != rtpTimeStartAt) {

                ALOGV("Ignoring malformed FU buffer.");



                // Delete the whole start of the FU.



                it = queue->begin();

                for (size_t i = 0; i <= totalCount; ++i) {

                    it = queue->erase(it);

                }



                mNextExpectedSeqNo = expectedSeqNo + 1;

                deleteUnitUnderSeq(queue, mNextExpectedSeqNo);



                return MALFORMED_PACKET;

            }
@@ -304,9 +450,16 @@ ARTPAssembler::AssemblyStatus AHEVCAssembler::addFragmentedNALUnit( 
            totalSize += size - 3;

            ++totalCount;



            expectedSeqNo = expectedSeqNo + 1;

            expectedSeqNo = buffer->int32Data() + 1;



            if (data[2] & 0x40) {

                if (pFrame && !recycleUnit(startSeqNo, expectedSeqNo,

                        connected, totalCount, 0.5f)) {

                    mNextExpectedSeqNo = expectedSeqNo;

                    deleteUnitUnderSeq(queue, mNextExpectedSeqNo);



                    return MALFORMED_PACKET;

                }

                // This is the last fragment.

                complete = true;

                break;
@@ -335,6 +488,7 @@ ARTPAssembler::AssemblyStatus AHEVCAssembler::addFragmentedNALUnit( 
    unit->data()[1] = tid;



    size_t offset = 2;

    int32_t cvo = -1;

    List<sp<ABuffer> >::iterator it = queue->begin();

    for (size_t i = 0; i < totalCount; ++i) {

        const sp<ABuffer> &buffer = *it;
@@ -345,6 +499,7 @@ ARTPAssembler::AssemblyStatus AHEVCAssembler::addFragmentedNALUnit( 
#endif



        memcpy(unit->data() + offset, buffer->data() + 3, buffer->size() - 3);

        buffer->meta()->findInt32("cvo", &cvo);

        offset += buffer->size() - 3;



        it = queue->erase(it);
@@ -352,6 +507,10 @@ ARTPAssembler::AssemblyStatus AHEVCAssembler::addFragmentedNALUnit( 


    unit->setRange(0, totalSize);



    if (cvo >= 0) {

        unit->meta()->setInt32("cvo", cvo);

    }



    addSingleNALUnit(unit);



    ALOGV("successfully assembled a NAL unit from fragments.");
@@ -372,6 +531,7 @@ void AHEVCAssembler::submitAccessUnit() { 


    sp<ABuffer> accessUnit = new ABuffer(totalSize);

    size_t offset = 0;

    int32_t cvo = -1;

    for (List<sp<ABuffer> >::iterator it = mNALUnits.begin();

         it != mNALUnits.end(); ++it) {

        memcpy(accessUnit->data() + offset, "\x00\x00\x00\x01", 4);
@@ -380,6 +540,7 @@ void AHEVCAssembler::submitAccessUnit() { 
        sp<ABuffer> nal = *it;

        memcpy(accessUnit->data() + offset, nal->data(), nal->size());

        offset += nal->size();

        nal->meta()->findInt32("cvo", &cvo);

    }



    CopyTimes(accessUnit, *mNALUnits.begin());
@@ -388,6 +549,9 @@ void AHEVCAssembler::submitAccessUnit() { 
    printf(mAccessUnitDamaged ? "X" : ".");

    fflush(stdout);

#endif

    if (cvo >= 0) {

        accessUnit->meta()->setInt32("cvo", cvo);

    }



    if (mAccessUnitDamaged) {

        accessUnit->meta()->setInt32("damaged", true);
@@ -401,22 +565,80 @@ void AHEVCAssembler::submitAccessUnit() { 
    msg->post();

}



int32_t AHEVCAssembler::pickProperSeq(const Q *q, uint32_t jit, int64_t play) {

    sp<ABuffer> buffer = *(q->begin());

    uint32_t rtpTime;

    int32_t nextSeqNo = buffer->int32Data();



    Q::const_iterator it = q->begin();

    while (it != q->end()) {

        CHECK((*it)->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));

        // if pkt in time exists, that should be the next pivot

        if (rtpTime + jit >= play) {

            nextSeqNo = (*it)->int32Data();

            break;

        }

        it++;

    }

    return nextSeqNo;

}



bool AHEVCAssembler::recycleUnit(uint32_t start, uint32_t end,  uint32_t connected,

         size_t avail, float goodRatio) {

    float total = end - start;

    float valid = connected;

    float exist = avail;

    bool isRecycle = (valid / total) >= goodRatio;



    ALOGV("checking p-frame losses.. recvBufs %f valid %f diff %f recycle? %d",

            exist, valid, total, isRecycle);



    return isRecycle;

}



int32_t AHEVCAssembler::deleteUnitUnderSeq(Q *q, uint32_t seq) {

    int32_t initSize = q->size();

    Q::iterator it = q->begin();

    while (it != q->end()) {

        if ((uint32_t)(*it)->int32Data() >= seq) {

            break;

        }

        it++;

    }

    q->erase(q->begin(), it);

    return initSize - q->size();

}



inline void AHEVCAssembler::printNowTimeUs(int64_t start, int64_t now, int64_t play) {

    ALOGD("start=%lld, now=%lld, played=%lld",

            (long long)start, (long long)now, (long long)play);

}



inline void AHEVCAssembler::printRTPTime(uint32_t rtp, int64_t play, uint32_t exp, bool isExp) {

    ALOGD("rtp-time(JB)=%u, played-rtp-time(JB)=%lld, expired-rtp-time(JB)=%u isExpired=%d",

            rtp, (long long)play, exp, isExp);

}





ARTPAssembler::AssemblyStatus AHEVCAssembler::assembleMore(

        const sp<ARTPSource> &source) {

    AssemblyStatus status = addNALUnit(source);

    if (status == MALFORMED_PACKET) {

        mAccessUnitDamaged = true;

        uint64_t timeAfterLastIFrame = (ALooper::GetNowUs() / 1000) - mLastIFrameProvidedAt;

        if (timeAfterLastIFrame > 1000) {

            ALOGV("request FIR to get a new I-Frame, time after "

                    "last I-Frame in miils %llu", (unsigned long long)timeAfterLastIFrame);

            source->onIssueFIRByAssembler();

        }

    }

    return status;

}



void AHEVCAssembler::packetLost() {

    CHECK(mNextExpectedSeqNoValid);

    ALOGV("packetLost (expected %d)", mNextExpectedSeqNo);

    ALOGD("packetLost (expected %d)", mNextExpectedSeqNo);



    ++mNextExpectedSeqNo;



    mAccessUnitDamaged = true;

}



void AHEVCAssembler::onByeReceived() {

media/libstagefright/rtsp/AHEVCAssembler.h

+17 −0
Original line number Diff line number Diff line
@@ -25,6 +25,8 @@ 


namespace android {



using Q = List<sp<ABuffer> >;



struct ABuffer;

struct AMessage;
@@ -45,8 +47,16 @@ private: 
    bool mNextExpectedSeqNoValid;

    uint32_t mNextExpectedSeqNo;

    bool mAccessUnitDamaged;

    bool mFirstIFrameProvided;

    uint64_t mLastIFrameProvidedAt;

    int32_t mWidth;

    int32_t mHeight;

    List<sp<ABuffer> > mNALUnits;



    int32_t addNack(const sp<ARTPSource> &source);

    void checkSpsUpdated(const sp<ABuffer> &buffer);

    void checkIFrameProvided(const sp<ABuffer> &buffer);

    bool dropFramesUntilIframe(const sp<ABuffer> &buffer);

    AssemblyStatus addNALUnit(const sp<ARTPSource> &source);

    void addSingleNALUnit(const sp<ABuffer> &buffer);

    AssemblyStatus addFragmentedNALUnit(List<sp<ABuffer> > *queue);
@@ -54,6 +64,13 @@ private: 


    void submitAccessUnit();



    int32_t pickProperSeq(const Q *q, uint32_t jit, int64_t play);

    bool recycleUnit(uint32_t start, uint32_t end, uint32_t conneceted,

             size_t avail, float goodRatio);

    int32_t deleteUnitUnderSeq(Q *q, uint32_t seq);

    void printNowTimeUs(int64_t start, int64_t now, int64_t play);

    void printRTPTime(uint32_t rtp, int64_t play, uint32_t exp, bool isExp);



    DISALLOW_EVIL_CONSTRUCTORS(AHEVCAssembler);

};