Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 4c23815c authored by James Dong's avatar James Dong
Browse files

Calculate audio media drift time from AudioSource 

The problem was that the time to receive an output buffer
from an audio encoder is different because the encoder does not
need to read from the source for all output buffers. This leads
to large fluctuation in terms of wall clock duration between two
neighboring audio sample outputs from the audio encoder. As a
result, the media time for the video track after adjustment using
the drifting changes wildly sometimes.

This patch addresses this issue by only updating the media drift
time when an audio source input buffer is read. the wall clock
for the audio track is also calculated at the same time when
the input audio buffer is read at AudioSource.

bug - 2959800

Change-Id: I3174aa182f744784b540f0a7198524d4eee8bd7b
parent 3424c02e

include/media/stagefright/AudioSource.h

+1 −0
Original line number Diff line number Diff line
@@ -72,6 +72,7 @@ private: 
    int64_t mPrevSampleTimeUs;

    int64_t mTotalLostFrames;

    int64_t mPrevLostBytes;

    int64_t mInitialReadTimeUs;



    MediaBufferGroup *mGroup;

include/media/stagefright/MPEG4Writer.h

+1 −1
Original line number Diff line number Diff line
@@ -132,7 +132,7 @@ private: 
    // Adjust other track media clock (presumably wall clock)

    // based on audio track media clock with the drift time.

    int64_t mDriftTimeUs;

    void addDriftTimeUs(int64_t driftTimeUs);

    void setDriftTimeUs(int64_t driftTimeUs);

    int64_t getDriftTimeUs();



    void lock();

include/media/stagefright/MetaData.h

+1 −0
Original line number Diff line number Diff line
@@ -48,6 +48,7 @@ enum { 
    kKeyTime              = 'time',  // int64_t (usecs)

    kKeyNTPTime           = 'ntpT',  // uint64_t (ntp-timestamp)

    kKeyTargetTime        = 'tarT',  // int64_t (usecs)

    kKeyDriftTime         = 'dftT',  // int64_t (usecs)

    kKeyDuration          = 'dura',  // int64_t (usecs)

    kKeyColorFormat       = 'colf',

    kKeyPlatformPrivate   = 'priv',  // pointer

media/libstagefright/AudioSource.cpp

+12 −3
Original line number Diff line number Diff line
@@ -84,6 +84,7 @@ status_t AudioSource::start(MetaData *params) { 


    mTrackMaxAmplitude = false;

    mMaxAmplitude = 0;

    mInitialReadTimeUs = 0;

    mStartTimeUs = 0;

    int64_t startTimeUs;

    if (params && params->findInt64(kKeyTime, &startTimeUs)) {
@@ -210,6 +211,7 @@ status_t AudioSource::read( 
        return NO_INIT;

    }



    int64_t readTimeUs = systemTime() / 1000;

    *out = NULL;



    MediaBuffer *buffer;
@@ -223,9 +225,10 @@ status_t AudioSource::read( 




        if (numFramesRecorded == 0 && mPrevSampleTimeUs == 0) {

            mInitialReadTimeUs = readTimeUs;

            // Initial delay

            if (mStartTimeUs > 0) {

                mStartTimeUs = systemTime() / 1000 - mStartTimeUs;

                mStartTimeUs = readTimeUs - mStartTimeUs;

            } else {

                // Assume latency is constant.

                mStartTimeUs += mRecord->latency() * 1000;
@@ -271,7 +274,10 @@ status_t AudioSource::read( 
            }

            memset(buffer->data(), 0, numLostBytes);

            buffer->set_range(0, numLostBytes);

            buffer->meta_data()->setInt64(kKeyTime, mPrevSampleTimeUs);

            if (numFramesRecorded == 0) {

                buffer->meta_data()->setInt64(kKeyTime, mStartTimeUs);

            }

            buffer->meta_data()->setInt64(kKeyDriftTime, readTimeUs - mInitialReadTimeUs);

            mPrevSampleTimeUs = timestampUs;

            *out = buffer;

            return OK;
@@ -309,7 +315,10 @@ status_t AudioSource::read( 
            trackMaxAmplitude((int16_t *) buffer->data(), n >> 1);

        }



        buffer->meta_data()->setInt64(kKeyTime, mPrevSampleTimeUs);

        if (numFramesRecorded == 0) {

            buffer->meta_data()->setInt64(kKeyTime, mStartTimeUs);

        }

        buffer->meta_data()->setInt64(kKeyDriftTime, readTimeUs - mInitialReadTimeUs);

        CHECK(timestampUs > mPrevSampleTimeUs);

        mPrevSampleTimeUs = timestampUs;

        LOGV("initial delay: %lld, sample rate: %d, timestamp: %lld",

media/libstagefright/MPEG4Writer.cpp

+9 −20
Original line number Diff line number Diff line
@@ -1430,9 +1430,6 @@ status_t MPEG4Writer::Track::threadEntry() { 
    int64_t previousPausedDurationUs = 0;

    int64_t timestampUs;



    int64_t wallClockTimeUs = 0;

    int64_t lastWallClockTimeUs = 0;



    sp<MetaData> meta_data;

    bool collectStats = collectStatisticalData();
@@ -1542,14 +1539,15 @@ status_t MPEG4Writer::Track::threadEntry() { 
            // of neighboring samples. This in turn helps reduce the track header size,

            // especially, the number of entries in the "stts" box.

            if (mNumSamples > 1) {

                int64_t durationUs = timestampUs + mOwner->getDriftTimeUs() - lastTimestampUs;

                int64_t currDriftTimeUs = mOwner->getDriftTimeUs();

                int64_t durationUs = timestampUs + currDriftTimeUs - lastTimestampUs;

                int64_t diffUs = (durationUs > lastDurationUs)

                            ? durationUs - lastDurationUs

                            : lastDurationUs - durationUs;

                if (diffUs <= 5000) {  // XXX: Magic number 5ms

                    timestampUs = lastTimestampUs + lastDurationUs;

                } else {

                    timestampUs += mOwner->getDriftTimeUs();

                    timestampUs += currDriftTimeUs;

                }

            }

        }
@@ -1557,12 +1555,6 @@ status_t MPEG4Writer::Track::threadEntry() { 
        if (mNumSamples > 1) {

            if (timestampUs <= lastTimestampUs) {

                LOGW("Frame arrives too late!");

#if 0

                // Drop the late frame.

                copy->release();

                copy = NULL;

                continue;

#else

                // Don't drop the late frame, since dropping a frame may cause

                // problems later during playback
@@ -1573,7 +1565,6 @@ status_t MPEG4Writer::Track::threadEntry() { 
                } else {

                    timestampUs = lastTimestampUs + (1000000LL + (mTimeScale >> 1)) / mTimeScale;

                }

#endif

            }

        }
@@ -1613,12 +1604,10 @@ status_t MPEG4Writer::Track::threadEntry() { 
        lastDurationTicks = currDurationTicks;

        lastTimestampUs = timestampUs;

        if (mIsRealTimeRecording && mIsAudio) {

            wallClockTimeUs = systemTime() / 1000;

            int64_t wallClockDurationUs = wallClockTimeUs - lastWallClockTimeUs;

            if (mNumSamples > 2) {

                mOwner->addDriftTimeUs(lastDurationUs - wallClockDurationUs);

            int64_t driftTimeUs = 0;

            if (meta_data->findInt64(kKeyDriftTime, &driftTimeUs)) {

                mOwner->setDriftTimeUs(driftTimeUs);

            }

            lastWallClockTimeUs = wallClockTimeUs;

        }



        if (isSync != 0) {
@@ -1851,10 +1840,10 @@ void MPEG4Writer::Track::logStatisticalData(bool isAudio) { 
    }

}



void MPEG4Writer::addDriftTimeUs(int64_t driftTimeUs) {

    LOGV("addDriftTimeUs: %lld us", driftTimeUs);

void MPEG4Writer::setDriftTimeUs(int64_t driftTimeUs) {

    LOGV("setDriftTimeUs: %lld us", driftTimeUs);

    Mutex::Autolock autolock(mLock);

    mDriftTimeUs += driftTimeUs;

    mDriftTimeUs = driftTimeUs;

}



int64_t MPEG4Writer::getDriftTimeUs() {