Resilent media time stamp adjustment

static const int64_t kMax32BitFileSize = 0x007fffffffLL;

static const uint8_t kNalUnitTypeSeqParamSet = 0x07;

static const uint8_t kNalUnitTypePicParamSet = 0x08;

static const int64_t kVideoMediaTimeAdjustPeriodTimeUs = 10000000LL;  // 10s

class MPEG4Writer::Track {

public:

    int64_t mPreviousTrackTimeUs;

    int64_t mTrackEveryTimeDurationUs;

    // Has the media time adjustment for video started?

    bool    mIsMediaTimeAdjustmentOn;

    // The time stamp when previous media time adjustment period starts

    int64_t mPrevMediaTimeAdjustTimestampUs;

    // Number of vidoe frames whose time stamp may be adjusted

    int64_t mMediaTimeAdjustNumFrames;

    // The sample number when previous meida time adjustmnet period starts

    int64_t mPrevMediaTimeAdjustSample;

    // The total accumulated drift time within a period of

    // kVideoMediaTimeAdjustPeriodTimeUs.

    int64_t mTotalDriftTimeToAdjustUs;

    // The total accumalated drift time since the start of the recording

    // excluding the current time adjustment period

    int64_t mPrevTotalAccumDriftTimeUs;

    // Update the audio track's drift information.

    void updateDriftTime(const sp<MetaData>& meta);

    // Adjust the time stamp of the video track according to

    // the drift time information from the audio track.

    void adjustMediaTime(int64_t *timestampUs);

    static void *ThreadWrapper(void *me);

    status_t threadEntry();

        size = MAX_MOOV_BOX_SIZE;

    }

    LOGI("limits: %lld/%lld bytes/us, bit rate: %d bps and the estimated"

    LOGV("limits: %lld/%lld bytes/us, bit rate: %d bps and the estimated"

         " moov size %lld bytes",

         mMaxFileSizeLimitBytes, mMaxFileDurationLimitUs, bitRate, size);

    return factor * size;

        // If file size is set to be larger than the 32 bit file

        // size limit, treat it as an error.

        if (mMaxFileSizeLimitBytes > kMax32BitFileSize) {

            LOGW("32-bi file size limit (%lld bytes) too big. "

            LOGW("32-bit file size limit (%lld bytes) too big. "

                 "It is changed to %lld bytes",

                mMaxFileSizeLimitBytes, kMax32BitFileSize);

            mMaxFileSizeLimitBytes = kMax32BitFileSize;

    mNumStscTableEntries = 0;

    mNumSttsTableEntries = 0;

    mMdatSizeBytes = 0;

    mIsMediaTimeAdjustmentOn = false;

    mPrevMediaTimeAdjustTimestampUs = 0;

    mMediaTimeAdjustNumFrames = 0;

    mPrevMediaTimeAdjustSample = 0;

    mTotalDriftTimeToAdjustUs = 0;

    mPrevTotalAccumDriftTimeUs = 0;

    pthread_create(&mThread, &attr, ThreadWrapper, this);

    pthread_attr_destroy(&attr);

    return OK;

}

/*

* The video track's media time adjustment for real-time applications

* is described as follows:

*

* First, the media time adjustment is done for every period of

* kVideoMediaTimeAdjustPeriodTimeUs. kVideoMediaTimeAdjustPeriodTimeUs

* is currently a fixed value chosen heuristically. The value of

* kVideoMediaTimeAdjustPeriodTimeUs should not be very large or very small

* for two considerations: on one hand, a relatively large value

* helps reduce large fluctuation of drift time in the audio encoding

* path; while on the other hand, a relatively small value helps keep

* restoring synchronization in audio/video more frequently. Note for the

* very first period of kVideoMediaTimeAdjustPeriodTimeUs, there is

* no media time adjustment for the video track.

*

* Second, the total accumulated audio track time drift found

* in a period of kVideoMediaTimeAdjustPeriodTimeUs is distributed

* over a stream of incoming video frames. The number of video frames

* affected is determined based on the number of recorded video frames

* within the past kVideoMediaTimeAdjustPeriodTimeUs period.

* We choose to distribute the drift time over only a portion

* (rather than all) of the total number of recorded video frames

* in order to make sure that the video track media time adjustment is

* completed for the current period before the next video track media

* time adjustment period starts. Currently, the portion chosen is a

* half (0.5).

*

* Last, various additional checks are performed to ensure that

* the actual audio encoding path does not have too much drift.

* In particular, 1) we want to limit the average incremental time

* adjustment for each video frame to be less than a threshold

* for a single period of kVideoMediaTimeAdjustPeriodTimeUs.

* Currently, the threshold is set to 5 ms. If the average incremental

* media time adjustment for a video frame is larger than the

* threshold, the audio encoding path has too much time drift.

* 2) We also want to limit the total time drift in the audio

* encoding path to be less than a threshold for a period of

* kVideoMediaTimeAdjustPeriodTimeUs. Currently, the threshold

* is 0.5% of kVideoMediaTimeAdjustPeriodTimeUs. If the time drift of

* the audio encoding path is larger than the threshold, the audio

* encoding path has too much time drift. We treat the large time

* drift of the audio encoding path as errors, since there is no

* way to keep audio/video in synchronization for real-time

* applications if the time drift is too large unless we drop some

* video frames, which has its own problems that we don't want

* to get into for the time being.

*/

void MPEG4Writer::Track::adjustMediaTime(int64_t *timestampUs) {

    if (*timestampUs - mPrevMediaTimeAdjustTimestampUs >=

        kVideoMediaTimeAdjustPeriodTimeUs) {

        LOGV("New media time adjustment period at %lld us", *timestampUs);

        mIsMediaTimeAdjustmentOn = true;

        mMediaTimeAdjustNumFrames =

                (mNumSamples - mPrevMediaTimeAdjustSample) >> 1;

        mPrevMediaTimeAdjustTimestampUs = *timestampUs;

        mPrevMediaTimeAdjustSample = mNumSamples;

        int64_t totalAccumDriftTimeUs = mOwner->getDriftTimeUs();

        mTotalDriftTimeToAdjustUs =

                totalAccumDriftTimeUs - mPrevTotalAccumDriftTimeUs;

        mPrevTotalAccumDriftTimeUs = totalAccumDriftTimeUs;

        // Check on incremental adjusted time per frame

        int64_t adjustTimePerFrameUs =

                mTotalDriftTimeToAdjustUs / mMediaTimeAdjustNumFrames;

        if (adjustTimePerFrameUs < 0) {

            adjustTimePerFrameUs = -adjustTimePerFrameUs;

        }

        if (adjustTimePerFrameUs >= 5000) {

            LOGE("Adjusted time per video frame is %lld us",

                adjustTimePerFrameUs);

            CHECK(!"Video frame time adjustment is too large!");

        }

        // Check on total accumulated time drift within a period of

        // kVideoMediaTimeAdjustPeriodTimeUs.

        int64_t driftPercentage = (mTotalDriftTimeToAdjustUs * 1000)

                / kVideoMediaTimeAdjustPeriodTimeUs;

        if (driftPercentage < 0) {

            driftPercentage = -driftPercentage;

        }

        if (driftPercentage > 5) {

            LOGE("Audio track has time drift %lld us over %lld us",

                mTotalDriftTimeToAdjustUs,

                kVideoMediaTimeAdjustPeriodTimeUs);

            CHECK(!"The audio track media time drifts too much!");

        }

    }

    if (mIsMediaTimeAdjustmentOn) {

        if (mNumSamples - mPrevMediaTimeAdjustSample <=

            mMediaTimeAdjustNumFrames) {

            // Do media time incremental adjustment

            int64_t incrementalAdjustTimeUs =

                        (mTotalDriftTimeToAdjustUs *

                            (mNumSamples - mPrevMediaTimeAdjustSample))

                                / mMediaTimeAdjustNumFrames;

            *timestampUs +=

                (incrementalAdjustTimeUs + mPrevTotalAccumDriftTimeUs);

            LOGV("Incremental video frame media time adjustment: %lld us",

                (incrementalAdjustTimeUs + mPrevTotalAccumDriftTimeUs));

        } else {

            // Within the remaining adjustment period,

            // no incremental adjustment is needed.

            *timestampUs +=

                (mTotalDriftTimeToAdjustUs + mPrevTotalAccumDriftTimeUs);

            LOGV("Fixed video frame media time adjustment: %lld us",

                (mTotalDriftTimeToAdjustUs + mPrevTotalAccumDriftTimeUs));

        }

    }

}

/*

 * Updates the drift time from the audio track so that

 * the video track can get the updated drift time information

 * from the file writer. The fluctuation of the drift time of the audio

 * encoding path is smoothed out with a simple filter by giving a larger

 * weight to more recently drift time. The filter coefficients, 0.5 and 0.5,

 * are heuristically determined.

 */

void MPEG4Writer::Track::updateDriftTime(const sp<MetaData>& meta) {

    int64_t driftTimeUs = 0;

    if (meta->findInt64(kKeyDriftTime, &driftTimeUs)) {

        int64_t prevDriftTimeUs = mOwner->getDriftTimeUs();

        int64_t timeUs = (driftTimeUs + prevDriftTimeUs) >> 1;

        mOwner->setDriftTimeUs(timeUs);

    }

}

status_t MPEG4Writer::Track::threadEntry() {

    int32_t count = 0;

    const int64_t interleaveDurationUs = mOwner->interleaveDuration();

        timestampUs -= previousPausedDurationUs;

        CHECK(timestampUs >= 0);

        if (mIsRealTimeRecording && !mIsAudio) {

            // The minor adjustment on the timestamp is heuristic/experimental

            // We are adjusting the timestamp to reduce the fluctuation of the duration

            // of neighboring samples. This in turn helps reduce the track header size,

            // especially, the number of entries in the "stts" box.

            if (mNumSamples > 1) {

                int64_t currDriftTimeUs = mOwner->getDriftTimeUs();

                int64_t durationUs = timestampUs + currDriftTimeUs - lastTimestampUs;

                int64_t diffUs = (durationUs > lastDurationUs)

                            ? durationUs - lastDurationUs

                            : lastDurationUs - durationUs;

                if (diffUs <= 5000) {  // XXX: Magic number 5ms

                    timestampUs = lastTimestampUs + lastDurationUs;

        // Media time adjustment for real-time applications

        if (mIsRealTimeRecording) {

            if (mIsAudio) {

                updateDriftTime(meta_data);

            } else {

                    timestampUs += currDriftTimeUs;

                }

                adjustMediaTime(&timestampUs);

            }

        }

        CHECK(timestampUs >= 0);

        if (mNumSamples > 1) {

            if (timestampUs <= lastTimestampUs) {

        lastDurationUs = timestampUs - lastTimestampUs;

        lastDurationTicks = currDurationTicks;

        lastTimestampUs = timestampUs;

        if (mIsRealTimeRecording && mIsAudio) {

            int64_t driftTimeUs = 0;

            if (meta_data->findInt64(kKeyDriftTime, &driftTimeUs)) {

                mOwner->setDriftTimeUs(driftTimeUs);

            }

        }

        if (isSync != 0) {

            addOneStssTableEntry(mNumSamples);

    mReachedEOS = true;

    LOGI("Received total/0-length (%d/%d) buffers and encoded %d frames. - %s",

            count, nZeroLengthFrames, mNumSamples, mIsAudio? "audio": "video");

    if (mIsAudio) {

        LOGI("Audio track drift time: %lld us", mOwner->getDriftTimeUs());

    }

    if (err == ERROR_END_OF_STREAM) {

        return OK;