Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 4f1f19d3 authored by Gopalakrishnan Nallasamy's avatar Gopalakrishnan Nallasamy
Browse files

MPEG4Writer+MPEG4Extractor:EditList to handle tracks' start times 

1) Handle tracks (video with bframes and without bframes) starting with different start
   offsets through edit list entries. Empty edit list entries for positive offsets.
   Normal edit list entries for negative offsets.
2) Handle audio tracks with start timestamps < 0, without affecting gapless playback.
3) Replaced the logic of adjusting first sample's delta/duration for video tracks
   without B-frames and non-video tracks inorder to maintain AV sync with edit list
   entries.
4) Adjusted CTTS offsets to reflect only the offset needed for BFrames reordering.
   To maintain AV sync start offset of tracks with BFrames were added to these offsets
   earlier.

Bug: 135030072
Bug: 142580952
Bug: 141653301

Test: 1) atest CtsMediaTestCases -- \
         --module-arg CtsMediaTestCases:size:small
      2) atest android.media.cts.MediaMuxerTest
      3) gapless playback clips are playing fine.

Change-Id: If3fa71431de199b02a17323a05af46759b20b905
parent 990cc725

media/extractors/mp4/MPEG4Extractor.cpp

+124 −74
Original line number Diff line number Diff line
@@ -83,7 +83,8 @@ public: 
                const Trex *trex,

                off64_t firstMoofOffset,

                const sp<ItemTable> &itemTable,

                uint64_t elstShiftStartTicks);

                uint64_t elstShiftStartTicks,

                uint64_t elstInitialEmptyEditTicks);

    virtual status_t init();



    virtual media_status_t start();
@@ -151,6 +152,7 @@ private: 
    // Start offset from composition time to presentation time.

    // Support shift only for video tracks through mElstShiftStartTicks for now.

    uint64_t mElstShiftStartTicks;

    uint64_t mElstInitialEmptyEditTicks;



    size_t parseNALSize(const uint8_t *data) const;

    status_t parseChunk(off64_t *offset);
@@ -484,12 +486,11 @@ media_status_t MPEG4Extractor::getTrackMetaData( 
        int64_t duration;

        int32_t samplerate;

        // Only for audio track.

        if (track->has_elst && mHeaderTimescale != 0 &&

        if (track->elst_needs_processing && mHeaderTimescale != 0 &&

            AMediaFormat_getInt64(track->meta, AMEDIAFORMAT_KEY_DURATION, &duration) &&

            AMediaFormat_getInt32(track->meta, AMEDIAFORMAT_KEY_SAMPLE_RATE, &samplerate)) {



            // Elst has to be processed only the first time this function is called.

            track->has_elst = false;

            track->elst_needs_processing = false;



            if (track->elst_segment_duration > INT64_MAX) {

                return;
@@ -1098,10 +1099,11 @@ status_t MPEG4Extractor::parseChunk(off64_t *offset, int depth) { 
                            track_b->sampleTable = mLastTrack->sampleTable;

                            track_b->includes_expensive_metadata = mLastTrack->includes_expensive_metadata;

                            track_b->skipTrack = mLastTrack->skipTrack;

                            track_b->has_elst = mLastTrack->has_elst;

                            track_b->elst_needs_processing = mLastTrack->elst_needs_processing;

                            track_b->elst_media_time = mLastTrack->elst_media_time;

                            track_b->elst_segment_duration = mLastTrack->elst_segment_duration;

                            track_b->elstShiftStartTicks = mLastTrack->elstShiftStartTicks;

                            track_b->elst_shift_start_ticks = mLastTrack->elst_shift_start_ticks;

                            track_b->elst_initial_empty_edit_ticks = mLastTrack->elst_initial_empty_edit_ticks;

                            track_b->subsample_encryption = mLastTrack->subsample_encryption;



                            track_b->mTx3gBuffer = mLastTrack->mTx3gBuffer;
@@ -1204,21 +1206,20 @@ status_t MPEG4Extractor::parseChunk(off64_t *offset, int depth) { 
                return ERROR_IO;

            }



            if (entry_count != 1) {

                // we only support a single entry at the moment, for gapless playback

                // or start offset

            if (entry_count > 2) {

                /* We support a single entry for gapless playback or negating offset for

                 * reordering B frames, two entries (empty edit) for start offset at the moment.

                 */

                ALOGW("ignoring edit list with %d entries", entry_count);

            } else {

                off64_t entriesoffset = data_offset + 8;

                uint64_t segment_duration;

                int64_t media_time;



                if (version == 1) {

                    if (!mDataSource->getUInt64(entriesoffset, &segment_duration) ||

                            !mDataSource->getUInt64(entriesoffset + 8, (uint64_t*)&media_time)) {

                        return ERROR_IO;

                    }

                } else if (version == 0) {

                uint64_t empty_edit_ticks = 0;

                bool empty_edit_present = false;

                for (int i = 0; i < entry_count; ++i) {

                    switch (version) {

                    case 0: {

                        uint32_t sd;

                        int32_t mt;

                        if (!mDataSource->getUInt32(entriesoffset, &sd) ||
@@ -1227,16 +1228,64 @@ status_t MPEG4Extractor::parseChunk(off64_t *offset, int depth) { 
                        }

                        segment_duration = sd;

                        media_time = mt;

                } else {

                        // 4(segment duration) + 4(media time) + 4(media rate)

                        entriesoffset += 12;

                        break;

                    }

                    case 1: {

                        if (!mDataSource->getUInt64(entriesoffset, &segment_duration) ||

                            !mDataSource->getUInt64(entriesoffset + 8, (uint64_t*)&media_time)) {

                            return ERROR_IO;

                        }



                        // 8(segment duration) + 8(media time) + 4(media rate)

                        entriesoffset += 20;

                        break;

                    }

                    default:

                        return ERROR_IO;

                        break;

                    }

                    // Empty edit entry would have to be first entry.

                    if (media_time == -1 && i == 0) {

                        int64_t durationUs;

                        if (AMediaFormat_getInt64(mFileMetaData, AMEDIAFORMAT_KEY_DURATION,

                                                  &durationUs)) {

                            empty_edit_ticks = segment_duration;

                            ALOGV("initial empty edit ticks: %" PRIu64, empty_edit_ticks);

                            empty_edit_present = true;

                        }

                    }

                    // Process second entry only when the first entry was an empty edit entry.

                    if (empty_edit_present && i == 1) {

                        int64_t durationUs;

                        if (AMediaFormat_getInt64(mLastTrack->meta, AMEDIAFORMAT_KEY_DURATION,

                                                  &durationUs) &&

                            mHeaderTimescale != 0) {

                            // Support only segment_duration<=track_duration and media_time==0 case.

                            uint64_t segmentDurationUs =

                                    segment_duration * 1000000 / mHeaderTimescale;

                            if (segmentDurationUs == 0 || segmentDurationUs > durationUs ||

                                media_time != 0) {

                                ALOGW("for now, unsupported second entry in empty edit list");

                            }

                        }

                    }

                }

                // save these for later, because the elst atom might precede

                // the atoms that actually gives us the duration and sample rate

                // needed to calculate the padding and delay values

                mLastTrack->has_elst = true;

                mLastTrack->elst_needs_processing = true;

                if (empty_edit_present) {

                    /* In movie header timescale, and needs to be converted to media timescale once

                     * we get that from a track's 'mdhd' atom, which at times come after 'elst'.

                     */

                    mLastTrack->elst_initial_empty_edit_ticks = empty_edit_ticks;

                } else {

                    mLastTrack->elst_media_time = media_time;

                    mLastTrack->elst_segment_duration = segment_duration;

                    ALOGV("segment_duration: %" PRIu64 " media_time: %" PRId64, segment_duration,

                          media_time);

                }

            }

            break;

        }
@@ -4275,15 +4324,28 @@ MediaTrackHelper *MPEG4Extractor::getTrack(size_t index) { 
        }

    }



    if (track->has_elst and !strncasecmp("video/", mime, 6) and track->elst_media_time > 0) {

        track->elstShiftStartTicks = track->elst_media_time;

        ALOGV("video track->elstShiftStartTicks :%" PRIu64, track->elstShiftStartTicks);

    // media_time is in media timescale as are STTS/CTTS entries.

    track->elst_shift_start_ticks = track->elst_media_time;

    ALOGV("track->elst_shift_start_ticks :%" PRIu64, track->elst_shift_start_ticks);

    if (mHeaderTimescale != 0) {

        // Convert empty_edit_ticks from movie timescale to media timescale.

        uint64_t elst_initial_empty_edit_ticks_mul = 0, elst_initial_empty_edit_ticks_add = 0;

        if (__builtin_mul_overflow(track->elst_initial_empty_edit_ticks, track->timescale,

                                   &elst_initial_empty_edit_ticks_mul) ||

            __builtin_add_overflow(elst_initial_empty_edit_ticks_mul, (mHeaderTimescale / 2),

                                   &elst_initial_empty_edit_ticks_add)) {

            ALOGE("track->elst_initial_empty_edit_ticks overflow");

            return nullptr;

        }

        track->elst_initial_empty_edit_ticks = elst_initial_empty_edit_ticks_add / mHeaderTimescale;

        ALOGV("track->elst_initial_empty_edit_ticks :%" PRIu64,

              track->elst_initial_empty_edit_ticks);

    }



    MPEG4Source *source =  new MPEG4Source(

            track->meta, mDataSource, track->timescale, track->sampleTable,

    MPEG4Source* source =

            new MPEG4Source(track->meta, mDataSource, track->timescale, track->sampleTable,

                            mSidxEntries, trex, mMoofOffset, itemTable,

            track->elstShiftStartTicks);

                            track->elst_shift_start_ticks, track->elst_initial_empty_edit_ticks);

    if (source->init() != OK) {

        delete source;

        return NULL;
@@ -4776,7 +4838,8 @@ MPEG4Source::MPEG4Source( 
        const Trex *trex,

        off64_t firstMoofOffset,

        const sp<ItemTable> &itemTable,

        uint64_t elstShiftStartTicks)

        uint64_t elstShiftStartTicks,

        uint64_t elstInitialEmptyEditTicks)

    : mFormat(format),

      mDataSource(dataSource),

      mTimescale(timeScale),
@@ -4806,7 +4869,8 @@ MPEG4Source::MPEG4Source( 
      mSrcBuffer(NULL),

      mIsHeif(itemTable != NULL),

      mItemTable(itemTable),

      mElstShiftStartTicks(elstShiftStartTicks) {

      mElstShiftStartTicks(elstShiftStartTicks),

      mElstInitialEmptyEditTicks(elstInitialEmptyEditTicks) {



    memset(&mTrackFragmentHeaderInfo, 0, sizeof(mTrackFragmentHeaderInfo));
@@ -4925,35 +4989,14 @@ MPEG4Source::MPEG4Source( 
    }



    CHECK(AMediaFormat_getInt32(format, AMEDIAFORMAT_KEY_TRACK_ID, &mTrackId));



}



status_t MPEG4Source::init() {

    status_t err = OK;

    const char *mime;

    CHECK(AMediaFormat_getString(mFormat, AMEDIAFORMAT_KEY_MIME, &mime));

    if (mFirstMoofOffset != 0) {

        off64_t offset = mFirstMoofOffset;

        err = parseChunk(&offset);

        if(err == OK && !strncasecmp("video/", mime, 6)

            && !mCurrentSamples.isEmpty()) {

            // Start offset should be less or equal to composition time of first sample.

            // ISO : sample_composition_time_offset, version 0 (unsigned) for major brands.

            mElstShiftStartTicks = std::min(mElstShiftStartTicks,

                                            (uint64_t)(*mCurrentSamples.begin()).compositionOffset);

        }

        return err;

        return parseChunk(&offset);

    }



    if (!strncasecmp("video/", mime, 6)) {

        uint64_t firstSampleCTS = 0;

        err = mSampleTable->getMetaDataForSample(0, NULL, NULL, &firstSampleCTS);

        // Start offset should be less or equal to composition time of first sample.

        // Composition time stamp of first sample cannot be negative.

        mElstShiftStartTicks = std::min(mElstShiftStartTicks, firstSampleCTS);

    }



    return err;

    return OK;

}



MPEG4Source::~MPEG4Source() {
@@ -5801,6 +5844,7 @@ media_status_t MPEG4Source::read( 


    int64_t seekTimeUs;

    ReadOptions::SeekMode mode;



    if (options && options->getSeekTo(&seekTimeUs, &mode)) {



        if (mIsHeif) {
@@ -5842,6 +5886,8 @@ media_status_t MPEG4Source::read( 
            }

            if( mode != ReadOptions::SEEK_FRAME_INDEX) {

                seekTimeUs += ((long double)mElstShiftStartTicks * 1000000) / mTimescale;

                ALOGV("shifted seekTimeUs :%" PRId64 ", mElstShiftStartTicks:%" PRIu64, seekTimeUs,

                      mElstShiftStartTicks);

            }



            uint32_t sampleIndex;
@@ -5916,7 +5962,8 @@ media_status_t MPEG4Source::read( 


    off64_t offset = 0;

    size_t size = 0;

    uint64_t cts, stts;

    int64_t cts;

    uint64_t stts;

    bool isSyncSample;

    bool newBuffer = false;

    if (mBuffer == NULL) {
@@ -5924,14 +5971,15 @@ media_status_t MPEG4Source::read( 


        status_t err;

        if (!mIsHeif) {

            err = mSampleTable->getMetaDataForSample(

                    mCurrentSampleIndex, &offset, &size, &cts, &isSyncSample, &stts);

            err = mSampleTable->getMetaDataForSample(mCurrentSampleIndex, &offset, &size,

                                                    (uint64_t*)&cts, &isSyncSample, &stts);

            if(err == OK) {

                /* Composition Time Stamp cannot be negative. Some files have video Sample

                * Time(STTS)delta with zero value(b/117402420).  Hence subtract only

                * min(cts, mElstShiftStartTicks), so that audio tracks can be played.

                */

                cts -= std::min(cts, mElstShiftStartTicks);

                if (mElstInitialEmptyEditTicks > 0) {

                    cts += mElstInitialEmptyEditTicks;

                } else {

                    // cts can be negative. for example, initial audio samples for gapless playback.

                    cts -= (int64_t)mElstShiftStartTicks;

                }

            }



        } else {
@@ -6272,7 +6320,7 @@ media_status_t MPEG4Source::fragmentedRead( 


    off64_t offset = 0;

    size_t size = 0;

    uint64_t cts = 0;

    int64_t cts = 0;

    bool isSyncSample = false;

    bool newBuffer = false;

    if (mBuffer == NULL || mCurrentSampleIndex >= mCurrentSamples.size()) {
@@ -6304,11 +6352,13 @@ media_status_t MPEG4Source::fragmentedRead( 
        offset = smpl->offset;

        size = smpl->size;

        cts = mCurrentTime + smpl->compositionOffset;

        /* Composition Time Stamp cannot be negative. Some files have video Sample

        * Time(STTS)delta with zero value(b/117402420).  Hence subtract only

        * min(cts, mElstShiftStartTicks), so that audio tracks can be played.

        */

        cts -= std::min(cts, mElstShiftStartTicks);



        if (mElstInitialEmptyEditTicks > 0) {

            cts += mElstInitialEmptyEditTicks;

        } else {

            // cts can be negative. for example, initial audio samples for gapless playback.

            cts -= (int64_t)mElstShiftStartTicks;

        }



        mCurrentTime += smpl->duration;

        isSyncSample = (mCurrentSampleIndex == 0);

media/extractors/mp4/MPEG4Extractor.h

+9 −5
Original line number Diff line number Diff line
@@ -82,14 +82,16 @@ private: 
        sp<SampleTable> sampleTable;

        bool includes_expensive_metadata;

        bool skipTrack;

        bool has_elst;

        bool elst_needs_processing;

        /* signed int, ISO Spec allows media_time = -1 for other use cases.

         * but we don't support empty edits for now.

         */

        int64_t elst_media_time;

        uint64_t elst_segment_duration;

        // unsigned int, shift start offset only when media_time > 0.

        uint64_t elstShiftStartTicks;

        // Shift start offset only when media_time > 0.

        uint64_t elst_shift_start_ticks;

        // Initial start offset, empty edit list entry.

        uint64_t elst_initial_empty_edit_ticks;

        bool subsample_encryption;



        uint8_t *mTx3gBuffer;
@@ -102,9 +104,11 @@ private: 
            timescale = 0;

            includes_expensive_metadata = false;

            skipTrack = false;

            has_elst = false;

            elst_needs_processing = false;

            elst_media_time = 0;

            elstShiftStartTicks = 0;

            elst_segment_duration = 0;

            elst_shift_start_ticks = 0;

            elst_initial_empty_edit_ticks = 0;

            subsample_encryption = false;

            mTx3gBuffer = NULL;

            mTx3gSize = mTx3gFilled = 0;

media/libstagefright/MPEG4Writer.cpp

+145 −54

File changed.

Preview size limit exceeded, changes collapsed.

media/libstagefright/include/media/stagefright/MPEG4Writer.h

+1 −1
Original line number Diff line number Diff line
@@ -106,7 +106,7 @@ private: 
    uint32_t mInterleaveDurationUs;

    int32_t mTimeScale;

    int64_t mStartTimestampUs;

    int32_t mStartTimeOffsetBFramesUs; // Start time offset when B Frames are present

    int32_t mStartTimeOffsetBFramesUs;  // Longest offset needed for reordering tracks with B Frames

    int mLatitudex10000;

    int mLongitudex10000;

    bool mAreGeoTagsAvailable;