Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 764d945b authored by Hangyu Kuang's avatar Hangyu Kuang
Browse files

media: Support accurate pause/resume/stop in GraphicBufferSource. 

Process all buffers that have been queued to the encoder on
StagefrightRecorder::pause()/stop(). Furthermore, add timestamp support for
pause/stop. Buffers willi keep getting encoded until 1) a buffer with a higher
timestamp is queued, or 2) the bufferqueue is abandoned by the producer.

Test: Recording with hacking GoogleCamera timestamp. And Mediarecorder CTS test.
Bug:32765698
Change-Id: I9ec33d635aef269836d7a5d9f4b906cb41b46a0d
parent 4441393a

include/media/stagefright/MediaCodecSource.h

+21 −4
Original line number Diff line number Diff line
@@ -54,7 +54,7 @@ struct MediaCodecSource : public MediaSource, 
    // MediaSource

    virtual status_t start(MetaData *params = NULL);

    virtual status_t stop();

    virtual status_t pause();

    virtual status_t pause(MetaData *params = NULL);

    virtual sp<MetaData> getFormat();

    virtual status_t read(

            MediaBuffer **buffer,
@@ -66,6 +66,12 @@ struct MediaCodecSource : public MediaSource, 
    // for AHandlerReflector

    void onMessageReceived(const sp<AMessage> &msg);



    // Set GraphicBufferSource stop time. GraphicBufferSource will stop

    // after receiving a buffer with timestamp larger or equal than stopTimeUs.

    // All the buffers with timestamp larger or equal to stopTimeUs will be

    // discarded. stopTimeUs uses SYSTEM_TIME_MONOTONIC time base.

    status_t setStopStimeUs(int64_t stopTimeUs);



protected:

    virtual ~MediaCodecSource();
@@ -79,6 +85,7 @@ private: 
        kWhatStop,

        kWhatPause,

        kWhatSetInputBufferTimeOffset,

        kWhatSetStopTimeOffset,

        kWhatGetFirstSampleSystemTimeUs,

        kWhatStopStalled,

    };
@@ -91,13 +98,23 @@ private: 
            uint32_t flags = 0);



    status_t onStart(MetaData *params);

    void onPause();



    // Pause the source at pauseStartTimeUs. For non-surface input,

    // buffers will be dropped immediately. For surface input, buffers

    // with timestamp smaller than pauseStartTimeUs will still be encoded.

    // Buffers with timestamp larger or queal to pauseStartTimeUs will be

    // dropped. pauseStartTimeUs uses SYSTEM_TIME_MONOTONIC time base.

    void onPause(int64_t pauseStartTimeUs);



    status_t init();

    status_t initEncoder();

    void releaseEncoder();

    status_t feedEncoderInputBuffers();

    void suspend();

    void resume(int64_t skipFramesBeforeUs = -1ll);

    // Resume GraphicBufferSource at resumeStartTimeUs. Buffers

    // from GraphicBufferSource with timestamp larger or equal to

    // resumeStartTimeUs will be encoded. resumeStartTimeUs uses

    // SYSTEM_TIME_MONOTONIC time base.

    void resume(int64_t resumeStartTimeUs = -1ll);

    void signalEOS(status_t err = ERROR_END_OF_STREAM);

    bool reachedEOS();

    status_t postSynchronouslyAndReturnError(const sp<AMessage> &msg);

media/libmedia/aidl/android/IGraphicBufferSource.aidl

+2 −1
Original line number Diff line number Diff line
@@ -25,11 +25,12 @@ import android.IOMXNode; 
 */

interface IGraphicBufferSource {

    void configure(IOMXNode omxNode, int dataSpace);

    void setSuspend(boolean suspend);

    void setSuspend(boolean suspend, long suspendTimeUs);

    void setRepeatPreviousFrameDelayUs(long repeatAfterUs);

    void setMaxFps(float maxFps);

    void setTimeLapseConfig(long timePerFrameUs, long timePerCaptureUs);

    void setStartTimeUs(long startTimeUs);

    void setStopTimeUs(long stopTimeUs);

    void setColorAspects(int aspects);

    void setTimeOffsetUs(long timeOffsetsUs);

    void signalEndOfInputStream();

media/libmediaplayerservice/StagefrightRecorder.cpp

+17 −5
Original line number Diff line number Diff line
@@ -1812,15 +1812,17 @@ status_t StagefrightRecorder::pause() { 
        return OK;

    }



    mPauseStartTimeUs = systemTime() / 1000;

    sp<MetaData> meta = new MetaData;

    meta->setInt64(kKeyTime, mPauseStartTimeUs);



    if (mAudioEncoderSource != NULL) {

        mAudioEncoderSource->pause();

    }

    if (mVideoEncoderSource != NULL) {

        mVideoEncoderSource->pause();

        mVideoEncoderSource->pause(meta.get());

    }



    mPauseStartTimeUs = systemTime() / 1000;



    return OK;

}
@@ -1835,6 +1837,8 @@ status_t StagefrightRecorder::resume() { 
        return OK;

    }



    int64_t resumeStartTimeUs = systemTime() / 1000;



    int64_t bufferStartTimeUs = 0;

    bool allSourcesStarted = true;

    for (const auto &source : { mAudioEncoderSource, mVideoEncoderSource }) {
@@ -1855,18 +1859,20 @@ status_t StagefrightRecorder::resume() { 
            mPauseStartTimeUs = bufferStartTimeUs;

        }

        // 30 ms buffer to avoid timestamp overlap

        mTotalPausedDurationUs += (systemTime() / 1000) - mPauseStartTimeUs - 30000;

        mTotalPausedDurationUs += resumeStartTimeUs - mPauseStartTimeUs - 30000;

    }

    double timeOffset = -mTotalPausedDurationUs;

    if (mCaptureFpsEnable) {

        timeOffset *= mCaptureFps / mFrameRate;

    }

    sp<MetaData> meta = new MetaData;

    meta->setInt64(kKeyTime, resumeStartTimeUs);

    for (const auto &source : { mAudioEncoderSource, mVideoEncoderSource }) {

        if (source == nullptr) {

            continue;

        }

        source->setInputBufferTimeOffset((int64_t)timeOffset);

        source->start();

        source->start(meta.get());

    }

    mPauseStartTimeUs = 0;
@@ -1883,6 +1889,12 @@ status_t StagefrightRecorder::stop() { 
        mCameraSourceTimeLapse = NULL;

    }



    if (mVideoEncoderSource != NULL) {

        int64_t stopTimeUs = systemTime() / 1000;

        sp<MetaData> meta = new MetaData;

        err = mVideoEncoderSource->setStopStimeUs(stopTimeUs);

    }



    if (mWriter != NULL) {

        err = mWriter->stop();

        mWriter.clear();

media/libstagefright/ACodec.cpp

+20 −2
Original line number Diff line number Diff line
@@ -6553,7 +6553,7 @@ status_t ACodec::LoadedState::setupInputSurface() { 


    if (mCodec->mCreateInputBuffersSuspended) {

        err = statusFromBinderStatus(

                mCodec->mGraphicBufferSource->setSuspend(true));

                mCodec->mGraphicBufferSource->setSuspend(true, -1));



        if (err != OK) {

            ALOGE("[%s] Unable to configure option to suspend (err %d)",
@@ -7117,8 +7117,10 @@ status_t ACodec::setParameters(const sp<AMessage> &params) { 
            return INVALID_OPERATION;

        }



        int64_t suspendStartTimeUs = -1;

        (void) params->findInt64("drop-start-time-us", &suspendStartTimeUs);

        status_t err = statusFromBinderStatus(

                mGraphicBufferSource->setSuspend(dropInputFrames != 0));

                mGraphicBufferSource->setSuspend(dropInputFrames != 0, suspendStartTimeUs));



        if (err != OK) {

            ALOGE("Failed to set parameter 'drop-input-frames' (err %d)", err);
@@ -7126,6 +7128,22 @@ status_t ACodec::setParameters(const sp<AMessage> &params) { 
        }

    }



    int64_t stopTimeUs;

    if (params->findInt64("stop-time-us", &stopTimeUs)) {

        if (mGraphicBufferSource == NULL) {

            ALOGE("[%s] Invalid to set stop time without surface",

                    mComponentName.c_str());

            return INVALID_OPERATION;

        }

        status_t err = statusFromBinderStatus(

                mGraphicBufferSource->setStopTimeUs(stopTimeUs));



        if (err != OK) {

            ALOGE("Failed to set parameter 'stop-time-us' (err %d)", err);

            return err;

        }

    }



    int32_t dummy;

    if (params->findInt32("request-sync", &dummy)) {

        status_t err = requestIDRFrame();

media/libstagefright/MediaCodecSource.cpp

+66 −25
Original line number Diff line number Diff line
@@ -363,8 +363,20 @@ status_t MediaCodecSource::stop() { 
    return postSynchronouslyAndReturnError(msg);

}



status_t MediaCodecSource::pause() {

    (new AMessage(kWhatPause, mReflector))->post();



status_t MediaCodecSource::setStopStimeUs(int64_t stopTimeUs) {

    if (!(mFlags & FLAG_USE_SURFACE_INPUT)) {

        return OK;

    }

    sp<AMessage> msg = new AMessage(kWhatSetStopTimeOffset, mReflector);

    msg->setInt64("stop-time-us", stopTimeUs);

    return postSynchronouslyAndReturnError(msg);

}



status_t MediaCodecSource::pause(MetaData* params) {

    sp<AMessage> msg = new AMessage(kWhatPause, mReflector);

    msg->setObject("meta", params);

    msg->post();

    return OK;

}
@@ -624,22 +636,13 @@ void MediaCodecSource::signalEOS(status_t err) { 
    }

}



void MediaCodecSource::suspend() {

    CHECK(mFlags & FLAG_USE_SURFACE_INPUT);

    if (mEncoder != NULL) {

        sp<AMessage> params = new AMessage;

        params->setInt32("drop-input-frames", true);

        mEncoder->setParameters(params);

    }

}



void MediaCodecSource::resume(int64_t skipFramesBeforeUs) {

void MediaCodecSource::resume(int64_t resumeStartTimeUs) {

    CHECK(mFlags & FLAG_USE_SURFACE_INPUT);

    if (mEncoder != NULL) {

        sp<AMessage> params = new AMessage;

        params->setInt32("drop-input-frames", false);

        if (skipFramesBeforeUs > 0) {

            params->setInt64("skip-frames-before", skipFramesBeforeUs);

        if (resumeStartTimeUs > 0) {

            params->setInt64("drop-start-time-us", resumeStartTimeUs);

        }

        mEncoder->setParameters(params);

    }
@@ -661,7 +664,7 @@ status_t MediaCodecSource::feedEncoderInputBuffers() { 
                mFirstSampleSystemTimeUs = systemTime() / 1000;

                if (mPausePending) {

                    mPausePending = false;

                    onPause();

                    onPause(mFirstSampleSystemTimeUs);

                    mbuf->release();

                    mAvailEncoderInputIndices.push_back(bufferIndex);

                    return OK;
@@ -728,6 +731,10 @@ status_t MediaCodecSource::onStart(MetaData *params) { 
        ALOGE("Failed to start while we're stopping");

        return INVALID_OPERATION;

    }

    int64_t startTimeUs;

    if (params == NULL || !params->findInt64(kKeyTime, &startTimeUs)) {

        startTimeUs = -1ll;

    }



    if (mStarted) {

        ALOGI("MediaCodecSource (%s) resuming", mIsVideo ? "video" : "audio");
@@ -739,7 +746,7 @@ status_t MediaCodecSource::onStart(MetaData *params) { 
            mEncoder->requestIDRFrame();

        }

        if (mFlags & FLAG_USE_SURFACE_INPUT) {

            resume();

            resume(startTimeUs);

        } else {

            CHECK(mPuller != NULL);

            mPuller->resume();
@@ -752,11 +759,14 @@ status_t MediaCodecSource::onStart(MetaData *params) { 
    status_t err = OK;



    if (mFlags & FLAG_USE_SURFACE_INPUT) {

        int64_t startTimeUs;

        if (!params || !params->findInt64(kKeyTime, &startTimeUs)) {

            startTimeUs = -1ll;

        if (mEncoder != NULL) {

            sp<AMessage> params = new AMessage;

            params->setInt32("drop-input-frames", false);

            if (startTimeUs >= 0) {

                params->setInt64("skip-frames-before", startTimeUs);

            }

            mEncoder->setParameters(params);

        }

        resume(startTimeUs);

    } else {

        CHECK(mPuller != NULL);

        sp<MetaData> meta = params;
@@ -781,9 +791,12 @@ status_t MediaCodecSource::onStart(MetaData *params) { 
    return OK;

}



void MediaCodecSource::onPause() {

    if (mFlags & FLAG_USE_SURFACE_INPUT) {

        suspend();

void MediaCodecSource::onPause(int64_t pauseStartTimeUs) {

    if ((mFlags & FLAG_USE_SURFACE_INPUT) && (mEncoder != NULL)) {

        sp<AMessage> params = new AMessage;

        params->setInt32("drop-input-frames", true);

        params->setInt64("drop-start-time-us", pauseStartTimeUs);

        mEncoder->setParameters(params);

    } else {

        CHECK(mPuller != NULL);

        mPuller->pause();
@@ -871,7 +884,7 @@ void MediaCodecSource::onMessageReceived(const sp<AMessage> &msg) { 
                            mFirstSampleSystemTimeUs = systemTime() / 1000;

                            if (mPausePending) {

                                mPausePending = false;

                                onPause();

                                onPause(mFirstSampleSystemTimeUs);

                                mbuf->release();

                                break;

                            }
@@ -1000,6 +1013,7 @@ void MediaCodecSource::onMessageReceived(const sp<AMessage> &msg) { 
            ALOGV("source (%s) stopped", mIsVideo ? "video" : "audio");

        }

        signalEOS();

        break;

    }



    case kWhatPause:
@@ -1007,7 +1021,14 @@ void MediaCodecSource::onMessageReceived(const sp<AMessage> &msg) { 
        if (mFirstSampleSystemTimeUs < 0) {

            mPausePending = true;

        } else {

            onPause();

            sp<RefBase> obj;

            CHECK(msg->findObject("meta", &obj));

            MetaData *params = static_cast<MetaData *>(obj.get());

            int64_t pauseStartTimeUs = -1;

            if (params == NULL || !params->findInt64(kKeyTime, &pauseStartTimeUs)) {

                pauseStartTimeUs = -1ll;

            }

            onPause(pauseStartTimeUs);

        }

        break;

    }
@@ -1030,6 +1051,26 @@ void MediaCodecSource::onMessageReceived(const sp<AMessage> &msg) { 
        response->postReply(replyID);

        break;

    }

    case kWhatSetStopTimeOffset:

    {

        sp<AReplyToken> replyID;

        CHECK(msg->senderAwaitsResponse(&replyID));

        status_t err = OK;

        int64_t stopTimeUs;

        CHECK(msg->findInt64("stop-time-us", &stopTimeUs));



        // Propagate the timestamp offset to GraphicBufferSource.

        if (mFlags & FLAG_USE_SURFACE_INPUT) {

            sp<AMessage> params = new AMessage;

            params->setInt64("stop-time-us", stopTimeUs);

            err = mEncoder->setParameters(params);

        }



        sp<AMessage> response = new AMessage;

        response->setInt32("err", err);

        response->postReply(replyID);

        break;

    }

    case kWhatGetFirstSampleSystemTimeUs:

    {

        sp<AReplyToken> replyID;