Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit eb24cbb1 authored by Praveen Chavan's avatar Praveen Chavan Committed by Gerrit - the friendly Code Review server
Browse files

NuPlayer: drop temporal-layers to achieve smooth fast-play 

For multi-layer-encoded content (with hierarchical referencing),
drop input-frames from selective enhancement layers while
playing back at faster speeds.
This scheme avoids the decoder from decoding *all* the frames,
most of which, would be dropped by the renderer.

Bug: 27596987
Change-Id: I824c699f3c2b80bb516c1b44493eda72239291ac
parent b2264390

media/libmediaplayerservice/nuplayer/GenericSource.cpp

+7 −0
Original line number Diff line number Diff line
@@ -1306,6 +1306,13 @@ sp<ABuffer> NuPlayer::GenericSource::mediaBufferToABuffer( 
    }

#endif



    if (trackType == MEDIA_TRACK_TYPE_VIDEO) {

        int32_t layerId;

        if (mb->meta_data()->findInt32(kKeyTemporalLayerId, &layerId)) {

            meta->setInt32("temporal-layer-id", layerId);

        }

    }



    if (trackType == MEDIA_TRACK_TYPE_TIMEDTEXT) {

        const char *mime;

        CHECK(mTimedTextTrack.mSource != NULL

media/libmediaplayerservice/nuplayer/NuPlayer.cpp

+20 −0
Original line number Diff line number Diff line
@@ -790,6 +790,10 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { 
                    sp<AMessage> params = new AMessage();

                    params->setFloat("operating-rate", rate * mPlaybackSettings.mSpeed);

                    mVideoDecoder->setParameters(params);



                    params = new AMessage();

                    params->setFloat("playback-speed", mPlaybackSettings.mSpeed);

                    mVideoDecoder->setParameters(params);

                }

            }
@@ -1695,6 +1699,22 @@ status_t NuPlayer::instantiateDecoder( 
            return err;

        }

    }



    if (!audio) {

        sp<MetaData> fileMeta = getFileMeta();

        if (fileMeta == NULL) {

            ALOGW("source has video meta but not file meta");

            return -1;

        }



        int32_t videoTemporalLayerCount = 0;

        if (fileMeta->findInt32(kKeyTemporalLayerCount, &videoTemporalLayerCount)

                && videoTemporalLayerCount > 0) {

            sp<AMessage> params = new AMessage();

            params->setInt32("temporal-layer-count", videoTemporalLayerCount);

            (*decoder)->setParameters(params);

        }

    }

    return OK;

}

media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp

+45 −0
Original line number Diff line number Diff line
@@ -71,6 +71,8 @@ NuPlayer::Decoder::Decoder( 
      mIsSecure(false),

      mFormatChangePending(false),

      mTimeChangePending(false),

      mPlaybackSpeed(1.0f),

      mVideoTemporalLayerCount(0),

      mResumePending(false),

      mComponentName("decoder") {

    mCodecLooper = new ALooper;
@@ -338,6 +340,11 @@ void NuPlayer::Decoder::onSetParameters(const sp<AMessage> &params) { 
        ALOGW("onSetParameters called before codec is created.");

        return;

    }

    if (params->findFloat("playback-speed", &mPlaybackSpeed)) {

        return;

    } else if (params->findInt32("temporal-layer-count", &mVideoTemporalLayerCount)) {

        return;

    }

    mCodec->setParameters(params);

}
@@ -759,7 +766,44 @@ status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) { 
        }



        dropAccessUnit = false;

        int32_t layerId = 0;

        if (!mIsAudio

                && !mIsSecure

                && mPlaybackSpeed > 1.0f

                && accessUnit->meta()->findInt32("temporal-layer-id", &layerId)) {



                if ((layerId + 1) > mVideoTemporalLayerCount) {

                    mVideoTemporalLayerCount = layerId + 1;

                }



            /*

                For content encoded with hierarchical layers,

                drop input frames from selective enhancement layers when

                playing back at faster speeds.



                speed = 1x (decode all layers)

                layer-d       0   2   1   2   0   2   1   2   0

                decode      | 0 |33 |66 |99 |133|166|199|233|266|

                render      | 0 |33 |66 |99 |133|166|199|233|266|



                speed = 2x (drop layer-2)

                layer-d       0   2   1   2   0   2   1   2   0

                decode      | 0 |   |66 |   |133|   |199|   |266|

                render      | 0 |66 |133|199|266|



                speed = 4x (drop layer-2 and layer-1)

                layer-d       0   2   1   2   0   2   1   2   0

                decode      | 0 |           |133|          |266|

                render      | 0 |133|266|

            */

            int32_t dropLayerThreshold = mVideoTemporalLayerCount - (uint32_t)log2f(mPlaybackSpeed) - 1;

            dropLayerThreshold = dropLayerThreshold < 0 ? 0 : dropLayerThreshold;

            dropAccessUnit = layerId > dropLayerThreshold;

            if (dropAccessUnit) {

                ALOGV("dropping layer=%d [@speed=%g, will drop layers with id > %d]",

                        layerId, mPlaybackSpeed, dropLayerThreshold);

            }

        } else if (!mIsAudio

                && !mIsSecure

                && mRenderer->getVideoLateByUs() > 100000ll

                && mIsVideoAVC
@@ -767,6 +811,7 @@ status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) { 
            dropAccessUnit = true;

            ++mNumInputFramesDropped;

        }



    } while (dropAccessUnit);



    // ALOGV("returned a valid buffer of %s data", mIsAudio ? "mIsAudio" : "video");

media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h

+2 −0
Original line number Diff line number Diff line
@@ -90,6 +90,8 @@ protected: 
    bool mIsSecure;

    bool mFormatChangePending;

    bool mTimeChangePending;

    float mPlaybackSpeed;

    int32_t mVideoTemporalLayerCount;



    bool mResumePending;

    AString mComponentName;