Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit bbaa1446 authored by Praveen Chavan's avatar Praveen Chavan Committed by Wei Jia
Browse files

NuPlayer: adjust temporal-layers to achieve smooth play 

For multi-layer-encoded content (with hierarchical referencing),
select a subset of layers based on playback speed.

Bug: 27596987
Change-Id: I824c699f3c2b80bb516c1b44493eda72239291ac
(cherry picked from commit 64ad0e0b2b5f992052d287e82ebd79f9f892dcbf)
parent 0093efba

media/libmediaplayerservice/nuplayer/GenericSource.cpp

+7 −0
Original line number Diff line number Diff line
@@ -1298,6 +1298,13 @@ sp<ABuffer> NuPlayer::GenericSource::mediaBufferToABuffer( 
    }

#endif



    if (trackType == MEDIA_TRACK_TYPE_VIDEO) {

        int32_t layerId;

        if (mb->meta_data()->findInt32(kKeyTemporalLayerId, &layerId)) {

            meta->setInt32("temporal-layer-id", layerId);

        }

    }



    if (trackType == MEDIA_TRACK_TYPE_TIMEDTEXT) {

        const char *mime;

        CHECK(mTimedTextTrack.mSource != NULL

media/libmediaplayerservice/nuplayer/NuPlayer.cpp

+24 −6
Original line number Diff line number Diff line
@@ -784,13 +784,10 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { 
            }



            if (mVideoDecoder != NULL) {

                float rate = getFrameRate();

                if (rate > 0) {

                sp<AMessage> params = new AMessage();

                    params->setFloat("operating-rate", rate * mPlaybackSettings.mSpeed);

                params->setFloat("playback-speed", mPlaybackSettings.mSpeed);

                mVideoDecoder->setParameters(params);

            }

            }



            sp<AMessage> response = new AMessage;

            response->setInt32("err", err);
@@ -1680,6 +1677,27 @@ status_t NuPlayer::instantiateDecoder( 
            return err;

        }

    }



    if (!audio) {

        sp<AMessage> params = new AMessage();

        float rate = getFrameRate();

        if (rate > 0) {

            params->setFloat("frame-rate-total", rate);

        }



        sp<MetaData> fileMeta = getFileMeta();

        if (fileMeta != NULL) {

            int32_t videoTemporalLayerCount;

            if (fileMeta->findInt32(kKeyTemporalLayerCount, &videoTemporalLayerCount)

                    && videoTemporalLayerCount > 0) {

                params->setInt32("temporal-layer-count", videoTemporalLayerCount);

            }

        }



        if (params->countEntries() > 0) {

            (*decoder)->setParameters(params);

        }

    }

    return OK;

}

media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp

+99 −11
Original line number Diff line number Diff line
@@ -41,6 +41,12 @@ 


namespace android {



static float kDisplayRefreshingRate = 60.f;



// The default total video frame rate of a stream when that info is not available from

// the source.

static float kDefaultVideoFrameRateTotal = 30.f;



static inline bool getAudioDeepBufferSetting() {

    return property_get_bool("media.stagefright.audio.deep", false /* default_value */);

}
@@ -69,11 +75,17 @@ NuPlayer::Decoder::Decoder( 
      mIsSecure(false),

      mFormatChangePending(false),

      mTimeChangePending(false),

      mFrameRateTotal(kDefaultVideoFrameRateTotal),

      mPlaybackSpeed(1.0f),

      mNumVideoTemporalLayerTotal(1),

      mNumVideoTemporalLayerAllowed(1),

      mCurrentMaxVideoTemporalLayerId(0),

      mResumePending(false),

      mComponentName("decoder") {

    mCodecLooper = new ALooper;

    mCodecLooper->setName("NPDecoder-CL");

    mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);

    mVideoTemporalLayerAggregateFps[0] = mFrameRateTotal;

}



NuPlayer::Decoder::~Decoder() {
@@ -329,11 +341,73 @@ void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) { 
}



void NuPlayer::Decoder::onSetParameters(const sp<AMessage> &params) {

    bool needAdjustLayers = false;

    float frameRateTotal;

    if (params->findFloat("frame-rate-total", &frameRateTotal)

            && mFrameRateTotal != frameRateTotal) {

        needAdjustLayers = true;

        mFrameRateTotal = frameRateTotal;

    }



    int32_t numVideoTemporalLayerTotal;

    if (params->findInt32("temporal-layer-count", &numVideoTemporalLayerTotal)

            && numVideoTemporalLayerTotal > 0

            && numVideoTemporalLayerTotal <= kMaxNumVideoTemporalLayers

            && mNumVideoTemporalLayerTotal != numVideoTemporalLayerTotal) {

        needAdjustLayers = true;

        mNumVideoTemporalLayerTotal = numVideoTemporalLayerTotal;

    }



    if (needAdjustLayers) {

        // TODO: For now, layer fps is calculated for some specific architectures.

        // But it really should be extracted from the stream.

        mVideoTemporalLayerAggregateFps[0] =

            mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - 1));

        for (int32_t i = 1; i < mNumVideoTemporalLayerTotal; ++i) {

            mVideoTemporalLayerAggregateFps[i] =

                mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - i))

                + mVideoTemporalLayerAggregateFps[i - 1];

        }

    }



    float playbackSpeed;

    if (params->findFloat("playback-speed", &playbackSpeed)

            && mPlaybackSpeed != playbackSpeed) {

        needAdjustLayers = true;

        mPlaybackSpeed = playbackSpeed;

    }



    if (needAdjustLayers) {

        int32_t layerId;

        for (layerId = 0; layerId < mNumVideoTemporalLayerTotal; ++layerId) {

            if (mVideoTemporalLayerAggregateFps[layerId] * mPlaybackSpeed

                    > kDisplayRefreshingRate) {

                --layerId;

                break;

            }

        }

        if (layerId < 0) {

            layerId = 0;

        } else if (layerId >= mNumVideoTemporalLayerTotal) {

            layerId = mNumVideoTemporalLayerTotal - 1;

        }

        mNumVideoTemporalLayerAllowed = layerId + 1;

        if (mCurrentMaxVideoTemporalLayerId > layerId) {

            mCurrentMaxVideoTemporalLayerId = layerId;

        }

        ALOGV("onSetParameters: allowed layers=%d, current max layerId=%d",

                mNumVideoTemporalLayerAllowed, mCurrentMaxVideoTemporalLayerId);



        if (mCodec == NULL) {

            ALOGW("onSetParameters called before codec is created.");

            return;

        }

    mCodec->setParameters(params);



        sp<AMessage> codecParams = new AMessage();

        codecParams->setFloat("operating-rate",

                mVideoTemporalLayerAggregateFps[layerId] * mPlaybackSpeed);

        mCodec->setParameters(codecParams);

    }

}



void NuPlayer::Decoder::onSetRenderer(const sp<Renderer> &renderer) {
@@ -742,14 +816,28 @@ status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) { 
        }



        dropAccessUnit = false;

        if (!mIsAudio

                && !mIsSecure

                && mRenderer->getVideoLateByUs() > 100000ll

        if (!mIsAudio && !mIsSecure) {

            int32_t layerId = 0;

            if (mRenderer->getVideoLateByUs() > 100000ll

                    && mIsVideoAVC

                    && !IsAVCReferenceFrame(accessUnit)) {

                dropAccessUnit = true;

            } else if (accessUnit->meta()->findInt32("temporal-layer-id", &layerId)) {

                // Add only one layer each time.

                if (layerId > mCurrentMaxVideoTemporalLayerId + 1

                        || layerId >= mNumVideoTemporalLayerAllowed) {

                    dropAccessUnit = true;

                    ALOGV("dropping layer(%d), speed=%g, allowed layer count=%d, max layerId=%d",

                            layerId, mPlaybackSpeed, mNumVideoTemporalLayerAllowed,

                            mCurrentMaxVideoTemporalLayerId);

                } else if (layerId > mCurrentMaxVideoTemporalLayerId) {

                    mCurrentMaxVideoTemporalLayerId = layerId;

                }

            }

            if (dropAccessUnit) {

                ++mNumInputFramesDropped;

            }

        }

    } while (dropAccessUnit);



    // ALOGV("returned a valid buffer of %s data", mIsAudio ? "mIsAudio" : "video");

media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h

+10 −0
Original line number Diff line number Diff line
@@ -57,6 +57,10 @@ private: 
        kWhatSetVideoSurface     = 'sSur'

    };



    enum {

        kMaxNumVideoTemporalLayers = 32,

    };



    sp<Surface> mSurface;



    sp<Source> mSource;
@@ -90,6 +94,12 @@ private: 
    bool mIsSecure;

    bool mFormatChangePending;

    bool mTimeChangePending;

    float mFrameRateTotal;

    float mPlaybackSpeed;

    int32_t mNumVideoTemporalLayerTotal;

    int32_t mNumVideoTemporalLayerAllowed;

    int32_t mCurrentMaxVideoTemporalLayerId;

    float mVideoTemporalLayerAggregateFps[kMaxNumVideoTemporalLayers];



    bool mResumePending;

    AString mComponentName;