Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 4a868846 authored by Wonsik Kim's avatar Wonsik Kim
Browse files

MediaCodec: fix output format change event timing 

In synchronous mode, process output format change at
dequeue, instead of at drain. Keep processing the change
at drain for asynchronous mode.

Fix redundant output format change logging.

Bug: 149751672
Test: atest CtsMediaTestCases:AdaptivePlaybackTest
Test: atest CtsMediaTestCases -- --module-arg CtsMediaTestCases:size:small
Change-Id: Ib373b71ba65f460edaf82365f8dd9be35348f635
parent 4a603cd0

media/codec2/sfplugin/CCodecBuffers.cpp

+8 −2
Original line number Diff line number Diff line
@@ -272,8 +272,6 @@ OutputBuffers::BufferAction OutputBuffers::popFromStashAndRegister( 


    // The output format can be processed without a registered slot.

    if (outputFormat) {

        ALOGD("[%s] popFromStashAndRegister: output format changed to %s",

                mName, outputFormat->debugString().c_str());

        updateSkipCutBuffer(outputFormat, entry.notify);

    }
@@ -301,6 +299,10 @@ OutputBuffers::BufferAction OutputBuffers::popFromStashAndRegister( 
    }



    if (!entry.notify) {

        if (outputFormat) {

            ALOGD("[%s] popFromStashAndRegister: output format changed to %s",

                    mName, outputFormat->debugString().c_str());

        }

        mPending.pop_front();

        return DISCARD;

    }
@@ -317,6 +319,10 @@ OutputBuffers::BufferAction OutputBuffers::popFromStashAndRegister( 
    // Append information from the front stash entry to outBuffer.

    (*outBuffer)->meta()->setInt64("timeUs", entry.timestamp);

    (*outBuffer)->meta()->setInt32("flags", entry.flags);

    if (outputFormat) {

        ALOGD("[%s] popFromStashAndRegister: output format changed to %s",

                mName, outputFormat->debugString().c_str());

    }

    ALOGV("[%s] popFromStashAndRegister: "

          "out buffer index = %zu [%p] => %p + %zu (%lld)",

          mName, *index, outBuffer->get(),

media/libstagefright/MediaCodec.cpp

+134 −111
Original line number Diff line number Diff line
@@ -2044,20 +2044,25 @@ bool MediaCodec::handleDequeueOutputBuffer(const sp<AReplyToken> &replyID, bool 
    } else if (mFlags & kFlagOutputBuffersChanged) {

        PostReplyWithError(replyID, INFO_OUTPUT_BUFFERS_CHANGED);

        mFlags &= ~kFlagOutputBuffersChanged;

    } else if (mFlags & kFlagOutputFormatChanged) {

        PostReplyWithError(replyID, INFO_FORMAT_CHANGED);

        mFlags &= ~kFlagOutputFormatChanged;

    } else {

        sp<AMessage> response = new AMessage;

        ssize_t index = dequeuePortBuffer(kPortIndexOutput);



        if (index < 0) {

            CHECK_EQ(index, -EAGAIN);

        BufferInfo *info = peekNextPortBuffer(kPortIndexOutput);

        if (!info) {

            return false;

        }



        const sp<MediaCodecBuffer> &buffer =

            mPortBuffers[kPortIndexOutput][index].mData;

        // In synchronous mode, output format change should be handled

        // at dequeue to put the event at the correct order.



        const sp<MediaCodecBuffer> &buffer = info->mData;

        handleOutputFormatChangeIfNeeded(buffer);

        if (mFlags & kFlagOutputFormatChanged) {

            PostReplyWithError(replyID, INFO_FORMAT_CHANGED);

            mFlags &= ~kFlagOutputFormatChanged;

            return true;

        }



        ssize_t index = dequeuePortBuffer(kPortIndexOutput);



        response->setSize("index", index);

        response->setSize("offset", buffer->offset());
@@ -2542,107 +2547,13 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) { 
                        break;

                    }



                    if (mFlags & kFlagIsAsync) {

                        sp<RefBase> obj;

                        CHECK(msg->findObject("buffer", &obj));

                        sp<MediaCodecBuffer> buffer = static_cast<MediaCodecBuffer *>(obj.get());



                    if (mOutputFormat != buffer->format()) {

                        if (mFlags & kFlagUseBlockModel) {

                            sp<AMessage> diff1 = mOutputFormat->changesFrom(buffer->format());

                            sp<AMessage> diff2 = buffer->format()->changesFrom(mOutputFormat);

                            std::set<std::string> keys;

                            size_t numEntries = diff1->countEntries();

                            AMessage::Type type;

                            for (size_t i = 0; i < numEntries; ++i) {

                                keys.emplace(diff1->getEntryNameAt(i, &type));

                            }

                            numEntries = diff2->countEntries();

                            for (size_t i = 0; i < numEntries; ++i) {

                                keys.emplace(diff2->getEntryNameAt(i, &type));

                            }

                            sp<WrapperObject<std::set<std::string>>> changedKeys{

                                new WrapperObject<std::set<std::string>>{std::move(keys)}};

                            buffer->meta()->setObject("changedKeys", changedKeys);

                        }

                        mOutputFormat = buffer->format();

                        ALOGV("[%s] output format changed to: %s",

                                mComponentName.c_str(), mOutputFormat->debugString(4).c_str());



                        if (mSoftRenderer == NULL &&

                                mSurface != NULL &&

                                (mFlags & kFlagUsesSoftwareRenderer)) {

                            AString mime;

                            CHECK(mOutputFormat->findString("mime", &mime));



                            // TODO: propagate color aspects to software renderer to allow better

                            // color conversion to RGB. For now, just mark dataspace for YUV

                            // rendering.

                            int32_t dataSpace;

                            if (mOutputFormat->findInt32("android._dataspace", &dataSpace)) {

                                ALOGD("[%s] setting dataspace on output surface to #%x",

                                        mComponentName.c_str(), dataSpace);

                                int err = native_window_set_buffers_data_space(

                                        mSurface.get(), (android_dataspace)dataSpace);

                                ALOGW_IF(err != 0, "failed to set dataspace on surface (%d)", err);

                            }

                            if (mOutputFormat->contains("hdr-static-info")) {

                                HDRStaticInfo info;

                                if (ColorUtils::getHDRStaticInfoFromFormat(mOutputFormat, &info)) {

                                    setNativeWindowHdrMetadata(mSurface.get(), &info);

                                }

                            }



                            sp<ABuffer> hdr10PlusInfo;

                            if (mOutputFormat->findBuffer("hdr10-plus-info", &hdr10PlusInfo)

                                    && hdr10PlusInfo != nullptr && hdr10PlusInfo->size() > 0) {

                                native_window_set_buffers_hdr10_plus_metadata(mSurface.get(),

                                        hdr10PlusInfo->size(), hdr10PlusInfo->data());

                            }



                            if (mime.startsWithIgnoreCase("video/")) {

                                mSurface->setDequeueTimeout(-1);

                                mSoftRenderer = new SoftwareRenderer(mSurface, mRotationDegrees);

                            }

                        }



                        requestCpuBoostIfNeeded();



                        if (mFlags & kFlagIsEncoder) {

                            // Before we announce the format change we should

                            // collect codec specific data and amend the output

                            // format as necessary.

                            int32_t flags = 0;

                            (void) buffer->meta()->findInt32("flags", &flags);

                            if ((flags & BUFFER_FLAG_CODECCONFIG) && !(mFlags & kFlagIsSecure)) {

                                status_t err =

                                    amendOutputFormatWithCodecSpecificData(buffer);



                                if (err != OK) {

                                    ALOGE("Codec spit out malformed codec "

                                          "specific data!");

                                }

                            }

                        }

                        if (mFlags & kFlagIsAsync) {

                            onOutputFormatChanged();

                        } else {

                            mFlags |= kFlagOutputFormatChanged;

                            postActivityNotificationIfPossible();

                        }



                        // Notify mCrypto of video resolution changes

                        if (mCrypto != NULL) {

                            int32_t left, top, right, bottom, width, height;

                            if (mOutputFormat->findRect("crop", &left, &top, &right, &bottom)) {

                                mCrypto->notifyResolution(right - left + 1, bottom - top + 1);

                            } else if (mOutputFormat->findInt32("width", &width)

                                    && mOutputFormat->findInt32("height", &height)) {

                                mCrypto->notifyResolution(width, height);

                            }

                        }

                    }



                    if (mFlags & kFlagIsAsync) {

                        // In asynchronous mode, output format change is processed immediately.

                        handleOutputFormatChangeIfNeeded(buffer);

                        onOutputBufferAvailable();

                    } else if (mFlags & kFlagDequeueOutputPending) {

                        CHECK(handleDequeueOutputBuffer(mDequeueOutputReplyID));
@@ -3473,6 +3384,106 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) { 
    }

}



void MediaCodec::handleOutputFormatChangeIfNeeded(const sp<MediaCodecBuffer> &buffer) {

    sp<AMessage> format = buffer->format();

    if (mOutputFormat == format) {

        return;

    }

    if (mFlags & kFlagUseBlockModel) {

        sp<AMessage> diff1 = mOutputFormat->changesFrom(format);

        sp<AMessage> diff2 = format->changesFrom(mOutputFormat);

        std::set<std::string> keys;

        size_t numEntries = diff1->countEntries();

        AMessage::Type type;

        for (size_t i = 0; i < numEntries; ++i) {

            keys.emplace(diff1->getEntryNameAt(i, &type));

        }

        numEntries = diff2->countEntries();

        for (size_t i = 0; i < numEntries; ++i) {

            keys.emplace(diff2->getEntryNameAt(i, &type));

        }

        sp<WrapperObject<std::set<std::string>>> changedKeys{

            new WrapperObject<std::set<std::string>>{std::move(keys)}};

        buffer->meta()->setObject("changedKeys", changedKeys);

    }

    mOutputFormat = format;

    ALOGV("[%s] output format changed to: %s",

            mComponentName.c_str(), mOutputFormat->debugString(4).c_str());



    if (mSoftRenderer == NULL &&

            mSurface != NULL &&

            (mFlags & kFlagUsesSoftwareRenderer)) {

        AString mime;

        CHECK(mOutputFormat->findString("mime", &mime));



        // TODO: propagate color aspects to software renderer to allow better

        // color conversion to RGB. For now, just mark dataspace for YUV

        // rendering.

        int32_t dataSpace;

        if (mOutputFormat->findInt32("android._dataspace", &dataSpace)) {

            ALOGD("[%s] setting dataspace on output surface to #%x",

                    mComponentName.c_str(), dataSpace);

            int err = native_window_set_buffers_data_space(

                    mSurface.get(), (android_dataspace)dataSpace);

            ALOGW_IF(err != 0, "failed to set dataspace on surface (%d)", err);

        }

        if (mOutputFormat->contains("hdr-static-info")) {

            HDRStaticInfo info;

            if (ColorUtils::getHDRStaticInfoFromFormat(mOutputFormat, &info)) {

                setNativeWindowHdrMetadata(mSurface.get(), &info);

            }

        }



        sp<ABuffer> hdr10PlusInfo;

        if (mOutputFormat->findBuffer("hdr10-plus-info", &hdr10PlusInfo)

                && hdr10PlusInfo != nullptr && hdr10PlusInfo->size() > 0) {

            native_window_set_buffers_hdr10_plus_metadata(mSurface.get(),

                    hdr10PlusInfo->size(), hdr10PlusInfo->data());

        }



        if (mime.startsWithIgnoreCase("video/")) {

            mSurface->setDequeueTimeout(-1);

            mSoftRenderer = new SoftwareRenderer(mSurface, mRotationDegrees);

        }

    }



    requestCpuBoostIfNeeded();



    if (mFlags & kFlagIsEncoder) {

        // Before we announce the format change we should

        // collect codec specific data and amend the output

        // format as necessary.

        int32_t flags = 0;

        (void) buffer->meta()->findInt32("flags", &flags);

        if ((flags & BUFFER_FLAG_CODECCONFIG) && !(mFlags & kFlagIsSecure)) {

            status_t err =

                amendOutputFormatWithCodecSpecificData(buffer);



            if (err != OK) {

                ALOGE("Codec spit out malformed codec "

                      "specific data!");

            }

        }

    }

    if (mFlags & kFlagIsAsync) {

        onOutputFormatChanged();

    } else {

        mFlags |= kFlagOutputFormatChanged;

        postActivityNotificationIfPossible();

    }



    // Notify mCrypto of video resolution changes

    if (mCrypto != NULL) {

        int32_t left, top, right, bottom, width, height;

        if (mOutputFormat->findRect("crop", &left, &top, &right, &bottom)) {

            mCrypto->notifyResolution(right - left + 1, bottom - top + 1);

        } else if (mOutputFormat->findInt32("width", &width)

                && mOutputFormat->findInt32("height", &height)) {

            mCrypto->notifyResolution(width, height);

        }

    }

}



void MediaCodec::extractCSD(const sp<AMessage> &format) {

    mCSD.clear();
@@ -3938,19 +3949,31 @@ status_t MediaCodec::onReleaseOutputBuffer(const sp<AMessage> &msg) { 
    return OK;

}



ssize_t MediaCodec::dequeuePortBuffer(int32_t portIndex) {

MediaCodec::BufferInfo *MediaCodec::peekNextPortBuffer(int32_t portIndex) {

    CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);



    List<size_t> *availBuffers = &mAvailPortBuffers[portIndex];



    if (availBuffers->empty()) {

        return nullptr;

    }



    return &mPortBuffers[portIndex][*availBuffers->begin()];

}



ssize_t MediaCodec::dequeuePortBuffer(int32_t portIndex) {

    CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);



    BufferInfo *info = peekNextPortBuffer(portIndex);

    if (!info) {

        return -EAGAIN;

    }



    List<size_t> *availBuffers = &mAvailPortBuffers[portIndex];

    size_t index = *availBuffers->begin();

    CHECK_EQ(info, &mPortBuffers[portIndex][index]);

    availBuffers->erase(availBuffers->begin());



    BufferInfo *info = &mPortBuffers[portIndex][index];

    CHECK(!info->mOwnedByClient);

    {

        Mutex::Autolock al(mBufferLock);

media/libstagefright/include/media/stagefright/MediaCodec.h

+2 −0
Original line number Diff line number Diff line
@@ -452,6 +452,7 @@ private: 
    size_t updateBuffers(int32_t portIndex, const sp<AMessage> &msg);

    status_t onQueueInputBuffer(const sp<AMessage> &msg);

    status_t onReleaseOutputBuffer(const sp<AMessage> &msg);

    BufferInfo *peekNextPortBuffer(int32_t portIndex);

    ssize_t dequeuePortBuffer(int32_t portIndex);



    status_t getBufferAndFormat(
@@ -483,6 +484,7 @@ private: 
    status_t onSetParameters(const sp<AMessage> &params);



    status_t amendOutputFormatWithCodecSpecificData(const sp<MediaCodecBuffer> &buffer);

    void handleOutputFormatChangeIfNeeded(const sp<MediaCodecBuffer> &buffer);

    bool isExecuting() const;



    uint64_t getGraphicBufferSize();