Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit b5986a36 authored by Michael Gonzalez's avatar Michael Gonzalez Committed by Shuzhen Wang
Browse files

Camera: Heic: Handle out-of-order buffer outputs 

During reprocess usecases, the Camera HAL may return JPEG Appsegment
frames out of order. This leads to a scenario where
mAppSegmentConsumer->lockNextBuffer may not refer to the currently muxed
frame.

To work around this, the number of lockable appsegment buffers has been
increased.

Test: Vendor testing
Bug: 141169323
Change-Id: Ia498f6ddaba588b66cbaeee94fe8bdd7314bb90c
parent 3d00ee51

services/camera/libcameraservice/api2/HeicCompositeStream.cpp

+33 −23
Original line number Diff line number Diff line
@@ -60,12 +60,12 @@ HeicCompositeStream::HeicCompositeStream(wp<CameraDeviceBase> device, 
        mUseGrid(false),

        mAppSegmentStreamId(-1),

        mAppSegmentSurfaceId(-1),

        mAppSegmentBufferAcquired(false),

        mMainImageStreamId(-1),

        mMainImageSurfaceId(-1),

        mYuvBufferAcquired(false),

        mProducerListener(new ProducerListener()),

        mDequeuedOutputBufferCnt(0),

        mLockedAppSegmentBufferCnt(0),

        mCodecOutputCounter(0),

        mGridTimestampUs(0) {

}
@@ -132,7 +132,7 @@ status_t HeicCompositeStream::createInternalStreams(const std::vector<sp<Surface 
    sp<IGraphicBufferProducer> producer;

    sp<IGraphicBufferConsumer> consumer;

    BufferQueue::createBufferQueue(&producer, &consumer);

    mAppSegmentConsumer = new CpuConsumer(consumer, 1);

    mAppSegmentConsumer = new CpuConsumer(consumer, kMaxAcquiredAppSegment);

    mAppSegmentConsumer->setFrameAvailableListener(this);

    mAppSegmentConsumer->setName(String8("Camera3-HeicComposite-AppSegmentStream"));

    mAppSegmentSurface = new Surface(producer);
@@ -527,12 +527,12 @@ void HeicCompositeStream::compilePendingInputLocked() { 
        mSettingsByTimestamp.erase(it);

    }



    while (!mInputAppSegmentBuffers.empty() && !mAppSegmentBufferAcquired) {

    while (!mInputAppSegmentBuffers.empty()) {

        CpuConsumer::LockedBuffer imgBuffer;

        auto it = mInputAppSegmentBuffers.begin();

        auto res = mAppSegmentConsumer->lockNextBuffer(&imgBuffer);

        if (res == NOT_ENOUGH_DATA) {

            // Canot not lock any more buffers.

            // Can not lock any more buffers.

            break;

        } else if ((res != OK) || (*it != imgBuffer.timestamp)) {

            if (res != OK) {
@@ -542,6 +542,7 @@ void HeicCompositeStream::compilePendingInputLocked() { 
                ALOGE("%s: Expecting JPEG_APP_SEGMENTS buffer with time stamp: %" PRId64

                        " received buffer with time stamp: %" PRId64, __FUNCTION__,

                        *it, imgBuffer.timestamp);

                mAppSegmentConsumer->unlockBuffer(imgBuffer);

            }

            mPendingInputFrames[*it].error = true;

            mInputAppSegmentBuffers.erase(it);
@@ -553,7 +554,7 @@ void HeicCompositeStream::compilePendingInputLocked() { 
            mAppSegmentConsumer->unlockBuffer(imgBuffer);

        } else {

            mPendingInputFrames[imgBuffer.timestamp].appSegmentBuffer = imgBuffer;

            mAppSegmentBufferAcquired = true;

            mLockedAppSegmentBufferCnt++;

        }

        mInputAppSegmentBuffers.erase(it);

    }
@@ -563,7 +564,7 @@ void HeicCompositeStream::compilePendingInputLocked() { 
        auto it = mInputYuvBuffers.begin();

        auto res = mMainImageConsumer->lockNextBuffer(&imgBuffer);

        if (res == NOT_ENOUGH_DATA) {

            // Canot not lock any more buffers.

            // Can not lock any more buffers.

            break;

        } else if (res != OK) {

            ALOGE("%s: Error locking YUV_888 image buffer: %s (%d)", __FUNCTION__,
@@ -797,13 +798,19 @@ status_t HeicCompositeStream::processInputFrame(nsecs_t timestamp, 
        }

    }



    if (inputFrame.appSegmentWritten && inputFrame.pendingOutputTiles == 0) {

    if (inputFrame.pendingOutputTiles == 0) {

        if (inputFrame.appSegmentWritten) {

            res = processCompletedInputFrame(timestamp, inputFrame);

            if (res != OK) {

                ALOGE("%s: Failed to process completed input frame: %s (%d)", __FUNCTION__,

                        strerror(-res), res);

                return res;

            }

        } else if (mLockedAppSegmentBufferCnt == kMaxAcquiredAppSegment) {

            ALOGE("%s: Out-of-order app segment buffers reaches limit %u", __FUNCTION__,

                    kMaxAcquiredAppSegment);

            return INVALID_OPERATION;

        }

    }



    return res;
@@ -936,11 +943,17 @@ status_t HeicCompositeStream::processAppSegment(nsecs_t timestamp, InputFrame &i 
                __FUNCTION__, strerror(-res), res);

        return res;

    }

    inputFrame.appSegmentWritten = true;



    ALOGV("%s: [%" PRId64 "]: appSegmentSize is %zu, width %d, height %d, app1Size %zu",

          __FUNCTION__, timestamp, appSegmentSize, inputFrame.appSegmentBuffer.width,

          inputFrame.appSegmentBuffer.height, app1Size);



    inputFrame.appSegmentWritten = true;

    // Release the buffer now so any pending input app segments can be processed

    mAppSegmentConsumer->unlockBuffer(inputFrame.appSegmentBuffer);

    inputFrame.appSegmentBuffer.data = nullptr;

    mLockedAppSegmentBufferCnt--;



    return OK;

}
@@ -1095,7 +1108,6 @@ void HeicCompositeStream::releaseInputFrameLocked(InputFrame *inputFrame /*out*/ 
    if (inputFrame->appSegmentBuffer.data != nullptr) {

        mAppSegmentConsumer->unlockBuffer(inputFrame->appSegmentBuffer);

        inputFrame->appSegmentBuffer.data = nullptr;

        mAppSegmentBufferAcquired = false;

    }



    while (!inputFrame->codecOutputBuffers.empty()) {
@@ -1133,11 +1145,13 @@ void HeicCompositeStream::releaseInputFrameLocked(InputFrame *inputFrame /*out*/ 
    }

}



void HeicCompositeStream::releaseInputFramesLocked(int64_t currentTs) {

void HeicCompositeStream::releaseInputFramesLocked() {

    auto it = mPendingInputFrames.begin();

    while (it != mPendingInputFrames.end()) {

        if (it->first <= currentTs) {

            releaseInputFrameLocked(&it->second);

        auto& inputFrame = it->second;

        if (inputFrame.error ||

            (inputFrame.appSegmentWritten && inputFrame.pendingOutputTiles == 0)) {

            releaseInputFrameLocked(&inputFrame);

            it = mPendingInputFrames.erase(it);

        } else {

            it++;
@@ -1541,7 +1555,7 @@ bool HeicCompositeStream::threadLoop() { 
            // In case we landed in error state, return any pending buffers and

            // halt all further processing.

            compilePendingInputLocked();

            releaseInputFramesLocked(currentTs);

            releaseInputFramesLocked();

            return false;

        }
@@ -1583,11 +1597,7 @@ bool HeicCompositeStream::threadLoop() { 
        mPendingInputFrames[currentTs].error = true;

    }



    if (mPendingInputFrames[currentTs].error ||

            (mPendingInputFrames[currentTs].appSegmentWritten &&

            mPendingInputFrames[currentTs].pendingOutputTiles == 0)) {

        releaseInputFramesLocked(currentTs);

    }

    releaseInputFramesLocked();



    return true;

}

services/camera/libcameraservice/api2/HeicCompositeStream.h

+5 −2
Original line number Diff line number Diff line
@@ -189,7 +189,7 @@ private: 
    status_t processCompletedInputFrame(nsecs_t timestamp, InputFrame &inputFrame);



    void releaseInputFrameLocked(InputFrame *inputFrame /*out*/);

    void releaseInputFramesLocked(int64_t currentTs);

    void releaseInputFramesLocked();



    size_t findAppSegmentsSize(const uint8_t* appSegmentBuffer, size_t maxSize,

            size_t* app1SegmentSize);
@@ -207,11 +207,13 @@ private: 
            static_cast<android_dataspace>(HAL_DATASPACE_JPEG_APP_SEGMENTS);

    static const android_dataspace kHeifDataSpace =

            static_cast<android_dataspace>(HAL_DATASPACE_HEIF);

    // Use the limit of pipeline depth in the API sepc as maximum number of acquired

    // app segment buffers.

    static const uint32_t kMaxAcquiredAppSegment = 8;



    int               mAppSegmentStreamId, mAppSegmentSurfaceId;

    sp<CpuConsumer>   mAppSegmentConsumer;

    sp<Surface>       mAppSegmentSurface;

    bool              mAppSegmentBufferAcquired;

    size_t            mAppSegmentMaxSize;

    CameraMetadata    mStaticInfo;
@@ -232,6 +234,7 @@ private: 


    // Keep all incoming APP segment Blob buffer pending further processing.

    std::vector<int64_t> mInputAppSegmentBuffers;

    int32_t           mLockedAppSegmentBufferCnt;



    // Keep all incoming HEIC blob buffer pending further processing.

    std::vector<CodecOutputBufferInfo> mCodecOutputBuffers;