Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit ba92d773 authored by Shuzhen Wang's avatar Shuzhen Wang
Browse files

Camera: re-space camera output for hardware texture target 

2 issues with overriding presentation timestamp for SurfaceTexture:
- We cannot override timestamp for SurfaceTexture, because the timestamp
  may be used for AV sync.
- TextureView always picks the latest buffer and display it right away
  without using the presentation timestamp.

Due to these reasons, the cameraservice re-spaces queuing of the buffer
based on capture time intervals rather than overriding timestamps.
The buffer timestamp is left untouched.

Test: Run Instagram, Snapchat, and Tiktok and observe improved preview
Test: Camera CTS
Bug: 195025014
Change-Id: I28b23d9f2b39b6a289b6d9f87968e4bc667870a4
parent 768f4c48

services/camera/libcameraservice/Android.bp

+1 −0
Original line number Diff line number Diff line
@@ -81,6 +81,7 @@ cc_library_shared { 
        "device3/Camera3OutputUtils.cpp",

        "device3/Camera3DeviceInjectionMethods.cpp",

        "device3/UHRCropAndMeteringRegionMapper.cpp",

        "device3/PreviewFrameSpacer.cpp",

        "device3/hidl/HidlCamera3Device.cpp",

        "device3/hidl/HidlCamera3OfflineSession.cpp",

        "device3/hidl/HidlCamera3OutputUtils.cpp",

services/camera/libcameraservice/device3/Camera3OutputStream.cpp

+44 −23
Original line number Diff line number Diff line
@@ -376,7 +376,17 @@ status_t Camera3OutputStream::returnBufferCheckedLocked( 
            dumpImageToDisk(timestamp, anwBuffer, anwReleaseFence);

        }



        nsecs_t captureTime = (mSyncToDisplay ? readoutTimestamp : timestamp) - mTimestampOffset;

        if (mPreviewFrameSpacer != nullptr) {

            res = mPreviewFrameSpacer->queuePreviewBuffer(timestamp - mTimestampOffset, transform,

                    anwBuffer, anwReleaseFence);

            if (res != OK) {

                ALOGE("%s: Stream %d: Error queuing buffer to preview buffer spacer: %s (%d)",

                        __FUNCTION__, mId, strerror(-res), res);

                return res;

            }

        } else {

            nsecs_t captureTime = (mSyncToDisplay ? readoutTimestamp : timestamp)

                    - mTimestampOffset;

            nsecs_t presentTime = mSyncToDisplay ?

                    syncTimestampToDisplayLocked(captureTime) : captureTime;
@@ -396,6 +406,7 @@ status_t Camera3OutputStream::returnBufferCheckedLocked( 
                      " %s (%d)", __FUNCTION__, mId, strerror(-res), res);

            }

        }

    }

    mLock.lock();



    // Once a valid buffer has been returned to the queue, can no longer
@@ -468,7 +479,7 @@ status_t Camera3OutputStream::configureQueueLocked() { 
        return res;

    }



    if ((res = configureConsumerQueueLocked(true /*allowDisplaySync*/)) != OK) {

    if ((res = configureConsumerQueueLocked(true /*allowPreviewRespace*/)) != OK) {

        return res;

    }
@@ -492,7 +503,7 @@ status_t Camera3OutputStream::configureQueueLocked() { 
    return OK;

}



status_t Camera3OutputStream::configureConsumerQueueLocked(bool allowDisplaySync) {

status_t Camera3OutputStream::configureConsumerQueueLocked(bool allowPreviewRespace) {

    status_t res;



    mTraceFirstBuffer = true;
@@ -582,20 +593,25 @@ status_t Camera3OutputStream::configureConsumerQueueLocked(bool allowDisplaySync 
    int timestampBase = getTimestampBase();

    bool isDefaultTimeBase = (timestampBase ==

            OutputConfiguration::TIMESTAMP_BASE_DEFAULT);

    if (allowDisplaySync)  {

        // We cannot distinguish between a SurfaceView and an ImageReader of

        // preview buffer format. Frames are synchronized to display in both

        // cases.

    if (allowPreviewRespace)  {

        bool forceChoreographer = (timestampBase ==

                OutputConfiguration::TIMESTAMP_BASE_CHOREOGRAPHER_SYNCED);

        bool defaultToChoreographer = (isDefaultTimeBase && isConsumedByHWComposer() &&

        bool defaultToChoreographer = (isDefaultTimeBase &&

                isConsumedByHWComposer() &&

                !property_get_bool("camera.disable_preview_scheduler", false));

        if (forceChoreographer || defaultToChoreographer) {

            mSyncToDisplay = true;

            mTotalBufferCount += kDisplaySyncExtraBuffer;

        } else if (isConsumedByHWTexture() && !isVideoStream()) {

            mPreviewFrameSpacer = new PreviewFrameSpacer(*this, mConsumer);

            mTotalBufferCount ++;

            res = mPreviewFrameSpacer->run(String8::format("PreviewSpacer-%d", mId).string());

            if (res != OK) {

                ALOGE("%s: Unable to start preview spacer", __FUNCTION__);

                return res;

            }

        }

    }



    mHandoutTotalBufferCount = 0;

    mFrameCount = 0;

    mLastTimestamp = 0;
@@ -1316,6 +1332,11 @@ nsecs_t Camera3OutputStream::syncTimestampToDisplayLocked(nsecs_t t) { 
    return expectedPresentT - vsyncEventData.frameInterval/3;

}



bool Camera3OutputStream::shouldLogError(status_t res) {

    Mutex::Autolock l(mLock);

    return shouldLogError(res, mState);

}



}; // namespace camera3



}; // namespace android

services/camera/libcameraservice/device3/Camera3OutputStream.h

+11 −4
Original line number Diff line number Diff line
@@ -28,6 +28,7 @@ 
#include "Camera3IOStreamBase.h"

#include "Camera3OutputStreamInterface.h"

#include "Camera3BufferManager.h"

#include "PreviewFrameSpacer.h"



namespace android {
@@ -250,6 +251,7 @@ class Camera3OutputStream : 
    static void applyZSLUsageQuirk(int format, uint64_t *consumerUsage /*inout*/);



    void setImageDumpMask(int mask) { mImageDumpMask = mask; }

    bool shouldLogError(status_t res);



  protected:

    Camera3OutputStream(int id, camera_stream_type_t type,
@@ -282,7 +284,7 @@ class Camera3OutputStream : 


    status_t getEndpointUsageForSurface(uint64_t *usage,

            const sp<Surface>& surface) const;

    status_t configureConsumerQueueLocked(bool allowDisplaySync);

    status_t configureConsumerQueueLocked(bool allowPreviewRespace);



    // Consumer as the output of camera HAL

    sp<Surface> mConsumer;
@@ -396,15 +398,14 @@ class Camera3OutputStream : 


    void returnPrefetchedBuffersLocked();



    // Synchronize camera timestamp to display, and the return value

    // can be used as presentation timestamp

    nsecs_t syncTimestampToDisplayLocked(nsecs_t t);



    static const int32_t kDequeueLatencyBinSize = 5; // in ms

    CameraLatencyHistogram mDequeueBufferLatency;



    int mImageDumpMask = 0;



    // Re-space frames by overriding timestamp to align with display Vsync.

    // Default is on for SurfaceView bound streams.

    nsecs_t mMinExpectedDuration = 0;

    bool mSyncToDisplay = false;

    DisplayEventReceiver mDisplayEventReceiver;
@@ -414,6 +415,12 @@ class Camera3OutputStream : 
    static constexpr size_t kDisplaySyncExtraBuffer = 2;

    static constexpr nsecs_t kSpacingResetIntervalNs = 1000000000LL; // 1 second

    static constexpr nsecs_t kTimelineThresholdNs = 1000000LL; // 1 millisecond

    nsecs_t syncTimestampToDisplayLocked(nsecs_t t);



    // Re-space frames by delaying queueBuffer so that frame delivery has

    // the same cadence as capture. Default is on for SurfaceTexture bound

    // streams.

    sp<PreviewFrameSpacer> mPreviewFrameSpacer;

}; // class Camera3OutputStream



} // namespace camera3

services/camera/libcameraservice/device3/Camera3SharedOutputStream.cpp

+1 −1
Original line number Diff line number Diff line
@@ -251,7 +251,7 @@ status_t Camera3SharedOutputStream::configureQueueLocked() { 
        return res;

    }



    res = configureConsumerQueueLocked(false/*allowDisplaySync*/);

    res = configureConsumerQueueLocked(false/*allowPreviewRespace*/);

    if (res != OK) {

        ALOGE("Failed to configureConsumerQueueLocked: %s(%d)", strerror(-res), res);

        return res;

services/camera/libcameraservice/device3/PreviewFrameSpacer.cpp

0 → 100644
+122 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2022 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



#define LOG_TAG "Camera3-PreviewFrameSpacer"

#define ATRACE_TAG ATRACE_TAG_CAMERA

//#define LOG_NDEBUG 0



#include <utils/Log.h>



#include "PreviewFrameSpacer.h"

#include "Camera3OutputStream.h"



namespace android {



namespace camera3 {



PreviewFrameSpacer::PreviewFrameSpacer(Camera3OutputStream& parent, sp<Surface> consumer) :

        mParent(parent),

        mConsumer(consumer) {

}



PreviewFrameSpacer::~PreviewFrameSpacer() {

    Thread::requestExitAndWait();

}



status_t PreviewFrameSpacer::queuePreviewBuffer(nsecs_t timestamp, int32_t transform,

        ANativeWindowBuffer* anwBuffer, int releaseFence) {

    Mutex::Autolock l(mLock);

    mPendingBuffers.emplace(timestamp, transform, anwBuffer, releaseFence);

    ALOGV("%s: mPendingBuffers size %zu, timestamp %" PRId64, __FUNCTION__,

            mPendingBuffers.size(), timestamp);



    mBufferCond.signal();

    return OK;

}



bool PreviewFrameSpacer::threadLoop() {

    Mutex::Autolock l(mLock);

    if (mPendingBuffers.size() == 0) {

        mBufferCond.waitRelative(mLock, kWaitDuration);

        return true;

    }



    nsecs_t currentTime = systemTime();

    auto buffer = mPendingBuffers.front();

    nsecs_t captureInterval = buffer.timestamp - mLastCameraCaptureTime;

    // If the capture interval exceeds threshold, directly queue

    // cached buffer.

    if (captureInterval >= kFrameIntervalThreshold) {

        mPendingBuffers.pop();

        queueBufferToClientLocked(buffer, currentTime);

        return true;

    }



    // Cache the frame to match capture time interval, for up to 33ms

    nsecs_t expectedQueueTime = mLastCameraPresentTime + captureInterval;

    nsecs_t frameWaitTime = std::min(kMaxFrameWaitTime, expectedQueueTime - currentTime);

    if (frameWaitTime > 0 && mPendingBuffers.size() < 2) {

        mBufferCond.waitRelative(mLock, frameWaitTime);

        if (exitPending()) {

            return true;

        }

        currentTime = systemTime();

    }

    ALOGV("%s: captureInterval %" PRId64 ", queueInterval %" PRId64 ", waited for %" PRId64

            ", timestamp %" PRId64, __FUNCTION__, captureInterval,

            currentTime - mLastCameraPresentTime, frameWaitTime, buffer.timestamp);

    mPendingBuffers.pop();

    queueBufferToClientLocked(buffer, currentTime);

    return true;

}



void PreviewFrameSpacer::requestExit() {

    // Call parent to set up shutdown

    Thread::requestExit();

    // Exit from other possible wait

    mBufferCond.signal();

}



void PreviewFrameSpacer::queueBufferToClientLocked(

        const BufferHolder& bufferHolder, nsecs_t currentTime) {

    mParent.setTransform(bufferHolder.transform, true/*mayChangeMirror*/);



    status_t res = native_window_set_buffers_timestamp(mConsumer.get(), bufferHolder.timestamp);

    if (res != OK) {

        ALOGE("%s: Preview Stream: Error setting timestamp: %s (%d)",

                __FUNCTION__, strerror(-res), res);

    }



    Camera3Stream::queueHDRMetadata(bufferHolder.anwBuffer.get()->handle, mConsumer,

            mParent.getDynamicRangeProfile());



    res = mConsumer->queueBuffer(mConsumer.get(), bufferHolder.anwBuffer.get(),

            bufferHolder.releaseFence);

    if (res != OK) {

        close(bufferHolder.releaseFence);

        if (mParent.shouldLogError(res)) {

            ALOGE("%s: Failed to queue buffer to client: %s(%d)", __FUNCTION__,

                    strerror(-res), res);

        }

    }



    mLastCameraPresentTime = currentTime;

    mLastCameraCaptureTime = bufferHolder.timestamp;

}



}; // namespace camera3



}; // namespace android