Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c25c6ed7 authored by TreeHugger Robot's avatar TreeHugger Robot Committed by Android (Google) Code Review
Browse files

Merge "Camera: Avoid latency accumulation when syncing preview to vsync" into tm-qpr-dev

parents 70c8179d 696e4da7

services/camera/libcameraservice/device3/Camera3Device.cpp

+23 −17
Original line number Diff line number Diff line
@@ -2679,7 +2679,7 @@ void Camera3Device::setErrorStateLockedV(const char *fmt, va_list args) { 
status_t Camera3Device::registerInFlight(uint32_t frameNumber,

        int32_t numBuffers, CaptureResultExtras resultExtras, bool hasInput,

        bool hasAppCallback, nsecs_t minExpectedDuration, nsecs_t maxExpectedDuration,

        const std::set<std::set<String8>>& physicalCameraIds,

        bool isFixedFps, const std::set<std::set<String8>>& physicalCameraIds,

        bool isStillCapture, bool isZslCapture, bool rotateAndCropAuto,

        const std::set<std::string>& cameraIdsWithZoom,

        const SurfaceMap& outputSurfaces, nsecs_t requestTimeNs) {
@@ -2688,7 +2688,7 @@ status_t Camera3Device::registerInFlight(uint32_t frameNumber, 


    ssize_t res;

    res = mInFlightMap.add(frameNumber, InFlightRequest(numBuffers, resultExtras, hasInput,

            hasAppCallback, minExpectedDuration, maxExpectedDuration, physicalCameraIds,

            hasAppCallback, minExpectedDuration, maxExpectedDuration, isFixedFps, physicalCameraIds,

            isStillCapture, isZslCapture, rotateAndCropAuto, cameraIdsWithZoom, requestTimeNs,

            outputSurfaces));

    if (res < 0) return res;
@@ -3248,16 +3248,18 @@ bool Camera3Device::RequestThread::sendRequestsBatch() { 
    return true;

}



std::pair<nsecs_t, nsecs_t> Camera3Device::RequestThread::calculateExpectedDurationRange(

Camera3Device::RequestThread::ExpectedDurationInfo

        Camera3Device::RequestThread::calculateExpectedDurationRange(

                const camera_metadata_t *request) {

    std::pair<nsecs_t, nsecs_t> expectedRange(

    ExpectedDurationInfo expectedDurationInfo = {

            InFlightRequest::kDefaultMinExpectedDuration,

            InFlightRequest::kDefaultMaxExpectedDuration);

            InFlightRequest::kDefaultMaxExpectedDuration,

            /*isFixedFps*/false};

    camera_metadata_ro_entry_t e = camera_metadata_ro_entry_t();

    find_camera_metadata_ro_entry(request,

            ANDROID_CONTROL_AE_MODE,

            &e);

    if (e.count == 0) return expectedRange;

    if (e.count == 0) return expectedDurationInfo;



    switch (e.data.u8[0]) {

        case ANDROID_CONTROL_AE_MODE_OFF:
@@ -3265,29 +3267,32 @@ std::pair<nsecs_t, nsecs_t> Camera3Device::RequestThread::calculateExpectedDurat 
                    ANDROID_SENSOR_EXPOSURE_TIME,

                    &e);

            if (e.count > 0) {

                expectedRange.first = e.data.i64[0];

                expectedRange.second = expectedRange.first;

                expectedDurationInfo.minDuration = e.data.i64[0];

                expectedDurationInfo.maxDuration = expectedDurationInfo.minDuration;

            }

            find_camera_metadata_ro_entry(request,

                    ANDROID_SENSOR_FRAME_DURATION,

                    &e);

            if (e.count > 0) {

                expectedRange.first = std::max(e.data.i64[0], expectedRange.first);

                expectedRange.second = expectedRange.first;

                expectedDurationInfo.minDuration =

                        std::max(e.data.i64[0], expectedDurationInfo.minDuration);

                expectedDurationInfo.maxDuration = expectedDurationInfo.minDuration;

            }

            expectedDurationInfo.isFixedFps = false;

            break;

        default:

            find_camera_metadata_ro_entry(request,

                    ANDROID_CONTROL_AE_TARGET_FPS_RANGE,

                    &e);

            if (e.count > 1) {

                expectedRange.first = 1e9 / e.data.i32[1];

                expectedRange.second = 1e9 / e.data.i32[0];

                expectedDurationInfo.minDuration = 1e9 / e.data.i32[1];

                expectedDurationInfo.maxDuration = 1e9 / e.data.i32[0];

            }

            expectedDurationInfo.isFixedFps = (e.data.i32[1] == e.data.i32[0]);

            break;

    }



    return expectedRange;

    return expectedDurationInfo;

}



bool Camera3Device::RequestThread::skipHFRTargetFPSUpdate(int32_t tag,
@@ -3902,13 +3907,14 @@ status_t Camera3Device::RequestThread::prepareHalRequests() { 
                isZslCapture = true;

            }

        }

        auto expectedDurationRange = calculateExpectedDurationRange(settings);

        auto expectedDurationInfo = calculateExpectedDurationRange(settings);

        res = parent->registerInFlight(halRequest->frame_number,

                totalNumBuffers, captureRequest->mResultExtras,

                /*hasInput*/halRequest->input_buffer != NULL,

                hasCallback,

                /*min*/expectedDurationRange.first,

                /*max*/expectedDurationRange.second,

                expectedDurationInfo.minDuration,

                expectedDurationInfo.maxDuration,

                expectedDurationInfo.isFixedFps,

                requestedPhysicalCameras, isStillCapture, isZslCapture,

                captureRequest->mRotateAndCropAuto, mPrevCameraIdsWithZoom,

                (mUseHalBufManager) ? uniqueSurfaceIdMap :

services/camera/libcameraservice/device3/Camera3Device.h

+11 −3
Original line number Diff line number Diff line
@@ -967,8 +967,13 @@ class Camera3Device : 
        // send request in mNextRequests to HAL in a batch. Return true = sucssess

        bool sendRequestsBatch();



        // Calculate the expected (minimum, maximum) duration range for a request

        std::pair<nsecs_t, nsecs_t> calculateExpectedDurationRange(

        // Calculate the expected (minimum, maximum, isFixedFps) duration info for a request

        struct ExpectedDurationInfo {

            nsecs_t minDuration;

            nsecs_t maxDuration;

            bool isFixedFps;

        };

        ExpectedDurationInfo calculateExpectedDurationRange(

                const camera_metadata_t *request);



        // Check and update latest session parameters based on the current request settings.
@@ -1087,7 +1092,7 @@ class Camera3Device : 
    status_t registerInFlight(uint32_t frameNumber,

            int32_t numBuffers, CaptureResultExtras resultExtras, bool hasInput,

            bool callback, nsecs_t minExpectedDuration, nsecs_t maxExpectedDuration,

            const std::set<std::set<String8>>& physicalCameraIds,

            bool isFixedFps, const std::set<std::set<String8>>& physicalCameraIds,

            bool isStillCapture, bool isZslCapture, bool rotateAndCropAuto,

            const std::set<std::string>& cameraIdsWithZoom, const SurfaceMap& outputSurfaces,

            nsecs_t requestTimeNs);
@@ -1339,6 +1344,9 @@ class Camera3Device : 


    // The current minimum expected frame duration based on AE_TARGET_FPS_RANGE

    nsecs_t mMinExpectedDuration = 0;

    // Whether the camera device runs at fixed frame rate based on AE_MODE and

    // AE_TARGET_FPS_RANGE

    bool mIsFixedFps = false;



    // Injection camera related methods.

    class Camera3DeviceInjectionMethods : public virtual RefBase {

services/camera/libcameraservice/device3/Camera3FakeStream.h

+1 −1
Original line number Diff line number Diff line
@@ -100,7 +100,7 @@ class Camera3FakeStream : 


    virtual status_t setBatchSize(size_t batchSize) override;



    virtual void onMinDurationChanged(nsecs_t /*duration*/) {}

    virtual void onMinDurationChanged(nsecs_t /*duration*/, bool /*fixedFps*/) {}

  protected:



    /**

services/camera/libcameraservice/device3/Camera3OfflineSession.h

+3 −0
Original line number Diff line number Diff line
@@ -248,6 +248,9 @@ class Camera3OfflineSession : 


    // The current minimum expected frame duration based on AE_TARGET_FPS_RANGE

    nsecs_t mMinExpectedDuration = 0;

    // Whether the camera device runs at fixed frame rate based on AE_MODE and

    // AE_TARGET_FPS_RANGE

    bool mIsFixedFps = false;



    // SetErrorInterface

    void setErrorState(const char *fmt, ...) override;

services/camera/libcameraservice/device3/Camera3OutputStream.cpp

+94 −18
Original line number Diff line number Diff line
@@ -1367,9 +1367,10 @@ status_t Camera3OutputStream::setBatchSize(size_t batchSize) { 
    return OK;

}



void Camera3OutputStream::onMinDurationChanged(nsecs_t duration) {

void Camera3OutputStream::onMinDurationChanged(nsecs_t duration, bool fixedFps) {

    Mutex::Autolock l(mLock);

    mMinExpectedDuration = duration;

    mFixedFps = fixedFps;

}



void Camera3OutputStream::returnPrefetchedBuffersLocked() {
@@ -1402,17 +1403,21 @@ nsecs_t Camera3OutputStream::syncTimestampToDisplayLocked(nsecs_t t) { 


    const VsyncEventData& vsyncEventData = parcelableVsyncEventData.vsync;

    nsecs_t currentTime = systemTime();

    nsecs_t minPresentT = mLastPresentTime + vsyncEventData.frameInterval / 2;



    // Reset capture to present time offset if:

    // - More than 1 second between frames.

    // - The frame duration deviates from multiples of vsync frame intervals.

    // Find the best presentation time without worrying about previous frame's

    // presentation time if capture interval is more than kSpacingResetIntervalNs.

    //

    // When frame interval is more than 50 ms apart (3 vsyncs for 60hz refresh rate),

    // there is little risk in starting over and finding the earliest vsync to latch onto.

    // - Update captureToPresentTime offset to be used for later frames.

    // - Example use cases:

    //   - when frame rate drops down to below 20 fps, or

    //   - A new streaming session starts (stopPreview followed by

    //   startPreview)

    //

    nsecs_t captureInterval = t - mLastCaptureTime;

    float captureToVsyncIntervalRatio = 1.0f * captureInterval / vsyncEventData.frameInterval;

    float ratioDeviation = std::fabs(

            captureToVsyncIntervalRatio - std::roundf(captureToVsyncIntervalRatio));

    if (captureInterval > kSpacingResetIntervalNs ||

            ratioDeviation >= kMaxIntervalRatioDeviation) {

        nsecs_t minPresentT = mLastPresentTime + vsyncEventData.frameInterval / 2;

    if (captureInterval > kSpacingResetIntervalNs) {

        for (size_t i = 0; i < VsyncEventData::kFrameTimelinesLength; i++) {

            const auto& timeline = vsyncEventData.frameTimelines[i];

            if (timeline.deadlineTimestamp >= currentTime &&
@@ -1434,21 +1439,54 @@ nsecs_t Camera3OutputStream::syncTimestampToDisplayLocked(nsecs_t t) { 
    nsecs_t idealPresentT = t + mCaptureToPresentOffset;

    nsecs_t expectedPresentT = mLastPresentTime;

    nsecs_t minDiff = INT64_MAX;

    // Derive minimum intervals between presentation times based on minimal



    // In fixed FPS case, when frame durations are close to multiples of display refresh

    // rate, derive minimum intervals between presentation times based on minimal

    // expected duration. The minimum number of Vsyncs is:

    // - 0 if minFrameDuration in (0, 1.5] * vSyncInterval,

    // - 1 if minFrameDuration in (1.5, 2.5] * vSyncInterval,

    // - and so on.

    //

    // This spaces out the displaying of the frames so that the frame

    // presentations are roughly in sync with frame captures.

    int minVsyncs = (mMinExpectedDuration - vsyncEventData.frameInterval / 2) /

            vsyncEventData.frameInterval;

    if (minVsyncs < 0) minVsyncs = 0;

    nsecs_t minInterval = minVsyncs * vsyncEventData.frameInterval;



    // In fixed FPS case, if the frame duration deviates from multiples of

    // display refresh rate, find the closest Vsync without requiring a minimum

    // number of Vsync.

    //

    // Example: (24fps camera, 60hz refresh):

    //   capture readout:  |  t1  |  t1  | .. |  t1  | .. |  t1  | .. |  t1  |

    //   display VSYNC:      | t2 | t2 | ... | t2 | ... | t2 | ... | t2 |

    //   |  : 1 frame

    //   t1 : 41.67ms

    //   t2 : 16.67ms

    //   t1/t2 = 2.5

    //

    //   24fps is a commonly used video frame rate. Because the capture

    //   interval is 2.5 times of display refresh interval, the minVsyncs

    //   calculation will directly fall at the boundary condition. In this case,

    //   we should fall back to the basic logic of finding closest vsync

    //   timestamp without worrying about minVsyncs.

    float captureToVsyncIntervalRatio = 1.0f * mMinExpectedDuration / vsyncEventData.frameInterval;

    float ratioDeviation = std::fabs(

            captureToVsyncIntervalRatio - std::roundf(captureToVsyncIntervalRatio));

    bool captureDeviateFromVsync = ratioDeviation >= kMaxIntervalRatioDeviation;

    bool cameraDisplayInSync = (mFixedFps && !captureDeviateFromVsync);



    // Find best timestamp in the vsync timelines:

    // - Only use at most 3 timelines to avoid long latency

    // - closest to the ideal present time,

    // - Only use at most kMaxTimelines timelines to avoid long latency

    // - closest to the ideal presentation time,

    // - deadline timestamp is greater than the current time, and

    // - the candidate present time is at least minInterval in the future

    //   compared to last present time.

    // - For fixed FPS, if the capture interval doesn't deviate too much from refresh interval,

    //   the candidate presentation time is at least minInterval in the future compared to last

    //   presentation time.

    // - For variable FPS, or if the capture interval deviates from refresh

    //   interval for more than 5%, find a presentation time closest to the

    //   (lastPresentationTime + captureToPresentOffset) instead.

    int maxTimelines = std::min(kMaxTimelines, (int)VsyncEventData::kFrameTimelinesLength);

    float biasForShortDelay = 1.0f;

    for (int i = 0; i < maxTimelines; i ++) {
@@ -1461,12 +1499,50 @@ nsecs_t Camera3OutputStream::syncTimestampToDisplayLocked(nsecs_t t) { 
        }

        if (std::abs(vsyncTime.expectedPresentationTime - idealPresentT) < minDiff &&

                vsyncTime.deadlineTimestamp >= currentTime &&

                vsyncTime.expectedPresentationTime >

                mLastPresentTime + minInterval + biasForShortDelay * kTimelineThresholdNs) {

                ((!cameraDisplayInSync && vsyncTime.expectedPresentationTime > minPresentT) ||

                 (cameraDisplayInSync && vsyncTime.expectedPresentationTime >

                mLastPresentTime + minInterval + biasForShortDelay * kTimelineThresholdNs))) {

            expectedPresentT = vsyncTime.expectedPresentationTime;

            minDiff = std::abs(vsyncTime.expectedPresentationTime - idealPresentT);

        }

    }



    if (expectedPresentT == mLastPresentTime && expectedPresentT <=

            vsyncEventData.frameTimelines[maxTimelines].expectedPresentationTime) {

        // Couldn't find a reasonable presentation time. Using last frame's

        // presentation time would cause a frame drop. The best option now

        // is to use the next VSync as long as the last presentation time

        // doesn't already has the maximum latency, in which case dropping the

        // buffer is more desired than increasing latency.

        //

        // Example: (60fps camera, 59.9hz refresh):

        //   capture readout:  | t1 | t1 | .. | t1 | .. | t1 | .. | t1 |

        //                      \    \    \     \    \    \    \     \   \

        //   queue to BQ:       |    |    |     |    |    |    |      |    |

        //                      \    \    \     \    \     \    \      \    \

        //   display VSYNC:      | t2 | t2 | ... | t2 | ... | t2 | ... | t2 |

        //

        //   |: 1 frame

        //   t1 : 16.67ms

        //   t2 : 16.69ms

        //

        // It takes 833 frames for capture readout count and display VSYNC count to be off

        // by 1.

        //  - At frames [0, 832], presentationTime is set to timeline[0]

        //  - At frames [833, 833*2-1], presentationTime is set to timeline[1]

        //  - At frames [833*2, 833*3-1] presentationTime is set to timeline[2]

        //  - At frame 833*3, no presentation time is found because we only

        //    search for timeline[0..2].

        //  - Drop one buffer is better than further extend the presentation

        //    time.

        //

        // However, if frame 833*2 arrives 16.67ms early (right after frame

        // 833*2-1), no presentation time can be found because

        // getLatestVsyncEventData is called early. In that case, it's better to

        // set presentation time by offseting last presentation time.

        expectedPresentT += vsyncEventData.frameInterval;

    }



    mLastCaptureTime = t;

    mLastPresentTime = expectedPresentT;