Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 4c6d2b5c authored by Emilian Peev's avatar Emilian Peev
Browse files

Camera: Add support for dynamic depth if possible 

Composite dynamic depth streams will be supported
in case camera devices include:
- Dense depth streams which are not exclusive.
- Jpeg/Blob output with sizes that either match depth
  or are with similar aspect ratio.
Make a guesstimate regarding the dynamic depth minimum
frame and stall durations.

Bug: 109735087
Test: Manual using application,
Camera CTS

Change-Id: I8a89c7895cf57ce4408e41b1afae9c85d48c4e07
parent 45b005c6

camera/ndk/include/camera/NdkCameraMetadataTags.h

+77 −0
Original line number Diff line number Diff line
@@ -5549,6 +5549,73 @@ typedef enum acamera_metadata_tag { 
    ACAMERA_DEPTH_AVAILABLE_RECOMMENDED_DEPTH_STREAM_CONFIGURATIONS = 

                                                                // int32[n*5]

            ACAMERA_DEPTH_START + 5,

    /**

     * <p>The available dynamic depth dataspace stream

     * configurations that this camera device supports

     * (i.e. format, width, height, output/input stream).</p>

     *

     * <p>Type: int32[n*4] (acamera_metadata_enum_android_depth_available_dynamic_depth_stream_configurations_t)</p>

     *

     * <p>This tag may appear in:

     * <ul>

     *   <li>ACameraMetadata from ACameraManager_getCameraCharacteristics</li>

     * </ul></p>

     *

     * <p>These are output stream configurations for use with

     * dataSpace DYNAMIC_DEPTH. The configurations are

     * listed as <code>(format, width, height, input?)</code> tuples.</p>

     * <p>Only devices that support depth output for at least

     * the HAL_PIXEL_FORMAT_Y16 dense depth map along with

     * HAL_PIXEL_FORMAT_BLOB with the same size or size with

     * the same aspect ratio can have dynamic depth dataspace

     * stream configuration. ACAMERA_DEPTH_DEPTH_IS_EXCLUSIVE also

     * needs to be set to FALSE.</p>

     *

     * @see ACAMERA_DEPTH_DEPTH_IS_EXCLUSIVE

     */

    ACAMERA_DEPTH_AVAILABLE_DYNAMIC_DEPTH_STREAM_CONFIGURATIONS = 

                                                                // int32[n*4] (acamera_metadata_enum_android_depth_available_dynamic_depth_stream_configurations_t)

            ACAMERA_DEPTH_START + 6,

    /**

     * <p>This lists the minimum frame duration for each

     * format/size combination for dynamic depth output streams.</p>

     *

     * <p>Type: int64[4*n]</p>

     *

     * <p>This tag may appear in:

     * <ul>

     *   <li>ACameraMetadata from ACameraManager_getCameraCharacteristics</li>

     * </ul></p>

     *

     * <p>This should correspond to the frame duration when only that

     * stream is active, with all processing (typically in android.*.mode)

     * set to either OFF or FAST.</p>

     * <p>When multiple streams are used in a request, the minimum frame

     * duration will be max(individual stream min durations).</p>

     * <p>The minimum frame duration of a stream (of a particular format, size)

     * is the same regardless of whether the stream is input or output.</p>

     */

    ACAMERA_DEPTH_AVAILABLE_DYNAMIC_DEPTH_MIN_FRAME_DURATIONS = // int64[4*n]

            ACAMERA_DEPTH_START + 7,

    /**

     * <p>This lists the maximum stall duration for each

     * output format/size combination for dynamic depth streams.</p>

     *

     * <p>Type: int64[4*n]</p>

     *

     * <p>This tag may appear in:

     * <ul>

     *   <li>ACameraMetadata from ACameraManager_getCameraCharacteristics</li>

     * </ul></p>

     *

     * <p>A stall duration is how much extra time would get added

     * to the normal minimum frame duration for a repeating request

     * that has streams with non-zero stall.</p>

     * <p>All dynamic depth output streams may have a nonzero stall

     * duration.</p>

     */

    ACAMERA_DEPTH_AVAILABLE_DYNAMIC_DEPTH_STALL_DURATIONS =     // int64[4*n]

            ACAMERA_DEPTH_START + 8,

    ACAMERA_DEPTH_END,



    /**
@@ -8246,6 +8313,16 @@ typedef enum acamera_metadata_enum_acamera_depth_depth_is_exclusive { 


} acamera_metadata_enum_android_depth_depth_is_exclusive_t;



// ACAMERA_DEPTH_AVAILABLE_DYNAMIC_DEPTH_STREAM_CONFIGURATIONS

typedef enum acamera_metadata_enum_acamera_depth_available_dynamic_depth_stream_configurations {

    ACAMERA_DEPTH_AVAILABLE_DYNAMIC_DEPTH_STREAM_CONFIGURATIONS_OUTPUT

                                                                      = 0,



    ACAMERA_DEPTH_AVAILABLE_DYNAMIC_DEPTH_STREAM_CONFIGURATIONS_INPUT

                                                                      = 1,



} acamera_metadata_enum_android_depth_available_dynamic_depth_stream_configurations_t;





// ACAMERA_LOGICAL_MULTI_CAMERA_SENSOR_SYNC_TYPE

typedef enum acamera_metadata_enum_acamera_logical_multi_camera_sensor_sync_type {

services/camera/libcameraservice/common/CameraProviderManager.cpp

+248 −0
Original line number Diff line number Diff line
@@ -500,6 +500,248 @@ void CameraProviderManager::ProviderInfo::DeviceInfo3::queryPhysicalCameraIds() 
    }

}



void CameraProviderManager::ProviderInfo::DeviceInfo3::getSupportedSizes(

        const CameraMetadata& ch, uint32_t tag, android_pixel_format_t format,

        std::vector<std::tuple<size_t, size_t>> *sizes/*out*/) {

    if (sizes == nullptr) {

        return;

    }



    auto scalerDims = ch.find(tag);

    if (scalerDims.count > 0) {

        // Scaler entry contains 4 elements (format, width, height, type)

        for (size_t i = 0; i < scalerDims.count; i += 4) {

            if ((scalerDims.data.i32[i] == format) &&

                    (scalerDims.data.i32[i+3] ==

                     ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT)) {

                sizes->push_back(std::make_tuple(scalerDims.data.i32[i+1],

                            scalerDims.data.i32[i+2]));

            }

        }

    }

}



void CameraProviderManager::ProviderInfo::DeviceInfo3::getSupportedDurations(

        const CameraMetadata& ch, uint32_t tag, android_pixel_format_t format,

        const std::vector<std::tuple<size_t, size_t>>& sizes,

        std::vector<int64_t> *durations/*out*/) {

    if (durations == nullptr) {

        return;

    }



    auto availableDurations = ch.find(tag);

    if (availableDurations.count > 0) {

        // Duration entry contains 4 elements (format, width, height, duration)

        for (size_t i = 0; i < availableDurations.count; i += 4) {

            for (const auto& size : sizes) {

                int64_t width = std::get<0>(size);

                int64_t height = std::get<1>(size);

                if ((availableDurations.data.i64[i] == format) &&

                        (availableDurations.data.i64[i+1] == width) &&

                        (availableDurations.data.i64[i+2] == height)) {

                    durations->push_back(availableDurations.data.i64[i+3]);

                }

            }

        }

    }

}

void CameraProviderManager::ProviderInfo::DeviceInfo3::getSupportedDynamicDepthDurations(

        const std::vector<int64_t>& depthDurations, const std::vector<int64_t>& blobDurations,

        std::vector<int64_t> *dynamicDepthDurations /*out*/) {

    if ((dynamicDepthDurations == nullptr) || (depthDurations.size() != blobDurations.size())) {

        return;

    }



    // Unfortunately there is no direct way to calculate the dynamic depth stream duration.

    // Processing time on camera service side can vary greatly depending on multiple

    // variables which are not under our control. Make a guesstimate by taking the maximum

    // corresponding duration value from depth and blob.

    auto depthDuration = depthDurations.begin();

    auto blobDuration = blobDurations.begin();

    dynamicDepthDurations->reserve(depthDurations.size());

    while ((depthDuration != depthDurations.end()) && (blobDuration != blobDurations.end())) {

        dynamicDepthDurations->push_back(std::max(*depthDuration, *blobDuration));

        depthDuration++; blobDuration++;

    }

}



void CameraProviderManager::ProviderInfo::DeviceInfo3::getSupportedDynamicDepthSizes(

        const std::vector<std::tuple<size_t, size_t>>& blobSizes,

        const std::vector<std::tuple<size_t, size_t>>& depthSizes,

        std::vector<std::tuple<size_t, size_t>> *dynamicDepthSizes /*out*/,

        std::vector<std::tuple<size_t, size_t>> *internalDepthSizes /*out*/) {

    if (dynamicDepthSizes == nullptr || internalDepthSizes == nullptr) {

        return;

    }



    // The dynamic depth spec. does not mention how close the AR ratio should be.

    // Try using something appropriate.

    float ARTolerance = .01f;



    //TODO: Remove this before merging! This is for testing purposes only

    ARTolerance = 10.f;



    for (const auto& blobSize : blobSizes) {

        float jpegAR = static_cast<float> (std::get<0>(blobSize)) /

                static_cast<float>(std::get<1>(blobSize));

        bool found = false;

        for (const auto& depthSize : depthSizes) {

            if (depthSize == blobSize) {

                internalDepthSizes->push_back(depthSize);

                found = true;

                break;

            } else {

                float depthAR = static_cast<float> (std::get<0>(depthSize)) /

                    static_cast<float>(std::get<1>(depthSize));

                if (std::fabs(jpegAR - depthAR) <= ARTolerance) {

                    internalDepthSizes->push_back(depthSize);

                    found = true;

                    break;

                }

            }

        }



        if (found) {

            dynamicDepthSizes->push_back(blobSize);

        }

    }

}



status_t CameraProviderManager::ProviderInfo::DeviceInfo3::addDynamicDepthTags() {

    uint32_t depthExclTag = ANDROID_DEPTH_DEPTH_IS_EXCLUSIVE;

    uint32_t depthSizesTag = ANDROID_DEPTH_AVAILABLE_DEPTH_STREAM_CONFIGURATIONS;

    auto& c = mCameraCharacteristics;

    std::vector<std::tuple<size_t, size_t>> supportedBlobSizes, supportedDepthSizes,

            supportedDynamicDepthSizes, internalDepthSizes;

    auto chTags = c.find(ANDROID_REQUEST_AVAILABLE_CHARACTERISTICS_KEYS);

    if (chTags.count == 0) {

        ALOGE("%s: Supported camera characteristics is empty!", __FUNCTION__);

        return BAD_VALUE;

    }



    bool isDepthExclusivePresent = std::find(chTags.data.i32, chTags.data.i32 + chTags.count,

            depthExclTag) != (chTags.data.i32 + chTags.count);

    bool isDepthSizePresent = std::find(chTags.data.i32, chTags.data.i32 + chTags.count,

            depthExclTag) != (chTags.data.i32 + chTags.count);

    if (!(isDepthExclusivePresent && isDepthSizePresent)) {

        // No depth support, nothing more to do.

        return OK;

    }



    auto depthExclusiveEntry = c.find(depthExclTag);

    if (depthExclusiveEntry.count > 0) {

        if (depthExclusiveEntry.data.u8[0] != ANDROID_DEPTH_DEPTH_IS_EXCLUSIVE_FALSE) {

            // Depth support is exclusive, nothing more to do.

            return OK;

        }

    } else {

        ALOGE("%s: Advertised depth exclusive tag but value is not present!", __FUNCTION__);

        return BAD_VALUE;

    }



    getSupportedSizes(c, ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS, HAL_PIXEL_FORMAT_BLOB,

            &supportedBlobSizes);

    getSupportedSizes(c, depthSizesTag, HAL_PIXEL_FORMAT_Y16, &supportedDepthSizes);

    if (supportedBlobSizes.empty() || supportedDepthSizes.empty()) {

        // Nothing to do in this case.

        return OK;

    }



    getSupportedDynamicDepthSizes(supportedBlobSizes, supportedDepthSizes,

            &supportedDynamicDepthSizes, &internalDepthSizes);

    if (supportedDynamicDepthSizes.empty()) {

        ALOGE("%s: No dynamic depth size matched!", __func__);

        // Nothing more to do.

        return OK;

    }



    std::vector<int32_t> dynamicDepthEntries;

    for (const auto& it : supportedDynamicDepthSizes) {

        int32_t entry[4] = {HAL_PIXEL_FORMAT_BLOB, static_cast<int32_t> (std::get<0>(it)),

                static_cast<int32_t> (std::get<1>(it)),

                ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT };

        dynamicDepthEntries.insert(dynamicDepthEntries.end(), entry, entry + 4);

    }



    std::vector<int64_t> depthMinDurations, depthStallDurations;

    std::vector<int64_t> blobMinDurations, blobStallDurations;

    std::vector<int64_t> dynamicDepthMinDurations, dynamicDepthStallDurations;



    getSupportedDurations(c, ANDROID_DEPTH_AVAILABLE_DEPTH_MIN_FRAME_DURATIONS,

            HAL_PIXEL_FORMAT_Y16, internalDepthSizes, &depthMinDurations);

    getSupportedDurations(c, ANDROID_SCALER_AVAILABLE_MIN_FRAME_DURATIONS,

            HAL_PIXEL_FORMAT_BLOB, supportedDynamicDepthSizes, &blobMinDurations);

    if (blobMinDurations.empty() || depthMinDurations.empty() ||

            (depthMinDurations.size() != blobMinDurations.size())) {

        ALOGE("%s: Unexpected number of available depth min durations! %zu vs. %zu",

                __FUNCTION__, depthMinDurations.size(), blobMinDurations.size());

        return BAD_VALUE;

    }



    getSupportedDurations(c, ANDROID_DEPTH_AVAILABLE_DEPTH_STALL_DURATIONS,

            HAL_PIXEL_FORMAT_Y16, internalDepthSizes, &depthStallDurations);

    getSupportedDurations(c, ANDROID_SCALER_AVAILABLE_STALL_DURATIONS,

            HAL_PIXEL_FORMAT_BLOB, supportedDynamicDepthSizes, &blobStallDurations);

    if (blobStallDurations.empty() || depthStallDurations.empty() ||

            (depthStallDurations.size() != blobStallDurations.size())) {

        ALOGE("%s: Unexpected number of available depth stall durations! %zu vs. %zu",

                __FUNCTION__, depthStallDurations.size(), blobStallDurations.size());

        return BAD_VALUE;

    }



    getSupportedDynamicDepthDurations(depthMinDurations, blobMinDurations,

            &dynamicDepthMinDurations);

    getSupportedDynamicDepthDurations(depthStallDurations, blobStallDurations,

            &dynamicDepthStallDurations);

    if (dynamicDepthMinDurations.empty() || dynamicDepthStallDurations.empty() ||

            (dynamicDepthMinDurations.size() != dynamicDepthStallDurations.size())) {

        ALOGE("%s: Unexpected number of dynamic depth stall/min durations! %zu vs. %zu",

                __FUNCTION__, dynamicDepthMinDurations.size(), dynamicDepthStallDurations.size());

        return BAD_VALUE;

    }



    std::vector<int64_t> dynamicDepthMinDurationEntries;

    auto itDuration = dynamicDepthMinDurations.begin();

    auto itSize = supportedDynamicDepthSizes.begin();

    while (itDuration != dynamicDepthMinDurations.end()) {

        int64_t entry[4] = {HAL_PIXEL_FORMAT_BLOB, static_cast<int32_t> (std::get<0>(*itSize)),

                static_cast<int32_t> (std::get<1>(*itSize)), *itDuration};

        dynamicDepthMinDurationEntries.insert(dynamicDepthMinDurationEntries.end(), entry,

                entry + 4);

        itDuration++; itSize++;

    }



    std::vector<int64_t> dynamicDepthStallDurationEntries;

    itDuration = dynamicDepthStallDurations.begin();

    itSize = supportedDynamicDepthSizes.begin();

    while (itDuration != dynamicDepthStallDurations.end()) {

        int64_t entry[4] = {HAL_PIXEL_FORMAT_BLOB, static_cast<int32_t> (std::get<0>(*itSize)),

                static_cast<int32_t> (std::get<1>(*itSize)), *itDuration};

        dynamicDepthStallDurationEntries.insert(dynamicDepthStallDurationEntries.end(), entry,

                entry + 4);

        itDuration++; itSize++;

    }



    c.update(ANDROID_DEPTH_AVAILABLE_DYNAMIC_DEPTH_STREAM_CONFIGURATIONS,

            dynamicDepthEntries.data(), dynamicDepthEntries.size());

    c.update(ANDROID_DEPTH_AVAILABLE_DYNAMIC_DEPTH_MIN_FRAME_DURATIONS,

            dynamicDepthMinDurationEntries.data(), dynamicDepthMinDurationEntries.size());

    c.update(ANDROID_DEPTH_AVAILABLE_DYNAMIC_DEPTH_STALL_DURATIONS,

            dynamicDepthStallDurationEntries.data(), dynamicDepthStallDurationEntries.size());



    std::vector<int32_t> supportedChTags;

    supportedChTags.reserve(chTags.count + 3);

    supportedChTags.insert(supportedChTags.end(), chTags.data.i32,

            chTags.data.i32 + chTags.count);

    supportedChTags.push_back(ANDROID_DEPTH_AVAILABLE_DYNAMIC_DEPTH_STREAM_CONFIGURATIONS);

    supportedChTags.push_back(ANDROID_DEPTH_AVAILABLE_DYNAMIC_DEPTH_MIN_FRAME_DURATIONS);

    supportedChTags.push_back(ANDROID_DEPTH_AVAILABLE_DYNAMIC_DEPTH_STALL_DURATIONS);

    c.update(ANDROID_REQUEST_AVAILABLE_CHARACTERISTICS_KEYS, supportedChTags.data(),

            supportedChTags.size());



    return OK;

}



status_t CameraProviderManager::ProviderInfo::DeviceInfo3::fixupMonochromeTags() {

    status_t res = OK;

    auto& c = mCameraCharacteristics;
@@ -1442,6 +1684,12 @@ CameraProviderManager::ProviderInfo::DeviceInfo3::DeviceInfo3(const std::string& 
                __FUNCTION__, strerror(-res), res);

        return;

    }

    res = addDynamicDepthTags();

    if (OK != res) {

        ALOGE("%s: Failed appending dynamic depth tags: %s (%d)", __FUNCTION__, strerror(-res),

                res);

        return;

    }

    camera_metadata_entry flashAvailable =

            mCameraCharacteristics.find(ANDROID_FLASH_INFO_AVAILABLE);

    if (flashAvailable.count == 1 &&

services/camera/libcameraservice/common/CameraProviderManager.h

+16 −0
Original line number Diff line number Diff line
@@ -470,6 +470,22 @@ private: 
            std::unordered_map<std::string, CameraMetadata> mPhysicalCameraCharacteristics;

            void queryPhysicalCameraIds();

            status_t fixupMonochromeTags();

            status_t addDynamicDepthTags();

            static void getSupportedSizes(const CameraMetadata& ch, uint32_t tag,

                    android_pixel_format_t format,

                    std::vector<std::tuple<size_t, size_t>> *sizes /*out*/);

            void getSupportedDurations( const CameraMetadata& ch, uint32_t tag,

                    android_pixel_format_t format,

                    const std::vector<std::tuple<size_t, size_t>>& sizes,

                    std::vector<int64_t> *durations/*out*/);

            void getSupportedDynamicDepthDurations(const std::vector<int64_t>& depthDurations,

                    const std::vector<int64_t>& blobDurations,

                    std::vector<int64_t> *dynamicDepthDurations /*out*/);

            static void getSupportedDynamicDepthSizes(

                    const std::vector<std::tuple<size_t, size_t>>& blobSizes,

                    const std::vector<std::tuple<size_t, size_t>>& depthSizes,

                    std::vector<std::tuple<size_t, size_t>> *dynamicDepthSizes /*out*/,

                    std::vector<std::tuple<size_t, size_t>> *internalDepthSizes /*out*/);

            status_t removeAvailableKeys(CameraMetadata& c, const std::vector<uint32_t>& keys,

                    uint32_t keyTag);

        };