Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 4b08d5dc authored by Emilian Peev's avatar Emilian Peev
Browse files

Camera: Keep Depth EXIF orientation consistent

Depth and confidence maps should always use the same
EXIF orientation as the main color image.

Bug: 123699590
Test: Manual using application,
Camera CTS,
adb shell /data/nativetest64/cameraservice_test/cameraservice_test
--gtest_filter=DepthProcessorTest.*
Change-Id: I0d887798e8717cdff81aba10d595dc3ccfe99197
parent 8b0e6fa0
Loading
Loading
Loading
Loading
+3 −0
Original line number Diff line number Diff line
@@ -137,6 +137,7 @@ cc_library_shared {
    name: "libdepthphoto",

    srcs: [
        "utils/ExifUtils.cpp",
        "common/DepthPhotoProcessor.cpp",
    ],

@@ -150,6 +151,8 @@ cc_library_shared {
        "libcutils",
        "libjpeg",
        "libmemunreachable",
        "libexif",
        "libcamera_client",
    ],

    include_dirs: [
+64 −5
Original line number Diff line number Diff line
@@ -32,9 +32,12 @@
#include <dynamic_depth/profile.h>
#include <dynamic_depth/profiles.h>
#include <jpeglib.h>
#include <libexif/exif-data.h>
#include <libexif/exif-system.h>
#include <math.h>
#include <sstream>
#include <utils/Errors.h>
#include <utils/ExifUtils.h>
#include <utils/Log.h>
#include <xmpmeta/xmp_data.h>
#include <xmpmeta/xmp_writer.h>
@@ -61,8 +64,44 @@ using dynamic_depth::Profiles;
namespace android {
namespace camera3 {

ExifOrientation getExifOrientation(const unsigned char *jpegBuffer, size_t jpegBufferSize) {
    if ((jpegBuffer == nullptr) || (jpegBufferSize == 0)) {
        return ExifOrientation::ORIENTATION_UNDEFINED;
    }

    auto exifData = exif_data_new();
    exif_data_load_data(exifData, jpegBuffer, jpegBufferSize);
    ExifEntry *orientation = exif_content_get_entry(exifData->ifd[EXIF_IFD_0],
            EXIF_TAG_ORIENTATION);
    if ((orientation == nullptr) || (orientation->size != sizeof(ExifShort))) {
        ALOGV("%s: Orientation EXIF entry invalid!", __FUNCTION__);
        exif_data_unref(exifData);
        return ExifOrientation::ORIENTATION_0_DEGREES;
    }

    auto orientationValue = exif_get_short(orientation->data, exif_data_get_byte_order(exifData));
    ExifOrientation ret;
    switch (orientationValue) {
        case ExifOrientation::ORIENTATION_0_DEGREES:
        case ExifOrientation::ORIENTATION_90_DEGREES:
        case ExifOrientation::ORIENTATION_180_DEGREES:
        case ExifOrientation::ORIENTATION_270_DEGREES:
            ret = static_cast<ExifOrientation> (orientationValue);
            break;
        default:
            ALOGE("%s: Unexpected EXIF orientation value: %d, defaulting to 0 degrees",
                    __FUNCTION__, orientationValue);
            ret = ExifOrientation::ORIENTATION_0_DEGREES;
    }

    exif_data_unref(exifData);

    return ret;
}

status_t encodeGrayscaleJpeg(size_t width, size_t height, uint8_t *in, void *out,
        const size_t maxOutSize, uint8_t jpegQuality, size_t &actualSize) {
        const size_t maxOutSize, uint8_t jpegQuality, ExifOrientation exifOrientation,
        size_t &actualSize) {
    status_t ret;
    // libjpeg is a C library so we use C-style "inheritance" by
    // putting libjpeg's jpeg_destination_mgr first in our custom
@@ -151,6 +190,23 @@ status_t encodeGrayscaleJpeg(size_t width, size_t height, uint8_t *in, void *out

    jpeg_start_compress(&cinfo, TRUE);

    if (exifOrientation != ExifOrientation::ORIENTATION_UNDEFINED) {
        std::unique_ptr<ExifUtils> utils(ExifUtils::create());
        utils->initializeEmpty();
        utils->setImageWidth(width);
        utils->setImageHeight(height);
        utils->setOrientationValue(exifOrientation);

        if (utils->generateApp1()) {
            const uint8_t* exifBuffer = utils->getApp1Buffer();
            size_t exifBufferSize = utils->getApp1Length();
            jpeg_write_marker(&cinfo, JPEG_APP0 + 1, static_cast<const JOCTET*>(exifBuffer),
                    exifBufferSize);
        } else {
            ALOGE("%s: Unable to generate App1 buffer", __FUNCTION__);
        }
    }

    for (size_t i = 0; i < cinfo.image_height; i++) {
        auto currentRow  = static_cast<JSAMPROW>(in + i*width);
        jpeg_write_scanlines(&cinfo, &currentRow, /*num_lines*/1);
@@ -169,7 +225,7 @@ status_t encodeGrayscaleJpeg(size_t width, size_t height, uint8_t *in, void *out
}

std::unique_ptr<dynamic_depth::DepthMap> processDepthMapFrame(DepthPhotoInputFrame inputFrame,
                std::vector<std::unique_ptr<Item>> *items /*out*/) {
        ExifOrientation exifOrientation, std::vector<std::unique_ptr<Item>> *items /*out*/) {
    std::vector<float> points, confidence;

    size_t pointCount = inputFrame.mDepthMapWidth * inputFrame.mDepthMapHeight;
@@ -227,7 +283,7 @@ std::unique_ptr<dynamic_depth::DepthMap> processDepthMapFrame(DepthPhotoInputFra
    size_t actualJpegSize;
    auto ret = encodeGrayscaleJpeg(inputFrame.mDepthMapWidth, inputFrame.mDepthMapHeight,
            pointsQuantized.data(), depthParams.depth_image_data.data(), inputFrame.mMaxJpegSize,
            inputFrame.mJpegQuality, actualJpegSize);
            inputFrame.mJpegQuality, exifOrientation, actualJpegSize);
    if (ret != NO_ERROR) {
        ALOGE("%s: Depth map compression failed!", __FUNCTION__);
        return nullptr;
@@ -236,7 +292,7 @@ std::unique_ptr<dynamic_depth::DepthMap> processDepthMapFrame(DepthPhotoInputFra

    ret = encodeGrayscaleJpeg(inputFrame.mDepthMapWidth, inputFrame.mDepthMapHeight,
            confidenceQuantized.data(), depthParams.confidence_data.data(), inputFrame.mMaxJpegSize,
            inputFrame.mJpegQuality, actualJpegSize);
            inputFrame.mJpegQuality, exifOrientation, actualJpegSize);
    if (ret != NO_ERROR) {
        ALOGE("%s: Confidence map compression failed!", __FUNCTION__);
        return nullptr;
@@ -262,7 +318,10 @@ extern "C" int processDepthPhotoFrame(DepthPhotoInputFrame inputFrame, size_t de
        return BAD_VALUE;
    }

    cameraParams->depth_map = processDepthMapFrame(inputFrame, &items);
    ExifOrientation exifOrientation = getExifOrientation(
            reinterpret_cast<const unsigned char*> (inputFrame.mMainJpegBuffer),
            inputFrame.mMainJpegSize);
    cameraParams->depth_map = processDepthMapFrame(inputFrame, exifOrientation, &items);
    if (cameraParams->depth_map == nullptr) {
        ALOGE("%s: Depth map processing failed!", __FUNCTION__);
        return BAD_VALUE;
+4 −0
Original line number Diff line number Diff line
@@ -27,6 +27,8 @@ LOCAL_SHARED_LIBRARIES := \
    libcamera_client \
    libcamera_metadata \
    libutils \
    libjpeg \
    libexif \
    android.hardware.camera.common@1.0 \
    android.hardware.camera.provider@2.4 \
    android.hardware.camera.provider@2.5 \
@@ -36,6 +38,8 @@ LOCAL_SHARED_LIBRARIES := \

LOCAL_C_INCLUDES += \
    system/media/private/camera/include \
    external/dynamic_depth/includes \
    external/dynamic_depth/internal \

LOCAL_CFLAGS += -Wall -Wextra -Werror

+280 −0
Original line number Diff line number Diff line
/*
 * Copyright (C) 2019 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#define LOG_NDEBUG 0
#define LOG_TAG "DepthProcessorTest"

#include <array>
#include <random>

#include <dlfcn.h>
#include <gtest/gtest.h>

#include "../common/DepthPhotoProcessor.h"
#include "../utils/ExifUtils.h"
#include "NV12Compressor.h"

using namespace android;
using namespace android::camera3;

static const size_t kTestBufferWidth = 640;
static const size_t kTestBufferHeight = 480;
static const size_t kTestBufferNV12Size ((((kTestBufferWidth) * (kTestBufferHeight)) * 3) / 2);
static const size_t kTestBufferDepthSize (kTestBufferWidth * kTestBufferHeight);
static const size_t kSeed = 1234;

void linkToDepthPhotoLibrary(void **libHandle /*out*/,
        process_depth_photo_frame *processFrameFunc /*out*/) {
    ASSERT_NE(libHandle, nullptr);
    ASSERT_NE(processFrameFunc, nullptr);

    *libHandle = dlopen(kDepthPhotoLibrary, RTLD_NOW | RTLD_LOCAL);
    if (*libHandle != nullptr) {
        *processFrameFunc = reinterpret_cast<camera3::process_depth_photo_frame> (
                dlsym(*libHandle, kDepthPhotoProcessFunction));
        ASSERT_NE(*processFrameFunc, nullptr);
    }
}

void generateColorJpegBuffer(int jpegQuality, ExifOrientation orientationValue, bool includeExif,
        std::vector<uint8_t> *colorJpegBuffer /*out*/) {
    ASSERT_NE(colorJpegBuffer, nullptr);

    std::array<uint8_t, kTestBufferNV12Size> colorSourceBuffer;
    std::default_random_engine gen(kSeed);
    std::uniform_int_distribution<int> uniDist(0, UINT8_MAX - 1);
    for (size_t i = 0; i < colorSourceBuffer.size(); i++) {
        colorSourceBuffer[i] = uniDist(gen);
    }
    NV12Compressor jpegCompressor;
    if (includeExif) {
        ASSERT_TRUE(jpegCompressor.compressWithExifOrientation(
                reinterpret_cast<const unsigned char*> (colorSourceBuffer.data()),
                kTestBufferWidth, kTestBufferHeight, jpegQuality, orientationValue));
    } else {
        ASSERT_TRUE(jpegCompressor.compress(
                reinterpret_cast<const unsigned char*> (colorSourceBuffer.data()),
                kTestBufferWidth, kTestBufferHeight, jpegQuality));
    }

    *colorJpegBuffer = std::move(jpegCompressor.getCompressedData());
    ASSERT_FALSE(colorJpegBuffer->empty());
}

void generateDepth16Buffer(std::array<uint16_t, kTestBufferDepthSize> *depth16Buffer /*out*/) {
    ASSERT_NE(depth16Buffer, nullptr);
    std::default_random_engine gen(kSeed+1);
    std::uniform_int_distribution<int> uniDist(0, UINT16_MAX - 1);
    for (size_t i = 0; i < depth16Buffer->size(); i++) {
        (*depth16Buffer)[i] = uniDist(gen);
    }
}

TEST(DepthProcessorTest, LinkToLibray) {
    void *libHandle;
    process_depth_photo_frame processFunc;
    linkToDepthPhotoLibrary(&libHandle, &processFunc);
    if (libHandle != nullptr) {
        dlclose(libHandle);
    }
}

TEST(DepthProcessorTest, BadInput) {
    void *libHandle;
    int jpegQuality = 95;

    process_depth_photo_frame processFunc;
    linkToDepthPhotoLibrary(&libHandle, &processFunc);
    if (libHandle == nullptr) {
        // Depth library no present, nothing more to test.
        return;
    }

    DepthPhotoInputFrame inputFrame;
    // Worst case both depth and confidence maps have the same size as the main color image.
    inputFrame.mMaxJpegSize = inputFrame.mMainJpegSize * 3;

    std::vector<uint8_t> colorJpegBuffer;
    generateColorJpegBuffer(jpegQuality, ExifOrientation::ORIENTATION_UNDEFINED,
            /*includeExif*/ false, &colorJpegBuffer);

    std::array<uint16_t, kTestBufferDepthSize> depth16Buffer;
    generateDepth16Buffer(&depth16Buffer);

    std::vector<uint8_t> depthPhotoBuffer(inputFrame.mMaxJpegSize);
    size_t actualDepthPhotoSize = 0;

    inputFrame.mMainJpegWidth = kTestBufferWidth;
    inputFrame.mMainJpegHeight = kTestBufferHeight;
    inputFrame.mJpegQuality = jpegQuality;
    ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
                &actualDepthPhotoSize), 0);

    inputFrame.mMainJpegBuffer = reinterpret_cast<const char*> (colorJpegBuffer.data());
    inputFrame.mMainJpegSize = colorJpegBuffer.size();
    ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
                &actualDepthPhotoSize), 0);

    inputFrame.mDepthMapBuffer = depth16Buffer.data();
    inputFrame.mDepthMapWidth = inputFrame.mDepthMapStride = kTestBufferWidth;
    inputFrame.mDepthMapHeight = kTestBufferHeight;
    ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), nullptr,
                &actualDepthPhotoSize), 0);

    ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(), nullptr),
            0);

    dlclose(libHandle);
}

TEST(DepthProcessorTest, BasicDepthPhotoValidation) {
    void *libHandle;
    int jpegQuality = 95;

    process_depth_photo_frame processFunc;
    linkToDepthPhotoLibrary(&libHandle, &processFunc);
    if (libHandle == nullptr) {
        // Depth library no present, nothing more to test.
        return;
    }

    std::vector<uint8_t> colorJpegBuffer;
    generateColorJpegBuffer(jpegQuality, ExifOrientation::ORIENTATION_UNDEFINED,
            /*includeExif*/ false, &colorJpegBuffer);

    std::array<uint16_t, kTestBufferDepthSize> depth16Buffer;
    generateDepth16Buffer(&depth16Buffer);

    DepthPhotoInputFrame inputFrame;
    inputFrame.mMainJpegBuffer = reinterpret_cast<const char*> (colorJpegBuffer.data());
    inputFrame.mMainJpegSize = colorJpegBuffer.size();
    // Worst case both depth and confidence maps have the same size as the main color image.
    inputFrame.mMaxJpegSize = inputFrame.mMainJpegSize * 3;
    inputFrame.mMainJpegWidth = kTestBufferWidth;
    inputFrame.mMainJpegHeight = kTestBufferHeight;
    inputFrame.mJpegQuality = jpegQuality;
    inputFrame.mDepthMapBuffer = depth16Buffer.data();
    inputFrame.mDepthMapWidth = inputFrame.mDepthMapStride = kTestBufferWidth;
    inputFrame.mDepthMapHeight = kTestBufferHeight;

    std::vector<uint8_t> depthPhotoBuffer(inputFrame.mMaxJpegSize);
    size_t actualDepthPhotoSize = 0;
    ASSERT_EQ(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
                &actualDepthPhotoSize), 0);
    ASSERT_TRUE((actualDepthPhotoSize > 0) && (depthPhotoBuffer.size() >= actualDepthPhotoSize));

    // The final depth photo must consist of three jpeg images:
    //  - the main color image
    //  - the depth map image
    //  - the confidence map image
    size_t mainJpegSize = 0;
    ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data(), actualDepthPhotoSize,
                &mainJpegSize), OK);
    ASSERT_TRUE((mainJpegSize > 0) && (mainJpegSize < actualDepthPhotoSize));
    size_t depthMapSize = 0;
    ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data() + mainJpegSize,
                actualDepthPhotoSize - mainJpegSize, &depthMapSize), OK);
    ASSERT_TRUE((depthMapSize > 0) && (depthMapSize < (actualDepthPhotoSize - mainJpegSize)));

    dlclose(libHandle);
}

TEST(DepthProcessorTest, TestDepthPhotoExifOrientation) {
    void *libHandle;
    int jpegQuality = 95;

    process_depth_photo_frame processFunc;
    linkToDepthPhotoLibrary(&libHandle, &processFunc);
    if (libHandle == nullptr) {
        // Depth library no present, nothing more to test.
        return;
    }

    ExifOrientation exifOrientations[] = { ExifOrientation::ORIENTATION_UNDEFINED,
            ExifOrientation::ORIENTATION_0_DEGREES, ExifOrientation::ORIENTATION_90_DEGREES,
            ExifOrientation::ORIENTATION_180_DEGREES, ExifOrientation::ORIENTATION_270_DEGREES };
    for (auto exifOrientation : exifOrientations) {
        std::vector<uint8_t> colorJpegBuffer;
        generateColorJpegBuffer(jpegQuality, exifOrientation, /*includeExif*/ true,
                &colorJpegBuffer);
        if (exifOrientation != ExifOrientation::ORIENTATION_UNDEFINED) {
            auto jpegExifOrientation = ExifOrientation::ORIENTATION_UNDEFINED;
            ASSERT_EQ(NV12Compressor::getExifOrientation(
                        reinterpret_cast<const unsigned char*> (colorJpegBuffer.data()),
                        colorJpegBuffer.size(), &jpegExifOrientation), OK);
            ASSERT_EQ(exifOrientation, jpegExifOrientation);
        }

        std::array<uint16_t, kTestBufferDepthSize> depth16Buffer;
        generateDepth16Buffer(&depth16Buffer);

        DepthPhotoInputFrame inputFrame;
        inputFrame.mMainJpegBuffer = reinterpret_cast<const char*> (colorJpegBuffer.data());
        inputFrame.mMainJpegSize = colorJpegBuffer.size();
        // Worst case both depth and confidence maps have the same size as the main color image.
        inputFrame.mMaxJpegSize = inputFrame.mMainJpegSize * 3;
        inputFrame.mMainJpegWidth = kTestBufferWidth;
        inputFrame.mMainJpegHeight = kTestBufferHeight;
        inputFrame.mJpegQuality = jpegQuality;
        inputFrame.mDepthMapBuffer = depth16Buffer.data();
        inputFrame.mDepthMapWidth = inputFrame.mDepthMapStride = kTestBufferWidth;
        inputFrame.mDepthMapHeight = kTestBufferHeight;

        std::vector<uint8_t> depthPhotoBuffer(inputFrame.mMaxJpegSize);
        size_t actualDepthPhotoSize = 0;
        ASSERT_EQ(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
                &actualDepthPhotoSize), 0);
        ASSERT_TRUE((actualDepthPhotoSize > 0) &&
                (depthPhotoBuffer.size() >= actualDepthPhotoSize));

        size_t mainJpegSize = 0;
        ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data(), actualDepthPhotoSize,
                &mainJpegSize), OK);
        ASSERT_TRUE((mainJpegSize > 0) && (mainJpegSize < actualDepthPhotoSize));
        size_t depthMapSize = 0;
        ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data() + mainJpegSize,
                actualDepthPhotoSize - mainJpegSize, &depthMapSize), OK);
        ASSERT_TRUE((depthMapSize > 0) && (depthMapSize < (actualDepthPhotoSize - mainJpegSize)));
        size_t confidenceMapSize = actualDepthPhotoSize - (mainJpegSize + depthMapSize);

        //Depth and confidence images must have the same EXIF orientation as the source
        auto depthJpegExifOrientation = ExifOrientation::ORIENTATION_UNDEFINED;
        ASSERT_EQ(NV12Compressor::getExifOrientation(
                reinterpret_cast<const unsigned char*> (depthPhotoBuffer.data() + mainJpegSize),
                depthMapSize, &depthJpegExifOrientation), OK);
        if (exifOrientation == ORIENTATION_UNDEFINED) {
            // In case of undefined or missing EXIF orientation, always expect 0 degrees in the
            // depth map.
            ASSERT_EQ(depthJpegExifOrientation, ExifOrientation::ORIENTATION_0_DEGREES);
        } else {
            ASSERT_EQ(depthJpegExifOrientation, exifOrientation);
        }

        auto confidenceJpegExifOrientation = ExifOrientation::ORIENTATION_UNDEFINED;
        ASSERT_EQ(NV12Compressor::getExifOrientation(
                reinterpret_cast<const unsigned char*> (depthPhotoBuffer.data() + mainJpegSize +
                    depthMapSize), confidenceMapSize, &confidenceJpegExifOrientation), OK);
        if (exifOrientation == ORIENTATION_UNDEFINED) {
            // In case of undefined or missing EXIF orientation, always expect 0 degrees in the
            // confidence map.
            ASSERT_EQ(confidenceJpegExifOrientation, ExifOrientation::ORIENTATION_0_DEGREES);
        } else {
            ASSERT_EQ(confidenceJpegExifOrientation, exifOrientation);
        }
    }

    dlclose(libHandle);
}
+349 −0

File added.

Preview size limit exceeded, changes collapsed.

Loading