Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 4b08d5dc authored by Emilian Peev's avatar Emilian Peev
Browse files

Camera: Keep Depth EXIF orientation consistent 

Depth and confidence maps should always use the same
EXIF orientation as the main color image.

Bug: 123699590
Test: Manual using application,
Camera CTS,
adb shell /data/nativetest64/cameraservice_test/cameraservice_test
--gtest_filter=DepthProcessorTest.*
Change-Id: I0d887798e8717cdff81aba10d595dc3ccfe99197
parent 8b0e6fa0

services/camera/libcameraservice/Android.bp

+3 −0
Original line number Diff line number Diff line
@@ -137,6 +137,7 @@ cc_library_shared { 
    name: "libdepthphoto",



    srcs: [

        "utils/ExifUtils.cpp",

        "common/DepthPhotoProcessor.cpp",

    ],
@@ -150,6 +151,8 @@ cc_library_shared { 
        "libcutils",

        "libjpeg",

        "libmemunreachable",

        "libexif",

        "libcamera_client",

    ],



    include_dirs: [

services/camera/libcameraservice/common/DepthPhotoProcessor.cpp

+64 −5
Original line number Diff line number Diff line
@@ -32,9 +32,12 @@ 
#include <dynamic_depth/profile.h>

#include <dynamic_depth/profiles.h>

#include <jpeglib.h>

#include <libexif/exif-data.h>

#include <libexif/exif-system.h>

#include <math.h>

#include <sstream>

#include <utils/Errors.h>

#include <utils/ExifUtils.h>

#include <utils/Log.h>

#include <xmpmeta/xmp_data.h>

#include <xmpmeta/xmp_writer.h>
@@ -61,8 +64,44 @@ using dynamic_depth::Profiles; 
namespace android {

namespace camera3 {



ExifOrientation getExifOrientation(const unsigned char *jpegBuffer, size_t jpegBufferSize) {

    if ((jpegBuffer == nullptr) || (jpegBufferSize == 0)) {

        return ExifOrientation::ORIENTATION_UNDEFINED;

    }



    auto exifData = exif_data_new();

    exif_data_load_data(exifData, jpegBuffer, jpegBufferSize);

    ExifEntry *orientation = exif_content_get_entry(exifData->ifd[EXIF_IFD_0],

            EXIF_TAG_ORIENTATION);

    if ((orientation == nullptr) || (orientation->size != sizeof(ExifShort))) {

        ALOGV("%s: Orientation EXIF entry invalid!", __FUNCTION__);

        exif_data_unref(exifData);

        return ExifOrientation::ORIENTATION_0_DEGREES;

    }



    auto orientationValue = exif_get_short(orientation->data, exif_data_get_byte_order(exifData));

    ExifOrientation ret;

    switch (orientationValue) {

        case ExifOrientation::ORIENTATION_0_DEGREES:

        case ExifOrientation::ORIENTATION_90_DEGREES:

        case ExifOrientation::ORIENTATION_180_DEGREES:

        case ExifOrientation::ORIENTATION_270_DEGREES:

            ret = static_cast<ExifOrientation> (orientationValue);

            break;

        default:

            ALOGE("%s: Unexpected EXIF orientation value: %d, defaulting to 0 degrees",

                    __FUNCTION__, orientationValue);

            ret = ExifOrientation::ORIENTATION_0_DEGREES;

    }



    exif_data_unref(exifData);



    return ret;

}



status_t encodeGrayscaleJpeg(size_t width, size_t height, uint8_t *in, void *out,

        const size_t maxOutSize, uint8_t jpegQuality, size_t &actualSize) {

        const size_t maxOutSize, uint8_t jpegQuality, ExifOrientation exifOrientation,

        size_t &actualSize) {

    status_t ret;

    // libjpeg is a C library so we use C-style "inheritance" by

    // putting libjpeg's jpeg_destination_mgr first in our custom
@@ -151,6 +190,23 @@ status_t encodeGrayscaleJpeg(size_t width, size_t height, uint8_t *in, void *out 


    jpeg_start_compress(&cinfo, TRUE);



    if (exifOrientation != ExifOrientation::ORIENTATION_UNDEFINED) {

        std::unique_ptr<ExifUtils> utils(ExifUtils::create());

        utils->initializeEmpty();

        utils->setImageWidth(width);

        utils->setImageHeight(height);

        utils->setOrientationValue(exifOrientation);



        if (utils->generateApp1()) {

            const uint8_t* exifBuffer = utils->getApp1Buffer();

            size_t exifBufferSize = utils->getApp1Length();

            jpeg_write_marker(&cinfo, JPEG_APP0 + 1, static_cast<const JOCTET*>(exifBuffer),

                    exifBufferSize);

        } else {

            ALOGE("%s: Unable to generate App1 buffer", __FUNCTION__);

        }

    }



    for (size_t i = 0; i < cinfo.image_height; i++) {

        auto currentRow  = static_cast<JSAMPROW>(in + i*width);

        jpeg_write_scanlines(&cinfo, &currentRow, /*num_lines*/1);
@@ -169,7 +225,7 @@ status_t encodeGrayscaleJpeg(size_t width, size_t height, uint8_t *in, void *out 
}



std::unique_ptr<dynamic_depth::DepthMap> processDepthMapFrame(DepthPhotoInputFrame inputFrame,

                std::vector<std::unique_ptr<Item>> *items /*out*/) {

        ExifOrientation exifOrientation, std::vector<std::unique_ptr<Item>> *items /*out*/) {

    std::vector<float> points, confidence;



    size_t pointCount = inputFrame.mDepthMapWidth * inputFrame.mDepthMapHeight;
@@ -227,7 +283,7 @@ std::unique_ptr<dynamic_depth::DepthMap> processDepthMapFrame(DepthPhotoInputFra 
    size_t actualJpegSize;

    auto ret = encodeGrayscaleJpeg(inputFrame.mDepthMapWidth, inputFrame.mDepthMapHeight,

            pointsQuantized.data(), depthParams.depth_image_data.data(), inputFrame.mMaxJpegSize,

            inputFrame.mJpegQuality, actualJpegSize);

            inputFrame.mJpegQuality, exifOrientation, actualJpegSize);

    if (ret != NO_ERROR) {

        ALOGE("%s: Depth map compression failed!", __FUNCTION__);

        return nullptr;
@@ -236,7 +292,7 @@ std::unique_ptr<dynamic_depth::DepthMap> processDepthMapFrame(DepthPhotoInputFra 


    ret = encodeGrayscaleJpeg(inputFrame.mDepthMapWidth, inputFrame.mDepthMapHeight,

            confidenceQuantized.data(), depthParams.confidence_data.data(), inputFrame.mMaxJpegSize,

            inputFrame.mJpegQuality, actualJpegSize);

            inputFrame.mJpegQuality, exifOrientation, actualJpegSize);

    if (ret != NO_ERROR) {

        ALOGE("%s: Confidence map compression failed!", __FUNCTION__);

        return nullptr;
@@ -262,7 +318,10 @@ extern "C" int processDepthPhotoFrame(DepthPhotoInputFrame inputFrame, size_t de 
        return BAD_VALUE;

    }



    cameraParams->depth_map = processDepthMapFrame(inputFrame, &items);

    ExifOrientation exifOrientation = getExifOrientation(

            reinterpret_cast<const unsigned char*> (inputFrame.mMainJpegBuffer),

            inputFrame.mMainJpegSize);

    cameraParams->depth_map = processDepthMapFrame(inputFrame, exifOrientation, &items);

    if (cameraParams->depth_map == nullptr) {

        ALOGE("%s: Depth map processing failed!", __FUNCTION__);

        return BAD_VALUE;

services/camera/libcameraservice/tests/Android.mk

+4 −0
Original line number Diff line number Diff line
@@ -27,6 +27,8 @@ LOCAL_SHARED_LIBRARIES := \ 
    libcamera_client \

    libcamera_metadata \

    libutils \

    libjpeg \

    libexif \

    android.hardware.camera.common@1.0 \

    android.hardware.camera.provider@2.4 \

    android.hardware.camera.provider@2.5 \
@@ -36,6 +38,8 @@ LOCAL_SHARED_LIBRARIES := \ 


LOCAL_C_INCLUDES += \

    system/media/private/camera/include \

    external/dynamic_depth/includes \

    external/dynamic_depth/internal \



LOCAL_CFLAGS += -Wall -Wextra -Werror

services/camera/libcameraservice/tests/DepthProcessorTest.cpp

0 → 100644
+280 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2019 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



#define LOG_NDEBUG 0

#define LOG_TAG "DepthProcessorTest"



#include <array>

#include <random>



#include <dlfcn.h>

#include <gtest/gtest.h>



#include "../common/DepthPhotoProcessor.h"

#include "../utils/ExifUtils.h"

#include "NV12Compressor.h"



using namespace android;

using namespace android::camera3;



static const size_t kTestBufferWidth = 640;

static const size_t kTestBufferHeight = 480;

static const size_t kTestBufferNV12Size ((((kTestBufferWidth) * (kTestBufferHeight)) * 3) / 2);

static const size_t kTestBufferDepthSize (kTestBufferWidth * kTestBufferHeight);

static const size_t kSeed = 1234;



void linkToDepthPhotoLibrary(void **libHandle /*out*/,

        process_depth_photo_frame *processFrameFunc /*out*/) {

    ASSERT_NE(libHandle, nullptr);

    ASSERT_NE(processFrameFunc, nullptr);



    *libHandle = dlopen(kDepthPhotoLibrary, RTLD_NOW | RTLD_LOCAL);

    if (*libHandle != nullptr) {

        *processFrameFunc = reinterpret_cast<camera3::process_depth_photo_frame> (

                dlsym(*libHandle, kDepthPhotoProcessFunction));

        ASSERT_NE(*processFrameFunc, nullptr);

    }

}



void generateColorJpegBuffer(int jpegQuality, ExifOrientation orientationValue, bool includeExif,

        std::vector<uint8_t> *colorJpegBuffer /*out*/) {

    ASSERT_NE(colorJpegBuffer, nullptr);



    std::array<uint8_t, kTestBufferNV12Size> colorSourceBuffer;

    std::default_random_engine gen(kSeed);

    std::uniform_int_distribution<int> uniDist(0, UINT8_MAX - 1);

    for (size_t i = 0; i < colorSourceBuffer.size(); i++) {

        colorSourceBuffer[i] = uniDist(gen);

    }

    NV12Compressor jpegCompressor;

    if (includeExif) {

        ASSERT_TRUE(jpegCompressor.compressWithExifOrientation(

                reinterpret_cast<const unsigned char*> (colorSourceBuffer.data()),

                kTestBufferWidth, kTestBufferHeight, jpegQuality, orientationValue));

    } else {

        ASSERT_TRUE(jpegCompressor.compress(

                reinterpret_cast<const unsigned char*> (colorSourceBuffer.data()),

                kTestBufferWidth, kTestBufferHeight, jpegQuality));

    }



    *colorJpegBuffer = std::move(jpegCompressor.getCompressedData());

    ASSERT_FALSE(colorJpegBuffer->empty());

}



void generateDepth16Buffer(std::array<uint16_t, kTestBufferDepthSize> *depth16Buffer /*out*/) {

    ASSERT_NE(depth16Buffer, nullptr);

    std::default_random_engine gen(kSeed+1);

    std::uniform_int_distribution<int> uniDist(0, UINT16_MAX - 1);

    for (size_t i = 0; i < depth16Buffer->size(); i++) {

        (*depth16Buffer)[i] = uniDist(gen);

    }

}



TEST(DepthProcessorTest, LinkToLibray) {

    void *libHandle;

    process_depth_photo_frame processFunc;

    linkToDepthPhotoLibrary(&libHandle, &processFunc);

    if (libHandle != nullptr) {

        dlclose(libHandle);

    }

}



TEST(DepthProcessorTest, BadInput) {

    void *libHandle;

    int jpegQuality = 95;



    process_depth_photo_frame processFunc;

    linkToDepthPhotoLibrary(&libHandle, &processFunc);

    if (libHandle == nullptr) {

        // Depth library no present, nothing more to test.

        return;

    }



    DepthPhotoInputFrame inputFrame;

    // Worst case both depth and confidence maps have the same size as the main color image.

    inputFrame.mMaxJpegSize = inputFrame.mMainJpegSize * 3;



    std::vector<uint8_t> colorJpegBuffer;

    generateColorJpegBuffer(jpegQuality, ExifOrientation::ORIENTATION_UNDEFINED,

            /*includeExif*/ false, &colorJpegBuffer);



    std::array<uint16_t, kTestBufferDepthSize> depth16Buffer;

    generateDepth16Buffer(&depth16Buffer);



    std::vector<uint8_t> depthPhotoBuffer(inputFrame.mMaxJpegSize);

    size_t actualDepthPhotoSize = 0;



    inputFrame.mMainJpegWidth = kTestBufferWidth;

    inputFrame.mMainJpegHeight = kTestBufferHeight;

    inputFrame.mJpegQuality = jpegQuality;

    ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),

                &actualDepthPhotoSize), 0);



    inputFrame.mMainJpegBuffer = reinterpret_cast<const char*> (colorJpegBuffer.data());

    inputFrame.mMainJpegSize = colorJpegBuffer.size();

    ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),

                &actualDepthPhotoSize), 0);



    inputFrame.mDepthMapBuffer = depth16Buffer.data();

    inputFrame.mDepthMapWidth = inputFrame.mDepthMapStride = kTestBufferWidth;

    inputFrame.mDepthMapHeight = kTestBufferHeight;

    ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), nullptr,

                &actualDepthPhotoSize), 0);



    ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(), nullptr),

            0);



    dlclose(libHandle);

}



TEST(DepthProcessorTest, BasicDepthPhotoValidation) {

    void *libHandle;

    int jpegQuality = 95;



    process_depth_photo_frame processFunc;

    linkToDepthPhotoLibrary(&libHandle, &processFunc);

    if (libHandle == nullptr) {

        // Depth library no present, nothing more to test.

        return;

    }



    std::vector<uint8_t> colorJpegBuffer;

    generateColorJpegBuffer(jpegQuality, ExifOrientation::ORIENTATION_UNDEFINED,

            /*includeExif*/ false, &colorJpegBuffer);



    std::array<uint16_t, kTestBufferDepthSize> depth16Buffer;

    generateDepth16Buffer(&depth16Buffer);



    DepthPhotoInputFrame inputFrame;

    inputFrame.mMainJpegBuffer = reinterpret_cast<const char*> (colorJpegBuffer.data());

    inputFrame.mMainJpegSize = colorJpegBuffer.size();

    // Worst case both depth and confidence maps have the same size as the main color image.

    inputFrame.mMaxJpegSize = inputFrame.mMainJpegSize * 3;

    inputFrame.mMainJpegWidth = kTestBufferWidth;

    inputFrame.mMainJpegHeight = kTestBufferHeight;

    inputFrame.mJpegQuality = jpegQuality;

    inputFrame.mDepthMapBuffer = depth16Buffer.data();

    inputFrame.mDepthMapWidth = inputFrame.mDepthMapStride = kTestBufferWidth;

    inputFrame.mDepthMapHeight = kTestBufferHeight;



    std::vector<uint8_t> depthPhotoBuffer(inputFrame.mMaxJpegSize);

    size_t actualDepthPhotoSize = 0;

    ASSERT_EQ(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),

                &actualDepthPhotoSize), 0);

    ASSERT_TRUE((actualDepthPhotoSize > 0) && (depthPhotoBuffer.size() >= actualDepthPhotoSize));



    // The final depth photo must consist of three jpeg images:

    //  - the main color image

    //  - the depth map image

    //  - the confidence map image

    size_t mainJpegSize = 0;

    ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data(), actualDepthPhotoSize,

                &mainJpegSize), OK);

    ASSERT_TRUE((mainJpegSize > 0) && (mainJpegSize < actualDepthPhotoSize));

    size_t depthMapSize = 0;

    ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data() + mainJpegSize,

                actualDepthPhotoSize - mainJpegSize, &depthMapSize), OK);

    ASSERT_TRUE((depthMapSize > 0) && (depthMapSize < (actualDepthPhotoSize - mainJpegSize)));



    dlclose(libHandle);

}



TEST(DepthProcessorTest, TestDepthPhotoExifOrientation) {

    void *libHandle;

    int jpegQuality = 95;



    process_depth_photo_frame processFunc;

    linkToDepthPhotoLibrary(&libHandle, &processFunc);

    if (libHandle == nullptr) {

        // Depth library no present, nothing more to test.

        return;

    }



    ExifOrientation exifOrientations[] = { ExifOrientation::ORIENTATION_UNDEFINED,

            ExifOrientation::ORIENTATION_0_DEGREES, ExifOrientation::ORIENTATION_90_DEGREES,

            ExifOrientation::ORIENTATION_180_DEGREES, ExifOrientation::ORIENTATION_270_DEGREES };

    for (auto exifOrientation : exifOrientations) {

        std::vector<uint8_t> colorJpegBuffer;

        generateColorJpegBuffer(jpegQuality, exifOrientation, /*includeExif*/ true,

                &colorJpegBuffer);

        if (exifOrientation != ExifOrientation::ORIENTATION_UNDEFINED) {

            auto jpegExifOrientation = ExifOrientation::ORIENTATION_UNDEFINED;

            ASSERT_EQ(NV12Compressor::getExifOrientation(

                        reinterpret_cast<const unsigned char*> (colorJpegBuffer.data()),

                        colorJpegBuffer.size(), &jpegExifOrientation), OK);

            ASSERT_EQ(exifOrientation, jpegExifOrientation);

        }



        std::array<uint16_t, kTestBufferDepthSize> depth16Buffer;

        generateDepth16Buffer(&depth16Buffer);



        DepthPhotoInputFrame inputFrame;

        inputFrame.mMainJpegBuffer = reinterpret_cast<const char*> (colorJpegBuffer.data());

        inputFrame.mMainJpegSize = colorJpegBuffer.size();

        // Worst case both depth and confidence maps have the same size as the main color image.

        inputFrame.mMaxJpegSize = inputFrame.mMainJpegSize * 3;

        inputFrame.mMainJpegWidth = kTestBufferWidth;

        inputFrame.mMainJpegHeight = kTestBufferHeight;

        inputFrame.mJpegQuality = jpegQuality;

        inputFrame.mDepthMapBuffer = depth16Buffer.data();

        inputFrame.mDepthMapWidth = inputFrame.mDepthMapStride = kTestBufferWidth;

        inputFrame.mDepthMapHeight = kTestBufferHeight;



        std::vector<uint8_t> depthPhotoBuffer(inputFrame.mMaxJpegSize);

        size_t actualDepthPhotoSize = 0;

        ASSERT_EQ(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),

                &actualDepthPhotoSize), 0);

        ASSERT_TRUE((actualDepthPhotoSize > 0) &&

                (depthPhotoBuffer.size() >= actualDepthPhotoSize));



        size_t mainJpegSize = 0;

        ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data(), actualDepthPhotoSize,

                &mainJpegSize), OK);

        ASSERT_TRUE((mainJpegSize > 0) && (mainJpegSize < actualDepthPhotoSize));

        size_t depthMapSize = 0;

        ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data() + mainJpegSize,

                actualDepthPhotoSize - mainJpegSize, &depthMapSize), OK);

        ASSERT_TRUE((depthMapSize > 0) && (depthMapSize < (actualDepthPhotoSize - mainJpegSize)));

        size_t confidenceMapSize = actualDepthPhotoSize - (mainJpegSize + depthMapSize);



        //Depth and confidence images must have the same EXIF orientation as the source

        auto depthJpegExifOrientation = ExifOrientation::ORIENTATION_UNDEFINED;

        ASSERT_EQ(NV12Compressor::getExifOrientation(

                reinterpret_cast<const unsigned char*> (depthPhotoBuffer.data() + mainJpegSize),

                depthMapSize, &depthJpegExifOrientation), OK);

        if (exifOrientation == ORIENTATION_UNDEFINED) {

            // In case of undefined or missing EXIF orientation, always expect 0 degrees in the

            // depth map.

            ASSERT_EQ(depthJpegExifOrientation, ExifOrientation::ORIENTATION_0_DEGREES);

        } else {

            ASSERT_EQ(depthJpegExifOrientation, exifOrientation);

        }



        auto confidenceJpegExifOrientation = ExifOrientation::ORIENTATION_UNDEFINED;

        ASSERT_EQ(NV12Compressor::getExifOrientation(

                reinterpret_cast<const unsigned char*> (depthPhotoBuffer.data() + mainJpegSize +

                    depthMapSize), confidenceMapSize, &confidenceJpegExifOrientation), OK);

        if (exifOrientation == ORIENTATION_UNDEFINED) {

            // In case of undefined or missing EXIF orientation, always expect 0 degrees in the

            // confidence map.

            ASSERT_EQ(confidenceJpegExifOrientation, ExifOrientation::ORIENTATION_0_DEGREES);

        } else {

            ASSERT_EQ(confidenceJpegExifOrientation, exifOrientation);

        }

    }



    dlclose(libHandle);

}

services/camera/libcameraservice/tests/NV12Compressor.cpp

0 → 100644
+349 −0

File added.

Preview size limit exceeded, changes collapsed.