Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e7f4b46a authored by Shuzhen Wang's avatar Shuzhen Wang
Browse files

Camera: Use capture result metadata to override EXIF 

Use static and dynamic metadata to override Exif tags.

Also added back a missing ATRACE_ASYNC_ENDs statement.

Test: Camera CTS
Test: ERROR_BUFFER of internal streams is propagated to app
Test: ERROR_RESULT only results in EXIF not being overridden
Bug: 124066183
Change-Id: Id2c69c6bee04ae724ff5f190b2dd96d0159700c9
parent c6aa5039

services/camera/libcameraservice/api2/HeicCompositeStream.cpp

+78 −14
Original line number Diff line number Diff line
@@ -136,6 +136,8 @@ status_t HeicCompositeStream::createInternalStreams(const std::vector<sp<Surface 
    mAppSegmentConsumer->setName(String8("Camera3-HeicComposite-AppSegmentStream"));

    mAppSegmentSurface = new Surface(producer);



    mStaticInfo = device->info();



    res = device->createStream(mAppSegmentSurface, mAppSegmentMaxSize, 1, format,

            kAppSegmentDataSpace, rotation, &mAppSegmentStreamId, physicalCameraId, surfaceIds);

    if (res == OK) {
@@ -606,9 +608,42 @@ void HeicCompositeStream::compilePendingInputLocked() { 
        mFrameNumberMap.erase(it);

    }



    // Heic composition doesn't depend on capture result, so no need to check

    // mErrorFrameNumbers. Just remove them.

    mErrorFrameNumbers.clear();

    while (!mCaptureResults.empty()) {

        auto it = mCaptureResults.begin();

        // Negative timestamp indicates that something went wrong during the capture result

        // collection process.

        if (it->first >= 0) {

            if (mPendingInputFrames[it->first].frameNumber == std::get<0>(it->second)) {

                mPendingInputFrames[it->first].result =

                        std::make_unique<CameraMetadata>(std::get<1>(it->second));

            } else {

                ALOGE("%s: Capture result frameNumber/timestamp mapping changed between "

                        "shutter and capture result!", __FUNCTION__);

            }

        }

        mCaptureResults.erase(it);

    }



    // mErrorFrameNumbers stores frame number of dropped buffers.

    auto it = mErrorFrameNumbers.begin();

    while (it != mErrorFrameNumbers.end()) {

        bool frameFound = false;

        for (auto &inputFrame : mPendingInputFrames) {

            if (inputFrame.second.frameNumber == *it) {

                inputFrame.second.error = true;

                frameFound = true;

                break;

            }

        }



        if (frameFound) {

            it = mErrorFrameNumbers.erase(it);

        } else {

            ALOGW("%s: Not able to find failing input with frame number: %" PRId64, __FUNCTION__,

                    *it);

            it++;

        }

    }



    // Distribute codec input buffers to be filled out from YUV output

    for (auto it = mPendingInputFrames.begin();
@@ -639,14 +674,14 @@ bool HeicCompositeStream::getNextReadyInputLocked(int64_t *currentTs /*out*/) { 


    bool newInputAvailable = false;

    for (const auto& it : mPendingInputFrames) {

        bool appSegmentBufferReady = (it.second.appSegmentBuffer.data != nullptr) &&

                !it.second.appSegmentWritten;

        bool appSegmentReady = (it.second.appSegmentBuffer.data != nullptr) &&

                !it.second.appSegmentWritten && it.second.result != nullptr;

        bool codecOutputReady = !it.second.codecOutputBuffers.empty();

        bool codecInputReady = (it.second.yuvBuffer.data != nullptr) &&

                (!it.second.codecInputBuffers.empty());

        if ((!it.second.error) &&

                (it.first < *currentTs) &&

                (appSegmentBufferReady || codecOutputReady || codecInputReady)) {

                (appSegmentReady || codecOutputReady || codecInputReady)) {

            *currentTs = it.first;

            newInputAvailable = true;

            break;
@@ -678,13 +713,13 @@ status_t HeicCompositeStream::processInputFrame(nsecs_t timestamp, 
    ATRACE_CALL();

    status_t res = OK;



    bool appSegmentBufferReady = inputFrame.appSegmentBuffer.data != nullptr &&

            !inputFrame.appSegmentWritten;

    bool appSegmentReady = inputFrame.appSegmentBuffer.data != nullptr &&

            !inputFrame.appSegmentWritten && inputFrame.result != nullptr;

    bool codecOutputReady = inputFrame.codecOutputBuffers.size() > 0;

    bool codecInputReady = inputFrame.yuvBuffer.data != nullptr &&

           !inputFrame.codecInputBuffers.empty();



    if (!appSegmentBufferReady && !codecOutputReady && !codecInputReady) {

    if (!appSegmentReady && !codecOutputReady && !codecInputReady) {

        ALOGW("%s: No valid appSegmentBuffer/codec input/outputBuffer available!", __FUNCTION__);

        return OK;

    }
@@ -710,7 +745,7 @@ status_t HeicCompositeStream::processInputFrame(nsecs_t timestamp, 
    }



    // Write JPEG APP segments data to the muxer.

    if (appSegmentBufferReady && inputFrame.muxer != nullptr) {

    if (appSegmentReady && inputFrame.muxer != nullptr) {

        res = processAppSegment(timestamp, inputFrame);

        if (res != OK) {

            ALOGE("%s: Failed to process JPEG APP segments: %s (%d)", __FUNCTION__,
@@ -829,10 +864,8 @@ status_t HeicCompositeStream::processAppSegment(nsecs_t timestamp, InputFrame &i 
        ALOGE("%s: Failed to initialize ExifUtils object!", __FUNCTION__);

        return BAD_VALUE;

    }

    //TODO: Use capture result metadata and static metadata to fill out the

    //rest.

    CameraMetadata dummyMeta;

    exifRes = exifUtils->setFromMetadata(dummyMeta, mOutputWidth, mOutputHeight);

    exifRes = exifUtils->setFromMetadata(*inputFrame.result, mStaticInfo,

            mOutputWidth, mOutputHeight);

    if (!exifRes) {

        ALOGE("%s: Failed to set Exif tags using metadata and main image sizes", __FUNCTION__);

        return BAD_VALUE;
@@ -1012,6 +1045,7 @@ status_t HeicCompositeStream::processCompletedInputFrame(nsecs_t timestamp, 
    }

    inputFrame.anb = nullptr;



    ATRACE_ASYNC_END("HEIC capture", inputFrame.frameNumber);

    return OK;

}
@@ -1497,6 +1531,36 @@ bool HeicCompositeStream::onStreamBufferError(const CaptureResultExtras& resultE 
    return res;

}



void HeicCompositeStream::onResultError(const CaptureResultExtras& resultExtras) {

    // For result error, since the APPS_SEGMENT buffer already contains EXIF,

    // simply skip using the capture result metadata to override EXIF.

    Mutex::Autolock l(mMutex);



    int64_t timestamp = -1;

    for (const auto& fn : mFrameNumberMap) {

        if (fn.first == resultExtras.frameNumber) {

            timestamp = fn.second;

            break;

        }

    }

    if (timestamp == -1) {

        for (const auto& inputFrame : mPendingInputFrames) {

            if (inputFrame.second.frameNumber == resultExtras.frameNumber) {

                timestamp = inputFrame.first;

                break;

            }

        }

    }



    if (timestamp == -1) {

        ALOGE("%s: Failed to find shutter timestamp for result error!", __FUNCTION__);

        return;

    }



    mCaptureResults.emplace(timestamp, std::make_tuple(resultExtras.frameNumber, CameraMetadata()));

    mInputReadyCondition.signal();

}



void HeicCompositeStream::CodecCallbackHandler::onMessageReceived(const sp<AMessage> &msg) {

    sp<HeicCompositeStream> parent = mParent.promote();

    if (parent == nullptr) return;

services/camera/libcameraservice/api2/HeicCompositeStream.h

+6 −4
Original line number Diff line number Diff line
@@ -77,7 +77,7 @@ protected: 


    bool threadLoop() override;

    bool onStreamBufferError(const CaptureResultExtras& resultExtras) override;

    void onResultError(const CaptureResultExtras& /*resultExtras*/) override {}

    void onResultError(const CaptureResultExtras& resultExtras) override;



private:

    //
@@ -147,6 +147,7 @@ private: 


        CpuConsumer::LockedBuffer          appSegmentBuffer;

        std::vector<CodecOutputBufferInfo> codecOutputBuffers;

        std::unique_ptr<CameraMetadata>    result;



        // Fields that are only applicable to HEVC tiling.

        CpuConsumer::LockedBuffer          yuvBuffer;
@@ -209,6 +210,7 @@ private: 
    sp<Surface>       mAppSegmentSurface;

    bool              mAppSegmentBufferAcquired;

    size_t            mAppSegmentMaxSize;

    CameraMetadata    mStaticInfo;



    int               mMainImageStreamId, mMainImageSurfaceId;

    sp<Surface>       mMainImageSurface;

services/camera/libcameraservice/utils/ExifUtils.cpp

+259 −220

File changed.

Preview size limit exceeded, changes collapsed.

services/camera/libcameraservice/utils/ExifUtils.h

+19 −85
Original line number Diff line number Diff line
@@ -52,78 +52,57 @@ public: 


    // Set all known fields from a metadata structure

    virtual bool setFromMetadata(const CameraMetadata& metadata,

            const CameraMetadata& staticInfo,

            const size_t imageWidth, const size_t imageHeight) = 0;



    // Sets the len aperture.

    // Returns false if memory allocation fails.

    virtual bool setAperture(uint32_t numerator, uint32_t denominator) = 0;

    virtual bool setAperture(float aperture) = 0;



    // Sets the value of brightness.

    // Returns false if memory allocation fails.

    virtual bool setBrightness(int32_t numerator, int32_t denominator) = 0;



    // Sets the color space.

    // sets the color space.

    // Returns false if memory allocation fails.

    virtual bool setColorSpace(uint16_t color_space) = 0;



    // Sets the information to compressed data.

    // Returns false if memory allocation fails.

    virtual bool setComponentsConfiguration(const std::string& components_configuration) = 0;



    // Sets the compression scheme used for the image data.

    // Returns false if memory allocation fails.

    virtual bool setCompression(uint16_t compression) = 0;



    // Sets image contrast.

    // Returns false if memory allocation fails.

    virtual bool setContrast(uint16_t contrast) = 0;



    // Sets the date and time of image last modified. It takes local time. The

    // name of the tag is DateTime in IFD0.

    // Returns false if memory allocation fails.

    virtual bool setDateTime(const struct tm& t) = 0;



    // Sets the image description.

    // Returns false if memory allocation fails.

    virtual bool setDescription(const std::string& description) = 0;



    // Sets the digital zoom ratio. If the numerator is 0, it means digital zoom

    // was not used.

    // Returns false if memory allocation fails.

    virtual bool setDigitalZoomRatio(uint32_t numerator, uint32_t denominator) = 0;

    virtual bool setDigitalZoomRatio(uint32_t crop_width, uint32_t crop_height,

            uint32_t sensor_width, uint32_t sensor_height) = 0;



    // Sets the exposure bias.

    // Returns false if memory allocation fails.

    virtual bool setExposureBias(int32_t numerator, int32_t denominator) = 0;

    virtual bool setExposureBias(int32_t ev,

            uint32_t ev_step_numerator, uint32_t ev_step_denominator) = 0;



    // Sets the exposure mode set when the image was shot.

    // Returns false if memory allocation fails.

    virtual bool setExposureMode(uint16_t exposure_mode) = 0;



    // Sets the program used by the camera to set exposure when the picture is

    // taken.

    // Returns false if memory allocation fails.

    virtual bool setExposureProgram(uint16_t exposure_program) = 0;

    virtual bool setExposureMode(uint8_t exposure_mode) = 0;



    // Sets the exposure time, given in seconds.

    // Returns false if memory allocation fails.

    virtual bool setExposureTime(uint32_t numerator, uint32_t denominator) = 0;

    virtual bool setExposureTime(float exposure_time) = 0;



    // Sets the status of flash.

    // Returns false if memory allocation fails.

    virtual bool setFlash(uint16_t flash) = 0;

    virtual bool setFlash(uint8_t flash_available, uint8_t flash_state, uint8_t ae_mode) = 0;



    // Sets the F number.

    // Returns false if memory allocation fails.

    virtual bool setFNumber(uint32_t numerator, uint32_t denominator) = 0;

    virtual bool setFNumber(float f_number) = 0;



    // Sets the focal length of lens used to take the image in millimeters.

    // Returns false if memory allocation fails.

    virtual bool setFocalLength(uint32_t numerator, uint32_t denominator) = 0;

    virtual bool setFocalLength(float focal_length) = 0;



    // Sets the degree of overall image gain adjustment.

    // Sets the focal length of lens for 35mm film used to take the image in millimeters.

    // Returns false if memory allocation fails.

    virtual bool setGainControl(uint16_t gain_control) = 0;

    virtual bool setFocalLengthIn35mmFilm(float focal_length,

            float sensor_size_x, float sensor_size_y) = 0;



    // Sets the altitude in meters.

    // Returns false if memory allocation fails.
@@ -157,45 +136,21 @@ public: 
    // Returns false if memory allocation fails.

    virtual bool setIsoSpeedRating(uint16_t iso_speed_ratings) = 0;



    // Sets the kind of light source.

    // Returns false if memory allocation fails.

    virtual bool setLightSource(uint16_t light_source) = 0;



    // Sets the smallest F number of the lens.

    // Returns false if memory allocation fails.

    virtual bool setMaxAperture(uint32_t numerator, uint32_t denominator) = 0;



    // Sets the metering mode.

    // Returns false if memory allocation fails.

    virtual bool setMeteringMode(uint16_t metering_mode) = 0;

    virtual bool setMaxAperture(float aperture) = 0;



    // Sets image orientation.

    // Returns false if memory allocation fails.

    virtual bool setOrientation(uint16_t orientation) = 0;



    // Sets the unit for measuring XResolution and YResolution.

    // Returns false if memory allocation fails.

    virtual bool setResolutionUnit(uint16_t resolution_unit) = 0;



    // Sets image saturation.

    // Returns false if memory allocation fails.

    virtual bool setSaturation(uint16_t saturation) = 0;



    // Sets the type of scene that was shot.

    // Returns false if memory allocation fails.

    virtual bool setSceneCaptureType(uint16_t type) = 0;



    // Sets image sharpness.

    // Returns false if memory allocation fails.

    virtual bool setSharpness(uint16_t sharpness) = 0;



    // Sets the shutter speed.

    // Returns false if memory allocation fails.

    virtual bool setShutterSpeed(int32_t numerator, int32_t denominator) = 0;

    virtual bool setShutterSpeed(float exposure_time) = 0;



    // Sets the distance to the subject, given in meters.

    // Returns false if memory allocation fails.

    virtual bool setSubjectDistance(uint32_t numerator, uint32_t denominator) = 0;

    virtual bool setSubjectDistance(float diopters) = 0;



    // Sets the fractions of seconds for the <DateTime> tag.

    // Returns false if memory allocation fails.
@@ -203,28 +158,7 @@ public: 


    // Sets the white balance mode set when the image was shot.

    // Returns false if memory allocation fails.

    virtual bool setWhiteBalance(uint16_t white_balance) = 0;



    // Sets the number of pixels per resolution unit in the image width.

    // Returns false if memory allocation fails.

    virtual bool setXResolution(uint32_t numerator, uint32_t denominator) = 0;



    // Sets the position of chrominance components in relation to the luminance

    // component.

    // Returns false if memory allocation fails.

    virtual bool setYCbCrPositioning(uint16_t ycbcr_positioning) = 0;



    // Sets the number of pixels per resolution unit in the image length.

    // Returns false if memory allocation fails.

    virtual bool setYResolution(uint32_t numerator, uint32_t denominator) = 0;



    // Sets the manufacturer of camera.

    // Returns false if memory allocation fails.

    virtual bool setMake(const std::string& make) = 0;



    // Sets the model number of camera.

    // Returns false if memory allocation fails.

    virtual bool setModel(const std::string& model) = 0;

    virtual bool setWhiteBalance(uint8_t white_blanace) = 0;



    // Generates APP1 segment.

    // Returns false if generating APP1 segment fails.