Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e7f4b46a authored by Shuzhen Wang's avatar Shuzhen Wang
Browse files

Camera: Use capture result metadata to override EXIF

Use static and dynamic metadata to override Exif tags.

Also added back a missing ATRACE_ASYNC_ENDs statement.

Test: Camera CTS
Test: ERROR_BUFFER of internal streams is propagated to app
Test: ERROR_RESULT only results in EXIF not being overridden
Bug: 124066183
Change-Id: Id2c69c6bee04ae724ff5f190b2dd96d0159700c9
parent c6aa5039
Loading
Loading
Loading
Loading
+78 −14
Original line number Diff line number Diff line
@@ -136,6 +136,8 @@ status_t HeicCompositeStream::createInternalStreams(const std::vector<sp<Surface
    mAppSegmentConsumer->setName(String8("Camera3-HeicComposite-AppSegmentStream"));
    mAppSegmentSurface = new Surface(producer);

    mStaticInfo = device->info();

    res = device->createStream(mAppSegmentSurface, mAppSegmentMaxSize, 1, format,
            kAppSegmentDataSpace, rotation, &mAppSegmentStreamId, physicalCameraId, surfaceIds);
    if (res == OK) {
@@ -606,9 +608,42 @@ void HeicCompositeStream::compilePendingInputLocked() {
        mFrameNumberMap.erase(it);
    }

    // Heic composition doesn't depend on capture result, so no need to check
    // mErrorFrameNumbers. Just remove them.
    mErrorFrameNumbers.clear();
    while (!mCaptureResults.empty()) {
        auto it = mCaptureResults.begin();
        // Negative timestamp indicates that something went wrong during the capture result
        // collection process.
        if (it->first >= 0) {
            if (mPendingInputFrames[it->first].frameNumber == std::get<0>(it->second)) {
                mPendingInputFrames[it->first].result =
                        std::make_unique<CameraMetadata>(std::get<1>(it->second));
            } else {
                ALOGE("%s: Capture result frameNumber/timestamp mapping changed between "
                        "shutter and capture result!", __FUNCTION__);
            }
        }
        mCaptureResults.erase(it);
    }

    // mErrorFrameNumbers stores frame number of dropped buffers.
    auto it = mErrorFrameNumbers.begin();
    while (it != mErrorFrameNumbers.end()) {
        bool frameFound = false;
        for (auto &inputFrame : mPendingInputFrames) {
            if (inputFrame.second.frameNumber == *it) {
                inputFrame.second.error = true;
                frameFound = true;
                break;
            }
        }

        if (frameFound) {
            it = mErrorFrameNumbers.erase(it);
        } else {
            ALOGW("%s: Not able to find failing input with frame number: %" PRId64, __FUNCTION__,
                    *it);
            it++;
        }
    }

    // Distribute codec input buffers to be filled out from YUV output
    for (auto it = mPendingInputFrames.begin();
@@ -639,14 +674,14 @@ bool HeicCompositeStream::getNextReadyInputLocked(int64_t *currentTs /*out*/) {

    bool newInputAvailable = false;
    for (const auto& it : mPendingInputFrames) {
        bool appSegmentBufferReady = (it.second.appSegmentBuffer.data != nullptr) &&
                !it.second.appSegmentWritten;
        bool appSegmentReady = (it.second.appSegmentBuffer.data != nullptr) &&
                !it.second.appSegmentWritten && it.second.result != nullptr;
        bool codecOutputReady = !it.second.codecOutputBuffers.empty();
        bool codecInputReady = (it.second.yuvBuffer.data != nullptr) &&
                (!it.second.codecInputBuffers.empty());
        if ((!it.second.error) &&
                (it.first < *currentTs) &&
                (appSegmentBufferReady || codecOutputReady || codecInputReady)) {
                (appSegmentReady || codecOutputReady || codecInputReady)) {
            *currentTs = it.first;
            newInputAvailable = true;
            break;
@@ -678,13 +713,13 @@ status_t HeicCompositeStream::processInputFrame(nsecs_t timestamp,
    ATRACE_CALL();
    status_t res = OK;

    bool appSegmentBufferReady = inputFrame.appSegmentBuffer.data != nullptr &&
            !inputFrame.appSegmentWritten;
    bool appSegmentReady = inputFrame.appSegmentBuffer.data != nullptr &&
            !inputFrame.appSegmentWritten && inputFrame.result != nullptr;
    bool codecOutputReady = inputFrame.codecOutputBuffers.size() > 0;
    bool codecInputReady = inputFrame.yuvBuffer.data != nullptr &&
           !inputFrame.codecInputBuffers.empty();

    if (!appSegmentBufferReady && !codecOutputReady && !codecInputReady) {
    if (!appSegmentReady && !codecOutputReady && !codecInputReady) {
        ALOGW("%s: No valid appSegmentBuffer/codec input/outputBuffer available!", __FUNCTION__);
        return OK;
    }
@@ -710,7 +745,7 @@ status_t HeicCompositeStream::processInputFrame(nsecs_t timestamp,
    }

    // Write JPEG APP segments data to the muxer.
    if (appSegmentBufferReady && inputFrame.muxer != nullptr) {
    if (appSegmentReady && inputFrame.muxer != nullptr) {
        res = processAppSegment(timestamp, inputFrame);
        if (res != OK) {
            ALOGE("%s: Failed to process JPEG APP segments: %s (%d)", __FUNCTION__,
@@ -829,10 +864,8 @@ status_t HeicCompositeStream::processAppSegment(nsecs_t timestamp, InputFrame &i
        ALOGE("%s: Failed to initialize ExifUtils object!", __FUNCTION__);
        return BAD_VALUE;
    }
    //TODO: Use capture result metadata and static metadata to fill out the
    //rest.
    CameraMetadata dummyMeta;
    exifRes = exifUtils->setFromMetadata(dummyMeta, mOutputWidth, mOutputHeight);
    exifRes = exifUtils->setFromMetadata(*inputFrame.result, mStaticInfo,
            mOutputWidth, mOutputHeight);
    if (!exifRes) {
        ALOGE("%s: Failed to set Exif tags using metadata and main image sizes", __FUNCTION__);
        return BAD_VALUE;
@@ -1012,6 +1045,7 @@ status_t HeicCompositeStream::processCompletedInputFrame(nsecs_t timestamp,
    }
    inputFrame.anb = nullptr;

    ATRACE_ASYNC_END("HEIC capture", inputFrame.frameNumber);
    return OK;
}

@@ -1497,6 +1531,36 @@ bool HeicCompositeStream::onStreamBufferError(const CaptureResultExtras& resultE
    return res;
}

void HeicCompositeStream::onResultError(const CaptureResultExtras& resultExtras) {
    // For result error, since the APPS_SEGMENT buffer already contains EXIF,
    // simply skip using the capture result metadata to override EXIF.
    Mutex::Autolock l(mMutex);

    int64_t timestamp = -1;
    for (const auto& fn : mFrameNumberMap) {
        if (fn.first == resultExtras.frameNumber) {
            timestamp = fn.second;
            break;
        }
    }
    if (timestamp == -1) {
        for (const auto& inputFrame : mPendingInputFrames) {
            if (inputFrame.second.frameNumber == resultExtras.frameNumber) {
                timestamp = inputFrame.first;
                break;
            }
        }
    }

    if (timestamp == -1) {
        ALOGE("%s: Failed to find shutter timestamp for result error!", __FUNCTION__);
        return;
    }

    mCaptureResults.emplace(timestamp, std::make_tuple(resultExtras.frameNumber, CameraMetadata()));
    mInputReadyCondition.signal();
}

void HeicCompositeStream::CodecCallbackHandler::onMessageReceived(const sp<AMessage> &msg) {
    sp<HeicCompositeStream> parent = mParent.promote();
    if (parent == nullptr) return;
+6 −4
Original line number Diff line number Diff line
@@ -77,7 +77,7 @@ protected:

    bool threadLoop() override;
    bool onStreamBufferError(const CaptureResultExtras& resultExtras) override;
    void onResultError(const CaptureResultExtras& /*resultExtras*/) override {}
    void onResultError(const CaptureResultExtras& resultExtras) override;

private:
    //
@@ -147,6 +147,7 @@ private:

        CpuConsumer::LockedBuffer          appSegmentBuffer;
        std::vector<CodecOutputBufferInfo> codecOutputBuffers;
        std::unique_ptr<CameraMetadata>    result;

        // Fields that are only applicable to HEVC tiling.
        CpuConsumer::LockedBuffer          yuvBuffer;
@@ -209,6 +210,7 @@ private:
    sp<Surface>       mAppSegmentSurface;
    bool              mAppSegmentBufferAcquired;
    size_t            mAppSegmentMaxSize;
    CameraMetadata    mStaticInfo;

    int               mMainImageStreamId, mMainImageSurfaceId;
    sp<Surface>       mMainImageSurface;
+259 −220

File changed.

Preview size limit exceeded, changes collapsed.

+19 −85
Original line number Diff line number Diff line
@@ -52,78 +52,57 @@ public:

    // Set all known fields from a metadata structure
    virtual bool setFromMetadata(const CameraMetadata& metadata,
            const CameraMetadata& staticInfo,
            const size_t imageWidth, const size_t imageHeight) = 0;

    // Sets the len aperture.
    // Returns false if memory allocation fails.
    virtual bool setAperture(uint32_t numerator, uint32_t denominator) = 0;
    virtual bool setAperture(float aperture) = 0;

    // Sets the value of brightness.
    // Returns false if memory allocation fails.
    virtual bool setBrightness(int32_t numerator, int32_t denominator) = 0;

    // Sets the color space.
    // sets the color space.
    // Returns false if memory allocation fails.
    virtual bool setColorSpace(uint16_t color_space) = 0;

    // Sets the information to compressed data.
    // Returns false if memory allocation fails.
    virtual bool setComponentsConfiguration(const std::string& components_configuration) = 0;

    // Sets the compression scheme used for the image data.
    // Returns false if memory allocation fails.
    virtual bool setCompression(uint16_t compression) = 0;

    // Sets image contrast.
    // Returns false if memory allocation fails.
    virtual bool setContrast(uint16_t contrast) = 0;

    // Sets the date and time of image last modified. It takes local time. The
    // name of the tag is DateTime in IFD0.
    // Returns false if memory allocation fails.
    virtual bool setDateTime(const struct tm& t) = 0;

    // Sets the image description.
    // Returns false if memory allocation fails.
    virtual bool setDescription(const std::string& description) = 0;

    // Sets the digital zoom ratio. If the numerator is 0, it means digital zoom
    // was not used.
    // Returns false if memory allocation fails.
    virtual bool setDigitalZoomRatio(uint32_t numerator, uint32_t denominator) = 0;
    virtual bool setDigitalZoomRatio(uint32_t crop_width, uint32_t crop_height,
            uint32_t sensor_width, uint32_t sensor_height) = 0;

    // Sets the exposure bias.
    // Returns false if memory allocation fails.
    virtual bool setExposureBias(int32_t numerator, int32_t denominator) = 0;
    virtual bool setExposureBias(int32_t ev,
            uint32_t ev_step_numerator, uint32_t ev_step_denominator) = 0;

    // Sets the exposure mode set when the image was shot.
    // Returns false if memory allocation fails.
    virtual bool setExposureMode(uint16_t exposure_mode) = 0;

    // Sets the program used by the camera to set exposure when the picture is
    // taken.
    // Returns false if memory allocation fails.
    virtual bool setExposureProgram(uint16_t exposure_program) = 0;
    virtual bool setExposureMode(uint8_t exposure_mode) = 0;

    // Sets the exposure time, given in seconds.
    // Returns false if memory allocation fails.
    virtual bool setExposureTime(uint32_t numerator, uint32_t denominator) = 0;
    virtual bool setExposureTime(float exposure_time) = 0;

    // Sets the status of flash.
    // Returns false if memory allocation fails.
    virtual bool setFlash(uint16_t flash) = 0;
    virtual bool setFlash(uint8_t flash_available, uint8_t flash_state, uint8_t ae_mode) = 0;

    // Sets the F number.
    // Returns false if memory allocation fails.
    virtual bool setFNumber(uint32_t numerator, uint32_t denominator) = 0;
    virtual bool setFNumber(float f_number) = 0;

    // Sets the focal length of lens used to take the image in millimeters.
    // Returns false if memory allocation fails.
    virtual bool setFocalLength(uint32_t numerator, uint32_t denominator) = 0;
    virtual bool setFocalLength(float focal_length) = 0;

    // Sets the degree of overall image gain adjustment.
    // Sets the focal length of lens for 35mm film used to take the image in millimeters.
    // Returns false if memory allocation fails.
    virtual bool setGainControl(uint16_t gain_control) = 0;
    virtual bool setFocalLengthIn35mmFilm(float focal_length,
            float sensor_size_x, float sensor_size_y) = 0;

    // Sets the altitude in meters.
    // Returns false if memory allocation fails.
@@ -157,45 +136,21 @@ public:
    // Returns false if memory allocation fails.
    virtual bool setIsoSpeedRating(uint16_t iso_speed_ratings) = 0;

    // Sets the kind of light source.
    // Returns false if memory allocation fails.
    virtual bool setLightSource(uint16_t light_source) = 0;

    // Sets the smallest F number of the lens.
    // Returns false if memory allocation fails.
    virtual bool setMaxAperture(uint32_t numerator, uint32_t denominator) = 0;

    // Sets the metering mode.
    // Returns false if memory allocation fails.
    virtual bool setMeteringMode(uint16_t metering_mode) = 0;
    virtual bool setMaxAperture(float aperture) = 0;

    // Sets image orientation.
    // Returns false if memory allocation fails.
    virtual bool setOrientation(uint16_t orientation) = 0;

    // Sets the unit for measuring XResolution and YResolution.
    // Returns false if memory allocation fails.
    virtual bool setResolutionUnit(uint16_t resolution_unit) = 0;

    // Sets image saturation.
    // Returns false if memory allocation fails.
    virtual bool setSaturation(uint16_t saturation) = 0;

    // Sets the type of scene that was shot.
    // Returns false if memory allocation fails.
    virtual bool setSceneCaptureType(uint16_t type) = 0;

    // Sets image sharpness.
    // Returns false if memory allocation fails.
    virtual bool setSharpness(uint16_t sharpness) = 0;

    // Sets the shutter speed.
    // Returns false if memory allocation fails.
    virtual bool setShutterSpeed(int32_t numerator, int32_t denominator) = 0;
    virtual bool setShutterSpeed(float exposure_time) = 0;

    // Sets the distance to the subject, given in meters.
    // Returns false if memory allocation fails.
    virtual bool setSubjectDistance(uint32_t numerator, uint32_t denominator) = 0;
    virtual bool setSubjectDistance(float diopters) = 0;

    // Sets the fractions of seconds for the <DateTime> tag.
    // Returns false if memory allocation fails.
@@ -203,28 +158,7 @@ public:

    // Sets the white balance mode set when the image was shot.
    // Returns false if memory allocation fails.
    virtual bool setWhiteBalance(uint16_t white_balance) = 0;

    // Sets the number of pixels per resolution unit in the image width.
    // Returns false if memory allocation fails.
    virtual bool setXResolution(uint32_t numerator, uint32_t denominator) = 0;

    // Sets the position of chrominance components in relation to the luminance
    // component.
    // Returns false if memory allocation fails.
    virtual bool setYCbCrPositioning(uint16_t ycbcr_positioning) = 0;

    // Sets the number of pixels per resolution unit in the image length.
    // Returns false if memory allocation fails.
    virtual bool setYResolution(uint32_t numerator, uint32_t denominator) = 0;

    // Sets the manufacturer of camera.
    // Returns false if memory allocation fails.
    virtual bool setMake(const std::string& make) = 0;

    // Sets the model number of camera.
    // Returns false if memory allocation fails.
    virtual bool setModel(const std::string& model) = 0;
    virtual bool setWhiteBalance(uint8_t white_blanace) = 0;

    // Generates APP1 segment.
    // Returns false if generating APP1 segment fails.