Merge "Camera: Use capture result metadata to override EXIF" (d0488712) · Commits · e / os / android_frameworks_av

services/camera/libcameraservice/api2/HeicCompositeStream.cpp

+78 −14

Original line number	Diff line number	Diff line
		@@ -136,6 +136,8 @@ status_t HeicCompositeStream::createInternalStreams(const std::vector<sp<Surface
		mAppSegmentConsumer->setName(String8("Camera3-HeicComposite-AppSegmentStream"));
		mAppSegmentSurface = new Surface(producer);

		mStaticInfo = device->info();

		res = device->createStream(mAppSegmentSurface, mAppSegmentMaxSize, 1, format,
		kAppSegmentDataSpace, rotation, &mAppSegmentStreamId, physicalCameraId, surfaceIds);
		if (res == OK) {
		@@ -606,9 +608,42 @@ void HeicCompositeStream::compilePendingInputLocked() {
		mFrameNumberMap.erase(it);
		}

		// Heic composition doesn't depend on capture result, so no need to check
		// mErrorFrameNumbers. Just remove them.
		mErrorFrameNumbers.clear();
		while (!mCaptureResults.empty()) {
		auto it = mCaptureResults.begin();
		// Negative timestamp indicates that something went wrong during the capture result
		// collection process.
		if (it->first >= 0) {
		if (mPendingInputFrames[it->first].frameNumber == std::get<0>(it->second)) {
		mPendingInputFrames[it->first].result =
		std::make_unique<CameraMetadata>(std::get<1>(it->second));
		} else {
		ALOGE("%s: Capture result frameNumber/timestamp mapping changed between "
		"shutter and capture result!", __FUNCTION__);
		}
		}
		mCaptureResults.erase(it);
		}

		// mErrorFrameNumbers stores frame number of dropped buffers.
		auto it = mErrorFrameNumbers.begin();
		while (it != mErrorFrameNumbers.end()) {
		bool frameFound = false;
		for (auto &inputFrame : mPendingInputFrames) {
		if (inputFrame.second.frameNumber == *it) {
		inputFrame.second.error = true;
		frameFound = true;
		break;
		}
		}

		if (frameFound) {
		it = mErrorFrameNumbers.erase(it);
		} else {
		ALOGW("%s: Not able to find failing input with frame number: %" PRId64, __FUNCTION__,
		*it);
		it++;
		}
		}

		// Distribute codec input buffers to be filled out from YUV output
		for (auto it = mPendingInputFrames.begin();
		@@ -639,14 +674,14 @@ bool HeicCompositeStream::getNextReadyInputLocked(int64_t currentTs /out*/) {

		bool newInputAvailable = false;
		for (const auto& it : mPendingInputFrames) {
		bool appSegmentBufferReady = (it.second.appSegmentBuffer.data != nullptr) &&
		!it.second.appSegmentWritten;
		bool appSegmentReady = (it.second.appSegmentBuffer.data != nullptr) &&
		!it.second.appSegmentWritten && it.second.result != nullptr;
		bool codecOutputReady = !it.second.codecOutputBuffers.empty();
		bool codecInputReady = (it.second.yuvBuffer.data != nullptr) &&
		(!it.second.codecInputBuffers.empty());
		if ((!it.second.error) &&
		(it.first < *currentTs) &&
		(appSegmentBufferReady \|\| codecOutputReady \|\| codecInputReady)) {
		(appSegmentReady \|\| codecOutputReady \|\| codecInputReady)) {
		*currentTs = it.first;
		newInputAvailable = true;
		break;
		@@ -678,13 +713,13 @@ status_t HeicCompositeStream::processInputFrame(nsecs_t timestamp,
		ATRACE_CALL();
		status_t res = OK;

		bool appSegmentBufferReady = inputFrame.appSegmentBuffer.data != nullptr &&
		!inputFrame.appSegmentWritten;
		bool appSegmentReady = inputFrame.appSegmentBuffer.data != nullptr &&
		!inputFrame.appSegmentWritten && inputFrame.result != nullptr;
		bool codecOutputReady = inputFrame.codecOutputBuffers.size() > 0;
		bool codecInputReady = inputFrame.yuvBuffer.data != nullptr &&
		!inputFrame.codecInputBuffers.empty();

		if (!appSegmentBufferReady && !codecOutputReady && !codecInputReady) {
		if (!appSegmentReady && !codecOutputReady && !codecInputReady) {
		ALOGW("%s: No valid appSegmentBuffer/codec input/outputBuffer available!", __FUNCTION__);
		return OK;
		}
		@@ -710,7 +745,7 @@ status_t HeicCompositeStream::processInputFrame(nsecs_t timestamp,
		}

		// Write JPEG APP segments data to the muxer.
		if (appSegmentBufferReady && inputFrame.muxer != nullptr) {
		if (appSegmentReady && inputFrame.muxer != nullptr) {
		res = processAppSegment(timestamp, inputFrame);
		if (res != OK) {
		ALOGE("%s: Failed to process JPEG APP segments: %s (%d)", __FUNCTION__,
		@@ -829,10 +864,8 @@ status_t HeicCompositeStream::processAppSegment(nsecs_t timestamp, InputFrame &i
		ALOGE("%s: Failed to initialize ExifUtils object!", __FUNCTION__);
		return BAD_VALUE;
		}
		//TODO: Use capture result metadata and static metadata to fill out the
		//rest.
		CameraMetadata dummyMeta;
		exifRes = exifUtils->setFromMetadata(dummyMeta, mOutputWidth, mOutputHeight);
		exifRes = exifUtils->setFromMetadata(*inputFrame.result, mStaticInfo,
		mOutputWidth, mOutputHeight);
		if (!exifRes) {
		ALOGE("%s: Failed to set Exif tags using metadata and main image sizes", __FUNCTION__);
		return BAD_VALUE;
		@@ -1012,6 +1045,7 @@ status_t HeicCompositeStream::processCompletedInputFrame(nsecs_t timestamp,
		}
		inputFrame.anb = nullptr;

		ATRACE_ASYNC_END("HEIC capture", inputFrame.frameNumber);
		return OK;
		}

		@@ -1497,6 +1531,36 @@ bool HeicCompositeStream::onStreamBufferError(const CaptureResultExtras& resultE
		return res;
		}

		void HeicCompositeStream::onResultError(const CaptureResultExtras& resultExtras) {
		// For result error, since the APPS_SEGMENT buffer already contains EXIF,
		// simply skip using the capture result metadata to override EXIF.
		Mutex::Autolock l(mMutex);

		int64_t timestamp = -1;
		for (const auto& fn : mFrameNumberMap) {
		if (fn.first == resultExtras.frameNumber) {
		timestamp = fn.second;
		break;
		}
		}
		if (timestamp == -1) {
		for (const auto& inputFrame : mPendingInputFrames) {
		if (inputFrame.second.frameNumber == resultExtras.frameNumber) {
		timestamp = inputFrame.first;
		break;
		}
		}
		}

		if (timestamp == -1) {
		ALOGE("%s: Failed to find shutter timestamp for result error!", __FUNCTION__);
		return;
		}

		mCaptureResults.emplace(timestamp, std::make_tuple(resultExtras.frameNumber, CameraMetadata()));
		mInputReadyCondition.signal();
		}

		void HeicCompositeStream::CodecCallbackHandler::onMessageReceived(const sp<AMessage> &msg) {
		sp<HeicCompositeStream> parent = mParent.promote();
		if (parent == nullptr) return;

services/camera/libcameraservice/api2/HeicCompositeStream.h

+6 −4

Original line number	Diff line number	Diff line
		@@ -77,7 +77,7 @@ protected:

		bool threadLoop() override;
		bool onStreamBufferError(const CaptureResultExtras& resultExtras) override;
		void onResultError(const CaptureResultExtras& /resultExtras/) override {}
		void onResultError(const CaptureResultExtras& resultExtras) override;

		private:
		//
		@@ -147,6 +147,7 @@ private:

		CpuConsumer::LockedBuffer appSegmentBuffer;
		std::vector<CodecOutputBufferInfo> codecOutputBuffers;
		std::unique_ptr<CameraMetadata> result;

		// Fields that are only applicable to HEVC tiling.
		CpuConsumer::LockedBuffer yuvBuffer;
		@@ -209,6 +210,7 @@ private:
		sp<Surface> mAppSegmentSurface;
		bool mAppSegmentBufferAcquired;
		size_t mAppSegmentMaxSize;
		CameraMetadata mStaticInfo;

		int mMainImageStreamId, mMainImageSurfaceId;
		sp<Surface> mMainImageSurface;

services/camera/libcameraservice/utils/ExifUtils.cpp

+259 −220

File changed.

Preview size limit exceeded, changes collapsed.

services/camera/libcameraservice/utils/ExifUtils.h

+19 −85

Original line number	Diff line number	Diff line
		@@ -52,78 +52,57 @@ public:

		// Set all known fields from a metadata structure
		virtual bool setFromMetadata(const CameraMetadata& metadata,
		const CameraMetadata& staticInfo,
		const size_t imageWidth, const size_t imageHeight) = 0;

		// Sets the len aperture.
		// Returns false if memory allocation fails.
		virtual bool setAperture(uint32_t numerator, uint32_t denominator) = 0;
		virtual bool setAperture(float aperture) = 0;

		// Sets the value of brightness.
		// Returns false if memory allocation fails.
		virtual bool setBrightness(int32_t numerator, int32_t denominator) = 0;

		// Sets the color space.
		// sets the color space.
		// Returns false if memory allocation fails.
		virtual bool setColorSpace(uint16_t color_space) = 0;

		// Sets the information to compressed data.
		// Returns false if memory allocation fails.
		virtual bool setComponentsConfiguration(const std::string& components_configuration) = 0;

		// Sets the compression scheme used for the image data.
		// Returns false if memory allocation fails.
		virtual bool setCompression(uint16_t compression) = 0;

		// Sets image contrast.
		// Returns false if memory allocation fails.
		virtual bool setContrast(uint16_t contrast) = 0;

		// Sets the date and time of image last modified. It takes local time. The
		// name of the tag is DateTime in IFD0.
		// Returns false if memory allocation fails.
		virtual bool setDateTime(const struct tm& t) = 0;

		// Sets the image description.
		// Returns false if memory allocation fails.
		virtual bool setDescription(const std::string& description) = 0;

		// Sets the digital zoom ratio. If the numerator is 0, it means digital zoom
		// was not used.
		// Returns false if memory allocation fails.
		virtual bool setDigitalZoomRatio(uint32_t numerator, uint32_t denominator) = 0;
		virtual bool setDigitalZoomRatio(uint32_t crop_width, uint32_t crop_height,
		uint32_t sensor_width, uint32_t sensor_height) = 0;

		// Sets the exposure bias.
		// Returns false if memory allocation fails.
		virtual bool setExposureBias(int32_t numerator, int32_t denominator) = 0;
		virtual bool setExposureBias(int32_t ev,
		uint32_t ev_step_numerator, uint32_t ev_step_denominator) = 0;

		// Sets the exposure mode set when the image was shot.
		// Returns false if memory allocation fails.
		virtual bool setExposureMode(uint16_t exposure_mode) = 0;

		// Sets the program used by the camera to set exposure when the picture is
		// taken.
		// Returns false if memory allocation fails.
		virtual bool setExposureProgram(uint16_t exposure_program) = 0;
		virtual bool setExposureMode(uint8_t exposure_mode) = 0;

		// Sets the exposure time, given in seconds.
		// Returns false if memory allocation fails.
		virtual bool setExposureTime(uint32_t numerator, uint32_t denominator) = 0;
		virtual bool setExposureTime(float exposure_time) = 0;

		// Sets the status of flash.
		// Returns false if memory allocation fails.
		virtual bool setFlash(uint16_t flash) = 0;
		virtual bool setFlash(uint8_t flash_available, uint8_t flash_state, uint8_t ae_mode) = 0;

		// Sets the F number.
		// Returns false if memory allocation fails.
		virtual bool setFNumber(uint32_t numerator, uint32_t denominator) = 0;
		virtual bool setFNumber(float f_number) = 0;

		// Sets the focal length of lens used to take the image in millimeters.
		// Returns false if memory allocation fails.
		virtual bool setFocalLength(uint32_t numerator, uint32_t denominator) = 0;
		virtual bool setFocalLength(float focal_length) = 0;

		// Sets the degree of overall image gain adjustment.
		// Sets the focal length of lens for 35mm film used to take the image in millimeters.
		// Returns false if memory allocation fails.
		virtual bool setGainControl(uint16_t gain_control) = 0;
		virtual bool setFocalLengthIn35mmFilm(float focal_length,
		float sensor_size_x, float sensor_size_y) = 0;

		// Sets the altitude in meters.
		// Returns false if memory allocation fails.
		@@ -157,45 +136,21 @@ public:
		// Returns false if memory allocation fails.
		virtual bool setIsoSpeedRating(uint16_t iso_speed_ratings) = 0;

		// Sets the kind of light source.
		// Returns false if memory allocation fails.
		virtual bool setLightSource(uint16_t light_source) = 0;

		// Sets the smallest F number of the lens.
		// Returns false if memory allocation fails.
		virtual bool setMaxAperture(uint32_t numerator, uint32_t denominator) = 0;

		// Sets the metering mode.
		// Returns false if memory allocation fails.
		virtual bool setMeteringMode(uint16_t metering_mode) = 0;
		virtual bool setMaxAperture(float aperture) = 0;

		// Sets image orientation.
		// Returns false if memory allocation fails.
		virtual bool setOrientation(uint16_t orientation) = 0;

		// Sets the unit for measuring XResolution and YResolution.
		// Returns false if memory allocation fails.
		virtual bool setResolutionUnit(uint16_t resolution_unit) = 0;

		// Sets image saturation.
		// Returns false if memory allocation fails.
		virtual bool setSaturation(uint16_t saturation) = 0;

		// Sets the type of scene that was shot.
		// Returns false if memory allocation fails.
		virtual bool setSceneCaptureType(uint16_t type) = 0;

		// Sets image sharpness.
		// Returns false if memory allocation fails.
		virtual bool setSharpness(uint16_t sharpness) = 0;

		// Sets the shutter speed.
		// Returns false if memory allocation fails.
		virtual bool setShutterSpeed(int32_t numerator, int32_t denominator) = 0;
		virtual bool setShutterSpeed(float exposure_time) = 0;

		// Sets the distance to the subject, given in meters.
		// Returns false if memory allocation fails.
		virtual bool setSubjectDistance(uint32_t numerator, uint32_t denominator) = 0;
		virtual bool setSubjectDistance(float diopters) = 0;

		// Sets the fractions of seconds for the <DateTime> tag.
		// Returns false if memory allocation fails.
		@@ -203,28 +158,7 @@ public:

		// Sets the white balance mode set when the image was shot.
		// Returns false if memory allocation fails.
		virtual bool setWhiteBalance(uint16_t white_balance) = 0;

		// Sets the number of pixels per resolution unit in the image width.
		// Returns false if memory allocation fails.
		virtual bool setXResolution(uint32_t numerator, uint32_t denominator) = 0;

		// Sets the position of chrominance components in relation to the luminance
		// component.
		// Returns false if memory allocation fails.
		virtual bool setYCbCrPositioning(uint16_t ycbcr_positioning) = 0;

		// Sets the number of pixels per resolution unit in the image length.
		// Returns false if memory allocation fails.
		virtual bool setYResolution(uint32_t numerator, uint32_t denominator) = 0;

		// Sets the manufacturer of camera.
		// Returns false if memory allocation fails.
		virtual bool setMake(const std::string& make) = 0;

		// Sets the model number of camera.
		// Returns false if memory allocation fails.
		virtual bool setModel(const std::string& model) = 0;
		virtual bool setWhiteBalance(uint8_t white_blanace) = 0;

		// Generates APP1 segment.
		// Returns false if generating APP1 segment fails.