Camera3Device: support shutter after result (43e69a67) · Commits · e / os / android_frameworks_av

services/camera/libcameraservice/device3/Camera3Device.cpp

+190 −136

Original line number	Original line	Diff line number	Diff line
	@@ -427,7 +427,7 @@ status_t Camera3Device::dump(int fd, const Vector<String16> &args) {
	InFlightRequest r = mInFlightMap.valueAt(i);		InFlightRequest r = mInFlightMap.valueAt(i);
	lines.appendFormat(" Frame %d \| Timestamp: %" PRId64 ", metadata"		lines.appendFormat(" Frame %d \| Timestamp: %" PRId64 ", metadata"
	" arrived: %s, buffers left: %d\n", mInFlightMap.keyAt(i),		" arrived: %s, buffers left: %d\n", mInFlightMap.keyAt(i),
	r.captureTimestamp, r.haveResultMetadata ? "true" : "false",		r.shutterTimestamp, r.haveResultMetadata ? "true" : "false",
	r.numBuffersLeft);		r.numBuffersLeft);
	}		}
	}		}
	@@ -1880,6 +1880,131 @@ bool Camera3Device::insert3AResult(CameraMetadata& result, int32_t tag,
	return true;		return true;
	}		}


			void Camera3Device::returnOutputBuffers(
			const camera3_stream_buffer_t *outputBuffers, size_t numBuffers,
			nsecs_t timestamp) {
			for (size_t i = 0; i < numBuffers; i++)
			{
			Camera3Stream *stream = Camera3Stream::cast(outputBuffers[i].stream);
			status_t res = stream->returnBuffer(outputBuffers[i], timestamp);
			// Note: stream may be deallocated at this point, if this buffer was
			// the last reference to it.
			if (res != OK) {
			ALOGE("Can't return buffer to its stream: %s (%d)",
			strerror(-res), res);
			}
			}
			}


			void Camera3Device::removeInFlightRequestIfReadyLocked(int idx) {

			const InFlightRequest &request = mInFlightMap.valueAt(idx);
			const uint32_t frameNumber = mInFlightMap.keyAt(idx);

			nsecs_t sensorTimestamp = request.sensorTimestamp;
			nsecs_t shutterTimestamp = request.shutterTimestamp;

			// Check if it's okay to remove the request from InFlightMap:
			// In the case of a successful request:
			// all input and output buffers, all result metadata, shutter callback
			// arrived.
			// In the case of a unsuccessful request:
			// all input and output buffers arrived.
			if (request.numBuffersLeft == 0 &&
			(request.requestStatus != OK \|\|
			(request.haveResultMetadata && shutterTimestamp != 0))) {
			ATRACE_ASYNC_END("frame capture", frameNumber);

			// Sanity check - if sensor timestamp matches shutter timestamp
			if (request.requestStatus == OK &&
			sensorTimestamp != shutterTimestamp) {
			SET_ERR("sensor timestamp (%" PRId64
			") for frame %d doesn't match shutter timestamp (%" PRId64 ")",
			sensorTimestamp, frameNumber, shutterTimestamp);
			}

			// for an unsuccessful request, it may have pending output buffers to
			// return.
			assert(request.requestStatus != OK \|\|
			request.pendingOutputBuffers.size() == 0);
			returnOutputBuffers(request.pendingOutputBuffers.array(),
			request.pendingOutputBuffers.size(), 0);

			mInFlightMap.removeItemsAt(idx, 1);

			ALOGVV("%s: removed frame %d from InFlightMap", __FUNCTION__, frameNumber);
			}

			// Sanity check - if we have too many in-flight frames, something has
			// likely gone wrong
			if (mInFlightMap.size() > kInFlightWarnLimit) {
			CLOGE("In-flight list too large: %zu", mInFlightMap.size());
			}
			}


			void Camera3Device::sendCaptureResult(CameraMetadata &pendingMetadata,
			CaptureResultExtras &resultExtras,
			CameraMetadata &collectedPartialResult,
			uint32_t frameNumber) {
			if (pendingMetadata.isEmpty())
			return;

			Mutex::Autolock l(mOutputLock);

			// TODO: need to track errors for tighter bounds on expected frame number
			if (frameNumber < mNextResultFrameNumber) {
			SET_ERR("Out-of-order capture result metadata submitted! "
			"(got frame number %d, expecting %d)",
			frameNumber, mNextResultFrameNumber);
			return;
			}
			mNextResultFrameNumber = frameNumber + 1;

			CaptureResult captureResult;
			captureResult.mResultExtras = resultExtras;
			captureResult.mMetadata = pendingMetadata;

			if (captureResult.mMetadata.update(ANDROID_REQUEST_FRAME_COUNT,
			(int32_t*)&frameNumber, 1) != OK) {
			SET_ERR("Failed to set frame# in metadata (%d)",
			frameNumber);
			return;
			} else {
			ALOGVV("%s: Camera %d: Set frame# in metadata (%d)",
			__FUNCTION__, mId, frameNumber);
			}

			// Append any previous partials to form a complete result
			if (mUsePartialResult && !collectedPartialResult.isEmpty()) {
			captureResult.mMetadata.append(collectedPartialResult);
			}

			captureResult.mMetadata.sort();

			// Check that there's a timestamp in the result metadata
			camera_metadata_entry entry =
			captureResult.mMetadata.find(ANDROID_SENSOR_TIMESTAMP);
			if (entry.count == 0) {
			SET_ERR("No timestamp provided by HAL for frame %d!",
			frameNumber);
			return;
			}

			// Valid result, insert into queue
			List<CaptureResult>::iterator queuedResult =
			mResultQueue.insert(mResultQueue.end(), CaptureResult(captureResult));
			ALOGVV("%s: result requestId = %" PRId32 ", frameNumber = %" PRId64
			", burstId = %" PRId32, __FUNCTION__,
			queuedResult->mResultExtras.requestId,
			queuedResult->mResultExtras.frameNumber,
			queuedResult->mResultExtras.burstId);

			mResultSignal.signal();
			}

	/**		/**
	* Camera HAL device callback methods		* Camera HAL device callback methods
	*/		*/
	@@ -1914,11 +2039,14 @@ void Camera3Device::processCaptureResult(const camera3_capture_result *result) {
	CaptureResultExtras resultExtras;		CaptureResultExtras resultExtras;
	bool hasInputBufferInRequest = false;		bool hasInputBufferInRequest = false;

	// Get capture timestamp and resultExtras from list of in-flight requests,		// Get shutter timestamp and resultExtras from list of in-flight requests,
	// where it was added by the shutter notification for this frame.		// where it was added by the shutter notification for this frame. If the
	// Then update the in-flight status and remove the in-flight entry if		// shutter timestamp isn't received yet, append the output buffers to the
	// all result data has been received.		// in-flight request and they will be returned when the shutter timestamp
	nsecs_t timestamp = 0;		// arrives. Update the in-flight status and remove the in-flight entry if
			// all result data and shutter timestamp have been received.
			nsecs_t shutterTimestamp = 0;

	{		{
	Mutex::Autolock l(mInFlightLock);		Mutex::Autolock l(mInFlightLock);
	ssize_t idx = mInFlightMap.indexOfKey(frameNumber);		ssize_t idx = mInFlightMap.indexOfKey(frameNumber);
	@@ -1928,12 +2056,16 @@ void Camera3Device::processCaptureResult(const camera3_capture_result *result) {
	return;		return;
	}		}
	InFlightRequest &request = mInFlightMap.editValueAt(idx);		InFlightRequest &request = mInFlightMap.editValueAt(idx);
	ALOGVV("%s: got InFlightRequest requestId = %" PRId32 ", frameNumber = %" PRId64		ALOGVV("%s: got InFlightRequest requestId = %" PRId32
	", burstId = %" PRId32,		", frameNumber = %" PRId64 ", burstId = %" PRId32
	__FUNCTION__, request.resultExtras.requestId, request.resultExtras.frameNumber,		", partialResultCount = %d",
	request.resultExtras.burstId);		__FUNCTION__, request.resultExtras.requestId,
	// Always update the partial count to the latest one. When framework aggregates adjacent		request.resultExtras.frameNumber, request.resultExtras.burstId,
	// partial results into one, the latest partial count will be used.		result->partial_result);
			// Always update the partial count to the latest one if it's not 0
			// (buffers only). When framework aggregates adjacent partial results
			// into one, the latest partial count will be used.
			if (result->partial_result != 0)
	request.resultExtras.partialResultCount = result->partial_result;		request.resultExtras.partialResultCount = result->partial_result;

	// Check if this result carries only partial metadata		// Check if this result carries only partial metadata
	@@ -1978,22 +2110,9 @@ void Camera3Device::processCaptureResult(const camera3_capture_result *result) {
	}		}
	}		}

	timestamp = request.captureTimestamp;		shutterTimestamp = request.shutterTimestamp;
	resultExtras = request.resultExtras;
	hasInputBufferInRequest = request.hasInputBuffer;		hasInputBufferInRequest = request.hasInputBuffer;

	/**
	* One of the following must happen before it's legal to call process_capture_result,
	* unless partial metadata is being provided:
	* - CAMERA3_MSG_SHUTTER (expected during normal operation)
	* - CAMERA3_MSG_ERROR (expected during flush)
	*/
	if (request.requestStatus == OK && timestamp == 0 && !isPartialResult) {
	SET_ERR("Called before shutter notify for frame %d",
	frameNumber);
	return;
	}

	// Did we get the (final) result metadata for this capture?		// Did we get the (final) result metadata for this capture?
	if (result->result != NULL && !isPartialResult) {		if (result->result != NULL && !isPartialResult) {
	if (request.haveResultMetadata) {		if (request.haveResultMetadata) {
	@@ -2026,103 +2145,38 @@ void Camera3Device::processCaptureResult(const camera3_capture_result *result) {
	return;		return;
	}		}

	// Check if everything has arrived for this result (buffers and metadata), remove it from		camera_metadata_ro_entry_t entry;
	// InFlightMap if both arrived or HAL reports error for this request (i.e. during flush).		res = find_camera_metadata_ro_entry(result->result,
	// For per-frame error notifications, camera3.h requirements state that all the		ANDROID_SENSOR_TIMESTAMP, &entry);
	// buffer handles for a failed frame capture must be returned via process_capture_result()		if (res == OK && entry.count == 1) {
	// call(s). Hence, Camera3Device needs to ensure that the frame entry is not deleted from		request.sensorTimestamp = entry.data.i64[0];
	// mInFlightMap until all buffers for that frame have been returned by HAL.
	if ((request.numBuffersLeft == 0) &&
	((request.requestStatus != OK) \|\| (request.haveResultMetadata))) {
	ATRACE_ASYNC_END("frame capture", frameNumber);
	mInFlightMap.removeItemsAt(idx, 1);
	}

	// Sanity check - if we have too many in-flight frames, something has
	// likely gone wrong
	if (mInFlightMap.size() > kInFlightWarnLimit) {
	CLOGE("In-flight list too large: %zu", mInFlightMap.size());
	}		}

			// If shutter event isn't received yet, append the output buffers to
			// the in-flight request. Otherwise, return the output buffers to
			// streams.
			if (shutterTimestamp == 0) {
			request.pendingOutputBuffers.appendArray(result->output_buffers,
			result->num_output_buffers);
			} else {
			returnOutputBuffers(result->output_buffers,
			result->num_output_buffers, shutterTimestamp);
	}		}

	// Process the result metadata, if provided
	bool gotResult = false;
	if (result->result != NULL && !isPartialResult) {		if (result->result != NULL && !isPartialResult) {
	Mutex::Autolock l(mOutputLock);		if (shutterTimestamp == 0) {
			request.pendingMetadata = result->result;
	gotResult = true;		request.partialResult.collectedResult = collectedPartialResult;

	// TODO: need to track errors for tighter bounds on expected frame number
	if (frameNumber < mNextResultFrameNumber) {
	SET_ERR("Out-of-order capture result metadata submitted! "
	"(got frame number %d, expecting %d)",
	frameNumber, mNextResultFrameNumber);
	return;
	}
	mNextResultFrameNumber = frameNumber + 1;

	CaptureResult captureResult;
	captureResult.mResultExtras = resultExtras;
	captureResult.mMetadata = result->result;

	if (captureResult.mMetadata.update(ANDROID_REQUEST_FRAME_COUNT,
	(int32_t*)&frameNumber, 1) != OK) {
	SET_ERR("Failed to set frame# in metadata (%d)",
	frameNumber);
	gotResult = false;
	} else {		} else {
	ALOGVV("%s: Camera %d: Set frame# in metadata (%d)",		CameraMetadata metadata;
	__FUNCTION__, mId, frameNumber);		metadata = result->result;
	}		sendCaptureResult(metadata, request.resultExtras,
			collectedPartialResult, frameNumber);
	// Append any previous partials to form a complete result
	if (mUsePartialResult && !collectedPartialResult.isEmpty()) {
	captureResult.mMetadata.append(collectedPartialResult);
	}

	captureResult.mMetadata.sort();

	// Check that there's a timestamp in the result metadata

	camera_metadata_entry entry =
	captureResult.mMetadata.find(ANDROID_SENSOR_TIMESTAMP);
	if (entry.count == 0) {
	SET_ERR("No timestamp provided by HAL for frame %d!",
	frameNumber);
	gotResult = false;
	} else if (timestamp != entry.data.i64[0]) {
	SET_ERR("Timestamp mismatch between shutter notify and result"
	" metadata for frame %d (%" PRId64 " vs %" PRId64 " respectively)",
	frameNumber, timestamp, entry.data.i64[0]);
	gotResult = false;
	}		}

	if (gotResult) {
	// Valid result, insert into queue
	List<CaptureResult>::iterator queuedResult =
	mResultQueue.insert(mResultQueue.end(), CaptureResult(captureResult));
	ALOGVV("%s: result requestId = %" PRId32 ", frameNumber = %" PRId64
	", burstId = %" PRId32, __FUNCTION__,
	queuedResult->mResultExtras.requestId,
	queuedResult->mResultExtras.frameNumber,
	queuedResult->mResultExtras.burstId);
	}		}
	} // scope for mOutputLock

	// Return completed buffers to their streams with the timestamp		removeInFlightRequestIfReadyLocked(idx);
			} // scope for mInFlightLock
	for (size_t i = 0; i < result->num_output_buffers; i++) {
	Camera3Stream *stream =
	Camera3Stream::cast(result->output_buffers[i].stream);
	res = stream->returnBuffer(result->output_buffers[i], timestamp);
	// Note: stream may be deallocated at this point, if this buffer was the
	// last reference to it.
	if (res != OK) {
	ALOGE("Can't return buffer %zu for frame %d to its stream: "
	" %s (%d)", i, frameNumber, strerror(-res), res);
	}
	}

	if (result->input_buffer != NULL) {		if (result->input_buffer != NULL) {
	if (hasInputBufferInRequest) {		if (hasInputBufferInRequest) {
	@@ -2142,13 +2196,6 @@ void Camera3Device::processCaptureResult(const camera3_capture_result *result) {
	__FUNCTION__);		__FUNCTION__);
	}		}
	}		}

	// Finally, signal any waiters for new frames

	if (gotResult) {
	mResultSignal.signal();
	}

	}		}

	void Camera3Device::notify(const camera3_notify_msg *msg) {		void Camera3Device::notify(const camera3_notify_msg *msg) {
	@@ -2266,8 +2313,6 @@ void Camera3Device::notifyShutter(const camera3_shutter_msg_t &msg,
	mNextShutterFrameNumber = msg.frame_number + 1;		mNextShutterFrameNumber = msg.frame_number + 1;
	}		}

	CaptureResultExtras resultExtras;

	// Set timestamp for the request in the in-flight tracking		// Set timestamp for the request in the in-flight tracking
	// and get the request ID to send upstream		// and get the request ID to send upstream
	{		{
	@@ -2275,21 +2320,30 @@ void Camera3Device::notifyShutter(const camera3_shutter_msg_t &msg,
	idx = mInFlightMap.indexOfKey(msg.frame_number);		idx = mInFlightMap.indexOfKey(msg.frame_number);
	if (idx >= 0) {		if (idx >= 0) {
	InFlightRequest &r = mInFlightMap.editValueAt(idx);		InFlightRequest &r = mInFlightMap.editValueAt(idx);
	r.captureTimestamp = msg.timestamp;
	resultExtras = r.resultExtras;		ALOGVV("Camera %d: %s: Shutter fired for frame %d (id %d) at %" PRId64,
			mId, __FUNCTION__,
			msg.frame_number, r.resultExtras.requestId, msg.timestamp);
			// Call listener, if any
			if (listener != NULL) {
			listener->notifyShutter(r.resultExtras, msg.timestamp);
			}

			r.shutterTimestamp = msg.timestamp;

			// send pending result and buffers
			sendCaptureResult(r.pendingMetadata, r.resultExtras,
			r.partialResult.collectedResult, msg.frame_number);
			returnOutputBuffers(r.pendingOutputBuffers.array(),
			r.pendingOutputBuffers.size(), r.shutterTimestamp);
			r.pendingOutputBuffers.clear();

			removeInFlightRequestIfReadyLocked(idx);
	}		}
	}		}
	if (idx < 0) {		if (idx < 0) {
	SET_ERR("Shutter notification for non-existent frame number %d",		SET_ERR("Shutter notification for non-existent frame number %d",
	msg.frame_number);		msg.frame_number);
	return;
	}
	ALOGVV("Camera %d: %s: Shutter fired for frame %d (id %d) at %" PRId64,
	mId, __FUNCTION__,
	msg.frame_number, resultExtras.requestId, msg.timestamp);
	// Call listener, if any
	if (listener != NULL) {
	listener->notifyShutter(resultExtras, msg.timestamp);
	}		}
	}		}

services/camera/libcameraservice/device3/Camera3Device.h

+44 −5

Original line number	Original line	Diff line number	Diff line
	@@ -521,7 +521,9 @@ class Camera3Device :

	struct InFlightRequest {		struct InFlightRequest {
	// Set by notify() SHUTTER call.		// Set by notify() SHUTTER call.
	nsecs_t captureTimestamp;		nsecs_t shutterTimestamp;
			// Set by process_capture_result().
			nsecs_t sensorTimestamp;
	int requestStatus;		int requestStatus;
	// Set by process_capture_result call with valid metadata		// Set by process_capture_result call with valid metadata
	bool haveResultMetadata;		bool haveResultMetadata;
	@@ -532,6 +534,21 @@ class Camera3Device :
	// If this request has any input buffer		// If this request has any input buffer
	bool hasInputBuffer;		bool hasInputBuffer;


			// The last metadata that framework receives from HAL and
			// not yet send out because the shutter event hasn't arrived.
			// It's added by process_capture_result and sent when framework
			// receives the shutter event.
			CameraMetadata pendingMetadata;

			// Buffers are added by process_capture_result when output buffers
			// return from HAL but framework has not yet received the shutter
			// event. They will be returned to the streams when framework receives
			// the shutter event.
			Vector<camera3_stream_buffer_t> pendingOutputBuffers;



	// Fields used by the partial result only		// Fields used by the partial result only
	struct PartialResultInFlight {		struct PartialResultInFlight {
	// Set by process_capture_result once 3A has been sent to clients		// Set by process_capture_result once 3A has been sent to clients
	@@ -546,7 +563,8 @@ class Camera3Device :

	// Default constructor needed by KeyedVector		// Default constructor needed by KeyedVector
	InFlightRequest() :		InFlightRequest() :
	captureTimestamp(0),		shutterTimestamp(0),
			sensorTimestamp(0),
	requestStatus(OK),		requestStatus(OK),
	haveResultMetadata(false),		haveResultMetadata(false),
	numBuffersLeft(0),		numBuffersLeft(0),
	@@ -554,7 +572,8 @@ class Camera3Device :
	}		}

	InFlightRequest(int numBuffers) :		InFlightRequest(int numBuffers) :
	captureTimestamp(0),		shutterTimestamp(0),
			sensorTimestamp(0),
	requestStatus(OK),		requestStatus(OK),
	haveResultMetadata(false),		haveResultMetadata(false),
	numBuffersLeft(numBuffers),		numBuffersLeft(numBuffers),
	@@ -562,7 +581,8 @@ class Camera3Device :
	}		}

	InFlightRequest(int numBuffers, CaptureResultExtras extras) :		InFlightRequest(int numBuffers, CaptureResultExtras extras) :
	captureTimestamp(0),		shutterTimestamp(0),
			sensorTimestamp(0),
	requestStatus(OK),		requestStatus(OK),
	haveResultMetadata(false),		haveResultMetadata(false),
	numBuffersLeft(numBuffers),		numBuffersLeft(numBuffers),
	@@ -571,7 +591,8 @@ class Camera3Device :
	}		}

	InFlightRequest(int numBuffers, CaptureResultExtras extras, bool hasInput) :		InFlightRequest(int numBuffers, CaptureResultExtras extras, bool hasInput) :
	captureTimestamp(0),		shutterTimestamp(0),
			sensorTimestamp(0),
	requestStatus(OK),		requestStatus(OK),
	haveResultMetadata(false),		haveResultMetadata(false),
	numBuffersLeft(numBuffers),		numBuffersLeft(numBuffers),
	@@ -639,6 +660,24 @@ class Camera3Device :
	void notifyShutter(const camera3_shutter_msg_t &msg,		void notifyShutter(const camera3_shutter_msg_t &msg,
	NotificationListener *listener);		NotificationListener *listener);

			// helper function to return the output buffers to the streams.
			void returnOutputBuffers(const camera3_stream_buffer_t *outputBuffers,
			size_t numBuffers, nsecs_t timestamp);

			// Insert the capture result given the pending metadata, result extras,
			// partial results, and the frame number to the result queue.
			void sendCaptureResult(CameraMetadata &pendingMetadata,
			CaptureResultExtras &resultExtras,
			CameraMetadata &collectedPartialResult, uint32_t frameNumber);

			/** Scope for mInFlightLock **/

			// Remove the in-flight request of the given index from mInFlightMap
			// if it's no longer needed. It must only be called with mInFlightLock held.
			void removeInFlightRequestIfReadyLocked(int idx);

			/** End scope for mInFlightLock **/

	/**		/**
	* Static callback forwarding methods from HAL to instance		* Static callback forwarding methods from HAL to instance
	*/		*/