Camera: add batching support (bed3a947) · Commits · e / os / android_hardware_interfaces

camera/device/3.2/ICameraDeviceCallback.hal

+16 −9

Original line number	Diff line number	Diff line
		@@ -35,7 +35,8 @@ interface ICameraDeviceCallback {
		/**
		* processCaptureResult:
		*
		* Send results from a completed capture to the framework.
		* Send results from one or more completed or partially completed captures
		* to the framework.
		* processCaptureResult() may be invoked multiple times by the HAL in
		* response to a single capture request. This allows, for example, the
		* metadata and low-resolution buffers to be returned in one call, and
		@@ -61,11 +62,14 @@ interface ICameraDeviceCallback {
		* acceptable and expected that the buffer for request 5 for stream A may be
		* returned after the buffer for request 6 for stream B is. And it is
		* acceptable that the result metadata for request 6 for stream B is
		* returned before the buffer for request 5 for stream A is.
		* returned before the buffer for request 5 for stream A is. If multiple
		* capture results are included in a single call, camera framework must
		* process results sequentially from lower index to higher index, as if
		* these results were sent to camera framework one by one, from lower index
		* to higher index.
		*
		* The HAL retains ownership of result structure, which only needs to be
		* valid to access during this call. The framework must copy whatever it
		* needs before this call returns.
		* valid to access during this call.
		*
		* The output buffers do not need to be filled yet; the framework must wait
		* on the stream buffer release sync fence before reading the buffer
		@@ -93,20 +97,23 @@ interface ICameraDeviceCallback {
		*
		* Performance requirements:
		*
		* This is a non-blocking call. The framework must return this call in 5ms.
		* This is a non-blocking call. The framework must handle each CaptureResult
		* within 5ms.
		*
		* The pipeline latency (see S7 for definition) should be less than or equal to
		* 4 frame intervals, and must be less than or equal to 8 frame intervals.
		*
		*/
		processCaptureResult(CaptureResult result);
		processCaptureResult(vec<CaptureResult> results);

		/**
		* notify:
		*
		* Asynchronous notification callback from the HAL, fired for various
		* reasons. Only for information independent of frame capture, or that
		* require specific timing.
		* require specific timing. Multiple messages may be sent in one call; a
		* message with a higher index must be considered to have occurred after a
		* message with a lower index.
		*
		* Multiple threads may call notify() simultaneously.
		*
		@@ -119,8 +126,8 @@ interface ICameraDeviceCallback {
		* ------------------------------------------------------------------------
		* Performance requirements:
		*
		* This is a non-blocking call. The framework must return this call in 5ms.
		* This is a non-blocking call. The framework must handle each message in 5ms.
		*/
		notify(NotifyMsg msg);
		notify(vec<NotifyMsg> msgs);

		};

camera/device/3.2/ICameraDeviceSession.hal

+16 −10

Original line number	Diff line number	Diff line
		@@ -171,14 +171,16 @@ interface ICameraDeviceSession {
		/**
		* processCaptureRequest:
		*
		* Send a new capture request to the HAL. The HAL must not return from
		* this call until it is ready to accept the next request to process. Only
		* one call to processCaptureRequest() must be made at a time by the
		* framework, and the calls must all be from the same thread. The next call
		* to processCaptureRequest() must be made as soon as a new request and
		* its associated buffers are available. In a normal preview scenario, this
		* means the function is generally called again by the framework almost
		* instantly.
		* Send a list of capture requests to the HAL. The HAL must not return from
		* this call until it is ready to accept the next set of requests to
		* process. Only one call to processCaptureRequest() must be made at a time
		* by the framework, and the calls must all be from the same thread. The
		* next call to processCaptureRequest() must be made as soon as a new
		* request and its associated buffers are available. In a normal preview
		* scenario, this means the function is generally called again by the
		* framework almost instantly. If more than one request is provided by the
		* client, the HAL must process the requests in order of lowest index to
		* highest index.
		*
		* The actual request processing is asynchronous, with the results of
		* capture being returned by the HAL through the processCaptureResult()
		@@ -229,10 +231,14 @@ interface ICameraDeviceSession {
		* If the camera device has encountered a serious error. After this
		* error is returned, only the close() method can be successfully
		* called by the framework.
		* @return numRequestProcessed Number of requests successfully processed by
		* camera HAL. When status is OK, this must be equal to the size of
		* requests. When the call fails, this number is the number of requests
		* that HAL processed successfully before HAL runs into an error.
		*
		*/
		processCaptureRequest(CaptureRequest request)
		generates (Status status);
		processCaptureRequest(vec<CaptureRequest> requests)
		generates (Status status, uint32_t numRequestProcessed);

		/**
		* flush:

camera/device/3.2/default/CameraDevice.cpp

+12 −1

Original line number	Diff line number	Diff line
		@@ -229,7 +229,18 @@ Return<void> CameraDevice::open(const sp<ICameraDeviceCallback>& callback, open_
		return Void();
		}

		session = new CameraDeviceSession(device, callback);
		struct camera_info info;
		res = mModule->getCameraInfo(mCameraIdInt, &info);
		if (res != OK) {
		ALOGE("%s: Could not open camera: getCameraInfo failed", __FUNCTION__);
		device->common.close(&device->common);
		mLock.unlock();
		_hidl_cb(Status::ILLEGAL_ARGUMENT, nullptr);
		return Void();
		}

		session = new CameraDeviceSession(
		device, info.static_camera_characteristics, callback);
		if (session == nullptr) {
		ALOGE("%s: camera device session allocation failed", __FUNCTION__);
		mLock.unlock();

camera/device/3.2/default/CameraDeviceSession.cpp

+408 −28

File changed.

Preview size limit exceeded, changes collapsed.

camera/device/3.2/default/CameraDeviceSession.h

+111 −5

Original line number	Diff line number	Diff line
		@@ -17,6 +17,8 @@
		#ifndef ANDROID_HARDWARE_CAMERA_DEVICE_V3_2_CAMERADEVICE3SESSION_H
		#define ANDROID_HARDWARE_CAMERA_DEVICE_V3_2_CAMERADEVICE3SESSION_H

		#include <deque>
		#include <map>
		#include <unordered_map>
		#include "hardware/camera_common.h"
		#include "hardware/camera3.h"
		@@ -27,6 +29,7 @@
		#include <hidl/MQDescriptor.h>
		#include <include/convert.h>
		#include "HandleImporter.h"
		#include "CameraMetadata.h"

		namespace android {
		namespace hardware {
		@@ -64,7 +67,9 @@ extern "C" {

		struct CameraDeviceSession : public ICameraDeviceSession, private camera3_callback_ops {

		CameraDeviceSession(camera3_device_t*, const sp<ICameraDeviceCallback>&);
		CameraDeviceSession(camera3_device_t*,
		const camera_metadata_t* deviceInfo,
		const sp<ICameraDeviceCallback>&);
		~CameraDeviceSession();
		// Call by CameraDevice to dump active device states
		void dumpState(const native_handle_t* fd);
		@@ -75,9 +80,12 @@ struct CameraDeviceSession : public ICameraDeviceSession, private camera3_callba
		bool isClosed();

		// Methods from ::android::hardware::camera::device::V3_2::ICameraDeviceSession follow.
		Return<void> constructDefaultRequestSettings(RequestTemplate type, constructDefaultRequestSettings_cb _hidl_cb) override;
		Return<void> configureStreams(const StreamConfiguration& requestedConfiguration, configureStreams_cb _hidl_cb) override;
		Return<Status> processCaptureRequest(const CaptureRequest& request) override;
		Return<void> constructDefaultRequestSettings(
		RequestTemplate type, constructDefaultRequestSettings_cb _hidl_cb) override;
		Return<void> configureStreams(
		const StreamConfiguration& requestedConfiguration, configureStreams_cb _hidl_cb) override;
		Return<void> processCaptureRequest(
		const hidl_vec<CaptureRequest>& requests, processCaptureRequest_cb _hidl_cb) override;
		Return<Status> flush() override;
		Return<void> close() override;

		@@ -94,7 +102,6 @@ private:
		bool mDisconnected = false;

		camera3_device_t* mDevice;
		const sp<ICameraDeviceCallback> mCallback;
		// Stream ID -> Camera3Stream cache
		std::map<int, Camera3Stream> mStreamMap;

		@@ -114,6 +121,104 @@ private:
		static HandleImporter& sHandleImporter;

		bool mInitFail;

		common::V1_0::helper::CameraMetadata mDeviceInfo;

		class ResultBatcher {
		public:
		ResultBatcher(const sp<ICameraDeviceCallback>& callback);
		void setNumPartialResults(uint32_t n);
		void setBatchedStreams(const std::vector<int>& streamsToBatch);

		void registerBatch(const hidl_vec<CaptureRequest>& requests);
		void notify(NotifyMsg& msg);
		void processCaptureResult(CaptureResult& result);

		private:
		struct InflightBatch {
		// Protect access to entire struct. Acquire this lock before read/write any data or
		// calling any methods. processCaptureResult and notify will compete for this lock
		// HIDL IPCs might be issued while the lock is held
		Mutex mLock;

		bool allDelivered() const;

		uint32_t mFirstFrame;
		uint32_t mLastFrame;
		uint32_t mBatchSize;

		bool mShutterDelivered = false;
		std::vector<NotifyMsg> mShutterMsgs;

		struct BufferBatch {
		bool mDelivered = false;
		// This currently assumes every batched request will output to the batched stream
		// and since HAL must always send buffers in order, no frameNumber tracking is
		// needed
		std::vector<StreamBuffer> mBuffers;
		};
		// Stream ID -> VideoBatch
		std::unordered_map<int, BufferBatch> mBatchBufs;

		struct MetadataBatch {
		// (frameNumber, metadata)
		std::vector<std::pair<uint32_t, CameraMetadata>> mMds;
		};
		// Partial result IDs that has been delivered to framework
		uint32_t mNumPartialResults;
		uint32_t mPartialResultProgress = 0;
		// partialResult -> MetadataBatch
		std::map<uint32_t, MetadataBatch> mResultMds;

		// Set to true when batch is removed from mInflightBatches
		// processCaptureResult and notify must check this flag after acquiring mLock to make
		// sure this batch isn't removed while waiting for mLock
		bool mRemoved = false;
		};

		static const int NOT_BATCHED = -1;

		// Get the batch index and pointer to InflightBatch (nullptrt if the frame is not batched)
		// Caller must acquire the InflightBatch::mLock before accessing the InflightBatch
		// It's possible that the InflightBatch is removed from mInflightBatches before the
		// InflightBatch::mLock is acquired (most likely caused by an error notification), so
		// caller must check InflightBatch::mRemoved flag after the lock is acquried.
		// This method will hold ResultBatcher::mLock briefly
		std::pair<int, std::shared_ptr<InflightBatch>> getBatch(uint32_t frameNumber);

		// Check if the first batch in mInflightBatches is ready to be removed, and remove it if so
		// This method will hold ResultBatcher::mLock briefly
		void checkAndRemoveFirstBatch();

		// The following sendXXXX methods must be called while the InflightBatch::mLock is locked
		// HIDL IPC methods will be called during these methods.
		void sendBatchShutterCbsLocked(std::shared_ptr<InflightBatch> batch);
		// send buffers for all batched streams
		void sendBatchBuffersLocked(std::shared_ptr<InflightBatch> batch);
		// send buffers for specified streams
		void sendBatchBuffersLocked(
		std::shared_ptr<InflightBatch> batch, const std::vector<int>& streams);
		void sendBatchMetadataLocked(
		std::shared_ptr<InflightBatch> batch, uint32_t lastPartialResultIdx);
		// End of sendXXXX methods

		// helper methods
		void freeReleaseFences(hidl_vec<CaptureResult>&);
		void notifySingleMsg(NotifyMsg& msg);
		void processOneCaptureResult(CaptureResult& result);

		// Protect access to mInflightBatches, mNumPartialResults and mStreamsToBatch
		// processCaptureRequest, processCaptureResult, notify will compete for this lock
		// Do NOT issue HIDL IPCs while holding this lock (except when HAL reports error)
		mutable Mutex mLock;
		std::deque<std::shared_ptr<InflightBatch>> mInflightBatches;
		uint32_t mNumPartialResults;
		std::vector<int> mStreamsToBatch;
		const sp<ICameraDeviceCallback> mCallback;
		} mResultBatcher;

		std::vector<int> mVideoStreamIds;

		bool initialize();

		Status initStatus() const;
		@@ -129,6 +234,7 @@ private:

		void cleanupBuffersLocked(int id);

		Status processOneCaptureRequest(const CaptureRequest& request);
		/**
		* Static callback forwarding methods from HAL to instance
		*/