Merge "Camera: Add PreviewFrameScheduler to address preview jitter" (9ea3c686) · Commits · e / os / android_frameworks_av

services/camera/libcameraservice/Android.bp

+2 −0

Original line number	Diff line number	Diff line
		@@ -84,6 +84,7 @@ cc_library_shared {
		"device3/Camera3OutputUtils.cpp",
		"device3/Camera3DeviceInjectionMethods.cpp",
		"device3/UHRCropAndMeteringRegionMapper.cpp",
		"device3/PreviewFrameScheduler.cpp",
		"gui/RingBufferConsumer.cpp",
		"hidl/AidlCameraDeviceCallbacks.cpp",
		"hidl/AidlCameraServiceListener.cpp",
		@@ -107,6 +108,7 @@ cc_library_shared {
		],

		shared_libs: [
		"libandroid",
		"libbase",
		"libdl",
		"libexif",

services/camera/libcameraservice/device3/Camera3OutputStream.cpp

+45 −20

Original line number	Diff line number	Diff line
		@@ -22,6 +22,7 @@
		#include <fstream>

		#include <android-base/unique_fd.h>
		#include <cutils/properties.h>
		#include <ui/GraphicBuffer.h>
		#include <utils/Log.h>
		#include <utils/Trace.h>
		@@ -347,33 +348,40 @@ status_t Camera3OutputStream::returnBufferCheckedLocked(
		mTraceFirstBuffer = false;
		}

		if (transform != -1) {
		setTransformLocked(transform);
		// If this is a JPEG output, and image dump mask is set, save image to
		// disk.
		if (getFormat() == HAL_PIXEL_FORMAT_BLOB && getDataSpace() == HAL_DATASPACE_V0_JFIF &&
		mImageDumpMask) {
		dumpImageToDisk(timestamp, anwBuffer, anwReleaseFence);
		}

		/* Certain consumers (such as AudioSource or HardwareComposer) use
		* MONOTONIC time, causing time misalignment if camera timestamp is
		* in BOOTTIME. Do the conversion if necessary. */
		res = native_window_set_buffers_timestamp(mConsumer.get(),
		mUseMonoTimestamp ? timestamp - mTimestampOffset : timestamp);
		nsecs_t adjustedTs = mUseMonoTimestamp ? timestamp - mTimestampOffset : timestamp;
		if (mPreviewFrameScheduler != nullptr) {
		res = mPreviewFrameScheduler->queuePreviewBuffer(adjustedTs, transform,
		anwBuffer, anwReleaseFence);
		if (res != OK) {
		ALOGE("%s: Stream %d: Error setting timestamp: %s (%d)",
		ALOGE("%s: Stream %d: Error queuing buffer to preview buffer scheduler: %s (%d)",
		__FUNCTION__, mId, strerror(-res), res);
		return res;
		}
		// If this is a JPEG output, and image dump mask is set, save image to
		// disk.
		if (getFormat() == HAL_PIXEL_FORMAT_BLOB && getDataSpace() == HAL_DATASPACE_V0_JFIF &&
		mImageDumpMask) {
		dumpImageToDisk(timestamp, anwBuffer, anwReleaseFence);
		} else {
		setTransform(transform);
		res = native_window_set_buffers_timestamp(mConsumer.get(), adjustedTs);
		if (res != OK) {
		ALOGE("%s: Stream %d: Error setting timestamp: %s (%d)",
		__FUNCTION__, mId, strerror(-res), res);
		return res;
		}

		res = queueBufferToConsumer(currentConsumer, anwBuffer, anwReleaseFence, surface_ids);
		if (shouldLogError(res, state)) {
		ALOGE("%s: Stream %d: Error queueing buffer to native window:"
		" %s (%d)", __FUNCTION__, mId, strerror(-res), res);
		}
		}
		}
		mLock.lock();

		// Once a valid buffer has been returned to the queue, can no longer
		@@ -412,6 +420,9 @@ status_t Camera3OutputStream::setTransform(int transform) {

		status_t Camera3OutputStream::setTransformLocked(int transform) {
		status_t res = OK;

		if (transform == -1) return res;

		if (mState == STATE_ERROR) {
		ALOGE("%s: Stream in error state", __FUNCTION__);
		return INVALID_OPERATION;
		@@ -437,7 +448,7 @@ status_t Camera3OutputStream::configureQueueLocked() {
		return res;
		}

		if ((res = configureConsumerQueueLocked()) != OK) {
		if ((res = configureConsumerQueueLocked(true /allowPreviewScheduler/)) != OK) {
		return res;
		}

		@@ -461,7 +472,7 @@ status_t Camera3OutputStream::configureQueueLocked() {
		return OK;
		}

		status_t Camera3OutputStream::configureConsumerQueueLocked() {
		status_t Camera3OutputStream::configureConsumerQueueLocked(bool allowPreviewScheduler) {
		status_t res;

		mTraceFirstBuffer = true;
		@@ -547,6 +558,15 @@ status_t Camera3OutputStream::configureConsumerQueueLocked() {
		}

		mTotalBufferCount = maxConsumerBuffers + camera_stream::max_buffers;
		if (allowPreviewScheduler && isConsumedByHWComposer()) {
		// We cannot distinguish between a SurfaceView and an ImageReader of
		// preview buffer format. The PreviewFrameScheduler needs to handle both.
		if (!property_get_bool("camera.disable_preview_scheduler", false)) {
		mPreviewFrameScheduler = std::make_unique<PreviewFrameScheduler>(*this, mConsumer);
		mTotalBufferCount += PreviewFrameScheduler::kQueueDepthWatermark;
		}
		}

		mHandoutTotalBufferCount = 0;
		mFrameCount = 0;
		mLastTimestamp = 0;
		@@ -1185,6 +1205,11 @@ void Camera3OutputStream::returnPrefetchedBuffersLocked() {
		}
		}

		bool Camera3OutputStream::shouldLogError(status_t res) {
		Mutex::Autolock l(mLock);
		return shouldLogError(res, mState);
		}

		}; // namespace camera3

		}; // namespace android

services/camera/libcameraservice/device3/Camera3OutputStream.h

+5 −1

Original line number	Diff line number	Diff line
		@@ -27,6 +27,7 @@
		#include "Camera3IOStreamBase.h"
		#include "Camera3OutputStreamInterface.h"
		#include "Camera3BufferManager.h"
		#include "PreviewFrameScheduler.h"

		namespace android {

		@@ -229,6 +230,7 @@ class Camera3OutputStream :
		static void applyZSLUsageQuirk(int format, uint64_t consumerUsage /inout*/);

		void setImageDumpMask(int mask) { mImageDumpMask = mask; }
		bool shouldLogError(status_t res);

		protected:
		Camera3OutputStream(int id, camera_stream_type_t type,
		@@ -255,7 +257,7 @@ class Camera3OutputStream :

		status_t getEndpointUsageForSurface(uint64_t *usage,
		const sp<Surface>& surface) const;
		status_t configureConsumerQueueLocked();
		status_t configureConsumerQueueLocked(bool allowPreviewScheduler);

		// Consumer as the output of camera HAL
		sp<Surface> mConsumer;
		@@ -370,6 +372,8 @@ class Camera3OutputStream :

		int mImageDumpMask = 0;

		// The preview stream scheduler for re-timing frames
		std::unique_ptr<PreviewFrameScheduler> mPreviewFrameScheduler;
		}; // class Camera3OutputStream

		} // namespace camera3

services/camera/libcameraservice/device3/Camera3SharedOutputStream.cpp

+1 −1

Original line number	Diff line number	Diff line
		@@ -247,7 +247,7 @@ status_t Camera3SharedOutputStream::configureQueueLocked() {
		return res;
		}

		res = configureConsumerQueueLocked();
		res = configureConsumerQueueLocked(false/allowPreviewScheduler/);
		if (res != OK) {
		ALOGE("Failed to configureConsumerQueueLocked: %s(%d)", strerror(-res), res);
		return res;

services/camera/libcameraservice/device3/PreviewFrameScheduler.cpp

0 → 100644

+210 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2021 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#define LOG_TAG "Camera3-PreviewFrameScheduler"
		#define ATRACE_TAG ATRACE_TAG_CAMERA
		//#define LOG_NDEBUG 0

		#include <utils/Log.h>
		#include <utils/Trace.h>

		#include <android/looper.h>
		#include "PreviewFrameScheduler.h"
		#include "Camera3OutputStream.h"

		namespace android {

		namespace camera3 {

		/**
		* Internal Choreographer thread implementation for polling and handling callbacks
		*/

		// Callback function for Choreographer
		static void frameCallback(const AChoreographerFrameCallbackData* callbackData, void* data) {
		PreviewFrameScheduler* parent = static_cast<PreviewFrameScheduler*>(data);
		if (parent == nullptr) {
		ALOGE("%s: Invalid data for Choreographer callback!", __FUNCTION__);
		return;
		}

		size_t length = AChoreographerFrameCallbackData_getFrameTimelinesLength(callbackData);
		std::vector<nsecs_t> timeline(length);
		for (size_t i = 0; i < length; i++) {
		nsecs_t timestamp = AChoreographerFrameCallbackData_getFrameTimelineExpectedPresentTime(
		callbackData, i);
		timeline[i] = timestamp;
		}

		parent->onNewPresentationTime(timeline);

		AChoreographer_postExtendedFrameCallback(AChoreographer_getInstance(), frameCallback, data);
		}

		struct ChoreographerThread : public Thread {
		ChoreographerThread();
		status_t start(PreviewFrameScheduler* parent);
		virtual status_t readyToRun() override;
		virtual bool threadLoop() override;

		protected:
		virtual ~ChoreographerThread() {}

		private:
		ChoreographerThread &operator=(const ChoreographerThread &);

		// This only impacts the shutdown time. It won't impact the choreographer
		// callback frequency.
		static constexpr nsecs_t kPollingTimeoutMs = 5;
		PreviewFrameScheduler* mParent = nullptr;
		};

		ChoreographerThread::ChoreographerThread() : Thread(false /canCallJava/) {
		}

		status_t ChoreographerThread::start(PreviewFrameScheduler* parent) {
		mParent = parent;
		return run("PreviewChoreographer");
		}

		status_t ChoreographerThread::readyToRun() {
		ALooper_prepare(ALOOPER_PREPARE_ALLOW_NON_CALLBACKS);
		if (AChoreographer_getInstance() == NULL) {
		return NO_INIT;
		}

		AChoreographer_postExtendedFrameCallback(
		AChoreographer_getInstance(), frameCallback, mParent);
		return OK;
		}

		bool ChoreographerThread::threadLoop() {
		if (exitPending()) {
		return false;
		}
		ALooper_pollOnce(kPollingTimeoutMs, nullptr, nullptr, nullptr);
		return true;
		}

		/**
		* PreviewFrameScheduler implementation
		*/

		PreviewFrameScheduler::PreviewFrameScheduler(Camera3OutputStream& parent, sp<Surface> consumer) :
		mParent(parent),
		mConsumer(consumer),
		mChoreographerThread(new ChoreographerThread()) {
		}

		PreviewFrameScheduler::~PreviewFrameScheduler() {
		{
		Mutex::Autolock l(mLock);
		mChoreographerThread->requestExit();
		}
		mChoreographerThread->join();
		}

		status_t PreviewFrameScheduler::queuePreviewBuffer(nsecs_t timestamp, int32_t transform,
		ANativeWindowBuffer* anwBuffer, int releaseFence) {
		// Start choreographer thread if it's not already running.
		if (!mChoreographerThread->isRunning()) {
		status_t res = mChoreographerThread->start(this);
		if (res != OK) {
		ALOGE("%s: Failed to init choreographer thread!", __FUNCTION__);
		return res;
		}
		}

		{
		Mutex::Autolock l(mLock);
		mPendingBuffers.emplace(timestamp, transform, anwBuffer, releaseFence);

		// Queue buffer to client right away if pending buffers are more than
		// the queue depth watermark.
		if (mPendingBuffers.size() > kQueueDepthWatermark) {
		auto oldBuffer = mPendingBuffers.front();
		mPendingBuffers.pop();

		status_t res = queueBufferToClientLocked(oldBuffer, oldBuffer.timestamp);
		if (res != OK) {
		return res;
		}

		// Reset the last capture and presentation time
		mLastCameraCaptureTime = 0;
		mLastCameraPresentTime = 0;
		} else {
		ATRACE_INT(kPendingBufferTraceName, mPendingBuffers.size());
		}
		}
		return OK;
		}

		void PreviewFrameScheduler::onNewPresentationTime(const std::vector<nsecs_t>& timeline) {
		ATRACE_CALL();
		Mutex::Autolock l(mLock);
		if (mPendingBuffers.size() > 0) {
		auto nextBuffer = mPendingBuffers.front();
		mPendingBuffers.pop();

		// Find the best presentation time by finding the element in the
		// choreographer timeline that's closest to the ideal presentation time.
		// The ideal presentation time is the last presentation time + frame
		// interval.
		nsecs_t cameraInterval = nextBuffer.timestamp - mLastCameraCaptureTime;
		nsecs_t idealPresentTime = (cameraInterval < kSpacingResetIntervalNs) ?
		(mLastCameraPresentTime + cameraInterval) : nextBuffer.timestamp;
		nsecs_t presentTime = *std::min_element(timeline.begin(), timeline.end(),
		[idealPresentTime](nsecs_t p1, nsecs_t p2) {
		return std::abs(p1 - idealPresentTime) < std::abs(p2 - idealPresentTime);
		});

		status_t res = queueBufferToClientLocked(nextBuffer, presentTime);
		ATRACE_INT(kPendingBufferTraceName, mPendingBuffers.size());

		if (mParent.shouldLogError(res)) {
		ALOGE("%s: Preview Stream: Error queueing buffer to native window:"
		" %s (%d)", __FUNCTION__, strerror(-res), res);
		}

		mLastCameraCaptureTime = nextBuffer.timestamp;
		mLastCameraPresentTime = presentTime;
		}
		}

		status_t PreviewFrameScheduler::queueBufferToClientLocked(
		const BufferHolder& bufferHolder, nsecs_t timestamp) {
		mParent.setTransform(bufferHolder.transform);

		status_t res = native_window_set_buffers_timestamp(mConsumer.get(), timestamp);
		if (res != OK) {
		ALOGE("%s: Preview Stream: Error setting timestamp: %s (%d)",
		__FUNCTION__, strerror(-res), res);
		return res;
		}

		res = mConsumer->queueBuffer(mConsumer.get(), bufferHolder.anwBuffer.get(),
		bufferHolder.releaseFence);
		if (res != OK) {
		close(bufferHolder.releaseFence);
		}

		return res;
		}

		}; // namespace camera3

		}; // namespace android