Merge "Pass surface timestamp to the capture result" into main (927a0dd3) · Commits · e / os / android_frameworks_av

services/camera/virtualcamera/Android.bp

+1 −0

Original line number	Diff line number	Diff line
		@@ -25,6 +25,7 @@ cc_defaults {
		"libEGL",
		"libGLESv2",
		"libGLESv3",
		"android.companion.virtualdevice.flags-aconfig-cc",
		],
		static_libs: [
		"android.hardware.camera.common@1.0-helper",

services/camera/virtualcamera/VirtualCameraRenderThread.cc

+71 −33

Original line number	Diff line number	Diff line
		@@ -14,9 +14,12 @@
		* limitations under the License.
		*/

		// #define LOG_NDEBUG 0
		#define LOG_TAG "VirtualCameraRenderThread"
		#include "VirtualCameraRenderThread.h"

		#include <android_companion_virtualdevice_flags.h>

		#include <chrono>
		#include <cstdint>
		#include <cstring>
		@@ -46,13 +49,11 @@
		#include "android-base/thread_annotations.h"
		#include "android/binder_auto_utils.h"
		#include "android/hardware_buffer.h"
		#include "hardware/gralloc.h"
		#include "system/camera_metadata.h"
		#include "ui/GraphicBuffer.h"
		#include "ui/Rect.h"
		#include "util/EglFramebuffer.h"
		#include "util/JpegUtil.h"
		#include "util/MetadataUtil.h"
		#include "util/Util.h"
		#include "utils/Errors.h"

		@@ -91,6 +92,8 @@ overloaded(Ts...) -> overloaded<Ts...>;

		using namespace std::chrono_literals;

		namespace flags = ::android::companion::virtualdevice::flags;

		static constexpr std::chrono::milliseconds kAcquireFenceTimeout = 500ms;

		static constexpr size_t kJpegThumbnailBufferSize = 32 * 1024; // 32 KiB
		@@ -117,8 +120,8 @@ NotifyMsg createBufferErrorNotifyMsg(int frameNumber, int streamId) {

		NotifyMsg createRequestErrorNotifyMsg(int frameNumber) {
		NotifyMsg msg;
		msg.set<NotifyMsg::Tag::error>(ErrorMsg{
		.frameNumber = frameNumber,
		msg.set<NotifyMsg::Tag::error>(
		ErrorMsg{.frameNumber = frameNumber,
		// errorStreamId needs to be set to -1 for ERROR_REQUEST
		// (not tied to specific stream).
		.errorStreamId = -1,
		@@ -413,29 +416,8 @@ void VirtualCameraRenderThread::processTask(
		std::memory_order_relaxed));

		if (request.getRequestSettings().fpsRange) {
		const int maxFps =
		std::max(1, request.getRequestSettings().fpsRange->maxFps);
		const std::chrono::nanoseconds minFrameDuration(
		static_cast<uint64_t>(1e9 / maxFps));
		const std::chrono::nanoseconds frameDuration =
		timestamp - lastAcquisitionTimestamp;
		if (frameDuration < minFrameDuration) {
		// We're too fast for the configured maxFps, let's wait a bit.
		const std::chrono::nanoseconds sleepTime =
		minFrameDuration - frameDuration;
		ALOGV("Current frame duration would be %" PRIu64
		" ns corresponding to, "
		"sleeping for %" PRIu64
		" ns before updating texture to match maxFps %d",
		static_cast<uint64_t>(frameDuration.count()),
		static_cast<uint64_t>(sleepTime.count()), maxFps);

		std::this_thread::sleep_for(sleepTime);
		timestamp = std::chrono::duration_cast<std::chrono::nanoseconds>(
		std::chrono::steady_clock::now().time_since_epoch());
		mLastAcquisitionTimestampNanoseconds.store(timestamp.count(),
		std::memory_order_relaxed);
		}
		int maxFps = std::max(1, request.getRequestSettings().fpsRange->maxFps);
		timestamp = throttleRendering(maxFps, lastAcquisitionTimestamp, timestamp);
		}

		// Calculate the maximal amount of time we can afford to wait for next frame.
		@@ -463,6 +445,17 @@ void VirtualCameraRenderThread::processTask(
		}
		// Acquire new (most recent) image from the Surface.
		mEglSurfaceTexture->updateTexture();
		std::chrono::nanoseconds captureTimestamp = timestamp;

		if (flags::camera_timestamp_from_surface()) {
		std::chrono::nanoseconds surfaceTimestamp =
		getSurfaceTimestamp(elapsedDuration);
		if (surfaceTimestamp.count() > 0) {
		captureTimestamp = surfaceTimestamp;
		}
		ALOGV("%s captureTimestamp:%lld timestamp:%lld", __func__,
		captureTimestamp.count(), timestamp.count());
		}

		CaptureResult captureResult;
		captureResult.fmqResultSize = 0;
		@@ -472,7 +465,7 @@ void VirtualCameraRenderThread::processTask(
		captureResult.inputBuffer.streamId = -1;
		captureResult.physicalCameraMetadata.resize(0);
		captureResult.result = createCaptureResultMetadata(
		timestamp, request.getRequestSettings(), mReportedSensorSize);
		captureTimestamp, request.getRequestSettings(), mReportedSensorSize);

		const std::vector<CaptureRequestBuffer>& buffers = request.getBuffers();
		captureResult.outputBuffers.resize(buffers.size());
		@@ -506,7 +499,7 @@ void VirtualCameraRenderThread::processTask(
		}

		std::vector<NotifyMsg> notifyMsg{
		createShutterNotifyMsg(request.getFrameNumber(), timestamp)};
		createShutterNotifyMsg(request.getFrameNumber(), captureTimestamp)};
		for (const StreamBuffer& resBuffer : captureResult.outputBuffers) {
		if (resBuffer.status != BufferStatus::OK) {
		notifyMsg.push_back(createBufferErrorNotifyMsg(request.getFrameNumber(),
		@@ -535,6 +528,51 @@ void VirtualCameraRenderThread::processTask(
		ALOGV("%s: Successfully called processCaptureResult", __func__);
		}

		std::chrono::nanoseconds VirtualCameraRenderThread::throttleRendering(
		int maxFps, std::chrono::nanoseconds lastAcquisitionTimestamp,
		std::chrono::nanoseconds timestamp) {
		const std::chrono::nanoseconds minFrameDuration(
		static_cast<uint64_t>(1e9 / maxFps));
		const std::chrono::nanoseconds frameDuration =
		timestamp - lastAcquisitionTimestamp;
		if (frameDuration < minFrameDuration) {
		// We're too fast for the configured maxFps, let's wait a bit.
		const std::chrono::nanoseconds sleepTime = minFrameDuration - frameDuration;
		ALOGV("Current frame duration would be %" PRIu64
		" ns corresponding to, "
		"sleeping for %" PRIu64
		" ns before updating texture to match maxFps %d",
		static_cast<uint64_t>(frameDuration.count()),
		static_cast<uint64_t>(sleepTime.count()), maxFps);

		std::this_thread::sleep_for(sleepTime);
		timestamp = std::chrono::duration_cast<std::chrono::nanoseconds>(
		std::chrono::steady_clock::now().time_since_epoch());
		mLastAcquisitionTimestampNanoseconds.store(timestamp.count(),
		std::memory_order_relaxed);
		}
		return timestamp;
		}

		std::chrono::nanoseconds VirtualCameraRenderThread::getSurfaceTimestamp(
		std::chrono::nanoseconds timeSinceLastFrame) {
		std::chrono::nanoseconds surfaceTimestamp = mEglSurfaceTexture->getTimestamp();
		if (surfaceTimestamp.count() < 0) {
		uint64_t lastSurfaceTimestamp = mLastSurfaceTimestampNanoseconds.load();
		if (lastSurfaceTimestamp > 0) {
		// The timestamps were provided by the producer but we are
		// repeating the last frame, so we increase the previous timestamp by
		// the elapsed time sinced its capture, otherwise the camera framework
		// will discard the frame.
		surfaceTimestamp = std::chrono::nanoseconds(lastSurfaceTimestamp +
		timeSinceLastFrame.count());
		}
		}
		mLastSurfaceTimestampNanoseconds.store(surfaceTimestamp.count(),
		std::memory_order_relaxed);
		return surfaceTimestamp;
		}

		void VirtualCameraRenderThread::flushCaptureRequest(
		const ProcessCaptureRequestTask& request) {
		CaptureResult captureResult;

services/camera/virtualcamera/VirtualCameraRenderThread.h

+18 −0

Original line number	Diff line number	Diff line
		@@ -18,6 +18,7 @@
		#define ANDROID_COMPANION_VIRTUALCAMERA_VIRTUALCAMERARENDERTHREAD_H

		#include <atomic>
		#include <chrono>
		#include <cstdint>
		#include <deque>
		#include <future>
		@@ -188,6 +189,22 @@ class VirtualCameraRenderThread {
		EglFrameBuffer& framebuffer, sp<Fence> fence = nullptr,
		std::optional<Rect> viewport = std::nullopt);

		// Throttle the current thread to ensure that we are not rendering faster than
		// the provided maxFps.
		// maxFps: The maximum fps in the capture request
		// lastAcquisitionTimestamp: timestamp of the previous frame
		// timestamp: the current capture time
		// Returns the time at which the capture has happened after throttling.
		std::chrono::nanoseconds throttleRendering(
		int maxFps, std::chrono::nanoseconds lastAcquisitionTimestamp,
		std::chrono::nanoseconds timestamp);

		// Fetch the timestamp of the latest buffer from the EGL Surface
		// timeSinceLastFrame: The elapsed time since the last captured frame.
		// Return 0 if no timestamp has been associated to this surface by the producer.
		std::chrono::nanoseconds getSurfaceTimestamp(
		std::chrono::nanoseconds timeSinceLastFrame);

		// Camera callback
		const std::shared_ptr<
		::aidl::android::hardware::camera::device::ICameraDeviceCallback>
		@@ -209,6 +226,7 @@ class VirtualCameraRenderThread {

		// Acquisition timestamp of last frame.
		std::atomic<uint64_t> mLastAcquisitionTimestampNanoseconds;
		std::atomic<uint64_t> mLastSurfaceTimestampNanoseconds;

		// EGL helpers - constructed and accessed only from rendering thread.
		std::unique_ptr<EglDisplayContext> mEglDisplayContext;

services/camera/virtualcamera/util/EglSurfaceTexture.cc

+6 −0

Original line number	Diff line number	Diff line
		@@ -15,6 +15,8 @@
		*/

		// #define LOG_NDEBUG 0
		#include <chrono>

		#include "utils/Timers.h"
		#define LOG_TAG "EglSurfaceTexture"

		@@ -99,6 +101,10 @@ bool EglSurfaceTexture::waitForNextFrame(const std::chrono::nanoseconds timeout)
		static_cast<nsecs_t>(timeout.count()));
		}

		std::chrono::nanoseconds EglSurfaceTexture::getTimestamp() {
		return std::chrono::nanoseconds(mGlConsumer->getTimestamp());
		}

		GLuint EglSurfaceTexture::updateTexture() {
		int previousFrameId;
		int framesAdvance = 0;

services/camera/virtualcamera/util/EglSurfaceTexture.h

+4 −0

Original line number	Diff line number	Diff line
		@@ -82,6 +82,10 @@ class EglSurfaceTexture {
		// See SurfaceTexture.getTransformMatrix for more details.
		std::array<float, 16> getTransformMatrix();

		// Retrieves the timestamp associated with the texture image
		// set by the most recent call to updateTexture.
		std::chrono::nanoseconds getTimestamp();

		private:
		#if !COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ)
		sp<IGraphicBufferProducer> mBufferProducer;