Merge "Extend output format with downscalable resolutions" into main (8becaab2) · Commits · e / os / android_frameworks_av

services/camera/virtualcamera/VirtualCameraDevice.cc

+61 −12

Original line number	Diff line number	Diff line
		@@ -81,8 +81,6 @@ constexpr uint8_t kPipelineMaxDepth = 2;

		constexpr MetadataBuilder::ControlRegion kDefaultEmptyControlRegion{};

		constexpr float kAspectRatioEpsilon = 0.05;

		const std::array<Resolution, 5> kStandardJpegThumbnailSizes{
		Resolution(176, 144), Resolution(240, 144), Resolution(256, 144),
		Resolution(240, 160), Resolution(240, 180)};
		@@ -91,14 +89,15 @@ const std::array<PixelFormat, 3> kOutputFormats{
		PixelFormat::IMPLEMENTATION_DEFINED, PixelFormat::YCBCR_420_888,
		PixelFormat::BLOB};

		bool isApproximatellySameAspectRatio(const Resolution r1, const Resolution r2) {
		float aspectRatio1 =
		static_cast<float>(r1.width) / static_cast<float>(r1.height);
		float aspectRatio2 =
		static_cast<float>(r2.width) / static_cast<float>(r2.height);

		return abs(aspectRatio1 - aspectRatio2) < kAspectRatioEpsilon;
		}
		// The resolutions below will used to extend the set of supported output formats.
		// All resolutions with lower pixel count and same aspect ratio as some supported
		// input resolution will be added to the set of supported output resolutions.
		const std::array<Resolution, 10> kOutputResolutions{
		Resolution(320, 240), Resolution(640, 360), Resolution(640, 480),
		Resolution(720, 480), Resolution(720, 576), Resolution(800, 600),
		Resolution(1024, 576), Resolution(1280, 720), Resolution(1280, 960),
		Resolution(1280, 1080),
		};

		std::vector<Resolution> getSupportedJpegThumbnailSizes(
		const std::vector<SupportedStreamConfiguration>& configs) {
		@@ -180,6 +179,36 @@ std::map<Resolution, int> getResolutionToMaxFpsMap(
		}
		}

		std::map<Resolution, int> additionalResolutionToMaxFpsMap;
		// Add additional resolutions we can support by downscaling input streams with
		// same aspect ratio.
		for (const Resolution& outputResolution : kOutputResolutions) {
		for (const auto& [resolution, maxFps] : resolutionToMaxFpsMap) {
		if (resolutionToMaxFpsMap.find(outputResolution) !=
		resolutionToMaxFpsMap.end()) {
		// Resolution is already in the map, skip it.
		continue;
		}

		if (outputResolution < resolution &&
		isApproximatellySameAspectRatio(outputResolution, resolution)) {
		// Lower resolution with same aspect ratio, we can achieve this by
		// downscaling, let's add it to the map.
		ALOGD(
		"Extending set of output resolutions with %dx%d which has same "
		"aspect ratio as supported input %dx%d.",
		outputResolution.width, outputResolution.height, resolution.width,
		resolution.height);
		additionalResolutionToMaxFpsMap[outputResolution] = maxFps;
		break;
		}
		}
		}

		// Add all resolution we can achieve by downscaling to the map.
		resolutionToMaxFpsMap.insert(additionalResolutionToMaxFpsMap.begin(),
		additionalResolutionToMaxFpsMap.end());

		return resolutionToMaxFpsMap;
		}

		@@ -401,6 +430,22 @@ bool VirtualCameraDevice::isStreamCombinationSupported(
		return false;
		}

		const std::vector<Stream>& streams = streamConfiguration.streams;

		Resolution firstStreamResolution(streams[0].width, streams[0].height);
		auto isSameAspectRatioAsFirst = [firstStreamResolution](const Stream& stream) {
		return isApproximatellySameAspectRatio(
		firstStreamResolution, Resolution(stream.width, stream.height));
		};
		if (!std::all_of(streams.begin(), streams.end(), isSameAspectRatioAsFirst)) {
		ALOGW(
		"%s: Requested streams do not have same aspect ratio. Different aspect "
		"ratios are currently "
		"not supported by virtual camera. Stream configuration: %s",
		__func__, streamConfiguration.toString().c_str());
		return false;
		}

		int numberOfProcessedStreams = 0;
		int numberOfStallStreams = 0;
		for (const Stream& stream : streamConfiguration.streams) {
		@@ -423,9 +468,13 @@ bool VirtualCameraDevice::isStreamCombinationSupported(
		numberOfProcessedStreams++;
		}

		Resolution requestedResolution(stream.width, stream.height);
		auto matchesSupportedInputConfig =
		[&stream](const SupportedStreamConfiguration& config) {
		return stream.width == config.width && stream.height == config.height;
		[requestedResolution](const SupportedStreamConfiguration& config) {
		Resolution supportedInputResolution(config.width, config.height);
		return requestedResolution <= supportedInputResolution &&
		isApproximatellySameAspectRatio(requestedResolution,
		supportedInputResolution);
		};
		if (std::none_of(mSupportedInputConfigurations.begin(),
		mSupportedInputConfigurations.end(),

services/camera/virtualcamera/VirtualCameraSession.cc

+65 −12

Original line number	Diff line number	Diff line
		@@ -21,6 +21,7 @@
		#include <algorithm>
		#include <atomic>
		#include <chrono>
		#include <cmath>
		#include <cstddef>
		#include <cstdint>
		#include <cstring>
		@@ -39,6 +40,7 @@
		#include "VirtualCameraDevice.h"
		#include "VirtualCameraRenderThread.h"
		#include "VirtualCameraStream.h"
		#include "aidl/android/companion/virtualcamera/SupportedStreamConfiguration.h"
		#include "aidl/android/hardware/camera/common/Status.h"
		#include "aidl/android/hardware/camera/device/BufferCache.h"
		#include "aidl/android/hardware/camera/device/BufferStatus.h"
		@@ -48,6 +50,7 @@
		#include "aidl/android/hardware/camera/device/NotifyMsg.h"
		#include "aidl/android/hardware/camera/device/RequestTemplate.h"
		#include "aidl/android/hardware/camera/device/ShutterMsg.h"
		#include "aidl/android/hardware/camera/device/Stream.h"
		#include "aidl/android/hardware/camera/device/StreamBuffer.h"
		#include "aidl/android/hardware/camera/device/StreamConfiguration.h"
		#include "aidl/android/hardware/camera/device/StreamRotation.h"
		@@ -106,9 +109,6 @@ constexpr size_t kMetadataMsgQueueSize = 0;
		// Maximum number of buffers to use per single stream.
		constexpr size_t kMaxStreamBuffers = 2;

		constexpr int32_t kDefaultJpegQuality = 80;
		constexpr int32_t kDefaultJpegThumbnailQuality = 70;

		// Thumbnail size (0,0) correspods to disabling thumbnail.
		const Resolution kDefaultJpegThumbnailSize(0, 0);

		@@ -204,6 +204,55 @@ Stream getHighestResolutionStream(const std::vector<Stream>& streams) {
		}));
		}

		Resolution resolutionFromStream(const Stream& stream) {
		return Resolution(stream.width, stream.height);
		}

		Resolution resolutionFromInputConfig(
		const SupportedStreamConfiguration& inputConfig) {
		return Resolution(inputConfig.width, inputConfig.height);
		}

		std::optional<SupportedStreamConfiguration> pickInputConfigurationForStreams(
		const std::vector<Stream>& requestedStreams,
		const std::vector<SupportedStreamConfiguration>& supportedInputConfigs) {
		Stream maxResolutionStream = getHighestResolutionStream(requestedStreams);
		Resolution maxResolution = resolutionFromStream(maxResolutionStream);

		// Find best fitting stream to satisfy all requested streams:
		// Best fitting => same or higher resolution as input with lowest pixel count
		// difference and same aspect ratio.
		auto isBetterInputConfig = [maxResolution](
		const SupportedStreamConfiguration& configA,
		const SupportedStreamConfiguration& configB) {
		int maxResPixelCount = maxResolution.width * maxResolution.height;
		int pixelCountDiffA =
		std::abs((configA.width * configA.height) - maxResPixelCount);
		int pixelCountDiffB =
		std::abs((configB.width * configB.height) - maxResPixelCount);

		return pixelCountDiffA < pixelCountDiffB;
		};

		std::optional<SupportedStreamConfiguration> bestConfig;
		for (const SupportedStreamConfiguration& inputConfig : supportedInputConfigs) {
		Resolution inputConfigResolution = resolutionFromInputConfig(inputConfig);
		if (inputConfigResolution < maxResolution \|\|
		!isApproximatellySameAspectRatio(inputConfigResolution, maxResolution)) {
		// We don't want to upscale from lower resolution, or use different aspect
		// ratio, skip.
		continue;
		}

		if (!bestConfig.has_value() \|\|
		isBetterInputConfig(inputConfig, bestConfig.value())) {
		bestConfig = inputConfig;
		}
		}

		return bestConfig;
		}

		RequestSettings createSettingsFromMetadata(const CameraMetadata& metadata) {
		return RequestSettings{
		.jpegQuality = getJpegQuality(metadata).value_or(
		@@ -279,15 +328,13 @@ ndk::ScopedAStatus VirtualCameraSession::configureStreams(
		halStreams.clear();
		halStreams.resize(in_requestedConfiguration.streams.size());

		sp<Surface> inputSurface = nullptr;
		int inputWidth;
		int inputHeight;

		if (!virtualCamera->isStreamCombinationSupported(in_requestedConfiguration)) {
		ALOGE("%s: Requested stream configuration is not supported", __func__);
		return cameraStatus(Status::ILLEGAL_ARGUMENT);
		}

		sp<Surface> inputSurface = nullptr;
		std::optional<SupportedStreamConfiguration> inputConfig;
		{
		std::lock_guard<std::mutex> lock(mLock);
		for (int i = 0; i < in_requestedConfiguration.streams.size(); ++i) {
		@@ -297,14 +344,20 @@ ndk::ScopedAStatus VirtualCameraSession::configureStreams(
		}
		}

		Stream maxResStream = getHighestResolutionStream(streams);
		inputWidth = maxResStream.width;
		inputHeight = maxResStream.height;
		inputConfig = pickInputConfigurationForStreams(
		streams, virtualCamera->getInputConfigs());
		if (!inputConfig.has_value()) {
		ALOGE(
		"%s: Failed to pick any input configuration for stream configuration "
		"request: %s",
		__func__, in_requestedConfiguration.toString().c_str());
		return cameraStatus(Status::ILLEGAL_ARGUMENT);
		}
		if (mRenderThread == nullptr) {
		// If there's no client callback, start camera in test mode.
		const bool testMode = mVirtualCameraClientCallback == nullptr;
		mRenderThread = std::make_unique<VirtualCameraRenderThread>(
		mSessionContext, Resolution(inputWidth, inputHeight),
		mSessionContext, resolutionFromInputConfig(*inputConfig),
		virtualCamera->getMaxInputResolution(), mCameraDeviceCallback,
		testMode);
		mRenderThread->start();
		@@ -318,7 +371,7 @@ ndk::ScopedAStatus VirtualCameraSession::configureStreams(
		// create single texture.
		mVirtualCameraClientCallback->onStreamConfigured(
		/streamId=/0, aidl::android::view::Surface(inputSurface.get()),
		inputWidth, inputHeight, Format::YUV_420_888);
		inputConfig->width, inputConfig->height, inputConfig->pixelFormat);
		}

		return ndk::ScopedAStatus::ok();

services/camera/virtualcamera/tests/VirtualCameraDeviceTest.cc

+69 −28

Original line number	Diff line number	Diff line
		@@ -55,6 +55,10 @@ using metadata_stream_t =
		camera_metadata_enum_android_scaler_available_stream_configurations_t;

		constexpr int kCameraId = 42;
		constexpr int kQvgaWidth = 320;
		constexpr int kQvgaHeight = 240;
		constexpr int k360pWidth = 640;
		constexpr int k360pHeight = 360;
		constexpr int kVgaWidth = 640;
		constexpr int kVgaHeight = 480;
		constexpr int kHdWidth = 1280;
		@@ -79,7 +83,8 @@ struct AvailableStreamConfiguration {
		const int width;
		const int height;
		const int pixelFormat;
		const metadata_stream_t streamConfiguration;
		const metadata_stream_t streamConfiguration =
		ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT;
		};

		bool operator==(const AvailableStreamConfiguration& a,
		@@ -173,24 +178,33 @@ INSTANTIATE_TEST_SUITE_P(
		.lensFacing = LensFacing::FRONT},
		.expectedAvailableStreamConfigs =
		{AvailableStreamConfiguration{
		.width = kQvgaWidth,
		.height = kQvgaHeight,
		.pixelFormat =
		ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888},
		AvailableStreamConfiguration{
		.width = kQvgaWidth,
		.height = kQvgaHeight,
		.pixelFormat =
		ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED},
		AvailableStreamConfiguration{
		.width = kQvgaWidth,
		.height = kQvgaHeight,
		.pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_BLOB},
		AvailableStreamConfiguration{
		.width = kVgaWidth,
		.height = kVgaHeight,
		.pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888,
		.streamConfiguration =
		ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT},
		.pixelFormat =
		ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888},
		AvailableStreamConfiguration{
		.width = kVgaWidth,
		.height = kVgaHeight,
		.pixelFormat =
		ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED,
		.streamConfiguration =
		ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT},
		ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED},
		AvailableStreamConfiguration{
		.width = kVgaWidth,
		.height = kVgaHeight,
		.pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_BLOB,
		.streamConfiguration =
		ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT}}},
		.pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_BLOB}}},
		VirtualCameraConfigTestParam{
		.inputConfig =
		VirtualCameraConfiguration{
		@@ -209,44 +223,71 @@ INSTANTIATE_TEST_SUITE_P(
		.sensorOrientation = SensorOrientation::ORIENTATION_0,
		.lensFacing = LensFacing::BACK},
		.expectedAvailableStreamConfigs = {
		AvailableStreamConfiguration{
		.width = kQvgaWidth,
		.height = kQvgaHeight,
		.pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888},
		AvailableStreamConfiguration{
		.width = kQvgaWidth,
		.height = kQvgaHeight,
		.pixelFormat =
		ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED},
		AvailableStreamConfiguration{
		.width = kQvgaWidth,
		.height = kQvgaHeight,
		.pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_BLOB},
		AvailableStreamConfiguration{
		.width = 640,
		.height = 360,
		.pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888},
		AvailableStreamConfiguration{
		.width = 640,
		.height = 360,
		.pixelFormat =
		ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED},
		AvailableStreamConfiguration{
		.width = 640,
		.height = 360,
		.pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_BLOB},
		AvailableStreamConfiguration{
		.width = kVgaWidth,
		.height = kVgaHeight,
		.pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888,
		.streamConfiguration =
		ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT},
		.pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888},
		AvailableStreamConfiguration{
		.width = kVgaWidth,
		.height = kVgaHeight,
		.pixelFormat =
		ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED,
		.streamConfiguration =
		ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT},
		ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED},
		AvailableStreamConfiguration{
		.width = kVgaWidth,
		.height = kVgaHeight,
		.pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_BLOB,
		.streamConfiguration =
		ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT},
		.pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_BLOB},
		AvailableStreamConfiguration{
		.width = 1024,
		.height = 576,
		.pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888},
		AvailableStreamConfiguration{
		.width = 1024,
		.height = 576,
		.pixelFormat =
		ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED},
		AvailableStreamConfiguration{
		.width = 1024,
		.height = 576,
		.pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_BLOB},
		AvailableStreamConfiguration{
		.width = kHdWidth,
		.height = kHdHeight,
		.pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888,
		.streamConfiguration =
		ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT},
		.pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888},
		AvailableStreamConfiguration{
		.width = kHdWidth,
		.height = kHdHeight,
		.pixelFormat =
		ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED,
		.streamConfiguration =
		ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT},
		ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED},
		AvailableStreamConfiguration{
		.width = kHdWidth,
		.height = kHdHeight,
		.pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_BLOB,
		.streamConfiguration =
		ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT}}}));
		.pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_BLOB}}}));

		class VirtualCameraDeviceTest : public ::testing::Test {
		public:

services/camera/virtualcamera/tests/VirtualCameraSessionTest.cc

+124 −21

File changed.

Preview size limit exceeded, changes collapsed.

services/camera/virtualcamera/util/Util.h

+16 −0

Original line number	Diff line number	Diff line
		@@ -17,6 +17,7 @@
		#ifndef ANDROID_COMPANION_VIRTUALCAMERA_UTIL_H
		#define ANDROID_COMPANION_VIRTUALCAMERA_UTIL_H

		#include <cmath>
		#include <cstdint>
		#include <memory>

		@@ -129,6 +130,10 @@ struct Resolution {
		: pixCount < otherPixCount;
		}

		bool operator<=(const Resolution& other) const {
		return this == other \|\| this < other;
		}

		bool operator==(const Resolution& other) const {
		return width == other.width && height == other.height;
		}
		@@ -137,6 +142,17 @@ struct Resolution {
		int height = 0;
		};

		inline bool isApproximatellySameAspectRatio(const Resolution r1,
		const Resolution r2) {
		static constexpr float kAspectRatioEpsilon = 0.05;
		float aspectRatio1 =
		static_cast<float>(r1.width) / static_cast<float>(r1.height);
		float aspectRatio2 =
		static_cast<float>(r2.width) / static_cast<float>(r2.height);

		return std::abs(aspectRatio1 - aspectRatio2) < kAspectRatioEpsilon;
		}

		std::ostream& operator<<(std::ostream& os, const Resolution& resolution);

		} // namespace virtualcamera