Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 4c9bb1eb authored by Jan Sebechlebsky's avatar Jan Sebechlebsky
Browse files

Extend output format with downscalable resolutions 

Extends set of supported output formats with "standard"
resolutions with same aspect ratio as input which we can
downscale into.

Also fix bug, where the callback was always invoked with hardcoded
YUV format.

Add validation to ensure all streams in the session have same aspect
ratio (support for different ratios coming in follow-up).

Bug: 301023410
Test: atest virtual_camera_tests
Test: atest VirtualCameraTest
Change-Id: I42a46712c0b12e8a5ae3d75fa07e8f8ea47e8cc2
parent ec2aee4f

services/camera/virtualcamera/VirtualCameraDevice.cc

+61 −12
Original line number Diff line number Diff line
@@ -81,8 +81,6 @@ constexpr uint8_t kPipelineMaxDepth = 2; 


constexpr MetadataBuilder::ControlRegion kDefaultEmptyControlRegion{};



constexpr float kAspectRatioEpsilon = 0.05;



const std::array<Resolution, 5> kStandardJpegThumbnailSizes{

    Resolution(176, 144), Resolution(240, 144), Resolution(256, 144),

    Resolution(240, 160), Resolution(240, 180)};
@@ -91,14 +89,15 @@ const std::array<PixelFormat, 3> kOutputFormats{ 
    PixelFormat::IMPLEMENTATION_DEFINED, PixelFormat::YCBCR_420_888,

    PixelFormat::BLOB};



bool isApproximatellySameAspectRatio(const Resolution r1, const Resolution r2) {

  float aspectRatio1 =

      static_cast<float>(r1.width) / static_cast<float>(r1.height);

  float aspectRatio2 =

      static_cast<float>(r2.width) / static_cast<float>(r2.height);



  return abs(aspectRatio1 - aspectRatio2) < kAspectRatioEpsilon;

}

// The resolutions below will used to extend the set of supported output formats.

// All resolutions with lower pixel count and same aspect ratio as some supported

// input resolution will be added to the set of supported output resolutions.

const std::array<Resolution, 10> kOutputResolutions{

    Resolution(320, 240),   Resolution(640, 360),  Resolution(640, 480),

    Resolution(720, 480),   Resolution(720, 576),  Resolution(800, 600),

    Resolution(1024, 576),  Resolution(1280, 720), Resolution(1280, 960),

    Resolution(1280, 1080),

};



std::vector<Resolution> getSupportedJpegThumbnailSizes(

    const std::vector<SupportedStreamConfiguration>& configs) {
@@ -180,6 +179,36 @@ std::map<Resolution, int> getResolutionToMaxFpsMap( 
    }

  }



  std::map<Resolution, int> additionalResolutionToMaxFpsMap;

  // Add additional resolutions we can support by downscaling input streams with

  // same aspect ratio.

  for (const Resolution& outputResolution : kOutputResolutions) {

    for (const auto& [resolution, maxFps] : resolutionToMaxFpsMap) {

      if (resolutionToMaxFpsMap.find(outputResolution) !=

          resolutionToMaxFpsMap.end()) {

        // Resolution is already in the map, skip it.

        continue;

      }



      if (outputResolution < resolution &&

          isApproximatellySameAspectRatio(outputResolution, resolution)) {

        // Lower resolution with same aspect ratio, we can achieve this by

        // downscaling, let's add it to the map.

        ALOGD(

            "Extending set of output resolutions with %dx%d which has same "

            "aspect ratio as supported input %dx%d.",

            outputResolution.width, outputResolution.height, resolution.width,

            resolution.height);

        additionalResolutionToMaxFpsMap[outputResolution] = maxFps;

        break;

      }

    }

  }



  // Add all resolution we can achieve by downscaling to the map.

  resolutionToMaxFpsMap.insert(additionalResolutionToMaxFpsMap.begin(),

                               additionalResolutionToMaxFpsMap.end());



  return resolutionToMaxFpsMap;

}
@@ -401,6 +430,22 @@ bool VirtualCameraDevice::isStreamCombinationSupported( 
    return false;

  }



  const std::vector<Stream>& streams = streamConfiguration.streams;



  Resolution firstStreamResolution(streams[0].width, streams[0].height);

  auto isSameAspectRatioAsFirst = [firstStreamResolution](const Stream& stream) {

    return isApproximatellySameAspectRatio(

        firstStreamResolution, Resolution(stream.width, stream.height));

  };

  if (!std::all_of(streams.begin(), streams.end(), isSameAspectRatioAsFirst)) {

    ALOGW(

        "%s: Requested streams do not have same aspect ratio. Different aspect "

        "ratios are currently "

        "not supported by virtual camera. Stream configuration: %s",

        __func__, streamConfiguration.toString().c_str());

    return false;

  }



  int numberOfProcessedStreams = 0;

  int numberOfStallStreams = 0;

  for (const Stream& stream : streamConfiguration.streams) {
@@ -423,9 +468,13 @@ bool VirtualCameraDevice::isStreamCombinationSupported( 
      numberOfProcessedStreams++;

    }



    Resolution requestedResolution(stream.width, stream.height);

    auto matchesSupportedInputConfig =

        [&stream](const SupportedStreamConfiguration& config) {

          return stream.width == config.width && stream.height == config.height;

        [requestedResolution](const SupportedStreamConfiguration& config) {

          Resolution supportedInputResolution(config.width, config.height);

          return requestedResolution <= supportedInputResolution &&

                 isApproximatellySameAspectRatio(requestedResolution,

                                                 supportedInputResolution);

        };

    if (std::none_of(mSupportedInputConfigurations.begin(),

                     mSupportedInputConfigurations.end(),

services/camera/virtualcamera/VirtualCameraSession.cc

+65 −12
Original line number Diff line number Diff line
@@ -21,6 +21,7 @@ 
#include <algorithm>

#include <atomic>

#include <chrono>

#include <cmath>

#include <cstddef>

#include <cstdint>

#include <cstring>
@@ -39,6 +40,7 @@ 
#include "VirtualCameraDevice.h"

#include "VirtualCameraRenderThread.h"

#include "VirtualCameraStream.h"

#include "aidl/android/companion/virtualcamera/SupportedStreamConfiguration.h"

#include "aidl/android/hardware/camera/common/Status.h"

#include "aidl/android/hardware/camera/device/BufferCache.h"

#include "aidl/android/hardware/camera/device/BufferStatus.h"
@@ -48,6 +50,7 @@ 
#include "aidl/android/hardware/camera/device/NotifyMsg.h"

#include "aidl/android/hardware/camera/device/RequestTemplate.h"

#include "aidl/android/hardware/camera/device/ShutterMsg.h"

#include "aidl/android/hardware/camera/device/Stream.h"

#include "aidl/android/hardware/camera/device/StreamBuffer.h"

#include "aidl/android/hardware/camera/device/StreamConfiguration.h"

#include "aidl/android/hardware/camera/device/StreamRotation.h"
@@ -106,9 +109,6 @@ constexpr size_t kMetadataMsgQueueSize = 0; 
// Maximum number of buffers to use per single stream.

constexpr size_t kMaxStreamBuffers = 2;



constexpr int32_t kDefaultJpegQuality = 80;

constexpr int32_t kDefaultJpegThumbnailQuality = 70;



// Thumbnail size (0,0) correspods to disabling thumbnail.

const Resolution kDefaultJpegThumbnailSize(0, 0);
@@ -204,6 +204,55 @@ Stream getHighestResolutionStream(const std::vector<Stream>& streams) { 
                            }));

}



Resolution resolutionFromStream(const Stream& stream) {

  return Resolution(stream.width, stream.height);

}



Resolution resolutionFromInputConfig(

    const SupportedStreamConfiguration& inputConfig) {

  return Resolution(inputConfig.width, inputConfig.height);

}



std::optional<SupportedStreamConfiguration> pickInputConfigurationForStreams(

    const std::vector<Stream>& requestedStreams,

    const std::vector<SupportedStreamConfiguration>& supportedInputConfigs) {

  Stream maxResolutionStream = getHighestResolutionStream(requestedStreams);

  Resolution maxResolution = resolutionFromStream(maxResolutionStream);



  // Find best fitting stream to satisfy all requested streams:

  // Best fitting => same or higher resolution as input with lowest pixel count

  // difference and same aspect ratio.

  auto isBetterInputConfig = [maxResolution](

                                 const SupportedStreamConfiguration& configA,

                                 const SupportedStreamConfiguration& configB) {

    int maxResPixelCount = maxResolution.width * maxResolution.height;

    int pixelCountDiffA =

        std::abs((configA.width * configA.height) - maxResPixelCount);

    int pixelCountDiffB =

        std::abs((configB.width * configB.height) - maxResPixelCount);



    return pixelCountDiffA < pixelCountDiffB;

  };



  std::optional<SupportedStreamConfiguration> bestConfig;

  for (const SupportedStreamConfiguration& inputConfig : supportedInputConfigs) {

    Resolution inputConfigResolution = resolutionFromInputConfig(inputConfig);

    if (inputConfigResolution < maxResolution ||

        !isApproximatellySameAspectRatio(inputConfigResolution, maxResolution)) {

      // We don't want to upscale from lower resolution, or use different aspect

      // ratio, skip.

      continue;

    }



    if (!bestConfig.has_value() ||

        isBetterInputConfig(inputConfig, bestConfig.value())) {

      bestConfig = inputConfig;

    }

  }



  return bestConfig;

}



RequestSettings createSettingsFromMetadata(const CameraMetadata& metadata) {

  return RequestSettings{

      .jpegQuality = getJpegQuality(metadata).value_or(
@@ -279,15 +328,13 @@ ndk::ScopedAStatus VirtualCameraSession::configureStreams( 
  halStreams.clear();

  halStreams.resize(in_requestedConfiguration.streams.size());



  sp<Surface> inputSurface = nullptr;

  int inputWidth;

  int inputHeight;



  if (!virtualCamera->isStreamCombinationSupported(in_requestedConfiguration)) {

    ALOGE("%s: Requested stream configuration is not supported", __func__);

    return cameraStatus(Status::ILLEGAL_ARGUMENT);

  }



  sp<Surface> inputSurface = nullptr;

  std::optional<SupportedStreamConfiguration> inputConfig;

  {

    std::lock_guard<std::mutex> lock(mLock);

    for (int i = 0; i < in_requestedConfiguration.streams.size(); ++i) {
@@ -297,14 +344,20 @@ ndk::ScopedAStatus VirtualCameraSession::configureStreams( 
      }

    }



    Stream maxResStream = getHighestResolutionStream(streams);

    inputWidth = maxResStream.width;

    inputHeight = maxResStream.height;

    inputConfig = pickInputConfigurationForStreams(

        streams, virtualCamera->getInputConfigs());

    if (!inputConfig.has_value()) {

      ALOGE(

          "%s: Failed to pick any input configuration for stream configuration "

          "request: %s",

          __func__, in_requestedConfiguration.toString().c_str());

      return cameraStatus(Status::ILLEGAL_ARGUMENT);

    }

    if (mRenderThread == nullptr) {

      // If there's no client callback, start camera in test mode.

      const bool testMode = mVirtualCameraClientCallback == nullptr;

      mRenderThread = std::make_unique<VirtualCameraRenderThread>(

          mSessionContext, Resolution(inputWidth, inputHeight),

          mSessionContext, resolutionFromInputConfig(*inputConfig),

          virtualCamera->getMaxInputResolution(), mCameraDeviceCallback,

          testMode);

      mRenderThread->start();
@@ -318,7 +371,7 @@ ndk::ScopedAStatus VirtualCameraSession::configureStreams( 
    // create single texture.

    mVirtualCameraClientCallback->onStreamConfigured(

        /*streamId=*/0, aidl::android::view::Surface(inputSurface.get()),

        inputWidth, inputHeight, Format::YUV_420_888);

        inputConfig->width, inputConfig->height, inputConfig->pixelFormat);

  }



  return ndk::ScopedAStatus::ok();

services/camera/virtualcamera/tests/VirtualCameraDeviceTest.cc

+69 −28
Original line number Diff line number Diff line
@@ -55,6 +55,10 @@ using metadata_stream_t = 
    camera_metadata_enum_android_scaler_available_stream_configurations_t;



constexpr int kCameraId = 42;

constexpr int kQvgaWidth = 320;

constexpr int kQvgaHeight = 240;

constexpr int k360pWidth = 640;

constexpr int k360pHeight = 360;

constexpr int kVgaWidth = 640;

constexpr int kVgaHeight = 480;

constexpr int kHdWidth = 1280;
@@ -79,7 +83,8 @@ struct AvailableStreamConfiguration { 
  const int width;

  const int height;

  const int pixelFormat;

  const metadata_stream_t streamConfiguration;

  const metadata_stream_t streamConfiguration =

      ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT;

};



bool operator==(const AvailableStreamConfiguration& a,
@@ -173,24 +178,33 @@ INSTANTIATE_TEST_SUITE_P( 
                    .lensFacing = LensFacing::FRONT},

            .expectedAvailableStreamConfigs =

                {AvailableStreamConfiguration{

                     .width = kQvgaWidth,

                     .height = kQvgaHeight,

                     .pixelFormat =

                         ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888},

                 AvailableStreamConfiguration{

                     .width = kQvgaWidth,

                     .height = kQvgaHeight,

                     .pixelFormat =

                         ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED},

                 AvailableStreamConfiguration{

                     .width = kQvgaWidth,

                     .height = kQvgaHeight,

                     .pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_BLOB},

                 AvailableStreamConfiguration{

                     .width = kVgaWidth,

                     .height = kVgaHeight,

                     .pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888,

                     .streamConfiguration =

                         ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT},

                     .pixelFormat =

                         ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888},

                 AvailableStreamConfiguration{

                     .width = kVgaWidth,

                     .height = kVgaHeight,

                     .pixelFormat =

                         ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED,

                     .streamConfiguration =

                         ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT},

                         ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED},

                 AvailableStreamConfiguration{

                     .width = kVgaWidth,

                     .height = kVgaHeight,

                     .pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_BLOB,

                     .streamConfiguration =

                         ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT}}},

                     .pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_BLOB}}},

        VirtualCameraConfigTestParam{

            .inputConfig =

                VirtualCameraConfiguration{
@@ -209,44 +223,71 @@ INSTANTIATE_TEST_SUITE_P( 
                    .sensorOrientation = SensorOrientation::ORIENTATION_0,

                    .lensFacing = LensFacing::BACK},

            .expectedAvailableStreamConfigs = {

                AvailableStreamConfiguration{

                    .width = kQvgaWidth,

                    .height = kQvgaHeight,

                    .pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888},

                AvailableStreamConfiguration{

                    .width = kQvgaWidth,

                    .height = kQvgaHeight,

                    .pixelFormat =

                        ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED},

                AvailableStreamConfiguration{

                    .width = kQvgaWidth,

                    .height = kQvgaHeight,

                    .pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_BLOB},

                AvailableStreamConfiguration{

                    .width = 640,

                    .height = 360,

                    .pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888},

                AvailableStreamConfiguration{

                    .width = 640,

                    .height = 360,

                    .pixelFormat =

                        ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED},

                AvailableStreamConfiguration{

                    .width = 640,

                    .height = 360,

                    .pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_BLOB},

                AvailableStreamConfiguration{

                    .width = kVgaWidth,

                    .height = kVgaHeight,

                    .pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888,

                    .streamConfiguration =

                        ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT},

                    .pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888},

                AvailableStreamConfiguration{

                    .width = kVgaWidth,

                    .height = kVgaHeight,

                    .pixelFormat =

                        ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED,

                    .streamConfiguration =

                        ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT},

                        ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED},

                AvailableStreamConfiguration{

                    .width = kVgaWidth,

                    .height = kVgaHeight,

                    .pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_BLOB,

                    .streamConfiguration =

                        ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT},

                    .pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_BLOB},

                AvailableStreamConfiguration{

                    .width = 1024,

                    .height = 576,

                    .pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888},

                AvailableStreamConfiguration{

                    .width = 1024,

                    .height = 576,

                    .pixelFormat =

                        ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED},

                AvailableStreamConfiguration{

                    .width = 1024,

                    .height = 576,

                    .pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_BLOB},

                AvailableStreamConfiguration{

                    .width = kHdWidth,

                    .height = kHdHeight,

                    .pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888,

                    .streamConfiguration =

                        ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT},

                    .pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_YCbCr_420_888},

                AvailableStreamConfiguration{

                    .width = kHdWidth,

                    .height = kHdHeight,

                    .pixelFormat =

                        ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED,

                    .streamConfiguration =

                        ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT},

                        ANDROID_SCALER_AVAILABLE_FORMATS_IMPLEMENTATION_DEFINED},

                AvailableStreamConfiguration{

                    .width = kHdWidth,

                    .height = kHdHeight,

                    .pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_BLOB,

                    .streamConfiguration =

                        ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT}}}));

                    .pixelFormat = ANDROID_SCALER_AVAILABLE_FORMATS_BLOB}}}));



class VirtualCameraDeviceTest : public ::testing::Test {

 public:

services/camera/virtualcamera/tests/VirtualCameraSessionTest.cc

+124 −21

File changed.

Preview size limit exceeded, changes collapsed.

services/camera/virtualcamera/util/Util.h

+16 −0
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ 
#ifndef ANDROID_COMPANION_VIRTUALCAMERA_UTIL_H

#define ANDROID_COMPANION_VIRTUALCAMERA_UTIL_H



#include <cmath>

#include <cstdint>

#include <memory>
@@ -129,6 +130,10 @@ struct Resolution { 
                                     : pixCount < otherPixCount;

  }



  bool operator<=(const Resolution& other) const {

    return *this == other || *this < other;

  }



  bool operator==(const Resolution& other) const {

    return width == other.width && height == other.height;

  }
@@ -137,6 +142,17 @@ struct Resolution { 
  int height = 0;

};



inline bool isApproximatellySameAspectRatio(const Resolution r1,

                                            const Resolution r2) {

  static constexpr float kAspectRatioEpsilon = 0.05;

  float aspectRatio1 =

      static_cast<float>(r1.width) / static_cast<float>(r1.height);

  float aspectRatio2 =

      static_cast<float>(r2.width) / static_cast<float>(r2.height);



  return std::abs(aspectRatio1 - aspectRatio2) < kAspectRatioEpsilon;

}



std::ostream& operator<<(std::ostream& os, const Resolution& resolution);



}  // namespace virtualcamera