Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 042d1fb0 authored by Jan Sebechlebsky's avatar Jan Sebechlebsky
Browse files

Support RGBA input buffers. 

This change adds separate EGL shader for RGBA->YUV conversion
and modifies the JPEG compression to render input texture
into temporary framebuffer (doing the compression if necessary).

Bug: 301023410
Test: atest virtual_camera_tests
Test: atest VirtualCameraTest

Change-Id: Id3bd19d4c364691e2b1554fcf78d5f9940754314
parent ac166cfe

services/camera/virtualcamera/VirtualCameraDevice.cc

+3 −2
Original line number Diff line number Diff line
@@ -119,9 +119,10 @@ std::optional<CameraMetadata> initCameraCharacteristics( 
    const std::vector<SupportedStreamConfiguration>& supportedInputConfig) {

  if (!std::all_of(supportedInputConfig.begin(), supportedInputConfig.end(),

                   [](const SupportedStreamConfiguration& config) {

                     return config.pixelFormat == Format::YUV_420_888;

                     return isFormatSupportedForInput(

                         config.width, config.height, config.pixelFormat);

                   })) {

    ALOGE("%s: input configuration contains unsupported pixel format", __func__);

    ALOGE("%s: input configuration contains unsupported format", __func__);

    return std::nullopt;

  }

services/camera/virtualcamera/VirtualCameraRenderThread.cc

+94 −17
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ 


#include <chrono>

#include <cstddef>

#include <cstdint>

#include <future>

#include <memory>

#include <mutex>
@@ -38,6 +39,7 @@ 
#include "android-base/thread_annotations.h"

#include "android/binder_auto_utils.h"

#include "android/hardware_buffer.h"

#include "ui/GraphicBuffer.h"

#include "util/EglFramebuffer.h"

#include "util/JpegUtil.h"

#include "util/MetadataBuilder.h"
@@ -109,6 +111,45 @@ NotifyMsg createRequestErrorNotifyMsg(int frameNumber) { 
  return msg;

}



std::shared_ptr<EglFrameBuffer> allocateTemporaryFramebuffer(

    EGLDisplay eglDisplay, const uint width, const int height) {

  const AHardwareBuffer_Desc desc{

      .width = static_cast<uint32_t>(width),

      .height = static_cast<uint32_t>(height),

      .layers = 1,

      .format = AHARDWAREBUFFER_FORMAT_Y8Cb8Cr8_420,

      .usage = AHARDWAREBUFFER_USAGE_GPU_FRAMEBUFFER |

               AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN,

      .rfu0 = 0,

      .rfu1 = 0};



  AHardwareBuffer* hwBufferPtr;

  int status = AHardwareBuffer_allocate(&desc, &hwBufferPtr);

  if (status != NO_ERROR) {

    ALOGE(

        "%s: Failed to allocate hardware buffer for temporary framebuffer: %d",

        __func__, status);

    return nullptr;

  }



  return std::make_shared<EglFrameBuffer>(

      eglDisplay,

      std::shared_ptr<AHardwareBuffer>(hwBufferPtr, AHardwareBuffer_release));

}



bool isYuvFormat(const PixelFormat pixelFormat) {

  switch (static_cast<android_pixel_format_t>(pixelFormat)) {

    case HAL_PIXEL_FORMAT_YCBCR_422_I:

    case HAL_PIXEL_FORMAT_YCBCR_422_SP:

    case HAL_PIXEL_FORMAT_Y16:

    case HAL_PIXEL_FORMAT_YV12:

    case HAL_PIXEL_FORMAT_YCBCR_420_888:

      return true;

    default:

      return false;

  }

}



}  // namespace



CaptureRequestBuffer::CaptureRequestBuffer(int streamId, int bufferId,
@@ -218,7 +259,10 @@ void VirtualCameraRenderThread::threadLoop() { 
  ALOGV("Render thread starting");



  mEglDisplayContext = std::make_unique<EglDisplayContext>();

  mEglTextureProgram = std::make_unique<EglTextureProgram>();

  mEglTextureYuvProgram =

      std::make_unique<EglTextureProgram>(EglTextureProgram::TextureFormat::YUV);

  mEglTextureRgbProgram = std::make_unique<EglTextureProgram>(

      EglTextureProgram::TextureFormat::RGBA);

  mEglSurfaceTexture = std::make_unique<EglSurfaceTexture>(mInputSurfaceWidth,

                                                           mInputSurfaceHeight);

  mInputSurfacePromise.set_value(mEglSurfaceTexture->getSurface());
@@ -371,6 +415,22 @@ ndk::ScopedAStatus VirtualCameraRenderThread::renderIntoBlobStreamBuffer( 
    return cameraStatus(Status::INTERNAL_ERROR);

  }



  // Let's create YUV framebuffer and render the surface into this.

  // This will take care about rescaling as well as potential format conversion.

  std::shared_ptr<EglFrameBuffer> framebuffer = allocateTemporaryFramebuffer(

      mEglDisplayContext->getEglDisplay(), stream->width, stream->height);

  if (framebuffer == nullptr) {

    ALOGE("Failed to allocate temporary framebuffer for JPEG compression");

    return cameraStatus(Status::INTERNAL_ERROR);

  }



  // Render into temporary framebuffer.

  ndk::ScopedAStatus status = renderIntoEglFramebuffer(*framebuffer);

  if (!status.isOk()) {

    ALOGE("Failed to render input texture into temporary framebuffer");

    return status;

  }



  AHardwareBuffer_Planes planes_info;



  int32_t rawFence = fence != nullptr ? fence->get() : -1;
@@ -382,11 +442,15 @@ ndk::ScopedAStatus VirtualCameraRenderThread::renderIntoBlobStreamBuffer( 
    return cameraStatus(Status::INTERNAL_ERROR);

  }



  sp<GraphicBuffer> gBuffer = mEglSurfaceTexture->getCurrentBuffer();

  std::shared_ptr<AHardwareBuffer> inHwBuffer = framebuffer->getHardwareBuffer();

  GraphicBuffer* gBuffer = GraphicBuffer::fromAHardwareBuffer(inHwBuffer.get());



  bool compressionSuccess = true;

  if (gBuffer != nullptr) {

    android_ycbcr ycbcr;

    if (gBuffer->getPixelFormat() != HAL_PIXEL_FORMAT_YCbCr_420_888) {

      // This should never happen since we're allocating the temporary buffer

      // with YUV420 layout above.

      ALOGE("%s: Cannot compress non-YUV buffer (pixelFormat %d)", __func__,

            gBuffer->getPixelFormat());

      AHardwareBuffer_unlock(hwBuffer.get(), nullptr);
@@ -441,38 +505,51 @@ ndk::ScopedAStatus VirtualCameraRenderThread::renderIntoImageStreamBuffer( 
    return cameraStatus(Status::ILLEGAL_ARGUMENT);

  }



  ndk::ScopedAStatus status = renderIntoEglFramebuffer(*framebuffer, fence);



  const std::chrono::nanoseconds after =

      std::chrono::duration_cast<std::chrono::nanoseconds>(

          std::chrono::steady_clock::now().time_since_epoch());



  ALOGV("Rendering to buffer %d, stream %d took %lld ns", bufferId, streamId,

        after.count() - before.count());



  return ndk::ScopedAStatus::ok();

}



ndk::ScopedAStatus VirtualCameraRenderThread::renderIntoEglFramebuffer(

    EglFrameBuffer& framebuffer, sp<Fence> fence) {

  ALOGV("%s", __func__);

  // Wait for fence to clear.

  if (fence != nullptr && fence->isValid()) {

    status_t ret = fence->wait(kAcquireFenceTimeout.count());

    if (ret != 0) {

      ALOGE(

          "Timeout while waiting for the acquire fence for buffer %d"

          " for streamId %d",

          bufferId, streamId);

      ALOGE("Timeout while waiting for the acquire fence for buffer");

      return cameraStatus(Status::INTERNAL_ERROR);

    }

  }



  mEglDisplayContext->makeCurrent();

  framebuffer->beforeDraw();

  framebuffer.beforeDraw();



  if (mEglSurfaceTexture->getCurrentBuffer() == nullptr) {

  sp<GraphicBuffer> textureBuffer = mEglSurfaceTexture->getCurrentBuffer();

  if (textureBuffer == nullptr) {

    // If there's no current buffer, nothing was written to the surface and

    // texture is not initialized yet. Let's render the framebuffer black

    // instead of rendering the texture.

    glClearColor(0.0f, 0.5f, 0.5f, 0.0f);

    glClear(GL_COLOR_BUFFER_BIT);

  } else {

    mEglTextureProgram->draw(mEglSurfaceTexture->updateTexture());

    const bool renderSuccess =

        isYuvFormat(static_cast<PixelFormat>(textureBuffer->getPixelFormat()))

            ? mEglTextureYuvProgram->draw(mEglSurfaceTexture->updateTexture())

            : mEglTextureRgbProgram->draw(mEglSurfaceTexture->updateTexture());

    if (!renderSuccess) {

      ALOGE("%s: Failed to render texture", __func__);

      return cameraStatus(Status::INTERNAL_ERROR);

    }

  framebuffer->afterDraw();



  const std::chrono::nanoseconds after =

      std::chrono::duration_cast<std::chrono::nanoseconds>(

          std::chrono::steady_clock::now().time_since_epoch());



  ALOGV("Rendering to buffer %d, stream %d took %lld ns", bufferId, streamId,

        after.count() - before.count());

  }

  framebuffer.afterDraw();



  return ndk::ScopedAStatus::ok();

}

services/camera/virtualcamera/VirtualCameraRenderThread.h

+11 −1
Original line number Diff line number Diff line
@@ -24,7 +24,9 @@ 


#include "VirtualCameraSessionContext.h"

#include "aidl/android/hardware/camera/device/ICameraDeviceCallback.h"

#include "android/binder_auto_utils.h"

#include "util/EglDisplayContext.h"

#include "util/EglFramebuffer.h"

#include "util/EglProgram.h"

#include "util/EglSurfaceTexture.h"
@@ -135,6 +137,13 @@ class VirtualCameraRenderThread { 
  ndk::ScopedAStatus renderIntoImageStreamBuffer(int streamId, int bufferId,

                                                 sp<Fence> fence = nullptr);



  // Render current image into provided EglFramebuffer.

  // If fence is specified, this function will block until the fence is cleared

  // before writing to the buffer.

  // Always called on the render thread.

  ndk::ScopedAStatus renderIntoEglFramebuffer(EglFrameBuffer& framebuffer,

                                              sp<Fence> fence = nullptr);



  // Camera callback

  const std::shared_ptr<

      ::aidl::android::hardware::camera::device::ICameraDeviceCallback>
@@ -156,7 +165,8 @@ class VirtualCameraRenderThread { 


  // EGL helpers - constructed and accessed only from rendering thread.

  std::unique_ptr<EglDisplayContext> mEglDisplayContext;

  std::unique_ptr<EglTextureProgram> mEglTextureProgram;

  std::unique_ptr<EglTextureProgram> mEglTextureYuvProgram;

  std::unique_ptr<EglTextureProgram> mEglTextureRgbProgram;

  std::unique_ptr<EglSurfaceTexture> mEglSurfaceTexture;



  std::promise<sp<Surface>> mInputSurfacePromise;

services/camera/virtualcamera/aidl/android/companion/virtualcamera/Format.aidl

+1 −0
Original line number Diff line number Diff line
@@ -24,5 +24,6 @@ package android.companion.virtualcamera; 
@Backing(type="int")

enum Format {

    UNKNOWN = 0,

    RGBA_8888 = 1,

    YUV_420_888 = 0x23,

}

services/camera/virtualcamera/tests/EglUtilTest.cc

+14 −2
Original line number Diff line number Diff line
@@ -61,12 +61,24 @@ TEST_F(EglTest, EglTestPatternProgramSuccessfulInit) { 
  EXPECT_TRUE(eglTestPatternProgram.isInitialized());

}



TEST_F(EglTest, EglTextureProgramSuccessfulInit) {

TEST_F(EglTest, EglTextureProgramYuvSuccessfulInit) {

  if (!isGlExtensionSupported(kGlExtYuvTarget)) {

      GTEST_SKIP() << "Skipping test because of missing required GL extension " << kGlExtYuvTarget;

  }



  EglTextureProgram eglTextureProgram;

  EglTextureProgram eglTextureProgram(EglTextureProgram::TextureFormat::YUV);



  // Verify the shaders compiled and linked successfully.

  EXPECT_TRUE(eglTextureProgram.isInitialized());

}



TEST_F(EglTest, EglTextureProgramRgbaSuccessfulInit) {

  if (!isGlExtensionSupported(kGlExtYuvTarget)) {

      GTEST_SKIP() << "Skipping test because of missing required GL extension "

                   << kGlExtYuvTarget;

  }



  EglTextureProgram eglTextureProgram(EglTextureProgram::TextureFormat::RGBA);



  // Verify the shaders compiled and linked successfully.

  EXPECT_TRUE(eglTextureProgram.isInitialized());