Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit b3771312 authored by Jan Sebechlebsky's avatar Jan Sebechlebsky
Browse files

Add support for DCT-unaligned JPEG compression. 

This cl adds ability to specify viewport before rendering to buffer
and makes sure that the buffer always includes necessary padding
when rendering into buffer used for JPEG compression.

Bug: 301023410
Test: atest virtual_camera_tests
Test: Manually examining unaligned-sized thumbnails using exiftool
Change-Id: Id20cde4633ae396be9f9e8f9422b28d2b1475740
parent 22f70dbc

services/camera/virtualcamera/VirtualCameraRenderThread.cc

+30 −55
Original line number Diff line number Diff line
@@ -46,6 +46,7 @@ 
#include "android/hardware_buffer.h"

#include "system/camera_metadata.h"

#include "ui/GraphicBuffer.h"

#include "ui/Rect.h"

#include "util/EglFramebuffer.h"

#include "util/JpegUtil.h"

#include "util/MetadataUtil.h"
@@ -535,8 +536,9 @@ std::vector<uint8_t> VirtualCameraRenderThread::createThumbnail( 


  ALOGV("%s: Creating thumbnail with size %d x %d, quality %d", __func__,

        resolution.width, resolution.height, quality);

  Resolution bufferSize = roundTo2DctSize(resolution);

  std::shared_ptr<EglFrameBuffer> framebuffer = allocateTemporaryFramebuffer(

      mEglDisplayContext->getEglDisplay(), resolution.width, resolution.height);

      mEglDisplayContext->getEglDisplay(), bufferSize.width, bufferSize.height);

  if (framebuffer == nullptr) {

    ALOGE(

        "Failed to allocate temporary framebuffer for JPEG thumbnail "
@@ -547,37 +549,22 @@ std::vector<uint8_t> VirtualCameraRenderThread::createThumbnail( 
  // TODO(b/324383963) Add support for letterboxing if the thumbnail size

  // doesn't correspond

  //  to input texture aspect ratio.

  if (!renderIntoEglFramebuffer(*framebuffer).isOk()) {

  if (!renderIntoEglFramebuffer(*framebuffer, /*fence=*/nullptr,

                                Rect(resolution.width, resolution.height))

           .isOk()) {

    ALOGE(

        "Failed to render input texture into temporary framebuffer for JPEG "

        "thumbnail");

    return {};

  }



  std::shared_ptr<AHardwareBuffer> inHwBuffer = framebuffer->getHardwareBuffer();

  GraphicBuffer* gBuffer = GraphicBuffer::fromAHardwareBuffer(inHwBuffer.get());



  if (gBuffer->getPixelFormat() != HAL_PIXEL_FORMAT_YCbCr_420_888) {

    // This should never happen since we're allocating the temporary buffer

    // with YUV420 layout above.

    ALOGE("%s: Cannot compress non-YUV buffer (pixelFormat %d)", __func__,

          gBuffer->getPixelFormat());

    return {};

  }



  YCbCrLockGuard yCbCrLock(inHwBuffer, AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN);

  if (yCbCrLock.getStatus() != NO_ERROR) {

    ALOGE("%s: Failed to lock graphic buffer while generating thumbnail: %d",

          __func__, yCbCrLock.getStatus());

    return {};

  }



  std::vector<uint8_t> compressedThumbnail;

  compressedThumbnail.resize(kJpegThumbnailBufferSize);

  ALOGE("%s: Compressing thumbnail %d x %d", __func__, gBuffer->getWidth(),

        gBuffer->getHeight());

  std::optional<size_t> compressedSize = compressJpeg(

      gBuffer->getWidth(), gBuffer->getHeight(), quality, *yCbCrLock, {},

  ALOGE("%s: Compressing thumbnail %d x %d", __func__, resolution.width,

        resolution.height);

  std::optional<size_t> compressedSize =

      compressJpeg(resolution.width, resolution.height, quality,

                   framebuffer->getHardwareBuffer(), {},

                   compressedThumbnail.size(), compressedThumbnail.data());

  if (!compressedSize.has_value()) {

    ALOGE("%s: Failed to compress jpeg thumbnail", __func__);
@@ -609,15 +596,22 @@ ndk::ScopedAStatus VirtualCameraRenderThread::renderIntoBlobStreamBuffer( 


  // Let's create YUV framebuffer and render the surface into this.

  // This will take care about rescaling as well as potential format conversion.

  // The buffer dimensions need to be rounded to nearest multiple of JPEG DCT

  // size, however we pass the viewport corresponding to size of the stream so

  // the image will be only rendered to the area corresponding to the stream

  // size.

  Resolution bufferSize =

      roundTo2DctSize(Resolution(stream->width, stream->height));

  std::shared_ptr<EglFrameBuffer> framebuffer = allocateTemporaryFramebuffer(

      mEglDisplayContext->getEglDisplay(), stream->width, stream->height);

      mEglDisplayContext->getEglDisplay(), bufferSize.width, bufferSize.height);

  if (framebuffer == nullptr) {

    ALOGE("Failed to allocate temporary framebuffer for JPEG compression");

    return cameraStatus(Status::INTERNAL_ERROR);

  }



  // Render into temporary framebuffer.

  ndk::ScopedAStatus status = renderIntoEglFramebuffer(*framebuffer);

  ndk::ScopedAStatus status = renderIntoEglFramebuffer(

      *framebuffer, /*fence=*/nullptr, Rect(stream->width, stream->height));

  if (!status.isOk()) {

    ALOGE("Failed to render input texture into temporary framebuffer");

    return status;
@@ -629,38 +623,14 @@ ndk::ScopedAStatus VirtualCameraRenderThread::renderIntoBlobStreamBuffer( 
    return cameraStatus(Status::INTERNAL_ERROR);

  }



  std::shared_ptr<AHardwareBuffer> inHwBuffer = framebuffer->getHardwareBuffer();

  GraphicBuffer* gBuffer = GraphicBuffer::fromAHardwareBuffer(inHwBuffer.get());



  if (gBuffer == nullptr) {

    ALOGE(

        "%s: Encountered invalid temporary buffer while rendering JPEG "

        "into BLOB stream",

        __func__);

    return cameraStatus(Status::INTERNAL_ERROR);

  }



  if (gBuffer->getPixelFormat() != HAL_PIXEL_FORMAT_YCbCr_420_888) {

    // This should never happen since we're allocating the temporary buffer

    // with YUV420 layout above.

    ALOGE("%s: Cannot compress non-YUV buffer (pixelFormat %d)", __func__,

          gBuffer->getPixelFormat());

    return cameraStatus(Status::INTERNAL_ERROR);

  }



  YCbCrLockGuard yCbCrLock(inHwBuffer, AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN);

  if (yCbCrLock.getStatus() != OK) {

    return cameraStatus(Status::INTERNAL_ERROR);

  }



  std::vector<uint8_t> app1ExifData =

      createExif(Resolution(stream->width, stream->height), resultMetadata,

                 createThumbnail(requestSettings.thumbnailResolution,

                                 requestSettings.thumbnailJpegQuality));

  std::optional<size_t> compressedSize = compressJpeg(

      gBuffer->getWidth(), gBuffer->getHeight(), requestSettings.jpegQuality,

      *yCbCrLock, app1ExifData, stream->bufferSize - sizeof(CameraBlob),

      (*planesLock).planes[0].data);

      stream->width, stream->height, requestSettings.jpegQuality,

      framebuffer->getHardwareBuffer(), app1ExifData,

      stream->bufferSize - sizeof(CameraBlob), (*planesLock).planes[0].data);



  if (!compressedSize.has_value()) {

    ALOGE("%s: Failed to compress JPEG image", __func__);
@@ -714,7 +684,7 @@ ndk::ScopedAStatus VirtualCameraRenderThread::renderIntoImageStreamBuffer( 
}



ndk::ScopedAStatus VirtualCameraRenderThread::renderIntoEglFramebuffer(

    EglFrameBuffer& framebuffer, sp<Fence> fence) {

    EglFrameBuffer& framebuffer, sp<Fence> fence, std::optional<Rect> viewport) {

  ALOGV("%s", __func__);

  // Wait for fence to clear.

  if (fence != nullptr && fence->isValid()) {
@@ -728,6 +698,11 @@ ndk::ScopedAStatus VirtualCameraRenderThread::renderIntoEglFramebuffer( 
  mEglDisplayContext->makeCurrent();

  framebuffer.beforeDraw();



  Rect viewportRect =

      viewport.value_or(Rect(framebuffer.getWidth(), framebuffer.getHeight()));

  glViewport(viewportRect.leftTop().x, viewportRect.leftTop().y,

             viewportRect.getWidth(), viewportRect.getHeight());



  sp<GraphicBuffer> textureBuffer = mEglSurfaceTexture->getCurrentBuffer();

  if (textureBuffer == nullptr) {

    // If there's no current buffer, nothing was written to the surface and

services/camera/virtualcamera/VirtualCameraRenderThread.h

+3 −2
Original line number Diff line number Diff line
@@ -170,8 +170,9 @@ class VirtualCameraRenderThread { 
  // If fence is specified, this function will block until the fence is cleared

  // before writing to the buffer.

  // Always called on the render thread.

  ndk::ScopedAStatus renderIntoEglFramebuffer(EglFrameBuffer& framebuffer,

                                              sp<Fence> fence = nullptr);

  ndk::ScopedAStatus renderIntoEglFramebuffer(

      EglFrameBuffer& framebuffer, sp<Fence> fence = nullptr,

      std::optional<Rect> viewport = std::nullopt);



  // Camera callback

  const std::shared_ptr<

services/camera/virtualcamera/tests/Android.bp

+1 −0
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ cc_test { 
    ],

    srcs: [

        "EglUtilTest.cc",

        "JpegUtilTest.cc",

        "VirtualCameraDeviceTest.cc",

        "VirtualCameraProviderTest.cc",

        "VirtualCameraRenderThreadTest.cc",

services/camera/virtualcamera/tests/JpegUtilTest.cc

0 → 100644
+199 −0
Original line number Diff line number Diff line 
/*

 * Copyright 2023 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



#include <sys/types.h>



#include "system/graphics.h"

#define LOG_TAG "JpegUtilTest"



#include <array>

#include <cstdint>

#include <cstring>



#include "android/hardware_buffer.h"

#include "gmock/gmock.h"

#include "gtest/gtest.h"

#include "jpeglib.h"

#include "util/JpegUtil.h"

#include "util/Util.h"

#include "utils/Errors.h"



namespace android {

namespace companion {

namespace virtualcamera {

namespace {



using testing::Eq;

using testing::Gt;

using testing::Optional;

using testing::VariantWith;



constexpr int kOutputBufferSize = 1024 * 1024;  // 1 MiB.

constexpr int kJpegQuality = 80;



// Create black YUV420 buffer for testing purposes.

std::shared_ptr<AHardwareBuffer> createHardwareBufferForTest(const int width,

                                                             const int height) {

  const AHardwareBuffer_Desc desc{.width = static_cast<uint32_t>(width),

                                  .height = static_cast<uint32_t>(height),

                                  .layers = 1,

                                  .format = AHARDWAREBUFFER_FORMAT_Y8Cb8Cr8_420,

                                  .usage = AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN,

                                  .stride = 0,

                                  .rfu0 = 0,

                                  .rfu1 = 0};



  AHardwareBuffer* hwBufferPtr;

  int status = AHardwareBuffer_allocate(&desc, &hwBufferPtr);

  if (status != NO_ERROR) {

    ALOGE(

        "%s: Failed to allocate hardware buffer for temporary framebuffer: %d",

        __func__, status);

    return nullptr;

  }



  std::shared_ptr<AHardwareBuffer> hwBuffer(hwBufferPtr,

                                            AHardwareBuffer_release);



  YCbCrLockGuard yCbCrLock(hwBuffer, AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN);

  const android_ycbcr& ycbr = (*yCbCrLock);



  uint8_t* y = reinterpret_cast<uint8_t*>(ycbr.y);

  for (int r = 0; r < height; r++) {

    memset(y + r * ycbr.ystride, 0x00, width);

  }



  uint8_t* cb = reinterpret_cast<uint8_t*>(ycbr.cb);

  uint8_t* cr = reinterpret_cast<uint8_t*>(ycbr.cr);

  for (int r = 0; r < height / 2; r++) {

    for (int c = 0; c < width / 2; c++) {

      cb[r * ycbr.cstride + c * ycbr.chroma_step] = 0xff / 2;

      cr[r * ycbr.cstride + c * ycbr.chroma_step] = 0xff / 2;

    }

  }



  return hwBuffer;

}



// Decode JPEG header, return image resolution on success or error message on error.

std::variant<std::string, Resolution> verifyHeaderAndGetResolution(

    const uint8_t* data, int size) {

  struct jpeg_decompress_struct ctx;

  struct jpeg_error_mgr jerr;



  struct DecompressionError {

    bool success = true;

    std::string error;

  } result;



  ctx.client_data = &result;



  ctx.err = jpeg_std_error(&jerr);

  ctx.err->error_exit = [](j_common_ptr cinfo) {

    reinterpret_cast<DecompressionError*>(cinfo->client_data)->success = false;

  };

  ctx.err->output_message = [](j_common_ptr cinfo) {

    char buffer[JMSG_LENGTH_MAX];

    (*cinfo->err->format_message)(cinfo, buffer);

    reinterpret_cast<DecompressionError*>(cinfo->client_data)->error = buffer;

    ALOGE("libjpeg error: %s", buffer);

  };



  jpeg_create_decompress(&ctx);

  jpeg_mem_src(&ctx, data, size);

  jpeg_read_header(&ctx, /*require_image=*/true);



  if (!result.success) {

    jpeg_destroy_decompress(&ctx);

    return result.error;

  }



  Resolution resolution(ctx.image_width, ctx.image_height);

  jpeg_destroy_decompress(&ctx);

  return resolution;

}



TEST(JpegUtil, roundToDctSize) {

  EXPECT_THAT(roundTo2DctSize(Resolution(640, 480)), Eq(Resolution(640, 480)));

  EXPECT_THAT(roundTo2DctSize(Resolution(5, 5)), Eq(Resolution(16, 16)));

  EXPECT_THAT(roundTo2DctSize(Resolution(32, 32)), Eq(Resolution(32, 32)));

  EXPECT_THAT(roundTo2DctSize(Resolution(33, 32)), Eq(Resolution(48, 32)));

  EXPECT_THAT(roundTo2DctSize(Resolution(32, 33)), Eq(Resolution(32, 48)));

}



class JpegUtilTest : public ::testing::Test {

 public:

  void SetUp() override {

    std::fill(mOutputBuffer.begin(), mOutputBuffer.end(), 0);

  }



 protected:

  std::optional<size_t> compress(int imageWidth, int imageHeight,

                                 std::shared_ptr<AHardwareBuffer> inBuffer) {

    return compressJpeg(imageWidth, imageHeight, kJpegQuality, inBuffer,

                        /*app1ExifData=*/{}, mOutputBuffer.size(),

                        mOutputBuffer.data());

  }



  std::array<uint8_t, kOutputBufferSize> mOutputBuffer;

};



TEST_F(JpegUtilTest, compressImageSizeAlignedWithDctSucceeds) {

  std::shared_ptr<AHardwareBuffer> inBuffer =

      createHardwareBufferForTest(640, 480);



  std::optional<size_t> compressedSize = compress(640, 480, inBuffer);



  EXPECT_THAT(compressedSize, Optional(Gt(0)));

  EXPECT_THAT(verifyHeaderAndGetResolution(mOutputBuffer.data(),

                                           compressedSize.value()),

              VariantWith<Resolution>(Resolution(640, 480)));

}



TEST_F(JpegUtilTest, compressImageSizeNotAlignedWidthDctSucceeds) {

  std::shared_ptr<AHardwareBuffer> inBuffer =

      createHardwareBufferForTest(640, 480);



  std::optional<size_t> compressedSize = compress(630, 470, inBuffer);



  EXPECT_THAT(compressedSize, Optional(Gt(0)));

  EXPECT_THAT(verifyHeaderAndGetResolution(mOutputBuffer.data(),

                                           compressedSize.value()),

              VariantWith<Resolution>(Resolution(630, 470)));

}



TEST_F(JpegUtilTest, compressImageWithBufferNotAlignedWithDctFails) {

  std::shared_ptr<AHardwareBuffer> inBuffer =

      createHardwareBufferForTest(641, 480);



  std::optional<size_t> compressedSize = compress(640, 480, inBuffer);



  EXPECT_THAT(compressedSize, Eq(std::nullopt));

}



TEST_F(JpegUtilTest, compressImageWithBufferTooSmallFails) {

  std::shared_ptr<AHardwareBuffer> inBuffer =

      createHardwareBufferForTest(634, 464);



  std::optional<size_t> compressedSize = compress(640, 480, inBuffer);



  EXPECT_THAT(compressedSize, Eq(std::nullopt));

}



}  // namespace

}  // namespace virtualcamera

}  // namespace companion

}  // namespace android

services/camera/virtualcamera/tests/VirtualCameraServiceTest.cc

+0 −11
Original line number Diff line number Diff line
@@ -238,17 +238,6 @@ TEST_F(VirtualCameraServiceTest, ConfigurationWithTooHighResFails) { 
  EXPECT_THAT(getCameraIds(), IsEmpty());

}



TEST_F(VirtualCameraServiceTest, ConfigurationWithUnalignedResolutionFails) {

  bool aidlRet;

  VirtualCameraConfiguration config =

      createConfiguration(641, 481, Format::YUV_420_888, kMaxFps);



  ASSERT_FALSE(

      mCameraService->registerCamera(mNdkOwnerToken, config, &aidlRet).isOk());

  EXPECT_FALSE(aidlRet);

  EXPECT_THAT(getCameraIds(), IsEmpty());

}



TEST_F(VirtualCameraServiceTest, ConfigurationWithNegativeResolutionFails) {

  bool aidlRet;

  VirtualCameraConfiguration config =