Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 5a5b2e68 authored by Treehugger Robot's avatar Treehugger Robot Committed by Android (Google) Code Review
Browse files

Merge "Add support for DCT-unaligned JPEG compression." into main

parents b327c100 b3771312

services/camera/virtualcamera/VirtualCameraRenderThread.cc

+30 −55
Original line number Diff line number Diff line
@@ -46,6 +46,7 @@ 
#include "android/hardware_buffer.h"

#include "system/camera_metadata.h"

#include "ui/GraphicBuffer.h"

#include "ui/Rect.h"

#include "util/EglFramebuffer.h"

#include "util/JpegUtil.h"

#include "util/MetadataUtil.h"
@@ -535,8 +536,9 @@ std::vector<uint8_t> VirtualCameraRenderThread::createThumbnail( 


  ALOGV("%s: Creating thumbnail with size %d x %d, quality %d", __func__,

        resolution.width, resolution.height, quality);

  Resolution bufferSize = roundTo2DctSize(resolution);

  std::shared_ptr<EglFrameBuffer> framebuffer = allocateTemporaryFramebuffer(

      mEglDisplayContext->getEglDisplay(), resolution.width, resolution.height);

      mEglDisplayContext->getEglDisplay(), bufferSize.width, bufferSize.height);

  if (framebuffer == nullptr) {

    ALOGE(

        "Failed to allocate temporary framebuffer for JPEG thumbnail "
@@ -547,37 +549,22 @@ std::vector<uint8_t> VirtualCameraRenderThread::createThumbnail( 
  // TODO(b/324383963) Add support for letterboxing if the thumbnail size

  // doesn't correspond

  //  to input texture aspect ratio.

  if (!renderIntoEglFramebuffer(*framebuffer).isOk()) {

  if (!renderIntoEglFramebuffer(*framebuffer, /*fence=*/nullptr,

                                Rect(resolution.width, resolution.height))

           .isOk()) {

    ALOGE(

        "Failed to render input texture into temporary framebuffer for JPEG "

        "thumbnail");

    return {};

  }



  std::shared_ptr<AHardwareBuffer> inHwBuffer = framebuffer->getHardwareBuffer();

  GraphicBuffer* gBuffer = GraphicBuffer::fromAHardwareBuffer(inHwBuffer.get());



  if (gBuffer->getPixelFormat() != HAL_PIXEL_FORMAT_YCbCr_420_888) {

    // This should never happen since we're allocating the temporary buffer

    // with YUV420 layout above.

    ALOGE("%s: Cannot compress non-YUV buffer (pixelFormat %d)", __func__,

          gBuffer->getPixelFormat());

    return {};

  }



  YCbCrLockGuard yCbCrLock(inHwBuffer, AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN);

  if (yCbCrLock.getStatus() != NO_ERROR) {

    ALOGE("%s: Failed to lock graphic buffer while generating thumbnail: %d",

          __func__, yCbCrLock.getStatus());

    return {};

  }



  std::vector<uint8_t> compressedThumbnail;

  compressedThumbnail.resize(kJpegThumbnailBufferSize);

  ALOGE("%s: Compressing thumbnail %d x %d", __func__, gBuffer->getWidth(),

        gBuffer->getHeight());

  std::optional<size_t> compressedSize = compressJpeg(

      gBuffer->getWidth(), gBuffer->getHeight(), quality, *yCbCrLock, {},

  ALOGE("%s: Compressing thumbnail %d x %d", __func__, resolution.width,

        resolution.height);

  std::optional<size_t> compressedSize =

      compressJpeg(resolution.width, resolution.height, quality,

                   framebuffer->getHardwareBuffer(), {},

                   compressedThumbnail.size(), compressedThumbnail.data());

  if (!compressedSize.has_value()) {

    ALOGE("%s: Failed to compress jpeg thumbnail", __func__);
@@ -609,15 +596,22 @@ ndk::ScopedAStatus VirtualCameraRenderThread::renderIntoBlobStreamBuffer( 


  // Let's create YUV framebuffer and render the surface into this.

  // This will take care about rescaling as well as potential format conversion.

  // The buffer dimensions need to be rounded to nearest multiple of JPEG DCT

  // size, however we pass the viewport corresponding to size of the stream so

  // the image will be only rendered to the area corresponding to the stream

  // size.

  Resolution bufferSize =

      roundTo2DctSize(Resolution(stream->width, stream->height));

  std::shared_ptr<EglFrameBuffer> framebuffer = allocateTemporaryFramebuffer(

      mEglDisplayContext->getEglDisplay(), stream->width, stream->height);

      mEglDisplayContext->getEglDisplay(), bufferSize.width, bufferSize.height);

  if (framebuffer == nullptr) {

    ALOGE("Failed to allocate temporary framebuffer for JPEG compression");

    return cameraStatus(Status::INTERNAL_ERROR);

  }



  // Render into temporary framebuffer.

  ndk::ScopedAStatus status = renderIntoEglFramebuffer(*framebuffer);

  ndk::ScopedAStatus status = renderIntoEglFramebuffer(

      *framebuffer, /*fence=*/nullptr, Rect(stream->width, stream->height));

  if (!status.isOk()) {

    ALOGE("Failed to render input texture into temporary framebuffer");

    return status;
@@ -629,38 +623,14 @@ ndk::ScopedAStatus VirtualCameraRenderThread::renderIntoBlobStreamBuffer( 
    return cameraStatus(Status::INTERNAL_ERROR);

  }



  std::shared_ptr<AHardwareBuffer> inHwBuffer = framebuffer->getHardwareBuffer();

  GraphicBuffer* gBuffer = GraphicBuffer::fromAHardwareBuffer(inHwBuffer.get());



  if (gBuffer == nullptr) {

    ALOGE(

        "%s: Encountered invalid temporary buffer while rendering JPEG "

        "into BLOB stream",

        __func__);

    return cameraStatus(Status::INTERNAL_ERROR);

  }



  if (gBuffer->getPixelFormat() != HAL_PIXEL_FORMAT_YCbCr_420_888) {

    // This should never happen since we're allocating the temporary buffer

    // with YUV420 layout above.

    ALOGE("%s: Cannot compress non-YUV buffer (pixelFormat %d)", __func__,

          gBuffer->getPixelFormat());

    return cameraStatus(Status::INTERNAL_ERROR);

  }



  YCbCrLockGuard yCbCrLock(inHwBuffer, AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN);

  if (yCbCrLock.getStatus() != OK) {

    return cameraStatus(Status::INTERNAL_ERROR);

  }



  std::vector<uint8_t> app1ExifData =

      createExif(Resolution(stream->width, stream->height), resultMetadata,

                 createThumbnail(requestSettings.thumbnailResolution,

                                 requestSettings.thumbnailJpegQuality));

  std::optional<size_t> compressedSize = compressJpeg(

      gBuffer->getWidth(), gBuffer->getHeight(), requestSettings.jpegQuality,

      *yCbCrLock, app1ExifData, stream->bufferSize - sizeof(CameraBlob),

      (*planesLock).planes[0].data);

      stream->width, stream->height, requestSettings.jpegQuality,

      framebuffer->getHardwareBuffer(), app1ExifData,

      stream->bufferSize - sizeof(CameraBlob), (*planesLock).planes[0].data);



  if (!compressedSize.has_value()) {

    ALOGE("%s: Failed to compress JPEG image", __func__);
@@ -714,7 +684,7 @@ ndk::ScopedAStatus VirtualCameraRenderThread::renderIntoImageStreamBuffer( 
}



ndk::ScopedAStatus VirtualCameraRenderThread::renderIntoEglFramebuffer(

    EglFrameBuffer& framebuffer, sp<Fence> fence) {

    EglFrameBuffer& framebuffer, sp<Fence> fence, std::optional<Rect> viewport) {

  ALOGV("%s", __func__);

  // Wait for fence to clear.

  if (fence != nullptr && fence->isValid()) {
@@ -728,6 +698,11 @@ ndk::ScopedAStatus VirtualCameraRenderThread::renderIntoEglFramebuffer( 
  mEglDisplayContext->makeCurrent();

  framebuffer.beforeDraw();



  Rect viewportRect =

      viewport.value_or(Rect(framebuffer.getWidth(), framebuffer.getHeight()));

  glViewport(viewportRect.leftTop().x, viewportRect.leftTop().y,

             viewportRect.getWidth(), viewportRect.getHeight());



  sp<GraphicBuffer> textureBuffer = mEglSurfaceTexture->getCurrentBuffer();

  if (textureBuffer == nullptr) {

    // If there's no current buffer, nothing was written to the surface and

services/camera/virtualcamera/VirtualCameraRenderThread.h

+3 −2
Original line number Diff line number Diff line
@@ -170,8 +170,9 @@ class VirtualCameraRenderThread { 
  // If fence is specified, this function will block until the fence is cleared

  // before writing to the buffer.

  // Always called on the render thread.

  ndk::ScopedAStatus renderIntoEglFramebuffer(EglFrameBuffer& framebuffer,

                                              sp<Fence> fence = nullptr);

  ndk::ScopedAStatus renderIntoEglFramebuffer(

      EglFrameBuffer& framebuffer, sp<Fence> fence = nullptr,

      std::optional<Rect> viewport = std::nullopt);



  // Camera callback

  const std::shared_ptr<

services/camera/virtualcamera/tests/Android.bp

+1 −0
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ cc_test { 
    ],

    srcs: [

        "EglUtilTest.cc",

        "JpegUtilTest.cc",

        "VirtualCameraDeviceTest.cc",

        "VirtualCameraProviderTest.cc",

        "VirtualCameraRenderThreadTest.cc",

services/camera/virtualcamera/tests/JpegUtilTest.cc

0 → 100644
+199 −0
Original line number Diff line number Diff line 
/*

 * Copyright 2023 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



#include <sys/types.h>



#include "system/graphics.h"

#define LOG_TAG "JpegUtilTest"



#include <array>

#include <cstdint>

#include <cstring>



#include "android/hardware_buffer.h"

#include "gmock/gmock.h"

#include "gtest/gtest.h"

#include "jpeglib.h"

#include "util/JpegUtil.h"

#include "util/Util.h"

#include "utils/Errors.h"



namespace android {

namespace companion {

namespace virtualcamera {

namespace {



using testing::Eq;

using testing::Gt;

using testing::Optional;

using testing::VariantWith;



constexpr int kOutputBufferSize = 1024 * 1024;  // 1 MiB.

constexpr int kJpegQuality = 80;



// Create black YUV420 buffer for testing purposes.

std::shared_ptr<AHardwareBuffer> createHardwareBufferForTest(const int width,

                                                             const int height) {

  const AHardwareBuffer_Desc desc{.width = static_cast<uint32_t>(width),

                                  .height = static_cast<uint32_t>(height),

                                  .layers = 1,

                                  .format = AHARDWAREBUFFER_FORMAT_Y8Cb8Cr8_420,

                                  .usage = AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN,

                                  .stride = 0,

                                  .rfu0 = 0,

                                  .rfu1 = 0};



  AHardwareBuffer* hwBufferPtr;

  int status = AHardwareBuffer_allocate(&desc, &hwBufferPtr);

  if (status != NO_ERROR) {

    ALOGE(

        "%s: Failed to allocate hardware buffer for temporary framebuffer: %d",

        __func__, status);

    return nullptr;

  }



  std::shared_ptr<AHardwareBuffer> hwBuffer(hwBufferPtr,

                                            AHardwareBuffer_release);



  YCbCrLockGuard yCbCrLock(hwBuffer, AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN);

  const android_ycbcr& ycbr = (*yCbCrLock);



  uint8_t* y = reinterpret_cast<uint8_t*>(ycbr.y);

  for (int r = 0; r < height; r++) {

    memset(y + r * ycbr.ystride, 0x00, width);

  }



  uint8_t* cb = reinterpret_cast<uint8_t*>(ycbr.cb);

  uint8_t* cr = reinterpret_cast<uint8_t*>(ycbr.cr);

  for (int r = 0; r < height / 2; r++) {

    for (int c = 0; c < width / 2; c++) {

      cb[r * ycbr.cstride + c * ycbr.chroma_step] = 0xff / 2;

      cr[r * ycbr.cstride + c * ycbr.chroma_step] = 0xff / 2;

    }

  }



  return hwBuffer;

}



// Decode JPEG header, return image resolution on success or error message on error.

std::variant<std::string, Resolution> verifyHeaderAndGetResolution(

    const uint8_t* data, int size) {

  struct jpeg_decompress_struct ctx;

  struct jpeg_error_mgr jerr;



  struct DecompressionError {

    bool success = true;

    std::string error;

  } result;



  ctx.client_data = &result;



  ctx.err = jpeg_std_error(&jerr);

  ctx.err->error_exit = [](j_common_ptr cinfo) {

    reinterpret_cast<DecompressionError*>(cinfo->client_data)->success = false;

  };

  ctx.err->output_message = [](j_common_ptr cinfo) {

    char buffer[JMSG_LENGTH_MAX];

    (*cinfo->err->format_message)(cinfo, buffer);

    reinterpret_cast<DecompressionError*>(cinfo->client_data)->error = buffer;

    ALOGE("libjpeg error: %s", buffer);

  };



  jpeg_create_decompress(&ctx);

  jpeg_mem_src(&ctx, data, size);

  jpeg_read_header(&ctx, /*require_image=*/true);



  if (!result.success) {

    jpeg_destroy_decompress(&ctx);

    return result.error;

  }



  Resolution resolution(ctx.image_width, ctx.image_height);

  jpeg_destroy_decompress(&ctx);

  return resolution;

}



TEST(JpegUtil, roundToDctSize) {

  EXPECT_THAT(roundTo2DctSize(Resolution(640, 480)), Eq(Resolution(640, 480)));

  EXPECT_THAT(roundTo2DctSize(Resolution(5, 5)), Eq(Resolution(16, 16)));

  EXPECT_THAT(roundTo2DctSize(Resolution(32, 32)), Eq(Resolution(32, 32)));

  EXPECT_THAT(roundTo2DctSize(Resolution(33, 32)), Eq(Resolution(48, 32)));

  EXPECT_THAT(roundTo2DctSize(Resolution(32, 33)), Eq(Resolution(32, 48)));

}



class JpegUtilTest : public ::testing::Test {

 public:

  void SetUp() override {

    std::fill(mOutputBuffer.begin(), mOutputBuffer.end(), 0);

  }



 protected:

  std::optional<size_t> compress(int imageWidth, int imageHeight,

                                 std::shared_ptr<AHardwareBuffer> inBuffer) {

    return compressJpeg(imageWidth, imageHeight, kJpegQuality, inBuffer,

                        /*app1ExifData=*/{}, mOutputBuffer.size(),

                        mOutputBuffer.data());

  }



  std::array<uint8_t, kOutputBufferSize> mOutputBuffer;

};



TEST_F(JpegUtilTest, compressImageSizeAlignedWithDctSucceeds) {

  std::shared_ptr<AHardwareBuffer> inBuffer =

      createHardwareBufferForTest(640, 480);



  std::optional<size_t> compressedSize = compress(640, 480, inBuffer);



  EXPECT_THAT(compressedSize, Optional(Gt(0)));

  EXPECT_THAT(verifyHeaderAndGetResolution(mOutputBuffer.data(),

                                           compressedSize.value()),

              VariantWith<Resolution>(Resolution(640, 480)));

}



TEST_F(JpegUtilTest, compressImageSizeNotAlignedWidthDctSucceeds) {

  std::shared_ptr<AHardwareBuffer> inBuffer =

      createHardwareBufferForTest(640, 480);



  std::optional<size_t> compressedSize = compress(630, 470, inBuffer);



  EXPECT_THAT(compressedSize, Optional(Gt(0)));

  EXPECT_THAT(verifyHeaderAndGetResolution(mOutputBuffer.data(),

                                           compressedSize.value()),

              VariantWith<Resolution>(Resolution(630, 470)));

}



TEST_F(JpegUtilTest, compressImageWithBufferNotAlignedWithDctFails) {

  std::shared_ptr<AHardwareBuffer> inBuffer =

      createHardwareBufferForTest(641, 480);



  std::optional<size_t> compressedSize = compress(640, 480, inBuffer);



  EXPECT_THAT(compressedSize, Eq(std::nullopt));

}



TEST_F(JpegUtilTest, compressImageWithBufferTooSmallFails) {

  std::shared_ptr<AHardwareBuffer> inBuffer =

      createHardwareBufferForTest(634, 464);



  std::optional<size_t> compressedSize = compress(640, 480, inBuffer);



  EXPECT_THAT(compressedSize, Eq(std::nullopt));

}



}  // namespace

}  // namespace virtualcamera

}  // namespace companion

}  // namespace android

services/camera/virtualcamera/tests/VirtualCameraServiceTest.cc

+0 −11
Original line number Diff line number Diff line
@@ -238,17 +238,6 @@ TEST_F(VirtualCameraServiceTest, ConfigurationWithTooHighResFails) { 
  EXPECT_THAT(getCameraIds(), IsEmpty());

}



TEST_F(VirtualCameraServiceTest, ConfigurationWithUnalignedResolutionFails) {

  bool aidlRet;

  VirtualCameraConfiguration config =

      createConfiguration(641, 481, Format::YUV_420_888, kMaxFps);



  ASSERT_FALSE(

      mCameraService->registerCamera(mNdkOwnerToken, config, &aidlRet).isOk());

  EXPECT_FALSE(aidlRet);

  EXPECT_THAT(getCameraIds(), IsEmpty());

}



TEST_F(VirtualCameraServiceTest, ConfigurationWithNegativeResolutionFails) {

  bool aidlRet;

  VirtualCameraConfiguration config =