Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 18ac32cc authored by Jan Sebechlebsky's avatar Jan Sebechlebsky
Browse files

Trigger texture update on each submission of input buffer. 

When input queue is full (which will happen if there are
no capture requests coming from the camera client), it blocks
until there's buffer freed up.
This is not desired (for example, it causes video decoder
attached to the virtual camera surface to drop frames), and
causes that the subsequent capture request received by camera
won't get the most recent image from the virtual camera owner.

This cl fixes this behavior, by consuming the buffer from
the input queue immediatelly after it's submitted.

Bug: 344572859
Test: atest VirtualCameraCaptureTest
Change-Id: Ic7556921630d56f73bbd861d35c5218351217ee6
parent 328d19bf

services/camera/virtualcamera/VirtualCameraRenderThread.cc

+70 −8
Original line number Diff line number Diff line
@@ -24,6 +24,7 @@ 
#include <memory>

#include <mutex>

#include <thread>

#include <utility>

#include <vector>



#include "Exif.h"
@@ -78,6 +79,15 @@ using ::android::hardware::camera::common::helper::ExifUtils; 


namespace {



// helper type for the visitor

template <class... Ts>

struct overloaded : Ts... {

  using Ts::operator()...;

};

// explicit deduction guide (not needed as of C++20)

template <class... Ts>

overloaded(Ts...) -> overloaded<Ts...>;



using namespace std::chrono_literals;



static constexpr std::chrono::milliseconds kAcquireFenceTimeout = 500ms;
@@ -89,6 +99,8 @@ static constexpr uint8_t kPipelineDepth = 2; 


static constexpr size_t kJpegThumbnailBufferSize = 32 * 1024;  // 32 KiB



static constexpr UpdateTextureTask kUpdateTextureTask;



CameraMetadata createCaptureResultMetadata(

    const std::chrono::nanoseconds timestamp,

    const RequestSettings& requestSettings,
@@ -287,6 +299,21 @@ std::chrono::nanoseconds getMaxFrameDuration( 
      static_cast<uint64_t>(1e9 / VirtualCameraDevice::kMinFps));

}



class FrameAvailableListenerProxy : public ConsumerBase::FrameAvailableListener {

 public:

  FrameAvailableListenerProxy(std::function<void()> callback)

      : mOnFrameAvailableCallback(callback) {

  }



  virtual void onFrameAvailable(const BufferItem&) override {

    ALOGV("%s: onFrameAvailable", __func__);

    mOnFrameAvailableCallback();

  }



 private:

  std::function<void()> mOnFrameAvailableCallback;

};



}  // namespace



CaptureRequestBuffer::CaptureRequestBuffer(int streamId, int bufferId,
@@ -345,9 +372,25 @@ const RequestSettings& ProcessCaptureRequestTask::getRequestSettings() const { 
  return mRequestSettings;

}



void VirtualCameraRenderThread::requestTextureUpdate() {

  std::lock_guard<std::mutex> lock(mLock);

  // If queue is not empty, we don't need to set the mTextureUpdateRequested

  // flag, since the texture will be updated during ProcessCaptureRequestTask

  // processing anyway.

  if (mQueue.empty()) {

    mTextureUpdateRequested = true;

    mCondVar.notify_one();

  }

}



void VirtualCameraRenderThread::enqueueTask(

    std::unique_ptr<ProcessCaptureRequestTask> task) {

  std::lock_guard<std::mutex> lock(mLock);

  // When enqueving process capture request task, clear the

  // mTextureUpdateRequested flag. If this flag is set, the texture was not yet

  // updated and it will be updated when processing ProcessCaptureRequestTask

  // anyway.

  mTextureUpdateRequested = false;

  mQueue.emplace_back(std::move(task));

  mCondVar.notify_one();

}
@@ -377,8 +420,7 @@ sp<Surface> VirtualCameraRenderThread::getInputSurface() { 
  return mInputSurfaceFuture.get();

}



std::unique_ptr<ProcessCaptureRequestTask>

VirtualCameraRenderThread::dequeueTask() {

RenderThreadTask VirtualCameraRenderThread::dequeueTask() {

  std::unique_lock<std::mutex> lock(mLock);

  // Clang's thread safety analysis doesn't perform alias analysis,

  // so it doesn't support moveable std::unique_lock.
@@ -389,12 +431,20 @@ VirtualCameraRenderThread::dequeueTask() { 
  ScopedLockAssertion lockAssertion(mLock);



  mCondVar.wait(lock, [this]() REQUIRES(mLock) {

    return mPendingExit || !mQueue.empty();

    return mPendingExit || mTextureUpdateRequested || !mQueue.empty();

  });

  if (mPendingExit) {

    return nullptr;

    // Render thread task with null task signals render thread to terminate.

    return RenderThreadTask(nullptr);

  }

  std::unique_ptr<ProcessCaptureRequestTask> task = std::move(mQueue.front());

  if (mTextureUpdateRequested) {

    // If mTextureUpdateRequested, it's guaranteed the queue is empty, return

    // kUpdateTextureTask to signal we want render thread to update the texture

    // (consume buffer from the queue).

    mTextureUpdateRequested = false;

    return RenderThreadTask(kUpdateTextureTask);

  }

  RenderThreadTask task(std::move(mQueue.front()));

  mQueue.pop_front();

  return task;

}
@@ -409,11 +459,23 @@ void VirtualCameraRenderThread::threadLoop() { 
      EglTextureProgram::TextureFormat::RGBA);

  mEglSurfaceTexture = std::make_unique<EglSurfaceTexture>(

      mInputSurfaceSize.width, mInputSurfaceSize.height);

  sp<FrameAvailableListenerProxy> frameAvailableListener =

      sp<FrameAvailableListenerProxy>::make(

          [this]() { requestTextureUpdate(); });

  mEglSurfaceTexture->setFrameAvailableListener(frameAvailableListener);



  mInputSurfacePromise.set_value(mEglSurfaceTexture->getSurface());



  while (std::unique_ptr<ProcessCaptureRequestTask> task = dequeueTask()) {

    processCaptureRequest(*task);

  while (RenderThreadTask task = dequeueTask()) {

    std::visit(

        overloaded{[this](const std::unique_ptr<ProcessCaptureRequestTask>& t) {

                     processTask(*t);

                   },

                   [this](const UpdateTextureTask&) {

                     ALOGV("Idle update of the texture");

                     mEglSurfaceTexture->updateTexture();

                   }},

        task);

  }



  // Destroy EGL utilities still on the render thread.
@@ -425,7 +487,7 @@ void VirtualCameraRenderThread::threadLoop() { 
  ALOGV("Render thread exiting");

}



void VirtualCameraRenderThread::processCaptureRequest(

void VirtualCameraRenderThread::processTask(

    const ProcessCaptureRequestTask& request) {

  std::chrono::nanoseconds timestamp =

      std::chrono::duration_cast<std::chrono::nanoseconds>(

services/camera/virtualcamera/VirtualCameraRenderThread.h

+28 −3
Original line number Diff line number Diff line
@@ -23,6 +23,7 @@ 
#include <future>

#include <memory>

#include <thread>

#include <variant>

#include <vector>



#include "VirtualCameraDevice.h"
@@ -34,7 +35,6 @@ 
#include "util/EglFramebuffer.h"

#include "util/EglProgram.h"

#include "util/EglSurfaceTexture.h"

#include "util/MetadataUtil.h"

#include "util/Util.h"



namespace android {
@@ -94,6 +94,24 @@ class ProcessCaptureRequestTask { 
  const RequestSettings mRequestSettings;

};



struct UpdateTextureTask {};



struct RenderThreadTask

    : public std::variant<std::unique_ptr<ProcessCaptureRequestTask>,

                          UpdateTextureTask> {

  // Allow implicit conversion to bool.

  //

  // Returns false, if the RenderThreadTask consist of null

  // ProcessCaptureRequestTask, which signals that the thread should terminate.

  operator bool() const {

    const bool isExitSignal =

        std::holds_alternative<std::unique_ptr<ProcessCaptureRequestTask>>(

            *this) &&

        std::get<std::unique_ptr<ProcessCaptureRequestTask>>(*this) == nullptr;

    return !isExitSignal;

  }

};



// Wraps dedicated rendering thread and rendering business with corresponding

// input surface.

class VirtualCameraRenderThread {
@@ -120,6 +138,12 @@ class VirtualCameraRenderThread { 
  // Stop rendering thread.

  void stop();



  // Send request to render thread to update the texture.

  // Currently queued buffers in the input surface will be consumed and the most

  // recent buffer in the input surface will be attached to the texture), all

  // other buffers will be returned to the buffer queue.

  void requestTextureUpdate() EXCLUDES(mLock);



  // Equeue capture task for processing on render thread.

  void enqueueTask(std::unique_ptr<ProcessCaptureRequestTask> task)

      EXCLUDES(mLock);
@@ -131,13 +155,13 @@ class VirtualCameraRenderThread { 
  sp<Surface> getInputSurface();



 private:

  std::unique_ptr<ProcessCaptureRequestTask> dequeueTask() EXCLUDES(mLock);

  RenderThreadTask dequeueTask() EXCLUDES(mLock);



  // Rendering thread entry point.

  void threadLoop();



  // Process single capture request task (always called on render thread).

  void processCaptureRequest(const ProcessCaptureRequestTask& captureRequestTask);

  void processTask(const ProcessCaptureRequestTask& captureRequestTask);



  // Flush single capture request task returning the error status immediately.

  void flushCaptureRequest(const ProcessCaptureRequestTask& captureRequestTask);
@@ -192,6 +216,7 @@ class VirtualCameraRenderThread { 
  std::mutex mLock;

  std::deque<std::unique_ptr<ProcessCaptureRequestTask>> mQueue GUARDED_BY(mLock);

  std::condition_variable mCondVar;

  volatile bool mTextureUpdateRequested GUARDED_BY(mLock);

  volatile bool mPendingExit GUARDED_BY(mLock);



  // Acquisition timestamp of last frame.

services/camera/virtualcamera/util/EglSurfaceTexture.cc

+5 −0
Original line number Diff line number Diff line
@@ -64,6 +64,11 @@ sp<GraphicBuffer> EglSurfaceTexture::getCurrentBuffer() { 
  return mGlConsumer->getCurrentBuffer();

}



void EglSurfaceTexture::setFrameAvailableListener(

    const wp<ConsumerBase::FrameAvailableListener>& listener) {

  mGlConsumer->setFrameAvailableListener(listener);

}



bool EglSurfaceTexture::waitForNextFrame(const std::chrono::nanoseconds timeout) {

  return mSurface->waitForNextFrame(mGlConsumer->getFrameNumber(),

                                    static_cast<nsecs_t>(timeout.count()));

services/camera/virtualcamera/util/EglSurfaceTexture.h

+4 −0
Original line number Diff line number Diff line
@@ -21,6 +21,7 @@ 
#include <cstdint>



#include "GLES/gl.h"

#include "gui/ConsumerBase.h"

#include "gui/Surface.h"

#include "utils/RefBase.h"
@@ -58,6 +59,9 @@ class EglSurfaceTexture { 
  // Returns false on timeout, true if new frame was received before timeout.

  bool waitForNextFrame(std::chrono::nanoseconds timeout);



  void setFrameAvailableListener(

      const wp<ConsumerBase::FrameAvailableListener>& listener);



  // Update the texture with the most recent submitted buffer.

  // Most be called on thread with EGL context.

  //