Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 99492e3c authored by Jan Sebechlebsky's avatar Jan Sebechlebsky
Browse files

Apply transformation matrix to texture coordinates. 

SurfaceTexture provides transformation matrix which should
be used to transform texture coordinates before sampling
to account for possible coordinate differences in the
underlying native buffer.

This was causing the texture to be flipped when rendered.
Also fix the square coordinates which caused stretching.

Bug: 301023410
Test: atest virtual_camera_tests
Test: atest CtsVirtualCameraTestCases
Test: manual with sample app & canvas + video

Change-Id: Ib7fd2bb03e6686ada193ea67541f051d981f2bad
parent 44a866a5

services/camera/virtualcamera/VirtualCameraRenderThread.cc

+6 −2
Original line number Diff line number Diff line
@@ -542,8 +542,12 @@ ndk::ScopedAStatus VirtualCameraRenderThread::renderIntoEglFramebuffer( 
  } else {

    const bool renderSuccess =

        isYuvFormat(static_cast<PixelFormat>(textureBuffer->getPixelFormat()))

            ? mEglTextureYuvProgram->draw(mEglSurfaceTexture->updateTexture())

            : mEglTextureRgbProgram->draw(mEglSurfaceTexture->updateTexture());

            ? mEglTextureYuvProgram->draw(

                  mEglSurfaceTexture->getTextureId(),

                  mEglSurfaceTexture->getTransformMatrix())

            : mEglTextureRgbProgram->draw(

                  mEglSurfaceTexture->getTextureId(),

                  mEglSurfaceTexture->getTransformMatrix());

    if (!renderSuccess) {

      ALOGE("%s: Failed to render texture", __func__);

      return cameraStatus(Status::INTERNAL_ERROR);

services/camera/virtualcamera/util/EglProgram.cc

+16 −6
Original line number Diff line number Diff line
@@ -68,12 +68,13 @@ constexpr char kJuliaFractalFragmentShader[] = R"( 
    })";



constexpr char kExternalTextureVertexShader[] = R"(#version 300 es

  uniform mat4 aTextureTransformMatrix; // Transform matrix given by surface texture.

  in vec4 aPosition;

  in vec2 aTextureCoord;

  out vec2 vTextureCoord;

  void main() {

    gl_Position = aPosition;

    vTextureCoord = aTextureCoord;

    vTextureCoord = (aTextureTransformMatrix * vec4(aTextureCoord, 0.0, 1.0)).xy;

  })";



constexpr char kExternalYuvTextureFragmentShader[] = R"(#version 300 es
@@ -100,10 +101,12 @@ constexpr char kExternalRgbaTextureFragmentShader[] = R"(#version 300 es 
    })";



constexpr int kCoordsPerVertex = 3;

constexpr std::array<float, 12> kSquareCoords{-1.f, 1.0f, 0.0f,  // top left

                                              -1.f, -1.f, 0.0f,  // bottom left

                                              1.0f, -1.f, 0.0f,  // bottom right

                                              1.0f, 1.0f, 0.0f};  // top right



constexpr std::array<float, 12> kSquareCoords{

    -1.f, -1.0f, 0.0f,   // top left

    -1.f, 1.f,   0.0f,   // bottom left

    1.0f, 1.f,   0.0f,   // bottom right

    1.0f, -1.0f, 0.0f};  // top right



constexpr std::array<float, 8> kTextureCoords{0.0f, 1.0f,   // top left

                                              0.0f, 0.0f,   // bottom left
@@ -267,7 +270,8 @@ EglTextureProgram::EglTextureProgram(const TextureFormat textureFormat) { 
  }

}



bool EglTextureProgram::draw(GLuint textureId) {

bool EglTextureProgram::draw(GLuint textureId,

                             const std::array<float, 16>& transformMatrix) {

  // Load compiled shader.

  glUseProgram(mProgram);

  if (checkEglError("glUseProgram")) {
@@ -286,6 +290,12 @@ bool EglTextureProgram::draw(GLuint textureId) { 
  glVertexAttribPointer(textureCoordHandle, 2, GL_FLOAT, false,

                        kTextureCoords.size(), kTextureCoords.data());



  // Pass transformation matrix for the texture coordinates.

  int transformMatrixHandle =

      glGetUniformLocation(mProgram, "aTextureTransformMatrix");

  glUniformMatrix4fv(transformMatrixHandle, 1, /*transpose=*/GL_FALSE,

                     transformMatrix.data());



  // Configure texture for the shader.

  int textureHandle = glGetUniformLocation(mProgram, "uTexture");

  glActiveTexture(GL_TEXTURE0);

services/camera/virtualcamera/util/EglProgram.h

+11 −1
Original line number Diff line number Diff line
@@ -17,6 +17,8 @@ 
#ifndef ANDROID_COMPANION_VIRTUALCAMERA_EGLPROGRAM_H

#define ANDROID_COMPANION_VIRTUALCAMERA_EGLPROGRAM_H



#include <array>



#include "GLES/gl.h"



namespace android {
@@ -59,7 +61,15 @@ class EglTextureProgram : public EglProgram { 


  EglTextureProgram(TextureFormat textureFormat = TextureFormat::YUV);



  bool draw(GLuint textureId);

  // Draw texture over whole viewport, applying transformMatrix to texture

  // coordinates.

  //

  // Transform matrix is 4x4 matrix represented in column-major order and is

  // applied to texture coordinates in (s,t,0,1), s,t from <0,1> range prior to

  // sampling:

  //

  // textureCoord = transformMatrix * vec4(s,t,0,1).xy

  bool draw(GLuint textureId, const std::array<float, 16>& transformMatrix);

};



}  // namespace virtualcamera

services/camera/virtualcamera/util/EglSurfaceTexture.cc

+10 −0
Original line number Diff line number Diff line
@@ -68,6 +68,16 @@ GLuint EglSurfaceTexture::updateTexture() { 
  return mTextureId;

}



GLuint EglSurfaceTexture::getTextureId() const {

  return mTextureId;

}



std::array<float, 16> EglSurfaceTexture::getTransformMatrix() {

  std::array<float, 16> matrix;

  mGlConsumer->getTransformMatrix(matrix.data());

  return matrix;

}



uint32_t EglSurfaceTexture::getWidth() const {

  return mWidth;

}

services/camera/virtualcamera/util/EglSurfaceTexture.h

+11 −0
Original line number Diff line number Diff line
@@ -57,6 +57,17 @@ class EglSurfaceTexture { 
  // Returns EGL texture id of the texture.

  GLuint updateTexture();



  // Returns EGL texture id of the underlying texture.

  GLuint getTextureId() const;



  // Returns 4x4 transformation matrix in column-major order,

  // which should be applied to EGL texture coordinates

  // before sampling from the texture backed by android native buffer,

  // so the corresponding region of the underlying buffer is sampled.

  //

  // See SurfaceTexture.getTransformMatrix for more details.

  std::array<float, 16> getTransformMatrix();



 private:

  sp<IGraphicBufferProducer> mBufferProducer;

  sp<IGraphicBufferConsumer> mBufferConsumer;