Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 6bd90432 authored by Nick Deakin's avatar Nick Deakin
Browse files

A bunch of improvements to recoverymap calculations. 

* Add proper color space conversions for map generation
* Add proper luminance calculation for map generation
* Add PQ encode/decode support
* Add structs for handling JPEG/R metadata, including HDR10 stuff
* Fill in HDR10 metadata, except for ST2086 info
* Update decode output from P010 to RGBA 1010102
* Update "hdr ratio" to "range scaling factor"

Bug: 252835416
Test: builds
No-Typo-Check: incorrectly interpretting code as a comment
Change-Id: I67d7be6d2a7821108d5f7ed26dd65285684e80f4
parent 10c7a55f

libs/jpegrecoverymap/include/jpegrecoverymap/jpegrerrorcode.h

+1 −0
Original line number Diff line number Diff line
@@ -37,6 +37,7 @@ enum { 
    ERROR_JPEGR_INVALID_NULL_PTR        = JPEGR_IO_ERROR_BASE - 2,

    ERROR_JPEGR_RESOLUTION_MISMATCH     = JPEGR_IO_ERROR_BASE - 3,

    ERROR_JPEGR_BUFFER_TOO_SMALL        = JPEGR_IO_ERROR_BASE - 4,

    ERROR_JPEGR_INVALID_COLORGAMUT      = JPEGR_IO_ERROR_BASE - 5,



    JPEGR_RUNTIME_ERROR_BASE            = -20000,

    ERROR_JPEGR_ENCODE_ERROR            = JPEGR_RUNTIME_ERROR_BASE - 1,

libs/jpegrecoverymap/include/jpegrecoverymap/recoverymap.h

+93 −31
Original line number Diff line number Diff line
@@ -22,11 +22,17 @@ 
namespace android::recoverymap {



typedef enum {

  JPEGR_COLORSPACE_UNSPECIFIED,

  JPEGR_COLORSPACE_BT709,

  JPEGR_COLORSPACE_P3,

  JPEGR_COLORSPACE_BT2100,

} jpegr_color_space;

  JPEGR_COLORGAMUT_UNSPECIFIED,

  JPEGR_COLORGAMUT_BT709,

  JPEGR_COLORGAMUT_P3,

  JPEGR_COLORGAMUT_BT2100,

} jpegr_color_gamut;



// Transfer functions as defined for XMP metadata

typedef enum {

  JPEGR_TF_HLG = 0,

  JPEGR_TF_PQ = 1,

} jpegr_transfer_function;



/*

 * Holds information for uncompressed image or recovery map.
@@ -38,8 +44,8 @@ struct jpegr_uncompressed_struct { 
    int width;

    // Height of the recovery map or image in pixels.

    int height;

    // Color space.

    jpegr_color_space colorSpace;

    // Color gamut.

    jpegr_color_gamut colorGamut;

};



/*
@@ -50,8 +56,8 @@ struct jpegr_compressed_struct { 
    void* data;

    // Data length.

    int length;

    // Color space.

    jpegr_color_space colorSpace;

    // Color gamut.

    jpegr_color_gamut colorGamut;

};



/*
@@ -64,9 +70,51 @@ struct jpegr_exif_struct { 
    int length;

};



struct chromaticity_coord {

  float x;

  float y;

};





struct st2086_metadata {

  // xy chromaticity coordinate of the red primary of the mastering display

  chromaticity_coord redPrimary;

  // xy chromaticity coordinate of the green primary of the mastering display

  chromaticity_coord greenPrimary;

  // xy chromaticity coordinate of the blue primary of the mastering display

  chromaticity_coord bluePrimary;

  // xy chromaticity coordinate of the white point of the mastering display

  chromaticity_coord whitePoint;

  // Maximum luminance in nits of the mastering display

  uint32_t maxLuminance;

  // Minimum luminance in nits of the mastering display

  float minLuminance;

};



struct hdr10_metadata {

  // Mastering display color volume

  st2086_metadata st2086Metadata;

  // Max frame average light level in nits

  float maxFALL;

  // Max content light level in nits

  float maxCLL;

};



struct jpegr_metadata {

  // JPEG/R version

  uint32_t version;

  // Range scaling factor for the map

  float rangeScalingFactor;

  // The transfer function for decoding the HDR representation of the image

  jpegr_transfer_function transferFunction;

  // HDR10 metadata, only applicable for transferFunction of JPEGR_TF_PQ

  hdr10_metadata hdr10Metadata;

};



typedef struct jpegr_uncompressed_struct* jr_uncompressed_ptr;

typedef struct jpegr_compressed_struct* jr_compressed_ptr;

typedef struct jpegr_exif_struct* jr_exif_ptr;

typedef struct jpegr_metadata* jr_metadata_ptr;



class RecoveryMap {

public:
@@ -75,9 +123,10 @@ public: 
     *

     * Generate recovery map from the HDR and SDR inputs, compress SDR YUV to 8-bit JPEG and append

     * the recovery map to the end of the compressed JPEG. HDR and SDR inputs must be the same

     * resolution and color space.

     * resolution.

     * @param uncompressed_p010_image uncompressed HDR image in P010 color format

     * @param uncompressed_yuv_420_image uncompressed SDR image in YUV_420 color format

     * @param hdr_tf transfer function of the HDR image

     * @param dest destination of the compressed JPEGR image

     * @param quality target quality of the JPEG encoding, must be in range of 0-100 where 100 is

     *                the highest quality
@@ -86,6 +135,7 @@ public: 
     */

    status_t encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image,

                         jr_uncompressed_ptr uncompressed_yuv_420_image,

                         jpegr_transfer_function hdr_tf,

                         jr_compressed_ptr dest,

                         int quality,

                         jr_exif_ptr exif);
@@ -100,12 +150,14 @@ public: 
     * @param uncompressed_p010_image uncompressed HDR image in P010 color format

     * @param uncompressed_yuv_420_image uncompressed SDR image in YUV_420 color format

     * @param compressed_jpeg_image compressed 8-bit JPEG image

     * @param hdr_tf transfer function of the HDR image

     * @param dest destination of the compressed JPEGR image

     * @return NO_ERROR if encoding succeeds, error code if error occurs.

     */

    status_t encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image,

                         jr_uncompressed_ptr uncompressed_yuv_420_image,

                         jr_compressed_ptr compressed_jpeg_image,

                         jpegr_transfer_function hdr_tf,

                         jr_compressed_ptr dest);



    /*
@@ -115,27 +167,28 @@ public: 
     *

     * Decode the compressed 8-bit JPEG image to YUV SDR, generate recovery map from the HDR input

     * and the decoded SDR result, append the recovery map to the end of the compressed JPEG. HDR

     * and SDR inputs must be the same resolution and color space.

     * and SDR inputs must be the same resolution.

     * @param uncompressed_p010_image uncompressed HDR image in P010 color format

     * @param compressed_jpeg_image compressed 8-bit JPEG image

     * @param hdr_tf transfer function of the HDR image

     * @param dest destination of the compressed JPEGR image

     * @return NO_ERROR if encoding succeeds, error code if error occurs.

     */

    status_t encodeJPEGR(jr_uncompressed_ptr uncompressed_p010_image,

                         jr_compressed_ptr compressed_jpeg_image,

                         jpegr_transfer_function hdr_tf,

                         jr_compressed_ptr dest);



    /*

     * Decompress JPEGR image.

     *

     * The output JPEGR image is in RGBA_1010102 data format if decoding to HDR.

     * @param compressed_jpegr_image compressed JPEGR image

     * @param dest destination of the uncompressed JPEGR image

     * @param exif destination of the decoded EXIF metadata. Default value is nullptr where EXIF

     *             metadata will not be decoded.

     * @param request_sdr flag that request SDR output, default to false (request HDR output). If

     *                    set to true, decoder will only decode the primary image which is SDR.

     *                    Setting of request_sdr and input source (HDR or SDR) can be found in

     *                    the table below:

     * @param exif destination of the decoded EXIF metadata.

     * @param request_sdr flag that request SDR output. If set to true, decoder will only decode

     *                    the primary image which is SDR. Setting of request_sdr and input source

     *                    (HDR or SDR) can be found in the table below:

     *                    |  input source  |  request_sdr  |  output of decoding  |

     *                    |       HDR      |     true      |          SDR         |

     *                    |       HDR      |     false     |          HDR         |
@@ -145,8 +198,8 @@ public: 
     */

    status_t decodeJPEGR(jr_compressed_ptr compressed_jpegr_image,

                         jr_uncompressed_ptr dest,

                         jr_exif_ptr exif = nullptr,

                         bool request_sdr = false);

                         jr_exif_ptr exif,

                         bool request_sdr);

private:

    /*

     * This method is called in the decoding pipeline. It will decode the recovery map.
@@ -176,26 +229,32 @@ private: 
     * @param uncompressed_yuv_420_image uncompressed SDR image in YUV_420 color format

     * @param uncompressed_p010_image uncompressed HDR image in P010 color format

     * @param dest recovery map; caller responsible for memory of data

     * @param hdr_ratio HDR ratio will be updated in this method

     * @param metadata metadata provides the transfer function for the HDR

     *                 image; range_scaling_factor and hdr10 FALL and CLL will

     *                 be updated.

     * @return NO_ERROR if calculation succeeds, error code if error occurs.

     */

    status_t generateRecoveryMap(jr_uncompressed_ptr uncompressed_yuv_420_image,

                                 jr_uncompressed_ptr uncompressed_p010_image,

                                 jr_uncompressed_ptr dest,

                                 float &hdr_ratio);

                                 jr_metadata_ptr metadata,

                                 jr_uncompressed_ptr dest);



    /*

     * This method is called in the decoding pipeline. It will take the uncompressed (decoded)

     * 8-bit yuv image and the uncompressed (decoded) recovery map as input, and calculate the

     * 10-bit recovered image (in p010 color format).

     * 8-bit yuv image, the uncompressed (decoded) recovery map, and extracted JPEG/R metadata as

     * input, and calculate the 10-bit recovered image. The recovered output image is the same

     * color gamut as the SDR image, with the transfer function specified in the JPEG/R metadata,

     * and is in RGBA1010102 data format.

     *

     * @param uncompressed_yuv_420_image uncompressed SDR image in YUV_420 color format

     * @param uncompressed_recovery_map uncompressed recovery map

     * @param metadata JPEG/R metadata extracted from XMP.

     * @param dest reconstructed HDR image

     * @return NO_ERROR if calculation succeeds, error code if error occurs.

     */

    status_t applyRecoveryMap(jr_uncompressed_ptr uncompressed_yuv_420_image,

                              jr_uncompressed_ptr uncompressed_recovery_map,

                              jr_metadata_ptr metadata,

                              jr_uncompressed_ptr dest);



    /*
@@ -204,9 +263,12 @@ private: 
     *

     * @param compressed_jpegr_image compressed JPEGR image

     * @param dest destination of compressed recovery map

     * @param metadata destination of the recovery map metadata

     * @return NO_ERROR if calculation succeeds, error code if error occurs.

     */

    status_t extractRecoveryMap(jr_compressed_ptr compressed_jpegr_image, jr_compressed_ptr dest);

    status_t extractRecoveryMap(jr_compressed_ptr compressed_jpegr_image,

                                jr_compressed_ptr dest,

                                jr_metadata_ptr metadata);



    /*

     * This method is called in the encoding pipeline. It will take the standard 8-bit JPEG image
@@ -215,13 +277,13 @@ private: 
     *

     * @param compressed_jpeg_image compressed 8-bit JPEG image

     * @param compress_recovery_map compressed recover map

     * @param hdr_ratio HDR ratio

     * @param metadata JPEG/R metadata to encode in XMP of the jpeg

     * @param dest compressed JPEGR image

     * @return NO_ERROR if calculation succeeds, error code if error occurs.

     */

    status_t appendRecoveryMap(jr_compressed_ptr compressed_jpeg_image,

                               jr_compressed_ptr compressed_recovery_map,

                               float hdr_ratio,

                               jr_metadata_ptr metadata,

                               jr_compressed_ptr dest);



    /*
@@ -229,7 +291,7 @@ private: 
     *

     * below is an example of the XMP metadata that this function generates where

     * secondary_image_length = 1000

     * hdr_ratio = 1.25

     * range_scaling_factor = 1.25

     *

     * <x:xmpmeta

     *   xmlns:x="adobe:ns:meta/"
@@ -239,7 +301,7 @@ private: 
     *     <rdf:Description

     *       xmlns:GContainer="http://ns.google.com/photos/1.0/container/">

     *       <GContainer:Version>1</GContainer:Version>

     *       <GContainer:HdrRatio>1.25</GContainer:HdrRatio>

     *       <GContainer:RangeScalingFactor>1.25</GContainer:RangeScalingFactor>

     *       <GContainer:Directory>

     *         <rdf:Seq>

     *           <rdf:li>
@@ -260,10 +322,10 @@ private: 
     * </x:xmpmeta>

     *

     * @param secondary_image_length length of secondary image

     * @param hdr_ratio hdr ratio

     * @param metadata JPEG/R metadata to encode as XMP

     * @return XMP metadata in type of string

     */

    std::string generateXmp(int secondary_image_length, float hdr_ratio);

    std::string generateXmp(int secondary_image_length, jpegr_metadata& metadata);

};



} // namespace android::recoverymap

libs/jpegrecoverymap/include/jpegrecoverymap/recoverymapmath.h

+47 −1
Original line number Diff line number Diff line
@@ -43,6 +43,9 @@ struct Color { 
  };

};



typedef Color (*ColorTransformFn)(Color);

typedef float (*ColorCalculationFn)(Color);



inline Color operator+=(Color& lhs, const Color& rhs) {

  lhs.r += rhs.r;

  lhs.g += rhs.g;
@@ -138,7 +141,10 @@ Color srgbInvOetf(Color e_gamma); 
////////////////////////////////////////////////////////////////////////////////

// Display-P3 transformations





/*

 * Calculated the luminance of a linear RGB P3 pixel, according to EG 432-1.

 */

float p3Luminance(Color e);





////////////////////////////////////////////////////////////////////////////////
@@ -169,10 +175,43 @@ Color hlgOetf(Color e); 
 */

Color hlgInvOetf(Color e_gamma);



/*

 * Convert from scene luminance in nits to PQ.

 */

Color pqOetf(Color e);



/*

 * Convert from PQ to scene luminance in nits.

 */

Color pqInvOetf(Color e_gamma);





////////////////////////////////////////////////////////////////////////////////

// Color space conversions



/*

 * Convert between color spaces with linear RGB data, according to ITU-R BT.2407 and EG 432-1.

 *

 * All conversions are derived from multiplying the matrix for XYZ to output RGB color gamut by the

 * matrix for input RGB color gamut to XYZ. The matrix for converting from XYZ to an RGB gamut is

 * always the inverse of the RGB gamut to XYZ matrix.

 */

Color bt709ToP3(Color e);

Color bt709ToBt2100(Color e);

Color p3ToBt709(Color e);

Color p3ToBt2100(Color e);

Color bt2100ToBt709(Color e);

Color bt2100ToP3(Color e);



/*

 * Identity conversion.

 */

inline Color identityConversion(Color e) { return e; }



/*

 * Get the conversion to apply to the HDR image for recovery map generation

 */

ColorTransformFn getHdrConversionFn(jpegr_color_gamut sdr_gamut, jpegr_color_gamut hdr_gamut);





////////////////////////////////////////////////////////////////////////////////
@@ -220,6 +259,13 @@ Color sampleYuv420(jr_uncompressed_ptr map, size_t map_scale_factor, size_t x, s 
 */

Color sampleP010(jr_uncompressed_ptr map, size_t map_scale_factor, size_t x, size_t y);



/*

 * Convert from Color to RGBA1010102.

 *

 * Alpha always set to 1.0.

 */

uint32_t colorToRgba1010102(Color e_gamma);



} // namespace android::recoverymap



#endif // ANDROID_JPEGRECOVERYMAP_RECOVERYMAPMATH_H

libs/jpegrecoverymap/recoverymap.cpp

+136 −53

File changed.

Preview size limit exceeded, changes collapsed.

libs/jpegrecoverymap/recoverymapmath.cpp

+121 −1
Original line number Diff line number Diff line
@@ -64,6 +64,11 @@ Color srgbInvOetf(Color e_gamma) { 
////////////////////////////////////////////////////////////////////////////////

// Display-P3 transformations



static const float kP3R = 0.22897f, kP3G = 0.69174f, kP3B = 0.07929f;



float p3Luminance(Color e) {

  return kP3R * e.r + kP3G * e.g + kP3B * e.b;

}





////////////////////////////////////////////////////////////////////////////////
@@ -128,7 +133,7 @@ Color hlgOetf(Color e) { 
  return {{{ hlgOetf(e.r), hlgOetf(e.g), hlgOetf(e.b) }}};

}



float hlgInvOetf(float e_gamma) {

static float hlgInvOetf(float e_gamma) {

  if (e_gamma <= 0.5f) {

    return pow(e_gamma, 2.0f) / 3.0f;

  } else {
@@ -142,10 +147,118 @@ Color hlgInvOetf(Color e_gamma) { 
             hlgInvOetf(e_gamma.b) }}};

}



static const float kPqM1 = 2610.0f / 16384.0f, kPqM2 = 2523.0f / 4096.0f * 128.0f;

static const float kPqC1 = 3424.0f / 4096.0f, kPqC2 = 2413.0f / 4096.0f * 32.0f,

                   kPqC3 = 2392.0f / 4096.0f * 32.0f;



static float pqOetf(float e) {

  if (e < 0.0f) e = 0.0f;

  return pow((kPqC1 + kPqC2 * pow(e / 10000.0f, kPqM1)) / (1 + kPqC3 * pow(e / 10000.0f, kPqM1)),

             kPqM2);

}



Color pqOetf(Color e) {

  return {{{ pqOetf(e.r), pqOetf(e.g), pqOetf(e.b) }}};

}



static float pqInvOetf(float e_gamma) {

  static const float kPqInvOetfCoef = log2(-(pow(kPqM1, 1.0f / kPqM2) - kPqC1)

                                         / (kPqC3 * pow(kPqM1, 1.0f / kPqM2) - kPqC2));

  return kPqInvOetfCoef / log2(e_gamma * 10000.0f);

}



Color pqInvOetf(Color e_gamma) {

  return {{{ pqInvOetf(e_gamma.r),

             pqInvOetf(e_gamma.g),

             pqInvOetf(e_gamma.b) }}};

}





////////////////////////////////////////////////////////////////////////////////

// Color conversions



Color bt709ToP3(Color e) {

 return {{{ 0.82254f * e.r + 0.17755f * e.g + 0.00006f * e.b,

            0.03312f * e.r + 0.96684f * e.g + -0.00001f * e.b,

            0.01706f * e.r + 0.07240f * e.g + 0.91049f * e.b }}};

}



Color bt709ToBt2100(Color e) {

 return {{{ 0.62740f * e.r + 0.32930f * e.g + 0.04332f * e.b,

            0.06904f * e.r + 0.91958f * e.g + 0.01138f * e.b,

            0.01636f * e.r + 0.08799f * e.g + 0.89555f * e.b }}};

}



Color p3ToBt709(Color e) {

 return {{{ 1.22482f * e.r + -0.22490f * e.g + -0.00007f * e.b,

            -0.04196f * e.r + 1.04199f * e.g + 0.00001f * e.b,

            -0.01961f * e.r + -0.07865f * e.g + 1.09831f * e.b }}};

}



Color p3ToBt2100(Color e) {

 return {{{ 0.75378f * e.r + 0.19862f * e.g + 0.04754f * e.b,

            0.04576f * e.r + 0.94177f * e.g + 0.01250f * e.b,

            -0.00121f * e.r + 0.01757f * e.g + 0.98359f * e.b }}};

}



Color bt2100ToBt709(Color e) {

 return {{{ 1.66045f * e.r + -0.58764f * e.g + -0.07286f * e.b,

            -0.12445f * e.r + 1.13282f * e.g + -0.00837f * e.b,

            -0.01811f * e.r + -0.10057f * e.g + 1.11878f * e.b }}};

}



Color bt2100ToP3(Color e) {

 return {{{ 1.34369f * e.r + -0.28223f * e.g + -0.06135f * e.b,

            -0.06533f * e.r + 1.07580f * e.g + -0.01051f * e.b,

            0.00283f * e.r + -0.01957f * e.g + 1.01679f * e.b

 }}};

}



// TODO: confirm we always want to convert like this before calculating

// luminance.

ColorTransformFn getHdrConversionFn(jpegr_color_gamut sdr_gamut, jpegr_color_gamut hdr_gamut) {

    switch (sdr_gamut) {

    case JPEGR_COLORGAMUT_BT709:

      switch (hdr_gamut) {

        case JPEGR_COLORGAMUT_BT709:

          return identityConversion;

        case JPEGR_COLORGAMUT_P3:

          return p3ToBt709;

        case JPEGR_COLORGAMUT_BT2100:

          return bt2100ToBt709;

        case JPEGR_COLORGAMUT_UNSPECIFIED:

          return nullptr;

      }

      break;

    case JPEGR_COLORGAMUT_P3:

      switch (hdr_gamut) {

        case JPEGR_COLORGAMUT_BT709:

          return bt709ToP3;

        case JPEGR_COLORGAMUT_P3:

          return identityConversion;

        case JPEGR_COLORGAMUT_BT2100:

          return bt2100ToP3;

        case JPEGR_COLORGAMUT_UNSPECIFIED:

          return nullptr;

      }

      break;

    case JPEGR_COLORGAMUT_BT2100:

      switch (hdr_gamut) {

        case JPEGR_COLORGAMUT_BT709:

          return bt709ToBt2100;

        case JPEGR_COLORGAMUT_P3:

          return p3ToBt2100;

        case JPEGR_COLORGAMUT_BT2100:

          return identityConversion;

        case JPEGR_COLORGAMUT_UNSPECIFIED:

          return nullptr;

      }

      break;

    case JPEGR_COLORGAMUT_UNSPECIFIED:

      return nullptr;

  }

}





////////////////////////////////////////////////////////////////////////////////

// Recovery map calculations
@@ -257,4 +370,11 @@ Color sampleP010(jr_uncompressed_ptr image, size_t map_scale_factor, size_t x, s 
  return samplePixels(image, map_scale_factor, x, y, getP010Pixel);

}



uint32_t colorToRgba1010102(Color e_gamma) {

  return (0x3ff & static_cast<uint32_t>(e_gamma.r * 1023.0f))

       | ((0x3ff & static_cast<uint32_t>(e_gamma.g * 1023.0f)) << 10)

       | ((0x3ff & static_cast<uint32_t>(e_gamma.b * 1023.0f)) << 20)

       | (0x3 << 30);  // Set alpha to 1.0

}



} // namespace android::recoverymap