Merge "Implement initial version of tone mapping in Skia RenderEngine." (da4f3515) · Commits · e / os / android_frameworks_native-old

libs/renderengine/Android.bp

+1 −0

Original line number	Diff line number	Diff line
		@@ -78,6 +78,7 @@ filegroup {
		"skia/SkiaRenderEngine.cpp",
		"skia/SkiaGLRenderEngine.cpp",
		"skia/filters/BlurFilter.cpp",
		"skia/filters/LinearEffect.cpp",
		],
		}

libs/renderengine/skia/SkiaGLRenderEngine.cpp

+60 −9

Original line number	Diff line number	Diff line
		@@ -16,6 +16,9 @@

		//#define LOG_NDEBUG 0
		#include <cstdint>

		#include "SkImageInfo.h"
		#include "system/graphics-base-v1.0.h"
		#undef LOG_TAG
		#define LOG_TAG "RenderEngine"
		#define ATRACE_TAG ATRACE_TAG_GRAPHICS
		@@ -50,6 +53,7 @@
		#include "../gl/GLExtensions.h"
		#include "SkiaGLRenderEngine.h"
		#include "filters/BlurFilter.h"
		#include "filters/LinearEffect.h"

		extern "C" EGLAPI const char* eglQueryStringImplementationANDROID(EGLDisplay dpy, EGLint name);

		@@ -411,6 +415,32 @@ static SkColorMatrix toSkColorMatrix(const mat4& matrix) {
		matrix[3][3], 0);
		}

		static bool needsToneMapping(ui::Dataspace sourceDataspace, ui::Dataspace destinationDataspace) {
		int64_t sourceTransfer = sourceDataspace & HAL_DATASPACE_TRANSFER_MASK;
		int64_t destTransfer = destinationDataspace & HAL_DATASPACE_TRANSFER_MASK;

		// Treat unsupported dataspaces as srgb
		if (destTransfer != HAL_DATASPACE_TRANSFER_LINEAR &&
		destTransfer != HAL_DATASPACE_TRANSFER_HLG &&
		destTransfer != HAL_DATASPACE_TRANSFER_ST2084) {
		destTransfer = HAL_DATASPACE_TRANSFER_SRGB;
		}

		if (sourceTransfer != HAL_DATASPACE_TRANSFER_LINEAR &&
		sourceTransfer != HAL_DATASPACE_TRANSFER_HLG &&
		sourceTransfer != HAL_DATASPACE_TRANSFER_ST2084) {
		sourceTransfer = HAL_DATASPACE_TRANSFER_SRGB;
		}

		const bool isSourceLinear = sourceTransfer == HAL_DATASPACE_TRANSFER_LINEAR;
		const bool isSourceSRGB = sourceTransfer == HAL_DATASPACE_TRANSFER_SRGB;
		const bool isDestLinear = destTransfer == HAL_DATASPACE_TRANSFER_LINEAR;
		const bool isDestSRGB = destTransfer == HAL_DATASPACE_TRANSFER_SRGB;

		return !(isSourceLinear && isDestSRGB) && !(isSourceSRGB && isDestLinear) &&
		sourceTransfer != destTransfer;
		}

		void SkiaGLRenderEngine::unbindExternalTextureBuffer(uint64_t bufferId) {
		std::lock_guard<std::mutex> lock(mRenderingMutex);
		mImageCache.erase(bufferId);
		@@ -539,13 +569,19 @@ status_t SkiaGLRenderEngine::drawLayers(const DisplaySettings& display,
		if (iter != mImageCache.end()) {
		image = iter->second;
		} else {
		image = SkImage::MakeFromAHardwareBuffer(item.buffer->toAHardwareBuffer(),
		item.usePremultipliedAlpha
		? kPremul_SkAlphaType
		: kUnpremul_SkAlphaType,
		image = SkImage::MakeFromAHardwareBuffer(
		item.buffer->toAHardwareBuffer(),
		item.isOpaque ? kOpaque_SkAlphaType
		: (item.usePremultipliedAlpha ? kPremul_SkAlphaType
		: kUnpremul_SkAlphaType),
		mUseColorManagement
		? toColorSpace(
		layer->sourceDataspace)
		? (needsToneMapping(layer->sourceDataspace, display.outputDataspace)
		// If we need to map to linear space, then
		// mark the source image with the same
		// colorspace as the destination surface so
		// that Skia's color management is a no-op.
		? toColorSpace(display.outputDataspace)
		: toColorSpace(layer->sourceDataspace))
		: SkColorSpace::MakeSRGB());
		mImageCache.insert({item.buffer->getId(), image});
		}
		@@ -594,7 +630,22 @@ status_t SkiaGLRenderEngine::drawLayers(const DisplaySettings& display,

		matrix.postConcat(texMatrix);
		matrix.postScale(rotatedBufferWidth, rotatedBufferHeight);
		paint.setShader(image->makeShader(matrix));
		sk_sp<SkShader> shader = image->makeShader(matrix);

		if (mUseColorManagement &&
		needsToneMapping(layer->sourceDataspace, display.outputDataspace)) {
		LinearEffect effect = LinearEffect{.inputDataspace = layer->sourceDataspace,
		.outputDataspace = display.outputDataspace,
		.undoPremultipliedAlpha = !item.isOpaque &&
		item.usePremultipliedAlpha};
		sk_sp<SkRuntimeEffect> runtimeEffect = buildRuntimeEffect(effect);
		paint.setShader(createLinearEffectShader(shader, effect, runtimeEffect,
		display.maxLuminance,
		layer->source.buffer.maxMasteringLuminance,
		layer->source.buffer.maxContentLuminance));
		} else {
		paint.setShader(shader);
		}
		} else {
		ATRACE_NAME("DrawColor");
		const auto color = layer->source.solidColor;

libs/renderengine/skia/filters/LinearEffect.cpp

0 → 100644

+308 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright 2020 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#include "LinearEffect.h"

		#include <SkString.h>

		#include <optional>

		#include "log/log.h"
		#include "math/mat4.h"
		#include "ui/ColorSpace.h"

		namespace android {
		namespace renderengine {
		namespace skia {

		static void generateEOTF(ui::Dataspace dataspace, SkString& shader) {
		switch (dataspace & HAL_DATASPACE_TRANSFER_MASK) {
		case HAL_DATASPACE_TRANSFER_ST2084:
		shader.append(R"(

		float3 EOTF(float3 color) {
		float m1 = (2610.0 / 4096.0) / 4.0;
		float m2 = (2523.0 / 4096.0) * 128.0;
		float c1 = (3424.0 / 4096.0);
		float c2 = (2413.0 / 4096.0) * 32.0;
		float c3 = (2392.0 / 4096.0) * 32.0;

		float3 tmp = pow(clamp(color, 0.0, 1.0), 1.0 / float3(m2));
		tmp = max(tmp - c1, 0.0) / (c2 - c3 * tmp);
		return pow(tmp, 1.0 / float3(m1));
		}
		)");
		break;
		default:
		shader.append(R"(

		float EOTF_sRGB(float srgb) {
		return srgb <= 0.04045 ? srgb / 12.92 : pow((srgb + 0.055) / 1.055, 2.4);
		}

		float3 EOTF_sRGB(float3 srgb) {
		return float3(EOTF_sRGB(srgb.r), EOTF_sRGB(srgb.g), EOTF_sRGB(srgb.b));
		}

		float3 EOTF(float3 srgb) {
		return sign(srgb.rgb) * EOTF_sRGB(abs(srgb.rgb));
		}
		)");
		break;
		}
		}

		static void generateXYZTransforms(SkString& shader) {
		shader.append(R"(
		uniform float4x4 in_rgbToXyz;
		uniform float4x4 in_xyzToRgb;
		float3 ToXYZ(float3 rgb) {
		return clamp((in_rgbToXyz * float4(rgb, 1.0)).rgb, 0.0, 1.0);
		}

		float3 ToRGB(float3 xyz) {
		return clamp((in_xyzToRgb * float4(xyz, 1.0)).rgb, 0.0, 1.0);
		}
		)");
		}

		static void generateOOTF(ui::Dataspace inputDataspace, ui::Dataspace outputDataspace,
		SkString& shader) {
		shader.append(R"(
		uniform float in_displayMaxLuminance;
		uniform float in_inputMaxLuminance;
		uniform float in_maxContentLuminance;
		)");
		switch (inputDataspace & HAL_DATASPACE_TRANSFER_MASK) {
		case HAL_DATASPACE_TRANSFER_ST2084:
		shader.append(R"(
		float3 ScaleLuminance(float3 xyz) {
		return xyz * 10000.0;
		}
		)");

		switch (outputDataspace & HAL_DATASPACE_TRANSFER_MASK) {
		default:
		shader.append(R"(
		float3 ToneMap(float3 xyz) {
		float maxInLumi = in_inputMaxLuminance;
		float maxOutLumi = in_displayMaxLuminance;

		float nits = xyz.y;

		// clamp to max input luminance
		nits = clamp(nits, 0.0, maxInLumi);

		// scale [0.0, maxInLumi] to [0.0, maxOutLumi]
		if (maxInLumi <= maxOutLumi) {
		return xyz * (maxOutLumi / maxInLumi);
		} else {
		// three control points
		const float x0 = 10.0;
		const float y0 = 17.0;
		float x1 = maxOutLumi * 0.75;
		float y1 = x1;
		float x2 = x1 + (maxInLumi - x1) / 2.0;
		float y2 = y1 + (maxOutLumi - y1) * 0.75;

		// horizontal distances between the last three control points
		float h12 = x2 - x1;
		float h23 = maxInLumi - x2;
		// tangents at the last three control points
		float m1 = (y2 - y1) / h12;
		float m3 = (maxOutLumi - y2) / h23;
		float m2 = (m1 + m3) / 2.0;

		if (nits < x0) {
		// scale [0.0, x0] to [0.0, y0] linearly
		float slope = y0 / x0;
		return xyz * slope;
		} else if (nits < x1) {
		// scale [x0, x1] to [y0, y1] linearly
		float slope = (y1 - y0) / (x1 - x0);
		nits = y0 + (nits - x0) * slope;
		} else if (nits < x2) {
		// scale [x1, x2] to [y1, y2] using Hermite interp
		float t = (nits - x1) / h12;
		nits = (y1 * (1.0 + 2.0 * t) + h12 * m1 * t) * (1.0 - t) * (1.0 - t) +
		(y2 * (3.0 - 2.0 * t) + h12 * m2 * (t - 1.0)) * t * t;
		} else {
		// scale [x2, maxInLumi] to [y2, maxOutLumi] using Hermite interp
		float t = (nits - x2) / h23;
		nits = (y2 * (1.0 + 2.0 * t) + h23 * m2 * t) * (1.0 - t) * (1.0 - t) +
		(maxOutLumi * (3.0 - 2.0 * t) + h23 * m3 * (t - 1.0)) * t * t;
		}
		}

		// color.y is greater than x0 and is thus non-zero
		return xyz * (nits / xyz.y);
		}
		)");
		break;
		}
		break;
		default:
		shader.append(R"(
		float3 ScaleLuminance(float3 xyz) {
		return xyz * in_displayMaxLuminance;
		}

		float3 ToneMap(float3 xyz) {
		return xyz;
		}
		)");
		break;
		}

		switch (outputDataspace & HAL_DATASPACE_TRANSFER_MASK) {
		case HAL_DATASPACE_TRANSFER_ST2084:
		shader.append(R"(
		float3 NormalizeLuminance(float3 xyz) {
		return xyz / 10000.0;
		}
		)");
		break;
		default:
		shader.append(R"(
		float3 NormalizeLuminance(float3 xyz) {
		return xyz / in_displayMaxLuminance;
		}
		)");
		break;
		}

		shader.append(R"(
		float3 OOTF(float3 xyz) {
		return NormalizeLuminance(ToneMap(ScaleLuminance(xyz)));
		}
		)");
		}

		static void generateOETF(ui::Dataspace dataspace, SkString& shader) {
		switch (dataspace & HAL_DATASPACE_TRANSFER_MASK) {
		case HAL_DATASPACE_TRANSFER_ST2084:
		shader.append(R"(

		float3 OETF(float3 xyz) {
		float m1 = (2610.0 / 4096.0) / 4.0;
		float m2 = (2523.0 / 4096.0) * 128.0;
		float c1 = (3424.0 / 4096.0);
		float c2 = (2413.0 / 4096.0) * 32.0;
		float c3 = (2392.0 / 4096.0) * 32.0;

		float3 tmp = pow(xyz, float3(m1));
		tmp = (c1 + c2 * tmp) / (1.0 + c3 * tmp);
		return pow(tmp, float3(m2));
		}
		)");
		break;
		default:
		shader.append(R"(
		float OETF_sRGB(float linear) {
		return linear <= 0.0031308 ?
		linear * 12.92 : (pow(linear, 1.0 / 2.4) * 1.055) - 0.055;
		}

		float3 OETF_sRGB(float3 linear) {
		return float3(OETF_sRGB(linear.r), OETF_sRGB(linear.g), OETF_sRGB(linear.b));
		}

		float3 OETF(float3 linear) {
		return sign(linear.rgb) * OETF_sRGB(abs(linear.rgb));
		}
		)");
		break;
		}
		}

		static void generateEffectiveOOTF(bool undoPremultipliedAlpha, SkString& shader) {
		shader.append(R"(
		in shader input;
		half4 main(float2 xy) {
		float4 c = float4(sample(input, xy));
		)");
		if (undoPremultipliedAlpha) {
		shader.append(R"(
		c.rgb = c.rgb / (c.a + 0.0019);
		)");
		}
		shader.append(R"(
		c.rgb = OETF(ToRGB(OOTF(ToXYZ(EOTF(c.rgb)))));
		)");
		if (undoPremultipliedAlpha) {
		shader.append(R"(
		c.rgb = c.rgb * (c.a + 0.0019);
		)");
		}
		shader.append(R"(
		return c;
		}
		)");
		}
		static ColorSpace toColorSpace(ui::Dataspace dataspace) {
		switch (dataspace & HAL_DATASPACE_STANDARD_MASK) {
		case HAL_DATASPACE_STANDARD_BT709:
		return ColorSpace::sRGB();
		break;
		case HAL_DATASPACE_STANDARD_DCI_P3:
		return ColorSpace::DisplayP3();
		break;
		case HAL_DATASPACE_STANDARD_BT2020:
		return ColorSpace::BT2020();
		break;
		default:
		return ColorSpace::sRGB();
		break;
		}
		}

		sk_sp<SkRuntimeEffect> buildRuntimeEffect(const LinearEffect& linearEffect) {
		SkString shaderString;
		generateEOTF(linearEffect.inputDataspace, shaderString);
		generateXYZTransforms(shaderString);
		generateOOTF(linearEffect.inputDataspace, linearEffect.outputDataspace, shaderString);
		generateOETF(linearEffect.outputDataspace, shaderString);
		generateEffectiveOOTF(linearEffect.undoPremultipliedAlpha, shaderString);

		auto [shader, error] = SkRuntimeEffect::Make(shaderString);
		if (!shader) {
		LOG_ALWAYS_FATAL("LinearColorFilter construction error: %s", error.c_str());
		}
		return shader;
		}

		sk_sp<SkShader> createLinearEffectShader(sk_sp<SkShader> shader, const LinearEffect& linearEffect,
		sk_sp<SkRuntimeEffect> runtimeEffect,
		float maxDisplayLuminance, float maxMasteringLuminance,
		float maxContentLuminance) {
		SkRuntimeShaderBuilder effectBuilder(runtimeEffect);

		effectBuilder.child("input") = shader;

		ColorSpace inputColorSpace = toColorSpace(linearEffect.inputDataspace);
		ColorSpace outputColorSpace = toColorSpace(linearEffect.outputDataspace);

		effectBuilder.uniform("in_rgbToXyz") = mat4(inputColorSpace.getRGBtoXYZ());
		effectBuilder.uniform("in_xyzToRgb") = mat4(outputColorSpace.getXYZtoRGB());
		effectBuilder.uniform("in_displayMaxLuminance") = maxDisplayLuminance;
		effectBuilder.uniform("in_inputMaxLuminance") =
		std::min(maxMasteringLuminance, maxContentLuminance);
		return effectBuilder.makeShader(nullptr, false);
		}

		} // namespace skia
		} // namespace renderengine
		} // namespace android
		No newline at end of file

libs/renderengine/skia/filters/LinearEffect.h

0 → 100644

+83 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright 2020 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#pragma once

		#include <optional>

		#include "SkColorMatrix.h"
		#include "SkRuntimeEffect.h"
		#include "SkShader.h"
		#include "ui/GraphicTypes.h"

		namespace android {
		namespace renderengine {
		namespace skia {

		/**
		* Arguments for creating an effect that applies color transformations in linear XYZ space.
		* A linear effect is decomposed into the following steps when operating on an image:
		* 1. Electrical-Optical Transfer Function (EOTF) maps the input RGB signal into the intended
		* relative display brightness of the scene in nits for each RGB channel
		* 2. Transformation matrix from linear RGB brightness to linear XYZ, to operate on display
		* luminance.
		* 3. Opto-Optical Transfer Function (OOTF) applies a "rendering intent". This can include tone
		* mapping to display SDR content alongside HDR content, or any number of subjective transformations
		* 4. Transformation matrix from linear XYZ back to linear RGB brightness.
		* 5. Opto-Electronic Transfer Function (OETF) maps the display brightness of the scene back to
		* output RGB colors.
		*
		* For further reading, consult the recommendation in ITU-R BT.2390-4:
		* https://www.itu.int/dms_pub/itu-r/opb/rep/R-REP-BT.2390-4-2018-PDF-E.pdf
		*
		* Skia normally attempts to do its own simple tone mapping, i.e., the working color space is
		* intended to be the output surface. However, Skia does not support complex tone mapping such as
		* polynomial interpolation. As such, this filter assumes that tone mapping has not yet been applied
		* to the source colors. so that the tone mapping process is only applied once by this effect. Tone
		* mapping is applied when presenting HDR content (content with HLG or PQ transfer functions)
		* alongside other content, whereby maximum input luminance is mapped to maximum output luminance
		* and intermediate values are interpolated.
		*/
		struct LinearEffect {
		// Input dataspace of the source colors.
		const ui::Dataspace inputDataspace = ui::Dataspace::SRGB;

		// Working dataspace for the output surface, for conversion from linear space.
		const ui::Dataspace outputDataspace = ui::Dataspace::SRGB;

		// Sets whether alpha premultiplication must be undone.
		// This is required if the source colors use premultiplied alpha and is not opaque.
		const bool undoPremultipliedAlpha = false;
		};

		sk_sp<SkRuntimeEffect> buildRuntimeEffect(const LinearEffect& linearEffect);

		// Generates a shader resulting from applying the a linear effect created from
		// LinearEffectARgs::buildEffect to an inputShader. We also provide additional HDR metadata upon
		// creating the shader:
		// * The max display luminance is the max luminance of the physical display in nits
		// * The max mastering luminance is provided as the max luminance from the SMPTE 2086
		// standard.
		// * The max content luminance is provided as the max light level from the CTA 861.3
		// standard.
		sk_sp<SkShader> createLinearEffectShader(sk_sp<SkShader> inputShader,
		const LinearEffect& linearEffect,
		sk_sp<SkRuntimeEffect> runtimeEffect,
		float maxDisplayLuminance, float maxMasteringLuminance,
		float maxContentLuminance);
		} // namespace skia
		} // namespace renderengine
		} // namespace android