[RenderEngine] Polish shader pipeline for HDR support. am: a296b0c232 (159742d2) · Commits · e / os / android_frameworks_native-old

services/surfaceflinger/RenderEngine/Description.cpp

+23 −2

Original line number	Diff line number	Diff line
		@@ -52,9 +52,30 @@ void Description::setProjectionMatrix(const mat4& mtx) {
		}

		void Description::setColorMatrix(const mat4& mtx) {
		const mat4 identity;
		mColorMatrix = mtx;
		mColorMatrixEnabled = (mtx != identity);
		}

		void Description::setInputTransformMatrix(const mat3& matrix) {
		mInputTransformMatrix = matrix;
		}

		void Description::setOutputTransformMatrix(const mat4& matrix) {
		mOutputTransformMatrix = matrix;
		}

		bool Description::hasInputTransformMatrix() const {
		const mat3 identity;
		return mInputTransformMatrix != identity;
		}

		bool Description::hasOutputTransformMatrix() const {
		const mat4 identity;
		return mOutputTransformMatrix != identity;
		}

		bool Description::hasColorMatrix() const {
		const mat4 identity;
		return mColorMatrix != identity;
		}

		const mat4& Description::getColorMatrix() const {

services/surfaceflinger/RenderEngine/Description.h

+11 −5

Original line number	Diff line number	Diff line
		@@ -43,6 +43,11 @@ public:
		void setColor(const half4& color);
		void setProjectionMatrix(const mat4& mtx);
		void setColorMatrix(const mat4& mtx);
		void setInputTransformMatrix(const mat3& matrix);
		void setOutputTransformMatrix(const mat4& matrix);
		bool hasInputTransformMatrix() const;
		bool hasOutputTransformMatrix() const;
		bool hasColorMatrix() const;
		const mat4& getColorMatrix() const;

		void setY410BT2020(bool enable);
		@@ -72,11 +77,6 @@ private:

		// color used when texturing is disabled or when setting alpha.
		half4 mColor;
		// projection matrix
		mat4 mProjectionMatrix;

		bool mColorMatrixEnabled = false;
		mat4 mColorMatrix;

		// true if the sampled pixel values are in Y410/BT2020 rather than RGBA
		bool mY410BT2020 = false;
		@@ -86,6 +86,12 @@ private:
		TransferFunction mOutputTransferFunction = TransferFunction::LINEAR;

		float mDisplayMaxLuminance;

		// projection matrix
		mat4 mProjectionMatrix;
		mat4 mColorMatrix;
		mat3 mInputTransformMatrix;
		mat4 mOutputTransformMatrix;
		};

		} /* namespace android */

services/surfaceflinger/RenderEngine/GLES20RenderEngine.cpp

+131 −43

Original line number	Diff line number	Diff line
		@@ -131,15 +131,24 @@ GLES20RenderEngine::GLES20RenderEngine(uint32_t featureFlags)
		// mColorBlindnessCorrection = M;

		if (mPlatformHasWideColor) {
		// Compute sRGB to DisplayP3 color transform
		// NOTE: For now, we are limiting wide-color support to
		ColorSpace srgb(ColorSpace::sRGB());
		ColorSpace displayP3(ColorSpace::DisplayP3());
		ColorSpace bt2020(ColorSpace::BT2020());

		// Compute sRGB to Display P3 transform matrix.
		// NOTE: For now, we are limiting output wide color space support to
		// Display-P3 only.
		mSrgbToDisplayP3 = mat4(
		ColorSpaceConnector(ColorSpace::sRGB(), ColorSpace::DisplayP3()).getTransform());
		mSrgbToDisplayP3 = mat4(ColorSpaceConnector(srgb, displayP3).getTransform());

		// Compute Display P3 to sRGB transform matrix.
		mDisplayP3ToSrgb = mat4(ColorSpaceConnector(displayP3, srgb).getTransform());

		// Compute BT2020 to DisplayP3 color transform
		mBt2020ToDisplayP3 = mat4(
		ColorSpaceConnector(ColorSpace::BT2020(), ColorSpace::DisplayP3()).getTransform());
		// no chromatic adaptation needed since all color spaces use D65 for their white points.
		mSrgbToXyz = srgb.getRGBtoXYZ();
		mDisplayP3ToXyz = displayP3.getRGBtoXYZ();
		mBt2020ToXyz = bt2020.getRGBtoXYZ();
		mXyzToDisplayP3 = mat4(displayP3.getXYZtoRGB());
		mXyzToBt2020 = mat4(bt2020.getXYZtoRGB());
		}
		}

		@@ -307,44 +316,96 @@ void GLES20RenderEngine::drawMesh(const Mesh& mesh) {
		glVertexAttribPointer(Program::position, mesh.getVertexSize(), GL_FLOAT, GL_FALSE,
		mesh.getByteStride(), mesh.getPositions());

		// TODO(b/73825729) Refactor this code block to handle BT2020 color space properly.
		// DISPLAY_P3 is the only supported wide color output
		if (mPlatformHasWideColor && mOutputDataSpace == Dataspace::DISPLAY_P3) {
		// By default, DISPLAY_P3 is the only supported wide color output. However,
		// when HDR content is present, hardware composer may be able to handle
		// BT2020 data space, in that case, the output data space is set to be
		// BT2020_HLG or BT2020_PQ respectively. In GPU fall back we need
		// to respect this and convert non-HDR content to HDR format.
		if (mPlatformHasWideColor) {
		Description wideColorState = mState;
		switch (mDataSpace) {
		case Dataspace::DISPLAY_P3:
		// input matches output
		Dataspace inputStandard = static_cast<Dataspace>(mDataSpace & Dataspace::STANDARD_MASK);
		Dataspace inputTransfer = static_cast<Dataspace>(mDataSpace & Dataspace::TRANSFER_MASK);
		Dataspace outputStandard = static_cast<Dataspace>(mOutputDataSpace &
		Dataspace::STANDARD_MASK);
		Dataspace outputTransfer = static_cast<Dataspace>(mOutputDataSpace &
		Dataspace::TRANSFER_MASK);
		bool needsXYZConversion = needsXYZTransformMatrix();

		if (needsXYZConversion) {
		// The supported input color spaces are standard RGB, Display P3 and BT2020.
		switch (inputStandard) {
		case Dataspace::STANDARD_DCI_P3:
		wideColorState.setInputTransformMatrix(mDisplayP3ToXyz);
		break;
		case Dataspace::STANDARD_BT2020:
		wideColorState.setInputTransformMatrix(mBt2020ToXyz);
		break;
		default:
		wideColorState.setInputTransformMatrix(mSrgbToXyz);
		break;
		case Dataspace::BT2020_PQ:
		case Dataspace::BT2020_ITU_PQ:
		wideColorState.setColorMatrix(mState.getColorMatrix() * mBt2020ToDisplayP3);
		}

		// The supported output color spaces are Display P3 and BT2020.
		switch (outputStandard) {
		case Dataspace::STANDARD_BT2020:
		wideColorState.setOutputTransformMatrix(mXyzToBt2020);
		break;
		default:
		wideColorState.setOutputTransformMatrix(mXyzToDisplayP3);
		break;
		}
		} else if (inputStandard != outputStandard) {
		// At this point, the input data space and output data space could be both
		// HDR data spaces, but they match each other, we do nothing in this case.
		// In addition to the case above, the input data space could be
		// - scRGB linear
		// - scRGB non-linear
		// - sRGB
		// - Display P3
		// The output data spaces could be
		// - sRGB
		// - Display P3
		if (outputStandard == Dataspace::STANDARD_BT709) {
		wideColorState.setOutputTransformMatrix(mDisplayP3ToSrgb);
		} else if (outputStandard == Dataspace::STANDARD_DCI_P3) {
		wideColorState.setOutputTransformMatrix(mSrgbToDisplayP3);
		}
		}

		// we need to convert the RGB value to linear space and convert it back when:
		// - there is a color matrix that is not an identity matrix, or
		// - there is an output transform matrix that is not an identity matrix, or
		// - the input transfer function doesn't match the output transfer function.
		if (wideColorState.hasColorMatrix() \|\| wideColorState.hasOutputTransformMatrix() \|\|
		inputTransfer != outputTransfer) {
		switch (inputTransfer) {
		case Dataspace::TRANSFER_ST2084:
		wideColorState.setInputTransferFunction(Description::TransferFunction::ST2084);
		wideColorState.setOutputTransferFunction(Description::TransferFunction::SRGB);
		break;
		case Dataspace::BT2020_HLG:
		case Dataspace::BT2020_ITU_HLG:
		wideColorState.setColorMatrix(mState.getColorMatrix() * mBt2020ToDisplayP3);
		case Dataspace::TRANSFER_HLG:
		wideColorState.setInputTransferFunction(Description::TransferFunction::HLG);
		wideColorState.setOutputTransferFunction(Description::TransferFunction::SRGB);
		break;
		default:
		// treat all other dataspaces as sRGB
		wideColorState.setColorMatrix(mState.getColorMatrix() * mSrgbToDisplayP3);
		switch (static_cast<Dataspace>(mDataSpace & Dataspace::TRANSFER_MASK)) {
		case Dataspace::TRANSFER_LINEAR:
		wideColorState.setInputTransferFunction(
		Description::TransferFunction::LINEAR);
		wideColorState.setInputTransferFunction(Description::TransferFunction::LINEAR);
		break;
		default:
		// treat all other transfer functions as sRGB
		wideColorState.setInputTransferFunction(
		Description::TransferFunction::SRGB);
		wideColorState.setInputTransferFunction(Description::TransferFunction::SRGB);
		break;
		}

		switch (outputTransfer) {
		case Dataspace::TRANSFER_ST2084:
		wideColorState.setOutputTransferFunction(Description::TransferFunction::ST2084);
		break;
		case Dataspace::TRANSFER_HLG:
		wideColorState.setOutputTransferFunction(Description::TransferFunction::HLG);
		break;
		default:
		wideColorState.setOutputTransferFunction(Description::TransferFunction::SRGB);
		ALOGV("drawMesh: gamut transform applied");
		break;
		}
		}

		ProgramCache::getInstance().useProgram(wideColorState);

		glDrawArrays(mesh.getPrimitive(), 0, mesh.getVertexCount());
		@@ -373,6 +434,33 @@ void GLES20RenderEngine::dump(String8& result) {
		dataspaceDetails(static_cast<android_dataspace>(mOutputDataSpace)).c_str());
		}

		bool GLES20RenderEngine::isHdrDataSpace(const Dataspace dataSpace) const {
		const Dataspace standard = static_cast<Dataspace>(dataSpace & Dataspace::STANDARD_MASK);
		const Dataspace transfer = static_cast<Dataspace>(dataSpace & Dataspace::TRANSFER_MASK);
		return standard == Dataspace::STANDARD_BT2020 &&
		(transfer == Dataspace::TRANSFER_ST2084 \|\| transfer == Dataspace::TRANSFER_HLG);
		}

		// For convenience, we want to convert the input color space to XYZ color space first,
		// and then convert from XYZ color space to output color space when
		// - SDR and HDR contents are mixed, either SDR content will be converted to HDR or
		// HDR content will be tone-mapped to SDR; Or,
		// - there are HDR PQ and HLG contents presented at the same time, where we want to convert
		// HLG content to PQ content.
		// In either case above, we need to operate the Y value in XYZ color space. Thus, when either
		// input data space or output data space is HDR data space, and the input transfer function
		// doesn't match the output transfer function, we would enable an intermediate transfrom to
		// XYZ color space.
		bool GLES20RenderEngine::needsXYZTransformMatrix() const {
		const bool isInputHdrDataSpace = isHdrDataSpace(mDataSpace);
		const bool isOutputHdrDataSpace = isHdrDataSpace(mOutputDataSpace);
		const Dataspace inputTransfer = static_cast<Dataspace>(mDataSpace & Dataspace::TRANSFER_MASK);
		const Dataspace outputTransfer = static_cast<Dataspace>(mOutputDataSpace &
		Dataspace::TRANSFER_MASK);

		return (isInputHdrDataSpace \|\| isOutputHdrDataSpace) && inputTransfer != outputTransfer;
		}

		// ---------------------------------------------------------------------------
		} // namespace impl
		} // namespace RE

services/surfaceflinger/RenderEngine/GLES20RenderEngine.h

+12 −1

Original line number	Diff line number	Diff line
		@@ -98,7 +98,18 @@ protected:
		// Currently only supporting sRGB, BT2020 and DisplayP3 color spaces
		const bool mPlatformHasWideColor = false;
		mat4 mSrgbToDisplayP3;
		mat4 mBt2020ToDisplayP3;
		mat4 mDisplayP3ToSrgb;
		mat3 mSrgbToXyz;
		mat3 mBt2020ToXyz;
		mat3 mDisplayP3ToXyz;
		mat4 mXyzToDisplayP3;
		mat4 mXyzToBt2020;

		private:
		// A data space is considered HDR data space if it has BT2020 color space
		// with PQ or HLG transfer function.
		bool isHdrDataSpace(const ui::Dataspace dataSpace) const;
		bool needsXYZTransformMatrix() const;
		};

		// ---------------------------------------------------------------------------

services/surfaceflinger/RenderEngine/Program.cpp

+12 −4

Original line number	Diff line number	Diff line
		@@ -58,13 +58,13 @@ Program::Program(const ProgramCache::Key& /needs/, const char* vertex, const c
		mVertexShader = vertexId;
		mFragmentShader = fragmentId;
		mInitialized = true;

		mColorMatrixLoc = glGetUniformLocation(programId, "colorMatrix");
		mProjectionMatrixLoc = glGetUniformLocation(programId, "projection");
		mTextureMatrixLoc = glGetUniformLocation(programId, "texture");
		mSamplerLoc = glGetUniformLocation(programId, "sampler");
		mColorLoc = glGetUniformLocation(programId, "color");
		mDisplayMaxLuminanceLoc = glGetUniformLocation(programId, "displayMaxLuminance");
		mInputTransformMatrixLoc = glGetUniformLocation(programId, "inputTransformMatrix");
		mOutputTransformMatrixLoc = glGetUniformLocation(programId, "outputTransformMatrix");

		// set-up the default values for our uniforms
		glUseProgram(programId);
		@@ -134,8 +134,16 @@ void Program::setUniforms(const Description& desc) {
		const float color[4] = {desc.mColor.r, desc.mColor.g, desc.mColor.b, desc.mColor.a};
		glUniform4fv(mColorLoc, 1, color);
		}
		if (mColorMatrixLoc >= 0) {
		glUniformMatrix4fv(mColorMatrixLoc, 1, GL_FALSE, desc.mColorMatrix.asArray());
		if (mInputTransformMatrixLoc >= 0) {
		glUniformMatrix3fv(mInputTransformMatrixLoc, 1, GL_FALSE,
		desc.mInputTransformMatrix.asArray());
		}
		if (mOutputTransformMatrixLoc >= 0) {
		// The output transform matrix and color matrix can be combined as one matrix
		// that is applied right before applying OETF.
		mat4 outputTransformMatrix = desc.mColorMatrix * desc.mOutputTransformMatrix;
		glUniformMatrix4fv(mOutputTransformMatrixLoc, 1, GL_FALSE,
		outputTransformMatrix.asArray());
		}
		if (mDisplayMaxLuminanceLoc >= 0) {
		glUniform1f(mDisplayMaxLuminanceLoc, desc.mDisplayMaxLuminance);