Use colorModulateColorGen instead of disabling vertex color

Originally, vertex color array was disabled if colorGen CGEN_VERTEX or CGEN_ONE_MINUS_VERTEX or alphaGen CGEN_VERTEX or CGEN_ONE_MINUS_VERTEX were used,
resulting in the shader receiving the default OpenGL values for the disabled arrays (0.0, 0.0, 0.0, 1.0).

Now it will instead be set via setting a bit in `u_ColorModulateColorGen`, which allows skipping the vertex format change. It will also be necessary for the geometry cache.

Also fixes incorrect lighting with `tr.mapOverBrightBits = 3`.

Author: VReaperV

src/engine/renderer/Material.cpp

@@ -1107,12 +1107,6 @@ void ProcessMaterialGeneric3D( Material* material, shaderStage_t* pStage, drawSu
 	material->tcGen_Lightmap = pStage->tcGen_Lightmap;
 	material->deformIndex = pStage->deformIndex;
 
-	colorGen_t rgbGen = SetRgbGen( pStage );
-	alphaGen_t alphaGen = SetAlphaGen( pStage );
-
-	material->useAttrColor = rgbGen == colorGen_t::CGEN_VERTEX || rgbGen == colorGen_t::CGEN_ONE_MINUS_VERTEX
-		|| alphaGen == alphaGen_t::AGEN_VERTEX || alphaGen == alphaGen_t::AGEN_ONE_MINUS_VERTEX;
-
 	gl_genericShaderMaterial->SetTCGenEnvironment( pStage->tcGen_Environment );
 	gl_genericShaderMaterial->SetTCGenLightmap( pStage->tcGen_Lightmap );
 
@@ -1138,14 +1132,6 @@ void ProcessMaterialLightMapping( Material* material, shaderStage_t* pStage, dra
 
 	DAEMON_ASSERT( !( enableDeluxeMapping && enableGridDeluxeMapping ) );
 
-	// useAttrColor has no effect since the lightMapping shader has ATTR_COLOR forced to be always
-	// on (_requiredVertexAttribs). If we removed ATTR_COLOR from there, we would need to detect
-	// implicit vertex lighting as well, not only rgbgen (see SetLightDeluxeMode).
-	/* colorGen_t rgbGen = SetRgbGen( pStage );
-	alphaGen_t alphaGen = SetAlphaGen( pStage );
-	material->useAttrColor = rgbGen == colorGen_t::CGEN_VERTEX || rgbGen == colorGen_t::CGEN_ONE_MINUS_VERTEX
-		|| alphaGen == alphaGen_t::AGEN_VERTEX || alphaGen == alphaGen_t::AGEN_ONE_MINUS_VERTEX; */
-
 	material->enableDeluxeMapping = enableDeluxeMapping;
 	material->enableGridLighting = enableGridLighting;
 	material->enableGridDeluxeMapping = enableGridDeluxeMapping;
@@ -2138,12 +2124,6 @@ void MaterialSystem::RenderMaterial( Material& material, const uint32_t viewID )
 
 	backEnd.currentEntity = &tr.worldEntity;
 
-	if ( material.useAttrColor ) {
-		material.shader->AddVertexAttribBit( ATTR_COLOR );
-	} else {
-		material.shader->DelVertexAttribBit( ATTR_COLOR );
-	}
-
 	GL_State( stateBits );
 	if ( material.usePolygonOffset ) {
 		glEnable( GL_POLYGON_OFFSET_FILL );

src/engine/renderer/Material.h

@@ -100,8 +100,6 @@ struct Material {
 	bool enableSpecularMapping;
 	bool enablePhysicalMapping;
 
-	bool useAttrColor = false;
-
 	cullType_t cullType;
 
 	uint32_t sort;
@@ -120,8 +118,7 @@ struct Material {
 
 	bool operator==( const Material& other ) {
 		return program == other.program && stateBits == other.stateBits && vbo == other.vbo && ibo == other.ibo
-			&& fog == other.fog && cullType == other.cullType && usePolygonOffset == other.usePolygonOffset
-			&& useAttrColor == other.useAttrColor;
+			&& fog == other.fog && cullType == other.cullType && usePolygonOffset == other.usePolygonOffset;
 	}
 
 	void AddTexture( Texture* texture ) {

src/engine/renderer/gl_shader.cpp

@@ -2220,7 +2220,7 @@ void GLShader::WriteUniformsToBuffer( uint32_t* buffer ) {
 }
 
 GLShader_generic::GLShader_generic( GLShaderManager *manager ) :
-	GLShader( "generic", ATTR_POSITION | ATTR_TEXCOORD | ATTR_QTANGENT, manager ),
+	GLShader( "generic", ATTR_POSITION | ATTR_TEXCOORD | ATTR_QTANGENT | ATTR_COLOR, manager ),
 	u_ColorMap( this ),
 	u_DepthMap( this ),
 	u_TextureMatrix( this ),
@@ -2252,7 +2252,7 @@ void GLShader_generic::SetShaderProgramUniforms( shaderProgram_t *shaderProgram
 }
 
 GLShader_genericMaterial::GLShader_genericMaterial( GLShaderManager* manager ) :
-	GLShader( "genericMaterial", "generic", true, ATTR_POSITION | ATTR_TEXCOORD | ATTR_QTANGENT, manager ),
+	GLShader( "genericMaterial", "generic", true, ATTR_POSITION | ATTR_TEXCOORD | ATTR_QTANGENT | ATTR_COLOR, manager ),
 	u_ColorMap( this ),
 	u_DepthMap( this ),
 	u_TextureMatrix( this ),

src/engine/renderer/gl_shader.h

@@ -3664,7 +3664,8 @@ enum class ColorModulate {
 	COLOR_MINUS_ONE = BIT( 1 ),
 	COLOR_LIGHTFACTOR = BIT( 2 ),
 	ALPHA_ONE = BIT( 3 ),
-	ALPHA_MINUS_ONE = BIT( 4 )
+	ALPHA_MINUS_ONE = BIT( 4 ),
+	ALPHA_ADD_ONE = BIT( 5 )
 };
 
 class u_ColorModulateColorGen :
@@ -3694,7 +3695,7 @@ class u_ColorModulateColorGen :
 					// vertexOverbright is only needed for non-lightmapped cases. When there is a
 					// lightmap, this is done by multiplying with the overbright-scaled white image
 					colorModulate |= Util::ordinal( ColorModulate::COLOR_LIGHTFACTOR );
-					lightFactor = uint32_t( tr.mapLightFactor ) << 5;
+					lightFactor = uint32_t( tr.mapLightFactor ) << 6;
 				} else {
 					colorModulate |= Util::ordinal( ColorModulate::COLOR_ONE );
 				}
@@ -3711,10 +3712,10 @@ class u_ColorModulateColorGen :
 
 		if ( useMapLightFactor ) {
 			ASSERT_EQ( vertexOverbright, false );
-			lightFactor = uint32_t( tr.mapLightFactor ) << 5;
+			lightFactor = uint32_t( tr.mapLightFactor ) << 6;
 		}
 
-		colorModulate |= lightFactor ? lightFactor : 1 << 5;
+		colorModulate |= lightFactor ? lightFactor : 1 << 6;
 
 		switch ( alphaGen ) {
 			case alphaGen_t::AGEN_VERTEX:
@@ -3731,10 +3732,12 @@ class u_ColorModulateColorGen :
 				break;
 		}
 
-		if ( needAttrib ) {
-			_shader->AddVertexAttribBit( ATTR_COLOR );
-		} else {
-			_shader->DelVertexAttribBit( ATTR_COLOR );
+		if ( !needAttrib ) {
+			/* Originally, this controlled whether the vertex color array was used,
+			now it does the equivalent by setting the color in the shader in such way as if it was using
+			the default OpenGL values for the disabled arrays (0.0, 0.0, 0.0, 1.0)
+			This allows to skip the vertex format change */
+			colorModulate |= Util::ordinal( ColorModulate::ALPHA_ADD_ONE );
 		}
 		this->SetValue( colorModulate );
 	}

src/engine/renderer/glsl_source/common.glsl

@@ -48,7 +48,8 @@ Bit 1: color * ( -1 )
 Bit 2: color += lightFactor
 Bit 3: alpha * 1
 Bit 4: alpha * ( -1 )
-Bit 5-7: lightFactor */
+Bit 5: alpha = 1
+Bit 6-9: lightFactor */
 
 float colorModArray[3] = float[3] ( 0.0f, 1.0f, -1.0f );
 
@@ -65,5 +66,10 @@ vec4 ColorModulateToColor( const in uint colorMod, const in float lightFactor )
 }
 
 float ColorModulateToLightFactor( const in uint colorMod ) {
-	return ( colorMod >> 5 ) & 0x7;
+	return ( colorMod >> 6 ) & 0xF;
+}
+
+// This is used to skip vertex colours if the colorMod doesn't need them
+bool ColorModulateToVertexColor( const in uint colorMod ) {
+	return ( colorMod & 0xFF ) == 0xFF;
 }

src/engine/renderer/glsl_source/generic_vp.glsl

@@ -68,6 +68,7 @@ void	main()
 
 	VertexFetch( position, LB, color, texCoord, lmCoord );
 	float lightFactor = ColorModulateToLightFactor( u_ColorModulateColorGen );
+	color.a = ColorModulateToVertexColor( u_ColorModulateColorGen ) ? 1.0 : color.a;
 	color = color * ColorModulateToColor( u_ColorModulateColorGen, lightFactor )
 		+ unpackUnorm4x8( u_Color ) * vec4( lightFactor, lightFactor, lightFactor, 1.0 );