NUKE fog image but try to emulate it exactly in GLSL

I want to NUKE the fog image and replace it with a simple calculation in
GLSL. But first, to make sure I'm not making a mistake somewhere, I
wanted to try to replicate the old behavior as exactly as possible.
Turns out I had to emulate texture filtering to get it reasonably close;
just using FogFactor with the actual s and t values resulted in
substantial differences in brightness.

So this commit NUKES the fog image but replicates it with overly
complicated GLSL.

Fixes the artifacts where the fog plane intersects a model when viewed
from inside the fog.

Author: slipher

src.cmake

@@ -152,6 +152,7 @@ set(GLSLSOURCELIST
     # Common shader libraries
     ${ENGINE_DIR}/renderer/glsl_source/common.glsl
     ${ENGINE_DIR}/renderer/glsl_source/common_cp.glsl
+    ${ENGINE_DIR}/renderer/glsl_source/fogEquation_fp.glsl
     ${ENGINE_DIR}/renderer/glsl_source/shaderProfiler_vp.glsl
     ${ENGINE_DIR}/renderer/glsl_source/shaderProfiler_fp.glsl
 

src/engine/renderer/Material.cpp

@@ -1115,11 +1115,6 @@ void BindShaderFog( Material* material ) {
 	gl_fogQuake3ShaderMaterial->SetUniform_ModelViewProjectionMatrix( glState.modelViewProjectionMatrix[glState.stackIndex] );
 
 	gl_fogQuake3ShaderMaterial->SetUniform_Time( backEnd.refdef.floatTime - backEnd.currentEntity->e.shaderTime );
-
-	// bind u_ColorMap
-	gl_fogQuake3ShaderMaterial->SetUniform_FogMapBindless(
-		GL_BindToTMU( 0, tr.fogImage )
-	);
 }
 
 void ProcessMaterialNONE( Material*, shaderStage_t*, MaterialSurface* ) {

src/engine/renderer/gl_shader.cpp

@@ -2580,7 +2580,6 @@ GLShader_skyboxMaterial::GLShader_skyboxMaterial() :
 GLShader_fogQuake3::GLShader_fogQuake3() :
 	GLShader( "fogQuake3", ATTR_POSITION | ATTR_QTANGENT,
 		false, "fogQuake3", "fogQuake3" ),
-	u_FogMap( this ),
 	u_ModelMatrix( this ),
 	u_ModelViewProjectionMatrix( this ),
 	u_ColorGlobal_Float( this ),
@@ -2597,15 +2596,9 @@ GLShader_fogQuake3::GLShader_fogQuake3() :
 {
 }
 
-void GLShader_fogQuake3::SetShaderProgramUniforms( ShaderProgramDescriptor *shaderProgram )
-{
-	glUniform1i( glGetUniformLocation( shaderProgram->id, "u_FogMap" ), 0 );
-}
-
 GLShader_fogQuake3Material::GLShader_fogQuake3Material() :
 	GLShader( "fogQuake3Material", ATTR_POSITION | ATTR_QTANGENT,
 		true, "fogQuake3", "fogQuake3" ),
-	u_FogMap( this ),
 	u_ModelMatrix( this ),
 	u_ModelViewProjectionMatrix( this ),
 	u_ColorGlobal_Uint( this ),
@@ -2619,7 +2612,6 @@ GLShader_fogQuake3Material::GLShader_fogQuake3Material() :
 GLShader_fogGlobal::GLShader_fogGlobal() :
 	GLShader( "fogGlobal", ATTR_POSITION,
 		false, "screenSpace", "fogGlobal" ),
-	u_ColorMap( this ),
 	u_DepthMap( this ),
 	u_UnprojectMatrix( this ),
 	u_Color_Float( this ),

src/engine/renderer/gl_shader.h

@@ -1826,18 +1826,6 @@ class u_CloudMap :
 	}
 };
 
-class u_FogMap :
-	GLUniformSampler2D {
-	public:
-	u_FogMap( GLShader* shader ) :
-		GLUniformSampler2D( shader, "u_FogMap" ) {
-	}
-
-	void SetUniform_FogMapBindless( GLuint64 bindlessHandle ) {
-		this->SetValueBindless( bindlessHandle );
-	}
-};
-
 class u_DepthTile1 :
 	GLUniformSampler2D {
 	public:
@@ -3426,7 +3414,6 @@ class GLShader_skyboxMaterial :
 
 class GLShader_fogQuake3 :
 	public GLShader,
-	public u_FogMap,
 	public u_ModelMatrix,
 	public u_ModelViewProjectionMatrix,
 	public u_ColorGlobal_Float,
@@ -3443,12 +3430,10 @@ class GLShader_fogQuake3 :
 {
 public:
 	GLShader_fogQuake3();
-	void SetShaderProgramUniforms( ShaderProgramDescriptor *shaderProgram ) override;
 };
 
 class GLShader_fogQuake3Material :
 	public GLShader,
-	public u_FogMap,
 	public u_ModelMatrix,
 	public u_ModelViewProjectionMatrix,
 	public u_ColorGlobal_Uint,
@@ -3463,7 +3448,6 @@ class GLShader_fogQuake3Material :
 
 class GLShader_fogGlobal :
 	public GLShader,
-	public u_ColorMap,
 	public u_DepthMap,
 	public u_UnprojectMatrix,
 	public u_Color_Float,

src/engine/renderer/glsl_source/fogEquation_fp.glsl

@@ -0,0 +1,77 @@
+/*
+===========================================================================
+Copyright (C) 1999-2005 Id Software, Inc.
+Copyright (C) 2006-2011 Robert Beckebans <trebor_7@users.sourceforge.net>
+
+This file is part of Daemon source code.
+
+Daemon source code is free software; you can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation; either version 2 of the License,
+or (at your option) any later version.
+
+Daemon source code is distributed in the hope that it will be
+useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with Daemon source code; if not, write to the Free Software
+Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+===========================================================================
+*/
+
+float FogTable(float s)
+{
+	return sqrt(floor(s * 255.0) / 255);
+}
+
+float FogFactor(float s, float t)
+{
+	s -= 1.0f / 512.0f;
+
+	if ( s < 0 )
+	{
+		return 0;
+	}
+
+	if ( t < 1.0f / 32.0f )
+	{
+		return 0;
+	}
+
+	if ( t < 31.0f / 32.0f )
+	{
+		s *= ( t - 1.0f / 32.0f ) / ( 30.0f / 32.0f );
+	}
+
+	s *= 8;
+
+	if ( s > 1.0f )
+	{
+		s = 1.0f;
+	}
+
+	return FogTable(s);
+}
+
+float GetFogAlpha(float s, float t)
+{
+	float sfloor = floor(s * 256 + 0.5) - 0.5;
+	float sceil = sfloor + 1;
+	sfloor = clamp(sfloor, 0.5, 255.5) / 256;
+	sceil = clamp(sceil, 0.5, 255.5) / 256;
+	float smix = sfloor < sceil ? (s - sfloor) * 256 : 0.5;
+
+	float tfloor = floor(t * 32 + 0.5) - 0.5;
+	float tceil = tfloor + 1;
+	tfloor = clamp(tfloor, 0.5, 31.5) / 32;
+	tceil = clamp(tceil, 0.5, 31.5) / 32;
+	float tmix = tfloor < tceil ? (t - tfloor) * 32 : 0.5;
+
+	float f00 = FogFactor(sfloor, tfloor);
+	float f01 = FogFactor(sfloor, tceil);
+	float f10 = FogFactor(sceil, tfloor);
+	float f11 = FogFactor(sceil, tceil);
+	return mix(mix(f00, f01, tmix), mix(f10, f11, tmix), smix);
+}

src/engine/renderer/glsl_source/fogGlobal_fp.glsl

@@ -23,8 +23,8 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 /* fogGlobal_fp.glsl */
 
 #insert common
+#insert fogEquation_fp
 
-uniform sampler2D	u_ColorMap; // fog texture
 uniform sampler2D	u_DepthMap;
 
 uniform colorPack u_Color;
@@ -46,10 +46,9 @@ void	main()
 	P.xyz /= P.w;
 
 	// calculate the length in fog (t is always 1 if eye is in fog)
-	st.s = distance(u_ViewOrigin, P.xyz) * u_FogDensity;
-	st.t = 1.0;
+	float s = distance(u_ViewOrigin, P.xyz) * u_FogDensity;
 
-	vec4 color = texture2D(u_ColorMap, st);
+	vec4 color = vec4(1, 1, 1, GetFogAlpha(s, 1.0));
 
 	outputColor = UnpackColor( u_Color ) * color;
 }

src/engine/renderer/glsl_source/fogQuake3_fp.glsl

@@ -22,9 +22,9 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 
 /* fogQuake3_fp.glsl */
 
-#define FOGQUAKE3_GLSL
+#insert fogEquation_fp
 
-uniform sampler2D u_FogMap;
+#define FOGQUAKE3_GLSL
 
 uniform float u_FogEyeT;
 
@@ -52,7 +52,7 @@ void	main()
 
 	t = 1.0 / 32.0 + ( 30.0 / 32.0 ) * t;
 
-	vec4 color = texture2D(u_FogMap, vec2(s, t));
+	vec4 color = vec4(1, 1, 1, GetFogAlpha(s, t));
 
 	color *= var_Color;
 

src/engine/renderer/tr_backend.cpp

@@ -1410,11 +1410,6 @@ void RB_RenderGlobalFog()
 
 	gl_fogGlobalShader->SetUniform_UnprojectMatrix( backEnd.viewParms.unprojectionMatrix );
 
-	// bind u_ColorMap
-	gl_fogGlobalShader->SetUniform_ColorMapBindless(
-		GL_BindToTMU( 0, tr.fogImage ) 
-	);
-
 	// bind u_DepthMap
 	gl_fogGlobalShader->SetUniform_DepthMapBindless(
 		GL_BindToTMU( 1, tr.currentDepthImage )

src/engine/renderer/tr_image.cpp

@@ -2351,121 +2351,6 @@ image_t *R_FindCubeImage( const char *name, imageParams_t &imageParams )
 	return nullptr;
 }
 
-/*
-=================
-R_InitFogTable
-=================
-*/
-void R_InitFogTable()
-{
-	int   i;
-	float d;
-	float exp;
-
-	exp = 0.5;
-
-	for ( i = 0; i < FOG_TABLE_SIZE; i++ )
-	{
-		d = pow( ( float ) i / ( FOG_TABLE_SIZE - 1 ), exp );
-
-		tr.fogTable[ i ] = d;
-	}
-}
-
-/*
-================
-R_FogFactor
-
-Returns a 0.0 to 1.0 fog density value
-This is called for each texel of the fog texture on startup
-and for each vertex of transparent shaders in fog dynamically
-================
-*/
-float R_FogFactor( float s, float t )
-{
-	float d;
-
-	s -= 1.0f / 512.0f;
-
-	if ( s < 0 )
-	{
-		return 0;
-	}
-
-	if ( t < 1.0f / 32.0f )
-	{
-		return 0;
-	}
-
-	if ( t < 31.0f / 32.0f )
-	{
-		s *= ( t - 1.0f / 32.0f ) / ( 30.0f / 32.0f );
-	}
-
-	// we need to leave a lot of clamp range
-	s *= 8;
-
-	if ( s > 1.0f )
-	{
-		s = 1.0f;
-	}
-
-	d = tr.fogTable[( int )( s * ( FOG_TABLE_SIZE - 1 ) ) ];
-
-	return d;
-}
-
-/*
-================
-R_CreateFogImage
-================
-*/
-static void R_CreateFogImage()
-{
-	// Fog image is always created because disabling fog is cheat.
-
-	int   x, y;
-	byte  *data, *ptr;
-	float d;
-	float borderColor[ 4 ];
-
-	constexpr int FOG_S = 256;
-	constexpr int FOG_T = 32;
-
-	ptr = data = (byte*) ri.Hunk_AllocateTempMemory( FOG_S * FOG_T * 4 );
-
-	// S is distance, T is depth
-	for ( y = 0; y < FOG_T; y++ )
-	{
-		for ( x = 0; x < FOG_S; x++ )
-		{
-			d = R_FogFactor( ( x + 0.5f ) / FOG_S, ( y + 0.5f ) / FOG_T );
-
-			ptr[ 0 ] = ptr[ 1 ] = ptr[ 2 ] = 255;
-			ptr[ 3 ] = 255 * d;
-			ptr += 4;
-		}
-	}
-
-	// standard openGL clamping doesn't really do what we want -- it includes
-	// the border color at the edges.  OpenGL 1.2 has clamp-to-edge, which does
-	// what we want.
-	imageParams_t imageParams = {};
-	imageParams.bits = IF_NOPICMIP;
-	imageParams.filterType = filterType_t::FT_DEFAULT;
-	imageParams.wrapType = wrapTypeEnum_t::WT_CLAMP;
-
-	tr.fogImage = R_CreateImage( "_fog", ( const byte ** ) &data, FOG_S, FOG_T, 1, imageParams );
-	ri.Hunk_FreeTempMemory( data );
-
-	borderColor[ 0 ] = 1.0;
-	borderColor[ 1 ] = 1.0;
-	borderColor[ 2 ] = 1.0;
-	borderColor[ 3 ] = 1;
-
-	glTexParameterfv( GL_TEXTURE_2D, GL_TEXTURE_BORDER_COLOR, borderColor );
-}
-
 /*
 ==================
 R_CreateDefaultImage
@@ -2811,7 +2696,6 @@ void R_CreateBuiltinImages()
 		image = R_CreateImage( "_cinematic", ( const byte ** ) &dataPtr, 1, 1, 1, imageParams );
 	}
 
-	R_CreateFogImage();
 	R_CreateContrastRenderFBOImage();
 	R_CreateBloomRenderFBOImages();
 	R_CreateCurrentRenderImage();

src/engine/renderer/tr_init.cpp

@@ -1324,8 +1324,6 @@ ScreenshotCmd screenshotPNGRegistration("screenshotPNG", ssFormat_t::SSF_PNG, "p
 			}
 		}
 
-		R_InitFogTable();
-
 		R_NoiseInit();
 
 		R_Register();