Bloom and fxaa generate artifacts on transparent materials.

[7Arz]

Godot version

3.5 rc1-rc2

System information

GNU/Linux Pop Os 20.04 GPU: Nvidia GeForce GTX 1070ti.

Issue description

Bloom and fxaa generate artifacts on transparent materials. It is related to this issue #61359

Steps to reproduce

Make transparent material with alpha texture and use bloom or fxxa.

Minimal reproduction project

No response

[7Arz]

This solved the problem for me:

// GLOW
vec4 apply_glow(vec4 color, vec3 glow) { // apply glow using the selected blending mode
#ifdef USE_GLOW_REPLACE
	color.rgb = glow;
#endif

#ifdef USE_GLOW_SCREEN
	//need color clamping
	color.rgb = clamp(color.rgb, vec3(0.0f), vec3(1.0f));
	color.rgb = max((color.rgb + glow) - (color.rgb * glow), vec3(0.0));
#endif

#ifdef USE_GLOW_SOFTLIGHT
	//need color clamping
	color.rgb = clamp(color.rgb, vec3(0.0f), vec3(1.0));
	glow = glow * vec3(0.5f) + vec3(0.5f);

	color.r = (glow.r <= 0.5f) ? (color.r - (1.0f - 2.0f * glow.r) * color.r * (1.0f - color.r)) : (((glow.r > 0.5f) && (color.r <= 0.25f)) ? (color.r + (2.0f * glow.r - 1.0f) * (4.0f * color.r * (4.0f * color.r + 1.0f) * (color.r - 1.0f) + 7.0f * color.r)) : (color.r + (2.0f * glow.r - 1.0f) * (sqrt(color.r) - color.r)));
	color.g = (glow.g <= 0.5f) ? (color.g - (1.0f - 2.0f * glow.g) * color.g * (1.0f - color.g)) : (((glow.g > 0.5f) && (color.g <= 0.25f)) ? (color.g + (2.0f * glow.g - 1.0f) * (4.0f * color.g * (4.0f * color.g + 1.0f) * (color.g - 1.0f) + 7.0f * color.g)) : (color.g + (2.0f * glow.g - 1.0f) * (sqrt(color.g) - color.g)));
	color.b = (glow.b <= 0.5f) ? (color.b - (1.0f - 2.0f * glow.b) * color.b * (1.0f - color.b)) : (((glow.b > 0.5f) && (color.b <= 0.25f)) ? (color.b + (2.0f * glow.b - 1.0f) * (4.0f * color.b * (4.0f * color.b + 1.0f) * (color.b - 1.0f) + 7.0f * color.b)) : (color.b + (2.0f * glow.b - 1.0f) * (sqrt(color.b) - color.b)));
#endif

#if !defined(USE_GLOW_SCREEN) && !defined(USE_GLOW_SOFTLIGHT) && !defined(USE_GLOW_REPLACE) // no other selected -> additive
	color.rgb += glow;
#endif

// Fix.
#ifdef DISABLE_ALPHA
	color.a = 1.0;
#endif

#ifndef USE_GLOW_SOFTLIGHT // softlight has no effect on black color
	// compute the alpha from glow
	float a = max(max(glow.r, glow.g), glow.b);
	color.a = a + color.a * (1.0 - a);
	if (color.a == 0.0) {
		color.rgb = vec3(0.0);
	} else if (color.a < 1.0) {
		color.rgb /= color.a;
	}
#endif

	return color;
} 

// FXAA

vec4 apply_fxaa(vec4 color, float exposure, vec2 uv_interp, vec2 pixel_size) {
	const float FXAA_REDUCE_MIN = (1.0 / 128.0);
	const float FXAA_REDUCE_MUL = (1.0 / 8.0);
	const float FXAA_SPAN_MAX = 8.0;

	vec4 rgbNW = textureLod(source, uv_interp + vec2(-1.0, -1.0) * pixel_size, 0.0);
	vec4 rgbNE = textureLod(source, uv_interp + vec2(1.0, -1.0) * pixel_size, 0.0);
	vec4 rgbSW = textureLod(source, uv_interp + vec2(-1.0, 1.0) * pixel_size, 0.0);
	vec4 rgbSE = textureLod(source, uv_interp + vec2(1.0, 1.0) * pixel_size, 0.0);
	vec3 rgbM = color.rgb;
	vec3 luma = vec3(0.299, 0.587, 0.114);
	float lumaNW = dot(rgbNW.rgb * exposure, luma) - ((1.0 - rgbNW.a) / 8.0);
	float lumaNE = dot(rgbNE.rgb * exposure, luma) - ((1.0 - rgbNE.a) / 8.0);
	float lumaSW = dot(rgbSW.rgb * exposure, luma) - ((1.0 - rgbSW.a) / 8.0);
	float lumaSE = dot(rgbSE.rgb * exposure, luma) - ((1.0 - rgbSE.a) / 8.0);
	float lumaM = dot(rgbM * exposure, luma) - (color.a / 8.0);
	float lumaMin = min(lumaM, min(min(lumaNW, lumaNE), min(lumaSW, lumaSE)));
	float lumaMax = max(lumaM, max(max(lumaNW, lumaNE), max(lumaSW, lumaSE)));

	vec2 dir;
	dir.x = -((lumaNW + lumaNE) - (lumaSW + lumaSE));
	dir.y = ((lumaNW + lumaSW) - (lumaNE + lumaSE));

	float dirReduce = max((lumaNW + lumaNE + lumaSW + lumaSE) *
					(0.25 * FXAA_REDUCE_MUL),
			FXAA_REDUCE_MIN);

	float rcpDirMin = 1.0 / (min(abs(dir.x), abs(dir.y)) + dirReduce);
	dir = min(vec2(FXAA_SPAN_MAX, FXAA_SPAN_MAX),
				  max(vec2(-FXAA_SPAN_MAX, -FXAA_SPAN_MAX),
						  dir * rcpDirMin)) *
			pixel_size;

	vec4 rgbA = 0.5 * exposure * (textureLod(source, uv_interp + dir * (1.0 / 3.0 - 0.5), 0.0) + textureLod(source, uv_interp + dir * (2.0 / 3.0 - 0.5), 0.0));
	vec4 rgbB = rgbA * 0.5 + 0.25 * exposure * (textureLod(source, uv_interp + dir * -0.5, 0.0) + textureLod(source, uv_interp + dir * 0.5, 0.0));

	float lumaB = dot(rgbB.rgb, luma) - ((1.0 - rgbB.a) / 8.0);
	vec4 color_output = ((lumaB < lumaMin) || (lumaB > lumaMax)) ? rgbA : rgbB;

// Fix.
#ifdef DISABLE_ALPHA
	color_output.a = 1.0;
#endif

	if (color_output.a == 0.0) {
		color_output.rgb = vec3(0.0);
	} else if (color_output.a < 1.0) {
		color_output.rgb /= color_output.a;
	}
	return color_output;
}

[clayjohn]

I can't seem to reproduce your issue, but I am not suprised that bloom and FXAA are causing further issues. Both read from the alpha channel to calculate their respective effect. Since we can no longer count on the alpha channel coming from the screen to be clean, we need to force alpha to 1 when using those effects.

For Glow that can be done exactly as you have done, but for FXAA, it is going to require a slight change in the calculation, as you can see that the effect changes depending on the alpha value. In the DISABLE_ALPHA block, we can just use the old version of the FXAA shader that ignored alpha.

[7Arz]

I can't seem to reproduce your issue.

I uploaded minimal reproduction project. The bug is more appreciated in play mode.
TransparentBug.zip

[akien-mga]

Fixed by #61442.