vk: rt: skip loading zero-area polygon lights

These cause NaNs in lighting computation, which leads to visual glitches, like strong yellow or green lighting.

Fixes #461

ref/vk/TODO.md

@@ -1,3 +1,6 @@
+# 2024-01-11 E361
+- [x] fix zero-area polygon lights nanites, fixes #461
+
 # 2024-01-09 E360
 - [x] validate all intermediate and final outputs against invalid values, complain into log
 - [ ] brdf math surprising edge cases

ref/vk/shaders/light_polygon.glsl

@@ -51,6 +51,16 @@ vec4 getPolygonLightSampleSimple(vec3 P, vec3 view_dir, const PolygonLight poly)
 	const vec3 light_dir = baryMix(v[0], v[1], v[2], rnd) - P;
 	const vec3 light_dir_n = normalize(light_dir);
 	const float contrib = - poly.area * dot(light_dir_n, poly.plane.xyz ) / dot(light_dir, light_dir);
+
+#ifdef DEBUG_VALIDATE_EXTRA
+	if (IS_INVALID(contrib)) {
+		debugPrintfEXT("getPolygonLightSampleSimple: poly.area=%f light_dir=(%f,%f,%f) INVALID contrib=%f",
+			poly.area,
+			PRIVEC3(light_dir),
+			contrib);
+	}
+#endif
+
 	return vec4(light_dir_n, contrib);
 }
 
@@ -134,6 +144,15 @@ vec4 getPolygonLightSampleProjected(vec3 view_dir, SampleContext ctx, const Poly
 		return vec4(0.f);
 
 	const projected_solid_angle_polygon_t sap = prepare_projected_solid_angle_polygon_sampling(vertices_count, clipped);
+
+#ifdef DEBUG_VALIDATE_EXTRA
+	if (IS_INVALID(sap.projected_solid_angle)) {
+		debugPrintfEXT("getPolygonLightSampleProjected: vertices_count=%d v0=(%f,%f,%f) v1=(%f,%f,%f) v2=(%f,%f,%f) INVALID sap.projected_solid_angle = %f",
+			vertices_count, PRIVEC3(clipped[0]), PRIVEC3(clipped[1]), PRIVEC3(clipped[2]),
+			sap.projected_solid_angle);
+	}
+#endif
+
 	const float contrib = sap.projected_solid_angle;
 	if (contrib <= 0.f)
 		return vec4(0.f);
@@ -269,6 +288,11 @@ void sampleEmissiveSurfaces(vec3 P, vec3 N, vec3 view_dir, MaterialProperties ma
 					__LINE__, PRIVEC3(specular), index, PRIVEC3(emissive), estimate, PRIVEC3(poly_specular));
 				specular = vec3(0.);
 			}
+			if (IS_INVALID3(diffuse) || any(lessThan(diffuse,vec3(0.)))) {
+				debugPrintfEXT("%d INVALID diffuse=(%f,%f,%f) light=%d emissive=(%f,%f,%f) estimate=%f poly_diffuse=(%f,%f,%f)",
+					__LINE__, PRIVEC3(diffuse), index, PRIVEC3(emissive), estimate, PRIVEC3(poly_diffuse));
+				diffuse = vec3(0.);
+			}
 #endif
 		}
 	}

ref/vk/vk_brush.c

@@ -484,7 +484,7 @@ static void fillWaterSurfaces( fill_water_surfaces_args_t args ) {
 	R_GeometryRangeUnlock( &geom_lock );
 }
 
-static rt_light_add_polygon_t loadPolyLight(const model_t *mod, const int surface_index, const msurface_t *surf, const vec3_t emissive);
+static qboolean loadPolyLight(rt_light_add_polygon_t *out_polygon, const model_t *mod, const int surface_index, const msurface_t *surf, const vec3_t emissive);
 
 static qboolean doesTextureChainChange( const texture_t *const base ) {
 	const texture_t *cur = base;
@@ -904,10 +904,12 @@ void R_BrushModelDraw( const cl_entity_t *ent, int render_mode, float blend, con
 			// but there's no easy way to do it for now.
 			vec3_t *emissive = &bmodel->render_model.geometries[geom_index].emissive;
 			if (RT_GetEmissiveForTexture(*emissive, new_tex_id)) {
-				rt_light_add_polygon_t polylight = loadPolyLight(mod, surface_index, geom->surf_deprecate, *emissive);
-				polylight.dynamic = true;
-				polylight.transform_row = (const matrix3x4*)&transform;
-				RT_LightAddPolygon(&polylight);
+				rt_light_add_polygon_t polylight;
+				if (loadPolyLight(&polylight, mod, surface_index, geom->surf_deprecate, *emissive)) {
+					polylight.dynamic = true;
+					polylight.transform_row = (const matrix3x4*)&transform;
+					RT_LightAddPolygon(&polylight);
+				}
 			}
 
 			if (new_tex_id == geom->ye_olde_texture)
@@ -1723,25 +1725,45 @@ void R_BrushModelDestroyAll( void ) {
 	memset(g_brush.conn.vertices, 0, sizeof(*g_brush.conn.vertices) * g_brush.conn.vertices_capacity);
 }
 
-static rt_light_add_polygon_t loadPolyLight(const model_t *mod, const int surface_index, const msurface_t *surf, const vec3_t emissive) {
-	rt_light_add_polygon_t lpoly = {0};
-	lpoly.num_vertices = Q_min(7, surf->numedges);
+static float computeArea(vec3_t *vertices, int vertices_count) {
+		vec3_t normal = {0, 0, 0};
+
+		for (int i = 2; i < vertices_count; ++i) {
+			vec3_t e[2], lnormal;
+			VectorSubtract(vertices[i-0], vertices[0], e[0]);
+			VectorSubtract(vertices[i-1], vertices[0], e[1]);
+			CrossProduct(e[0], e[1], lnormal);
+			VectorAdd(lnormal, normal, normal);
+		}
+
+		return VectorLength(normal);
+}
+
+static qboolean loadPolyLight(rt_light_add_polygon_t *out_polygon, const model_t *mod, const int surface_index, const msurface_t *surf, const vec3_t emissive) {
+	(*out_polygon) = (rt_light_add_polygon_t){0};
+	out_polygon->num_vertices = Q_min(7, surf->numedges);
 
 	// TODO split, don't clip
 	if (surf->numedges > 7)
 		WARN_THROTTLED(10, "emissive surface %d has %d vertices; clipping to 7", surface_index, surf->numedges);
 
-	VectorCopy(emissive, lpoly.emissive);
+	VectorCopy(emissive, out_polygon->emissive);
 
-	for (int i = 0; i < lpoly.num_vertices; ++i) {
+	for (int i = 0; i < out_polygon->num_vertices; ++i) {
 		const int iedge = mod->surfedges[surf->firstedge + i];
 		const medge_t *edge = mod->edges + (iedge >= 0 ? iedge : -iedge);
 		const mvertex_t *vertex = mod->vertexes + (iedge >= 0 ? edge->v[0] : edge->v[1]);
-		VectorCopy(vertex->position, lpoly.vertices[i]);
+		VectorCopy(vertex->position, out_polygon->vertices[i]);
 	}
 
-	lpoly.surface = surf;
-	return lpoly;
+	const float area = computeArea(out_polygon->vertices, out_polygon->num_vertices);
+	if (area <= 0) {
+		ERR("%s: emissive surface=%d has area=%f, skipping", __FUNCTION__, surface_index, area);
+		return false;
+	}
+
+	out_polygon->surface = surf;
+	return true;
 }
 
 void R_VkBrushModelCollectEmissiveSurfaces( const struct model_s *mod, qboolean is_worldmodel ) {
@@ -1821,16 +1843,17 @@ void R_VkBrushModelCollectEmissiveSurfaces( const struct model_s *mod, qboolean
 	if (!is_static) {
 		if (bmodel->dynamic_polylights)
 			Mem_Free(bmodel->dynamic_polylights);
-		bmodel->dynamic_polylights_count = emissive_surfaces_count;
+		bmodel->dynamic_polylights_count = 0;
 		bmodel->dynamic_polylights = Mem_Malloc(vk_core.pool, sizeof(bmodel->dynamic_polylights[0]) * emissive_surfaces_count);
 	}
 
 	// Apply all emissive surfaces found
 	int geom_indices_count = 0;
 	for (int i = 0; i < emissive_surfaces_count; ++i) {
 		const emissive_surface_t* const s = emissive_surfaces + i;
-
-		rt_light_add_polygon_t polylight = loadPolyLight(mod, s->surface_index, s->surf, s->emissive);
+		rt_light_add_polygon_t polylight;
+		if (!loadPolyLight(&polylight, mod, s->surface_index, s->surf, s->emissive))
+			continue;
 
 		// func_any surfaces do not really belong to BSP+PVS system, so they can't be used
 		// for lights visibility calculation directly.
@@ -1863,7 +1886,8 @@ void R_VkBrushModelCollectEmissiveSurfaces( const struct model_s *mod, qboolean
 			}
 			*/
 		} else {
-			bmodel->dynamic_polylights[i] = polylight;
+			ASSERT(bmodel->dynamic_polylights_count < emissive_surfaces_count);
+			bmodel->dynamic_polylights[bmodel->dynamic_polylights_count++] = polylight;
 		}
 
 		// Assign the emissive value to the right geometry

ref/vk/vk_light.c

@@ -1110,6 +1110,11 @@ int RT_LightAddPolygon(const rt_light_add_polygon_t *addpoly) {
 		}
 
 		poly->area = VectorLength(normal);
+		if (poly->area <= 0) {
+			ERR("%s: Polygon light has zero area", __FUNCTION__);
+			return -1;
+		}
+
 		VectorM(1.f / poly->area, normal, poly->plane);
 		poly->plane[3] = -DotProduct(vertices[0], poly->plane);