fix(health): only consider non-empty health checks

For wildcard health checks, only consider wildcards with a non-empty
health check. Fixes argoproj#16905 (at least partially).

Signed-off-by: Blake Pettersson <blake.pettersson@gmail.com>

docs/operator-manual/health.md

@@ -98,20 +98,27 @@ data:
     return hs
 ```
 
-In order to prevent duplication of the custom health check for potentially multiple resources, it is also possible to specify a wildcard in the resource kind, and anywhere in the resource group, like this:
+In order to prevent duplication of custom health checks for potentially multiple resources, it is also possible to
+specify a wildcard in the resource kind, and anywhere in the resource group, like this:
 
 ```yaml
-  resource.customizations.health.ec2.aws.crossplane.io_*: |
-    ...
+  resource.customizations: |
+    ec2.aws.crossplane.io/*:
+      health.lua: |
+        ...
 ```
 
 ```yaml
-  resource.customizations.health.*.aws.crossplane.io_*: |
-    ...
+  # If a key _begins_ with a wildcard, please ensure that the GVK key is quoted.
+  resource.customizations: |
+    "*.aws.crossplane.io/*":
+      health.lua: |
+        ...
 ```
 
 !!!important
-    Please, note that there can be ambiguous resolution of wildcards, see [#16905](https://github.com/argoproj/argo-cd/issues/16905)
+Please, note that wildcards are only supported when using the `resource.customizations` key, the `resource.customizations.health.<group>_<kind>`
+style keys do not work since wildcards (`*`) are not supported in Kubernetes configmap keys.
 
 The `obj` is a global variable which contains the resource. The script must return an object with status and optional message field.
 The custom health check might return one of the following health statuses:
@@ -121,7 +128,7 @@ The custom health check might return one of the following health statuses:
   * `Degraded` - the resource is degraded
   * `Suspended` - the resource is suspended and waiting for some external event to resume (e.g. suspended CronJob or paused Deployment)
 
-By default health typically returns `Progressing` status.
+By default, health typically returns a `Progressing` status.
 
 NOTE: As a security measure, access to the standard Lua libraries will be disabled by default. Admins can control access by
 setting `resource.customizations.useOpenLibs.<group>_<kind>`. In the following example, standard libraries are enabled for health check of `cert-manager.io/Certificate`.

util/lua/lua.go

@@ -426,11 +426,14 @@ func GetConfigMapKey(gvk schema.GroupVersionKind) string {
 	return fmt.Sprintf("%s/%s", gvk.Group, gvk.Kind)
 }
 
+// GetWildcardConfigMapKey returns the first encountered resource override which matches the wildcard and has a
+// non-empty health script. Having multiple wildcards with non-empty health checks that can match the GVK is
+// non-deterministic.
 func GetWildcardConfigMapKey(vm VM, gvk schema.GroupVersionKind) string {
 	gvkKeyToMatch := GetConfigMapKey(gvk)
 
-	for key := range vm.ResourceOverrides {
-		if glob.Match(key, gvkKeyToMatch) {
+	for key, override := range vm.ResourceOverrides {
+		if glob.Match(key, gvkKeyToMatch) && override.HealthLua != "" {
 			return key
 		}
 	}

util/lua/lua_test.go

@@ -787,6 +787,11 @@ return hs`
 	const healthWildcardOverrideScript = `
  hs = {}
  hs.status = "Healthy"
+ return hs`
+
+	const healthWildcardOverrideScriptWithPriority = `
+ hs = {}
+ hs.status = "UnHealthy"
  return hs`
 
 	getHealthOverride := func(openLibs bool) ResourceHealthOverrides {
@@ -804,6 +809,15 @@ return hs`
 		},
 	}
 
+	getWildcardHealthOverrideWithPriority := ResourceHealthOverrides{
+		"*.aws.crossplane.io/*": appv1.ResourceOverride{
+			HealthLua: "",
+		},
+		"*.aws*": appv1.ResourceOverride{
+			HealthLua: healthWildcardOverrideScriptWithPriority,
+		},
+	}
+
 	t.Run("Enable Lua standard lib", func(t *testing.T) {
 		testObj := StrToUnstructured(testSA)
 		overrides := getHealthOverride(true)
@@ -837,6 +851,15 @@ return hs`
 		assert.Equal(t, expectedStatus, status)
 	})
 
+	t.Run("Get resource health for wildcard override with empty health.lua", func(t *testing.T) {
+		testObj := StrToUnstructured(ec2AWSCrossplaneObjJson)
+		overrides := getWildcardHealthOverrideWithPriority
+		status, err := overrides.GetResourceHealth(testObj)
+		require.NoError(t, err)
+		expectedStatus := &health.HealthStatus{Status: "Unknown", Message: "Lua returned an invalid health status"}
+		assert.Equal(t, expectedStatus, status)
+	})
+
 	t.Run("Resource health for wildcard override not found", func(t *testing.T) {
 		testObj := StrToUnstructured(testSA)
 		overrides := getWildcardHealthOverride