feat: update to using default scrapeclass for tls config

docs/configuration/uds-monitoring-metrics.md

@@ -4,10 +4,12 @@ type: docs
 weight: 1
 ---
 
-UDS Core leverages Pepr to handle setup of Prometheus scraping metrics endpoints, with the particular configuration necessary to work in a STRICT mTLS (Istio) environment. We handle this with both mutations of existing service monitors and generation of service monitors via the `Package` CR.
+UDS Core leverages Pepr to handle setup of Prometheus scraping metrics endpoints, with the particular configuration necessary to work in a STRICT mTLS (Istio) environment. We handle this via a default scrapeClass in prometheus to add the istio certs. When a monitor needs to be exempt from that tlsConfig a mutation is performed to leverage a plain scrape class without istio certs.
 
 ## Mutations
 
+Note: The below implementation has been deprecated in favor of a default `scrapeClass` with the file-based `tlsConfig` required for istio mTLS in prometheus automatically, supplemented with a mutation of `scrapeClass: exempt` that exempts monitors from the `tlsConfig` required for istio if the destination namespace is not istio injected (e.g. kube-system), unless the `uds/skip-sm-mutate` annotation is specified. The mutation behavior stated in the paragraph immediately below this section will be removed in a later release.
+
 All service monitors are mutated to set the scrape scheme to HTTPS and set the TLS Config to what is required for Istio mTLS scraping (see [this doc](https://istio.io/latest/docs/ops/integrations/prometheus/#tls-settings) for details). Beyond this, no other fields are mutated. Supporting existing service monitors is useful since some charts include service monitors by default with more advanced configurations, and it is in our best interest to enable those and use them where possible.
 
 Assumptions are made about STRICT mTLS here for simplicity, based on the `istio-injection` namespace label. Without making these assumptions we would need to query `PeerAuthentication` resources or another resource to determine the exact workload mTLS posture.
@@ -16,7 +18,7 @@ Note: This mutation is the default behavior for all service monitors but can be
 
 ## Package CR `monitor` field
 
-UDS Core also supports generating service monitors from the `monitor` list in the `Package` spec. Charts do not always support service monitors, so generating them can be useful. This also provides a simplified way for other users to create service monitors, similar to the way we handle `VirtualServices` today. A full example of this can be seen below:
+UDS Core also supports generating `ServiceMonitors` and/or `PodMonitors` from the `monitor` list in the `Package` spec. Charts do not always support monitors, so generating them can be useful. This also provides a simplified way for other users to create monitors, similar to the way we handle `VirtualServices` today. A full example of this can be seen below:
 
 ```yaml
 ...
@@ -28,9 +30,16 @@ spec:
       targetPort: 1234 # Corresponding target port on the pod/container (for network policy)
       # Optional properties depending on your application
       description: "Metrics" # Add to customize the service monitor name
+      kind: ServiceMonitor # optional, kind defaults to service monitor if not specified. PodMonitor is the other valid option.
       podSelector: # Add if pod labels are different than `selector` (for network policy)
         app: barfoo
       path: "/mymetrics" # Add if metrics are exposed on a different path than "/metrics"
+      authorization: # Add if authorization is required for the metrics endpoint
+        credentials:
+          key: "example-key"
+          name: "example-secret"
+          optional: false
+        type: "Bearer"
 ```
 
 This config is used to generate service monitors and corresponding network policies to setup scraping for your applications. The `ServiceMonitor`s will go through the mutation process to add `tlsConfig` and `scheme` to work in an istio environment.

packages/slim-dev/zarf.yaml

@@ -44,7 +44,7 @@ components:
   - name: pepr-uds-core
     required: true
     import:
-      path: ../../dist
+      path: ../../src/pepr
       name: module
 
   # Keycloak

packages/standard/zarf.yaml

@@ -44,7 +44,7 @@ components:
   - name: pepr-uds-core
     required: true
     import:
-      path: ../../dist
+      path: ../../src/pepr
       name: module
 
   # Metrics Server

src/pepr/operator/controllers/monitoring/common.ts

@@ -0,0 +1,12 @@
+import { Monitor } from "../../crd";
+import { sanitizeResourceName } from "../utils";
+
+export function generateMonitorName(pkgName: string, monitor: Monitor) {
+  const { selector, portName, description } = monitor;
+
+  // Ensure the resource name is valid
+  const nameSuffix = description || `${Object.values(selector)}-${portName}`;
+  const name = sanitizeResourceName(`${pkgName}-${nameSuffix}`);
+
+  return name;
+}

src/pepr/operator/controllers/monitoring/pod-monitor.spec.ts

@@ -0,0 +1,41 @@
+import { describe, expect, it } from "@jest/globals";
+import { Monitor } from "../../crd";
+import { generatePodMonitor } from "./pod-monitor";
+
+describe("test generate Pod monitor", () => {
+  it("should return a valid Pod Monitor object", () => {
+    const ownerRefs = [
+      {
+        apiVersion: "uds.dev/v1alpha1",
+        kind: "Package",
+        name: "test",
+        uid: "f50120aa-2713-4502-9496-566b102b1174",
+      },
+    ];
+    const portName = "http-metrics";
+    const metricsPath = "/test";
+    const selectorApp = "test";
+    const monitor: Monitor = {
+      portName: portName,
+      path: metricsPath,
+      targetPort: 1234,
+      selector: {
+        app: selectorApp,
+      },
+    };
+    const namespace = "test";
+    const pkgName = "test";
+    const generation = "1";
+    const payload = generatePodMonitor(monitor, namespace, pkgName, generation, ownerRefs);
+
+    expect(payload).toBeDefined();
+    expect(payload.metadata?.name).toEqual(`${pkgName}-${selectorApp}-${portName}`);
+    expect(payload.metadata?.namespace).toEqual(namespace);
+    expect(payload.spec?.podMetricsEndpoints).toBeDefined();
+    if (payload.spec?.podMetricsEndpoints) {
+      expect(payload.spec.podMetricsEndpoints[0].port).toEqual(portName);
+      expect(payload.spec.podMetricsEndpoints[0].path).toEqual(metricsPath);
+    }
+    expect(payload.spec?.selector.matchLabels).toHaveProperty("app", "test");
+  });
+});

src/pepr/operator/controllers/monitoring/pod-monitor.ts

@@ -0,0 +1,103 @@
+import { V1OwnerReference } from "@kubernetes/client-node";
+import { K8s } from "pepr";
+import { Component, setupLogger } from "../../../logger";
+import { Monitor, PrometheusPodMonitor, UDSPackage } from "../../crd";
+import { Kind } from "../../crd/generated/package-v1alpha1";
+import { getOwnerRef } from "../utils";
+import { generateMonitorName } from "./common";
+
+// configure subproject logger
+const log = setupLogger(Component.OPERATOR_MONITORING);
+
+/**
+ * Generate a pod monitor for a pod
+ *
+ * @param pkg UDS Package
+ * @param namespace
+ */
+export async function podMonitor(pkg: UDSPackage, namespace: string) {
+  const pkgName = pkg.metadata!.name!;
+  const generation = (pkg.metadata?.generation ?? 0).toString();
+  const ownerRefs = getOwnerRef(pkg);
+
+  log.debug(`Reconciling PodMonitors for ${pkgName}`);
+
+  // Get the list of monitored services
+  const monitorList = pkg.spec?.monitor ?? [];
+
+  // Create a list of generated PodMonitors
+  const payloads: PrometheusPodMonitor[] = [];
+
+  try {
+    for (const monitor of monitorList) {
+      if (monitor.kind === Kind.PodMonitor) {
+        const payload = generatePodMonitor(monitor, namespace, pkgName, generation, ownerRefs);
+
+        log.debug(payload, `Applying PodMonitor ${payload.metadata?.name}`);
+
+        // Apply the PodMonitor and force overwrite any existing policy
+        await K8s(PrometheusPodMonitor).Apply(payload, { force: true });
+
+        payloads.push(payload);
+      }
+    }
+
+    // Get all related PodMonitors in the namespace
+    const podMonitors = await K8s(PrometheusPodMonitor)
+      .InNamespace(namespace)
+      .WithLabel("uds/package", pkgName)
+      .Get();
+
+    // Find any orphaned PodMonitors (not matching the current generation)
+    const orphanedMonitor = podMonitors.items.filter(
+      m => m.metadata?.labels?.["uds/generation"] !== generation,
+    );
+
+    // Delete any orphaned PodMonitors
+    for (const m of orphanedMonitor) {
+      log.debug(m, `Deleting orphaned PodMonitor ${m.metadata!.name}`);
+      await K8s(PrometheusPodMonitor).Delete(m);
+    }
+  } catch (err) {
+    throw new Error(`Failed to process PodMonitors for ${pkgName}, cause: ${JSON.stringify(err)}`);
+  }
+
+  // Return the list of monitor names
+  return [...payloads.map(m => m.metadata!.name!)];
+}
+
+export function generatePodMonitor(
+  monitor: Monitor,
+  namespace: string,
+  pkgName: string,
+  generation: string,
+  ownerRefs: V1OwnerReference[],
+) {
+  const { selector, portName } = monitor;
+  const name = generateMonitorName(pkgName, monitor);
+  const payload: PrometheusPodMonitor = {
+    metadata: {
+      name,
+      namespace,
+      labels: {
+        "uds/package": pkgName,
+        "uds/generation": generation,
+      },
+      ownerReferences: ownerRefs,
+    },
+    spec: {
+      podMetricsEndpoints: [
+        {
+          port: portName,
+          path: monitor.path || "/metrics",
+          authorization: monitor.authorization,
+        },
+      ],
+      selector: {
+        matchLabels: selector,
+      },
+    },
+  };
+
+  return payload;
+}

src/pepr/operator/controllers/monitoring/service-monitor.spec.ts

@@ -1,6 +1,6 @@
 import { describe, expect, it } from "@jest/globals";
-import { generateServiceMonitor } from "./service-monitor";
 import { Monitor } from "../../crd";
+import { generateServiceMonitor } from "./service-monitor";
 
 describe("test generate service monitor", () => {
   it("should return a valid Service Monitor object", () => {

src/pepr/operator/controllers/monitoring/service-monitor.ts

@@ -2,8 +2,10 @@ import { K8s } from "pepr";
 
 import { V1OwnerReference } from "@kubernetes/client-node";
 import { Component, setupLogger } from "../../../logger";
-import { Monitor, Prometheus, UDSPackage } from "../../crd";
-import { getOwnerRef, sanitizeResourceName } from "../utils";
+import { Monitor, PrometheusServiceMonitor, UDSPackage } from "../../crd";
+import { Kind } from "../../crd/generated/package-v1alpha1";
+import { getOwnerRef } from "../utils";
+import { generateMonitorName } from "./common";
 
 // configure subproject logger
 const log = setupLogger(Component.OPERATOR_MONITORING);
@@ -25,35 +27,37 @@ export async function serviceMonitor(pkg: UDSPackage, namespace: string) {
   const monitorList = pkg.spec?.monitor ?? [];
 
   // Create a list of generated ServiceMonitors
-  const payloads: Prometheus.ServiceMonitor[] = [];
+  const payloads: PrometheusServiceMonitor[] = [];
 
   try {
     for (const monitor of monitorList) {
-      const payload = generateServiceMonitor(monitor, namespace, pkgName, generation, ownerRefs);
+      if (monitor.kind !== Kind.PodMonitor) {
+        const payload = generateServiceMonitor(monitor, namespace, pkgName, generation, ownerRefs);
 
-      log.debug(payload, `Applying ServiceMonitor ${payload.metadata?.name}`);
+        log.debug(payload, `Applying ServiceMonitor ${payload.metadata?.name}`);
 
-      // Apply the ServiceMonitor and force overwrite any existing policy
-      await K8s(Prometheus.ServiceMonitor).Apply(payload, { force: true });
+        // Apply the ServiceMonitor and force overwrite any existing policy
+        await K8s(PrometheusServiceMonitor).Apply(payload, { force: true });
 
-      payloads.push(payload);
+        payloads.push(payload);
+      }
     }
 
     // Get all related ServiceMonitors in the namespace
-    const serviceMonitors = await K8s(Prometheus.ServiceMonitor)
+    const serviceMonitors = await K8s(PrometheusServiceMonitor)
       .InNamespace(namespace)
       .WithLabel("uds/package", pkgName)
       .Get();
 
     // Find any orphaned ServiceMonitors (not matching the current generation)
-    const orphanedSM = serviceMonitors.items.filter(
-      sm => sm.metadata?.labels?.["uds/generation"] !== generation,
+    const orphanedMonitor = serviceMonitors.items.filter(
+      m => m.metadata?.labels?.["uds/generation"] !== generation,
     );
 
     // Delete any orphaned ServiceMonitors
-    for (const sm of orphanedSM) {
-      log.debug(sm, `Deleting orphaned ServiceMonitor ${sm.metadata!.name}`);
-      await K8s(Prometheus.ServiceMonitor).Delete(sm);
+    for (const m of orphanedMonitor) {
+      log.debug(m, `Deleting orphaned ServiceMonitor ${m.metadata!.name}`);
+      await K8s(PrometheusServiceMonitor).Delete(m);
     }
   } catch (err) {
     throw new Error(
@@ -62,17 +66,7 @@ export async function serviceMonitor(pkg: UDSPackage, namespace: string) {
   }
 
   // Return the list of monitor names
-  return [...payloads.map(sm => sm.metadata!.name!)];
-}
-
-export function generateSMName(pkgName: string, monitor: Monitor) {
-  const { selector, portName, description } = monitor;
-
-  // Ensure the resource name is valid
-  const nameSuffix = description || `${Object.values(selector)}-${portName}`;
-  const name = sanitizeResourceName(`${pkgName}-${nameSuffix}`);
-
-  return name;
+  return [...payloads.map(m => m.metadata!.name!)];
 }
 
 export function generateServiceMonitor(
@@ -83,8 +77,8 @@ export function generateServiceMonitor(
   ownerRefs: V1OwnerReference[],
 ) {
   const { selector, portName } = monitor;
-  const name = generateSMName(pkgName, monitor);
-  const payload: Prometheus.ServiceMonitor = {
+  const name = generateMonitorName(pkgName, monitor);
+  const payload: PrometheusServiceMonitor = {
     metadata: {
       name,
       namespace,
@@ -99,6 +93,7 @@ export function generateServiceMonitor(
         {
           port: portName,
           path: monitor.path || "/metrics",
+          authorization: monitor.authorization,
         },
       ],
       selector: {