Add setting for filtering empty containers on Prometheus queries

packages/core/src/common/cluster-types.ts

@@ -112,6 +112,10 @@ export interface ClusterPreferences extends ClusterPrometheusPreferences {
  defaultNamespace?: string;
 }
 
+interface QueryFilterOptions {
+ hideEmptyContainers: boolean;
+}
+
 /**
  * A cluster's prometheus settings (a subset of cluster settings)
  */
@@ -125,6 +129,7 @@ export interface ClusterPrometheusPreferences {
  prometheusProvider?: {
  type: string;
  };
+ prometheusQueryOptions?: QueryFilterOptions;
 }
 
 /**

packages/core/src/main/prometheus/operator-provider.injectable.ts.ts

@@ -9,6 +9,9 @@ import { getInjectable } from "@ogre-tools/injectable";
 
 export const getOperatorLikeQueryFor = ({ rateAccuracy }: { rateAccuracy: string }): PrometheusProvider["getQuery"] => (
  (opts, queryName) => {
+ const emptyContainerAndImageFilter = opts.hideEmptyContainers === "true" ? `container!="", image!="",` : "";
+ const emptyContainerFilter = opts.hideEmptyContainers === "true" ? `container!="",` : "";
+
  switch(opts.category) {
  case "cluster":
  switch (queryName) {
@@ -71,19 +74,19 @@ export const getOperatorLikeQueryFor = ({ rateAccuracy }: { rateAccuracy: string
  case "pods":
  switch (queryName) {
  case "cpuUsage":
- return `sum(rate(container_cpu_usage_seconds_total{pod=~"${opts.pods}", namespace="${opts.namespace}"}[${rateAccuracy}])) by (${opts.selector})`;
+ return `sum(rate(container_cpu_usage_seconds_total{${emptyContainerAndImageFilter} pod=~"${opts.pods}", namespace="${opts.namespace}"}[${rateAccuracy}])) by (${opts.selector})`;
  case "cpuRequests":
  return `sum(kube_pod_container_resource_requests{pod=~"${opts.pods}", resource="cpu", namespace="${opts.namespace}"}) by (${opts.selector})`;
  case "cpuLimits":
- return `sum(kube_pod_container_resource_limits{pod=~"${opts.pods}", resource="cpu", namespace="${opts.namespace}"}) by (${opts.selector})`;
+ return `sum(kube_pod_container_resource_limits{${emptyContainerAndImageFilter} pod=~"${opts.pods}", resource="cpu", namespace="${opts.namespace}"}) by (${opts.selector})`;
  case "memoryUsage":
  return `sum(container_memory_working_set_bytes{pod=~"${opts.pods}", namespace="${opts.namespace}"}) by (${opts.selector})`;
  case "memoryRequests":
- return `sum(kube_pod_container_resource_requests{pod=~"${opts.pods}", resource="memory", namespace="${opts.namespace}"}) by (${opts.selector})`;
+ return `sum(kube_pod_container_resource_requests{${emptyContainerFilter} pod=~"${opts.pods}", resource="memory", namespace="${opts.namespace}"}) by (${opts.selector})`;
  case "memoryLimits":
- return `sum(kube_pod_container_resource_limits{pod=~"${opts.pods}", resource="memory", namespace="${opts.namespace}"}) by (${opts.selector})`;
+ return `sum(kube_pod_container_resource_limits{${emptyContainerFilter} pod=~"${opts.pods}", resource="memory", namespace="${opts.namespace}"}) by (${opts.selector})`;
  case "fsUsage":
- return `sum(container_fs_usage_bytes{pod=~"${opts.pods}", namespace="${opts.namespace}"}) by (${opts.selector})`;
+ return `sum(container_fs_usage_bytes{${emptyContainerFilter} pod=~"${opts.pods}", namespace="${opts.namespace}"}) by (${opts.selector})`;
  case "fsWrites":
  return `sum(rate(container_fs_writes_bytes_total{pod=~"${opts.pods}", namespace="${opts.namespace}"}[${rateAccuracy}])) by (${opts.selector})`;
  case "fsReads":

packages/core/src/main/routes/metrics/add-metrics-route.injectable.ts

@@ -97,9 +97,13 @@ const addMetricsRouteInjectable = getRouteInjectable({
 
  if (isObject(payload)) {
  const data = payload as Record<string, Record<string, string>>;
+ const queryFilterPreferences: Record<string, string> = cluster.preferences.prometheusQueryOptions ?
+ Object.fromEntries(Object.entries(cluster.preferences.prometheusQueryOptions).map(([k, v]) => [k, String(v)]))
+ : {};
+
  const queries = object.entries(data)
  .map(([queryName, queryOpts]) => (
- provider.getQuery(queryOpts, queryName)
+ provider.getQuery({ ...queryOpts, ...queryFilterPreferences }, queryName)
  ));
 
  const result = await loadMetrics(queries, cluster, prometheusPath, queryParams);

packages/core/src/renderer/components/cluster-settings/prometheus-setting.tsx

@@ -16,6 +16,9 @@ import type { MetricProviderInfo, RequestMetricsProviders } from "../../../commo
 import { withInjectables } from "@ogre-tools/injectable-react";
 import requestMetricsProvidersInjectable from "../../../common/k8s-api/endpoints/metrics.api/request-providers.injectable";
 import productNameInjectable from "../../../common/vars/product-name.injectable";
+import { Checkbox } from "../checkbox";
+import Gutter from "../gutter/gutter";
+
 
 export interface ClusterPrometheusSettingProps {
  cluster: Cluster;
@@ -123,6 +126,9 @@ class NonInjectedClusterPrometheusSetting extends React.Component<ClusterPrometh
  };
 
  render() {
+ const providerType = this.props.cluster.preferences.prometheusProvider?.type;
+ const showQueryFilters = providerType === "operator" || !providerType;
+
  return (
  <>
  <section>
@@ -147,6 +153,28 @@ class NonInjectedClusterPrometheusSetting extends React.Component<ClusterPrometh
  )
  }
  </section>
+ {
+ showQueryFilters && (
+ <>
+ <Gutter />
+ <section>
+ <SubTitle title="Prometheus queries" />
+ <Checkbox
+ label="Filter empty containers on pod metrics"
+ value={this.props.cluster.preferences.prometheusQueryOptions?.hideEmptyContainers ?? false}
+ onChange={v => {
+ this.props.cluster.preferences.prometheusQueryOptions = {
+ hideEmptyContainers: v,
+ };
+ }}
+ />
+ <small className="hint">
+ In certain metric setups, pod metrics may be observed as double values. This filter can be helpful in ensuring accurate metric are shown.
+ </small>
+ </section>
+ </>
+ )
+ }
  {this.canEditPrometheusPath && (
  <>
  <hr />