prometheus en thanos working. minio (persistant data) not yet

probleem met hard link storage; sidecar naar minio toe. ;;

dit probleem; thanos-io/thanos#6811

src/DIYManagementAPI/Controllers/DIYController.cs

@@ -12,7 +12,7 @@ public class DIYController : ControllerBase
     {
 
         private static readonly Counter RequestCounter = Metrics
-      .CreateCounter("api_diytestmodels_requests_total", "Total number of requests to DiyTestModels API");
+      .CreateCounter("api_diytestmodels_requests_total", "Total number of requests to DiyTestModels API"); //!!!! ! !  ! !!  !! ! !
         private readonly DIYService _service;
 
         public DIYController(DIYService service)

src/README.md

@@ -19,7 +19,9 @@ TODO; dit netjes maken natuurlijk.en de docker configs enz weer netjes maken. Pr
 ![img_2.png](img_2.png)
 
 prometheus, ziedt wel de results, maar metrics zelf gaan na 5 min weg (rentention)
+
 ![img_3.png](img_3.png)
 
 thanos, door minio, blijven de metrics wel langer beschikbaar
+
 ![img_4.png](img_4.png)

src/docker-compose.yml

@@ -29,7 +29,7 @@ services:
       # Disable compaction by setting min and max block duration to the same value
       - --storage.tsdb.min-block-duration=1m
       - --storage.tsdb.max-block-duration=1m
-      - --storage.tsdb.retention.time=5m # wat is dit?
+      - --storage.tsdb.retention.time=5m
     ports:
       - "9090:9090"
     networks:

src/k8s/diymanagementapi.yaml

@@ -17,6 +17,10 @@ spec:
       labels:
         app: diymanagementapi
         version: "1.0"
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/path: "/metrics"
+        prometheus.io/port: "5300"
     spec:
       containers:
       - env:

src/k8s/monitoring-namespace.yaml

@@ -0,0 +1,5 @@
+# monitoring-namespace.yaml
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: monitoring

src/k8s/monitoring/minio-deployment.yaml

@@ -0,0 +1,62 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: minio
+  namespace: monitoring
+  labels:
+    app: minio
+spec:
+  type: NodePort  # Verander dit van 'ClusterIP' naar 'NodePort'
+  ports:
+    - port: 9000
+      targetPort: 9000
+      nodePort: 30902  # Je kunt een specifieke poort instellen of Kubernetes er een laten toewijzen
+      name: api
+    - port: 9001
+      targetPort: 9001
+      nodePort: 30901  # Je kunt een specifieke poort instellen of Kubernetes er een laten toewijzen
+      name: console
+  selector:
+    app: minio
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: minio
+  namespace: monitoring
+  labels:
+    app: minio
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: minio
+  template:
+    metadata:
+      labels:
+        app: minio
+    spec:
+      containers:
+        - name: minio
+          image: minio/minio
+          args:
+            - server
+            - /data
+            - "--console-address"
+            - ":9001"
+          env:
+            - name: MINIO_ROOT_USER
+              value: minio
+            - name: MINIO_ROOT_PASSWORD
+              value: minio123
+          ports:
+            - containerPort: 9000
+              name: api
+            - containerPort: 9001
+              name: console
+          volumeMounts:
+            - name: minio-data
+              mountPath: /data
+      volumes:
+        - name: minio-data
+          emptyDir: {}

src/k8s/monitoring/prometheus-configmap.yaml

@@ -0,0 +1,44 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: prometheus-config
+  namespace: monitoring
+data:
+  prometheus.yml: |
+    global:
+      scrape_interval: 10s
+      evaluation_interval: 10s
+      external_labels:
+        monitor: 'test-monitor'
+        cluster: 'test-cluster' # Nodig om fouten te voorkomen
+
+    scrape_configs:
+      - job_name: 'prometheus'
+        static_configs:
+          - targets: ['localhost:9090']
+
+      - job_name: 'diymanagementapi'
+        static_configs:
+          - targets: ['diymanagementapi.pitstop.svc.cluster.local:5300']
+        metrics_path: /metrics
+
+      - job_name: 'kubernetes-pods'
+        kubernetes_sd_configs:
+          - role: pod
+        relabel_configs:
+          - source_labels: [__meta_kubernetes_namespace]
+            action: keep
+            regex: pitstop
+          - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+            action: keep
+            regex: "true"
+          - source_labels:
+              [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
+            action: replace
+            regex: ([^:]+)(?::\d+)?;(\d+)
+            replacement: $1:$2
+            target_label: __address__
+          - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
+            action: replace
+            target_label: __metrics_path__
+            regex: (.+)

src/k8s/monitoring/prometheus-deployment.yaml

@@ -0,0 +1,64 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: prometheus
+  namespace: monitoring
+  labels:
+    app: prometheus
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: prometheus
+  template:
+    metadata:
+      labels:
+        app: prometheus
+    spec:
+      containers:
+        - name: prometheus
+          image: prom/prometheus:latest
+          args:
+            - "--config.file=/etc/prometheus/prometheus.yml"
+            - "--storage.tsdb.path=/prometheus"
+            - "--web.enable-lifecycle"
+            # Disable compaction by setting min and max block duration to the same value
+            - "--storage.tsdb.min-block-duration=1m"
+            - "--storage.tsdb.max-block-duration=1m"
+            - "--storage.tsdb.retention.time=5m"
+          ports:
+            - containerPort: 9090
+          volumeMounts:
+            - name: prometheus-config-volume
+              mountPath: /etc/prometheus/
+            - name: prometheus-data-volume
+              mountPath: /prometheus
+        - name: thanos-sidecar
+          image: quay.io/thanos/thanos:v0.34.0
+          args:
+            - sidecar
+            - "--prometheus.url=http://localhost:9090"
+            - "--grpc-address=0.0.0.0:10901"
+            - "--http-address=0.0.0.0:10902"
+            - "--tsdb.path=/prometheus"
+            - "--objstore.config-file=/etc/thanos/objstore.yml"
+            - "--shipper.allow-feature=unsafe-non-native-hardlink" # added, nu werkt het helemaal niet meer
+          ports:
+            - containerPort: 10901
+              name: grpc
+            - containerPort: 10902
+              name: http
+          volumeMounts:
+            - name: prometheus-data-volume
+              mountPath: /prometheus
+            - name: thanos-objstore-config
+              mountPath: /etc/thanos
+      volumes:
+        - name: prometheus-config-volume
+          configMap:
+            name: prometheus-config
+        - name: prometheus-data-volume
+          emptyDir: {}
+        - name: thanos-objstore-config
+          configMap:
+            name: thanos-objstore-config

src/k8s/monitoring/prometheus-service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: prometheus
+  namespace: monitoring
+spec:
+  type: NodePort
+  ports:
+    - port: 9090
+      targetPort: 9090
+      nodePort: 30900
+      protocol: TCP
+      name: http
+  selector:
+    app: prometheus

src/k8s/monitoring/prometheus.yml

@@ -0,0 +1,17 @@
+global:
+  scrape_interval: 10s
+  evaluation_interval: 10s
+
+  external_labels:
+    monitor: 'test-monitor'
+    cluster: 'test-cluster'
+
+scrape_configs:
+  - job_name: 'prometheus'
+    static_configs:
+      - targets: ['localhost:9090']
+
+  - job_name: 'diymanagementapi'
+    static_configs:
+      - targets: ['diymanagementapi:5300']
+    metrics_path: /metrics

src/k8s/monitoring/thanos-objstore-config.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: thanos-objstore-config
+  namespace: monitoring
+data:
+  objstore.yml: |
+    type: s3
+    config:
+      bucket: thanos-bucket
+      endpoint: minio.monitoring.svc.cluster.local:9000
+      access_key: minio
+      secret_key: minio123
+      insecure: true

src/k8s/monitoring/thanos-query-deployment.yaml

@@ -0,0 +1,31 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: thanos-query
+  namespace: monitoring
+  labels:
+    app: thanos-query
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: thanos-query
+  template:
+    metadata:
+      labels:
+        app: thanos-query
+    spec:
+      containers:
+        - name: thanos-query
+          image: quay.io/thanos/thanos:v0.32.2
+          args:
+            - query
+            - "--http-address=0.0.0.0:9090"
+            - "--grpc-address=0.0.0.0:10901"
+            - "--store=thanos-sidecar.monitoring.svc.cluster.local:10901"
+            - "--store=thanos-store.monitoring.svc.cluster.local:10901"
+          ports:
+            - containerPort: 9090
+              name: http
+            - containerPort: 10901
+              name: grpc

src/k8s/monitoring/thanos-query-service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: thanos-query
+  namespace: monitoring
+spec:
+  type: NodePort
+  ports:
+    - port: 9090
+      targetPort: 9090
+      nodePort: 30990
+      protocol: TCP
+      name: http
+  selector:
+    app: thanos-query

src/k8s/monitoring/thanos-sidecar-service.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: thanos-sidecar
+  namespace: monitoring
+spec:
+  ports:
+    - port: 10901
+      targetPort: 10901
+      protocol: TCP
+      name: grpc
+  selector:
+    app: prometheus

src/k8s/monitoring/thanos-store-deployment.yaml

@@ -0,0 +1,42 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: thanos-store
+  namespace: monitoring
+  labels:
+    app: thanos-store
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: thanos-store
+  template:
+    metadata:
+      labels:
+        app: thanos-store
+    spec:
+      containers:
+        - name: thanos-store
+          image: quay.io/thanos/thanos:v0.32.2
+          args:
+            - store
+            - "--grpc-address=0.0.0.0:10901"
+            - "--http-address=0.0.0.0:10902"
+            - "--data-dir=/data"
+            - "--objstore.config-file=/etc/thanos/objstore.yml"
+          ports:
+            - containerPort: 10901
+              name: grpc
+            - containerPort: 10902
+              name: http
+          volumeMounts:
+            - name: thanos-objstore-config
+              mountPath: /etc/thanos
+            - name: data
+              mountPath: /data
+      volumes:
+        - name: thanos-objstore-config
+          configMap:
+            name: thanos-objstore-config
+        - name: data
+          emptyDir: {}

src/k8s/monitoring/thanos-store-service.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: thanos-store
+  namespace: monitoring
+spec:
+  ports:
+    - port: 10901
+      targetPort: 10901
+      protocol: TCP
+      name: grpc
+  selector:
+    app: thanos-store