Setup Prometheus metrics with addition scrape configuration

Setup prometheus

https://github.com/prometheus-operator/prometheus-operator?tab=readme-ov-file#quickstart

git clone https://github.com/prometheus-operator/prometheus-operator.git
kubectl create -f bundle.yaml

Ensure prometheusagents have access to clusterrole minio-operator-role

k -n minio-operator edit clusterrole minio-operator-role
###
- apiGroups:
  - monitoring.coreos.com
  resources:
  - prometheuses
  - prometheusagents
  verbs:
  - '*'
###

Create prometheus resources

cat <<EOF | kubectl apply -f -
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: prometheus
rules:
- apiGroups: [""]
  resources:
  - nodes
  - nodes/metrics
  - services
  - endpoints
  - pods
  verbs: ["get", "list", "watch"]
- apiGroups: [""]
  resources:
  - configmaps
  verbs: ["get"]
- apiGroups:
  - networking.k8s.io
  resources:
  - ingresses
  verbs: ["get", "list", "watch"]
- nonResourceURLs: ["/metrics"]
  verbs: ["get"]
EOF

cat <<EOF | kubectl apply -f -
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: prometheus
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: prometheus
subjects:
- kind: ServiceAccount
  name: prometheus-operator
  namespace: default
EOF

cat <<EOF | kubectl apply -f -
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: service-monitor
  labels:
    v1.min.io/tenant: myminio
spec:
  selector:
    matchLabels:
      v1.min.io/tenant: myminio
  endpoints:
  - port: http-minio
EOF

cat <<EOF | kubectl apply -f -
apiVersion: monitoring.coreos.com/v1
kind: Prometheus
metadata:
  name: prometheus
spec:
  serviceAccountName: prometheus-operator
  serviceMonitorSelector:
    matchLabels:
      v1.min.io/tenant: myminio
EOF

cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
  name: prometheus
spec:
  type: NodePort
  ports:
  - name: web
    nodePort: 30082
    port: 9090
    protocol: TCP
    targetPort: web
  selector:
    prometheus: prometheus
EOF

Setup scrape

kubectl patch tenant -n minio-tenant myminio --type='merge' -p '{"spec":{"prometheusOperator": true, "prometheusOperatorScrapeMetricsPath": ["/minio/v2/metrics/bucket", "/minio/v2/metrics/cluster", "/minio/v2/metrics/node", "/minio/v2/metrics/resource"]}}'

Validate

kubectl get secret minio-prom-additional-scrape-config -o json | jq -r '.data."prometheus-additional.yaml"' | base64 -d

Access the metrics

http://<prometheus node and port>/api/v1/label/__name__/values