diff --git a/charts/spiderpool/files/grafana-rdma-workload.json b/charts/spiderpool/files/grafana-rdma-workload.json new file mode 100644 index 000000000..35e8211bf --- /dev/null +++ b/charts/spiderpool/files/grafana-rdma-workload.json @@ -0,0 +1,916 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 12, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 11, + "panels": [], + "title": "Summary by $kind", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (owner_name) (\n rate(rdma_rx_vport_rdma_unicast_bytes_total{owner_namespace=~\"$namespace\", owner_kind=~\"$kind\"}[1m])\n)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Bandwidth per $kind | Read ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (owner_name) (\n rate(rdma_tx_vport_rdma_unicast_bytes_total{owner_namespace=~\"$namespace\", owner_kind=~\"$kind\"}[1m])\n)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Bandwidth | Write | $kind", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 30, + "panels": [], + "title": "Pods", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (pod_name) (\n rate(rdma_rx_vport_rdma_unicast_bytes_total{owner_namespace=~\"$namespace\", owner_kind=~\"$kind\", owner_name=~\"$name\"}[1m])\n)", + "interval": "", + "legendFormat": "{{pod_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Bandwidth per Pod | Read", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (pod_name) (\n rate(rdma_tx_vport_rdma_unicast_bytes_total{owner_namespace=~\"$namespace\", owner_kind=~\"$kind\", owner_name=~\"$name\"}[1m])\n)", + "interval": "", + "legendFormat": "{{pod_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Bandwidth per Pod | Write", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 18 + }, + "id": 32, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.14", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (pod_name) (\n rate(rdma_rx_vport_rdma_unicast_packets_total{owner_namespace=~\"$namespace\", owner_kind=~\"$kind\", owner_name=~\"$name\"}[1m])\n)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{pod_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Packets per pod | unicast | Read", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 18 + }, + "id": 33, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.14", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (pod_name) (\n rate(rdma_tx_vport_rdma_unicast_packets_total{owner_namespace=~\"$namespace\", owner_kind=~\"$kind\", owner_name=~\"$name\"}[1m])\n)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{pod_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Packets per pod | unicast | Write", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 18 + }, + "id": 34, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.14", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (pod_name) (\n rate(rdma_tx_vport_rdma_multicast_packets_total{owner_namespace=~\"$namespace\", owner_kind=~\"$kind\", owner_name=~\"$name\"}[1m])\n)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{pod_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Packets per pod | multicast | Write", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 18 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.14", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (pod_name) (\n rate(rdma_tx_vport_rdma_multicast_packets_total{owner_namespace=~\"$namespace\", owner_kind=~\"$kind\", owner_name=~\"$name\"}[1m])\n)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{pod_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Packets per pod | multicast | Write", + "type": "timeseries" + } + ], + "refresh": false, + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 2, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "DaemonSet", + "value": "DaemonSet" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(rdma_vport_speed_mbps_total{}, owner_kind)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "kind", + "options": [], + "query": { + "query": "label_values(rdma_vport_speed_mbps_total{}, owner_kind)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(rdma_vport_speed_mbps_total{owner_kind=~\"$kind\"}, owner_namespace)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(rdma_vport_speed_mbps_total{owner_kind=~\"$kind\"}, owner_namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "rdma-test-gpu-tool", + "value": "rdma-test-gpu-tool" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(rdma_vport_speed_mbps_total{owner_kind=~\"$kind\", owner_namespace=~\"$namespace\"}, owner_name)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "name", + "options": [], + "query": { + "query": "label_values(rdma_vport_speed_mbps_total{owner_kind=~\"$kind\", owner_namespace=~\"$namespace\"}, owner_name)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "spiderpool RDMA Dashboard | sriov - Workload", + "uid": "AAT6f2ZNz", + "version": 33, + "weekStart": "" +} diff --git a/charts/spiderpool/templates/role_binding.yaml b/charts/spiderpool/templates/role_binding.yaml index f18317670..4f49acff5 100644 --- a/charts/spiderpool/templates/role_binding.yaml +++ b/charts/spiderpool/templates/role_binding.yaml @@ -19,4 +19,17 @@ subjects: name: {{ .Values.spiderpoolInit.name | trunc 63 | trimSuffix "-" }} namespace: {{ .Release.Namespace }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: spiderpool-read-all-resources +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: spiderpool-read-all-resources +subjects: +- kind: ServiceAccount + name: {{ .Values.spiderpoolAgent.name | trunc 63 | trimSuffix "-" }} + namespace: {{ .Release.Namespace }} \ No newline at end of file diff --git a/charts/spiderpool/templates/role_read_all.yaml b/charts/spiderpool/templates/role_read_all.yaml new file mode 100644 index 000000000..5abc399be --- /dev/null +++ b/charts/spiderpool/templates/role_read_all.yaml @@ -0,0 +1,10 @@ +# for rdma metrics exporter, read rdma pod owner's info +# for example, the rdma pod owner is a job, the job's owner is a cronjob +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: spiderpool-read-all-resources +rules: + - apiGroups: ["*"] + resources: ["*"] + verbs: ["get", "list", "watch"] diff --git a/cmd/spiderpool-agent/cmd/config.go b/cmd/spiderpool-agent/cmd/config.go index 45d981ae8..4ca927e4e 100644 --- a/cmd/spiderpool-agent/cmd/config.go +++ b/cmd/spiderpool-agent/cmd/config.go @@ -14,6 +14,7 @@ import ( "github.com/spf13/pflag" "go.uber.org/atomic" "gopkg.in/yaml.v3" + "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" "github.com/spidernet-io/spiderpool/api/v1/agent/client" @@ -121,6 +122,9 @@ type AgentContext struct { SubnetManager subnetmanager.SubnetManager KubevirtManager kubevirtmanager.KubevirtManager + // k8s client + ClientSet *kubernetes.Clientset + // handler HttpServer *server.Server UnixServer *server.Server diff --git a/cmd/spiderpool-agent/cmd/daemon.go b/cmd/spiderpool-agent/cmd/daemon.go index 3eeb485c8..3eef82dbc 100644 --- a/cmd/spiderpool-agent/cmd/daemon.go +++ b/cmd/spiderpool-agent/cmd/daemon.go @@ -20,6 +20,7 @@ import ( "go.uber.org/automaxprocs/maxprocs" "go.uber.org/zap" apiruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" @@ -147,6 +148,13 @@ func DaemonMain() { } agentContext.CRDManager = mgr + logger.Info("Begin to initialize k8s clientSet") + clientSet, err := initK8sClientSet() + if nil != err { + logger.Fatal(err.Error()) + } + agentContext.ClientSet = clientSet + logger.Info("Begin to initialize spiderpool-agent metrics HTTP server") initAgentMetricsServer(agentContext.InnerCtx) @@ -325,6 +333,16 @@ func waitAPIServerReady(ctx context.Context) error { return errors.New("failed to talk to API Server") } +// initK8sClientSet will new kubernetes ClientSet +func initK8sClientSet() (*kubernetes.Clientset, error) { + clientSet, err := kubernetes.NewForConfig(ctrl.GetConfigOrDie()) + if nil != err { + return nil, fmt.Errorf("failed to init K8s clientset: %v", err) + } + + return clientSet, nil +} + func initAgentServiceManagers(ctx context.Context) { logger.Debug("Begin to initialize Node manager") nodeManager, err := nodemanager.NewNodeManager( diff --git a/cmd/spiderpool-agent/cmd/metrics_server.go b/cmd/spiderpool-agent/cmd/metrics_server.go index 9d263baa1..610f197e6 100644 --- a/cmd/spiderpool-agent/cmd/metrics_server.go +++ b/cmd/spiderpool-agent/cmd/metrics_server.go @@ -8,8 +8,11 @@ import ( "fmt" "net/http" + "k8s.io/client-go/informers" + "github.com/spidernet-io/spiderpool/pkg/constant" "github.com/spidernet-io/spiderpool/pkg/metric" + "github.com/spidernet-io/spiderpool/pkg/podownercache" ) // initAgentMetricsServer will start an opentelemetry http server for spiderpool agent. @@ -23,7 +26,19 @@ func initAgentMetricsServer(ctx context.Context) { logger.Fatal(err.Error()) } - err = metric.InitSpiderpoolAgentMetrics(ctx, agentContext.Cfg.EnableRDMAMetric, agentContext.CRDManager.GetClient()) + var cache podownercache.CacheInterface + if agentContext.Cfg.EnableRDMAMetric { //nolint:golint + informerFactory := informers.NewSharedInformerFactory(agentContext.ClientSet, 0) + podInformer := informerFactory.Core().V1().Pods().Informer() + informerFactory.Start(ctx.Done()) + + cache, err = podownercache.New(ctx, podInformer, agentContext.CRDManager.GetClient()) + if err != nil { + logger.Fatal(err.Error()) + } + } + + err = metric.InitSpiderpoolAgentMetrics(ctx, cache) if nil != err { logger.Fatal(err.Error()) } diff --git a/docs/images/rdma/rdma-cluster.png b/docs/images/rdma/rdma-cluster.png new file mode 100644 index 000000000..c7323f853 Binary files /dev/null and b/docs/images/rdma/rdma-cluster.png differ diff --git a/docs/images/rdma/rdma-node1.png b/docs/images/rdma/rdma-node1.png new file mode 100644 index 000000000..551044019 Binary files /dev/null and b/docs/images/rdma/rdma-node1.png differ diff --git a/docs/images/rdma/rdma-node2.png b/docs/images/rdma/rdma-node2.png new file mode 100644 index 000000000..2ae97ef4a Binary files /dev/null and b/docs/images/rdma/rdma-node2.png differ diff --git a/docs/images/rdma/rdma-pod.png b/docs/images/rdma/rdma-pod.png new file mode 100644 index 000000000..e23c896af Binary files /dev/null and b/docs/images/rdma/rdma-pod.png differ diff --git a/docs/images/rdma/rdma-workload.png b/docs/images/rdma/rdma-workload.png new file mode 100644 index 000000000..31cdaa43b Binary files /dev/null and b/docs/images/rdma/rdma-workload.png differ diff --git a/docs/usage/rdma-metrics-zh_CN.md b/docs/usage/rdma-metrics-zh_CN.md index 833ea6377..3d5e83958 100644 --- a/docs/usage/rdma-metrics-zh_CN.md +++ b/docs/usage/rdma-metrics-zh_CN.md @@ -83,3 +83,19 @@ helm upgrade --install spiderpool spiderpool/spiderpool --reuse-values --wait -- | rx_vport_rdma_multicast_bytes | Counter | 接收到的多播 RDMA 包的字节数 | | | tx_vport_rdma_multicast_bytes | Counter | 发送的多播 RDMA 包的字节数 | | | vport_speed_mbps | 速度 | 端口的速度，以兆位每秒（Mbps）表示 | | + +## Grafana 监控面板 + +Grafana RDMA Cluster 监控面板，可以查看当前集群每个节点的 RDMA 监控。 +![RDMA Dashboard](../images/rdma/rdma-cluster.png) + +Grafana RDMA Node 监控面板，可以查看每个物理网卡的 RDMA 监控，以及该物理网卡的带宽利用率。同时提供了该节点宿主机的 vf 网卡统计，以及该节点 Pod 使用 RDMA 网卡的监控。 +![RDMA Dashboard](../images/rdma/rdma-node1.png) +
+![RDMA Dashboard](../images/rdma/rdma-node2.png) + +Grafana RDMA Pod 监控面板，可以查看 Pod 里面每张网卡的 RDMA 监控，同时提供了网卡错误统计信息，根据这些信息的统计可以排查问题。 +![RDMA Dashboard](../images/rdma/rdma-pod.png) + +Grafana RDMA Workload 监控面板。在进行 AI 推理和训练时，往往使用 Job, Deployment, KServer 等顶层资源下发 CR 启动一组 Pod 进行训练，可以查看每个顶层资源的 RDMA 监控。 +![RDMA Dashboard](../images/rdma/rdma-workload.png) diff --git a/docs/usage/rdma-metrics.md b/docs/usage/rdma-metrics.md index 5e54ddd0d..4ad45d2f8 100644 --- a/docs/usage/rdma-metrics.md +++ b/docs/usage/rdma-metrics.md @@ -83,3 +83,19 @@ Below is a table containing "Metric Name," "Metric Type," "Metric Meaning," and | rx_vport_rdma_multicast_bytes | Counter | Number of bytes received in multicast RDMA packets | | | tx_vport_rdma_multicast_bytes | Counter | Number of bytes transmitted in multicast RDMA packets | | | vport_speed_mbps | Speed | Speed of the port in Mbps | | + +## Grafana Monitoring Dashboard + +The Grafana RDMA Cluster monitoring dashboard provides a view of the RDMA metrics for each node in the current cluster. +![RDMA Dashboard](../images/rdma/rdma-cluster.png) + +The Grafana RDMA Node monitoring dashboard displays RDMA metrics for each physical NIC (Network Interface Card) and the bandwidth utilization of those NICs. It also includes statistics for VF NICs on the host node and monitoring metrics for Pods using RDMA NICs on that node. +![RDMA Dashboard](../images/rdma/rdma-node1.png) +
+![RDMA Dashboard](../images/rdma/rdma-node2.png) + +The Grafana RDMA Pod monitoring dashboard provides RDMA metrics for each NIC within a Pod, along with NIC error statistics. These metrics help in troubleshooting issues. +![RDMA Dashboard](../images/rdma/rdma-pod.png) + +The Grafana RDMA Workload monitoring dashboard is designed for monitoring RDMA metrics for top-level resources such as Jobs, Deployments, and KServers. These resources typically initiate a set of Pods for AI inference and training tasks. +![RDMA Dashboard](../images/rdma/rdma-workload.png) diff --git a/pkg/ippoolmanager/ippool_manager_suite_test.go b/pkg/ippoolmanager/ippool_manager_suite_test.go index 0e9f0aa6d..94fbc82a8 100644 --- a/pkg/ippoolmanager/ippool_manager_suite_test.go +++ b/pkg/ippoolmanager/ippool_manager_suite_test.go @@ -71,7 +71,7 @@ var _ = BeforeSuite(func() { Build() _, err = metric.InitMetric(context.TODO(), constant.SpiderpoolAgent, false, false) Expect(err).NotTo(HaveOccurred()) - err = metric.InitSpiderpoolAgentMetrics(context.TODO(), false, fakeClient) + err = metric.InitSpiderpoolAgentMetrics(context.TODO(), nil) Expect(err).NotTo(HaveOccurred()) tracker = k8stesting.NewObjectTracker(scheme, k8sscheme.Codecs.UniversalDecoder()) diff --git a/pkg/metric/metrics_instance.go b/pkg/metric/metrics_instance.go index 6c6e53532..6bf503d5f 100644 --- a/pkg/metric/metrics_instance.go +++ b/pkg/metric/metrics_instance.go @@ -9,9 +9,9 @@ import ( "go.opentelemetry.io/otel/attribute" api "go.opentelemetry.io/otel/metric" - "sigs.k8s.io/controller-runtime/pkg/client" "github.com/spidernet-io/spiderpool/pkg/lock" + "github.com/spidernet-io/spiderpool/pkg/podownercache" "github.com/spidernet-io/spiderpool/pkg/rdmametrics" ) @@ -221,10 +221,10 @@ func (a *asyncInt64Gauge) Record(value int64, attrs ...attribute.KeyValue) { } // InitSpiderpoolAgentMetrics serves for spiderpool agent metrics initialization -func InitSpiderpoolAgentMetrics(ctx context.Context, enableRDMAMetric bool, client client.Client) error { +func InitSpiderpoolAgentMetrics(ctx context.Context, cache podownercache.CacheInterface) error { // for rdma - if enableRDMAMetric { - err := rdmametrics.Register(ctx, meter, client) + if cache != nil { + err := rdmametrics.Register(ctx, meter, cache) if err != nil { return err } diff --git a/pkg/podownercache/pod_owner_cache.go b/pkg/podownercache/pod_owner_cache.go new file mode 100644 index 000000000..13f08372b --- /dev/null +++ b/pkg/podownercache/pod_owner_cache.go @@ -0,0 +1,172 @@ +// Copyright 2024 Authors of spidernet-io +// SPDX-License-Identifier: Apache-2.0 + +package podownercache + +import ( + "context" + "fmt" + "github.com/spidernet-io/spiderpool/pkg/lock" + "github.com/spidernet-io/spiderpool/pkg/logutils" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/cache" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type PodOwnerCache struct { + ctx context.Context + apiReader client.Reader + + cacheLock lock.RWMutex + pods map[types.NamespacedName]Pod + ipToPod map[string]types.NamespacedName +} + +type Pod struct { + types.NamespacedName + OwnerInfo OwnerInfo + IPs []string +} + +type OwnerInfo struct { + APIVersion string + Kind string + Namespace string + Name string +} + +type CacheInterface interface { + GetPodByIP(ip string) *Pod +} + +var logger *zap.Logger + +func New(ctx context.Context, podInformer cache.SharedIndexInformer, apiReader client.Reader) (CacheInterface, error) { + logger = logutils.Logger.Named("PodOwnerCache") + logger.Info("create PodOwnerCache informer") + + res := &PodOwnerCache{ + ctx: ctx, + apiReader: apiReader, + cacheLock: lock.RWMutex{}, + pods: make(map[types.NamespacedName]Pod), + ipToPod: make(map[string]types.NamespacedName), + } + + _, err := podInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: res.onPodAdd, + UpdateFunc: res.onPodUpdate, + DeleteFunc: res.onPodDel, + }) + if nil != err { + logger.Error(err.Error()) + return nil, err + } + + return res, nil +} + +func (s *PodOwnerCache) onPodAdd(obj interface{}) { + if pod, ok := obj.(*corev1.Pod); ok { + if pod.Spec.HostNetwork { + return + } + if len(pod.Status.PodIPs) > 0 { + ips := make([]string, 0, len(pod.Status.PodIPs)) + for _, p := range pod.Status.PodIPs { + ips = append(ips, p.IP) + } + owner, err := s.getFinalOwner(pod) + if err != nil { + logger.Warn("", zap.Error(err)) + return + } + s.cacheLock.Lock() + defer s.cacheLock.Unlock() + key := types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name} + s.pods[key] = Pod{ + NamespacedName: key, + OwnerInfo: *owner, + IPs: ips, + } + for _, ip := range ips { + s.ipToPod[ip] = key + } + } + } +} + +func (s *PodOwnerCache) onPodUpdate(oldObj, newObj interface{}) { + s.onPodAdd(newObj) +} + +func (s *PodOwnerCache) onPodDel(obj interface{}) { + if pod, ok := obj.(*corev1.Pod); ok { + s.cacheLock.Lock() + defer s.cacheLock.Unlock() + + key := types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name} + if _, ok := s.pods[key]; !ok { + return + } + for _, ip := range s.pods[key].IPs { + delete(s.ipToPod, ip) + } + delete(s.pods, key) + } +} + +func (s *PodOwnerCache) getFinalOwner(obj metav1.Object) (*OwnerInfo, error) { + var finalOwner *OwnerInfo + + for { + ownerRefs := obj.GetOwnerReferences() + if len(ownerRefs) == 0 { + break + } + + ownerRef := ownerRefs[0] // Assuming the first owner reference + finalOwner = &OwnerInfo{ + APIVersion: ownerRef.APIVersion, + Kind: ownerRef.Kind, + Namespace: obj.GetNamespace(), + Name: ownerRef.Name, + } + + // Prepare an empty object of the owner kind + ownerObj := &unstructured.Unstructured{} + ownerObj.SetAPIVersion(ownerRef.APIVersion) + ownerObj.SetKind(ownerRef.Kind) + + err := s.apiReader.Get(s.ctx, client.ObjectKey{ + Namespace: obj.GetNamespace(), + Name: ownerRef.Name, + }, ownerObj) + if err != nil { + return nil, fmt.Errorf("error fetching owner: %v", err) + } + + // Set obj to the current owner to continue the loop + obj = ownerObj + } + + return finalOwner, nil +} + +func (s *PodOwnerCache) GetPodByIP(ip string) *Pod { + s.cacheLock.RLock() + defer s.cacheLock.RUnlock() + item, exists := s.ipToPod[ip] + if !exists { + return nil + } + pod, exists := s.pods[item] + if !exists { + return nil + } + return &pod +} diff --git a/pkg/podownercache/pod_owner_cache_test.go b/pkg/podownercache/pod_owner_cache_test.go new file mode 100644 index 000000000..b024ca84c --- /dev/null +++ b/pkg/podownercache/pod_owner_cache_test.go @@ -0,0 +1,157 @@ +// Copyright 2024 Authors of spidernet-io +// SPDX-License-Identifier: Apache-2.0 + +package podownercache + +import ( + "context" + "fmt" + appv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" + kruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + "sigs.k8s.io/controller-runtime/pkg/client" + k8sfakecli "sigs.k8s.io/controller-runtime/pkg/client/fake" + "testing" + "time" +) + +// Label(K00002) + +func TestPodOwnerCache(t *testing.T) { + fakeCli := fake.NewSimpleClientset() + factory := informers.NewSharedInformerFactory(fakeCli, 0*time.Second) + informer := factory.Core().V1().Pods().Informer() + //indexer := informer.GetIndexer() + + pod := &corev1.Pod{ + ObjectMeta: v1.ObjectMeta{ + Name: "test-pod", + Namespace: "test-ns", + OwnerReferences: []v1.OwnerReference{ + { + APIVersion: "apps/v1", + Kind: "ReplicaSet", + Name: "test-rs", + }, + }, + }, + Status: corev1.PodStatus{ + PodIPs: []corev1.PodIP{ + { + IP: "10.6.1.20", + }, + }, + }, + } + + //err := indexer.Add() + //if err != nil { + // t.Fatal(err) + //} + + stopCh := make(chan struct{}) + defer close(stopCh) + go factory.Start(stopCh) + + scheme := kruntime.NewScheme() + err := corev1.AddToScheme(scheme) + if err != nil { + t.Fatal(err) + } + err = appv1.AddToScheme(scheme) + if err != nil { + t.Fatal(err) + } + + objs := getMockObjs() + cli := k8sfakecli.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build() + + cache, err := New(context.Background(), informer, cli) + if err != nil { + t.Fatal(err) + } + + // case add + _, err = fakeCli.CoreV1().Pods("test-ns").Create(context.Background(), pod, v1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } + + time.Sleep(time.Second * 6) + + res := cache.GetPodByIP("10.6.1.20") + if res == nil { + t.Fatal("res is nil") + } + if res.OwnerInfo.Namespace != "test-ns" && res.OwnerInfo.Name != "test-deployment" { + t.Fatal(fmt.Println("res is not equal to test-ns and test-deployment")) + } + + // case update + _, err = fakeCli.CoreV1().Pods("test-ns").Update(context.Background(), pod, v1.UpdateOptions{}) + if err != nil { + t.Fatal(err) + } + time.Sleep(time.Second * 6) + res = cache.GetPodByIP("10.6.1.20") + if res == nil { + t.Fatal("res is nil") + } + if res.OwnerInfo.Namespace != "test-ns" && res.OwnerInfo.Name != "test-deployment" { + t.Fatal(fmt.Println("res is not equal to test-ns and test-deployment")) + } + + // case delete + err = fakeCli.CoreV1().Pods("test-ns").Delete(context.Background(), "test-pod", v1.DeleteOptions{}) + if err != nil { + t.Fatal("res is nil") + } + time.Sleep(time.Second * 6) + res = cache.GetPodByIP("10.6.1.20") + if res != nil { + t.Fatal("res is not nil") + } + + // case for not exist ip + res = cache.GetPodByIP("10.6.1.29") + if res != nil { + t.Fatal("res is not nil") + } +} + +func getMockObjs() []client.Object { + return []client.Object{ + &appv1.ReplicaSet{ + TypeMeta: v1.TypeMeta{ + Kind: "ReplicaSet", + APIVersion: "apps/v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: "test-rs", + Namespace: "test-ns", + OwnerReferences: []v1.OwnerReference{ + { + APIVersion: "apps/v1", + Kind: "Deployment", + Name: "test-deployment", + UID: "test-uid", + }, + }, + }, + }, + + &appv1.Deployment{ + TypeMeta: v1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: "test-deployment", + Namespace: "test-ns", + }, + }, + } +} diff --git a/pkg/rdmametrics/metrics.go b/pkg/rdmametrics/metrics.go index 10268b989..17d1143fa 100644 --- a/pkg/rdmametrics/metrics.go +++ b/pkg/rdmametrics/metrics.go @@ -25,11 +25,10 @@ import ( "github.com/containernetworking/plugins/pkg/ns" "github.com/spidernet-io/spiderpool/pkg/lock" "github.com/spidernet-io/spiderpool/pkg/logutils" + "github.com/spidernet-io/spiderpool/pkg/podownercache" "github.com/spidernet-io/spiderpool/pkg/rdmametrics/ethtool" ) -var cli client.Client - const netnsPath = "/var/run/netns" var ( @@ -110,8 +109,7 @@ type RDMADevice struct { IsRoot bool } -func Register(ctx context.Context, meter metric.Meter, client client.Client) error { - cli = client +func Register(ctx context.Context, meter metric.Meter, cache podownercache.CacheInterface) error { log := logutils.Logger.Named("rdma-metrics-exporter") nodeName, err := os.Hostname() if err != nil { @@ -143,6 +141,7 @@ func Register(ctx context.Context, meter metric.Meter, client client.Client) err RdmaLinkList: netlink.RdmaLinkList, LinkList: netlink.LinkList, }, + cache: cache, } err = e.registerMetrics(meter) if err != nil { @@ -166,6 +165,7 @@ type exporter struct { registration metric.Registration waitToRegisterMetrics map[string]struct{} observableMap map[string]metric.Int64ObservableCounter + cache podownercache.CacheInterface } func (e *exporter) registerMetrics(meter metric.Meter) error { @@ -241,11 +241,6 @@ func (e *exporter) Callback(ctx context.Context, observer metric.Observer) error } list = append(list, "") - podIPMap, err := getIPToPodMap(ctx, cli) - if err != nil { - e.log.Error("failed to get IP to pod map", zap.Error(err)) - return fmt.Errorf("failed to get ip map pod: %w", err) - } unRegistrationMetric := make([]string, 0) getObservable := func(key string) (metric.Int64ObservableCounter, bool) { if val, ok := e.observableMap[rdmaMetricsPrefix+key]; ok { @@ -260,7 +255,7 @@ func (e *exporter) Callback(ctx context.Context, observer metric.Observer) error return fmt.Errorf("failed to get node guid net device name map: %w", err) } for _, netNsID := range list { - if err := e.processNetNS(netNsID, podIPMap, nodeGuidNetDeviceNameMap, observer, getObservable); err != nil { + if err := e.processNetNS(netNsID, nodeGuidNetDeviceNameMap, observer, getObservable); err != nil { e.log.Error("failed to process net ns", zap.String("net_ns_id", netNsID), zap.Error(err)) continue } @@ -287,7 +282,7 @@ func (e *exporter) updateUnregisteredMetrics(unRegistrationMetric []string) { } } -func (e *exporter) processNetNS(netNsID string, ipMapPod map[string]types.NamespacedName, +func (e *exporter) processNetNS(netNsID string, nodeGuidNetDeviceNameMap map[string]string, observer metric.Observer, getObservable GetObservable) error { podPrimaryIP, statsList, err := getRDMAStats(netNsID, e.netns, nodeGuidNetDeviceNameMap, e.netlinkImpl, e.exec, e.ethtool) @@ -299,12 +294,21 @@ func (e *exporter) processNetNS(netNsID string, ipMapPod map[string]types.Namesp return nil } - var attributeNamespace, attributeName *attribute.KeyValue - if item, ok := ipMapPod[podPrimaryIP]; ok { - attributeNamespace, attributeName = getPodAttributes(item) + attributes := []*attribute.KeyValue{ + e.nodeName, + } + + if pod := e.cache.GetPodByIP(podPrimaryIP); pod != nil { + namespace := attribute.String("pod_namespace", pod.Namespace) + name := attribute.String("pod_name", pod.Name) + ownerAPIVersion := attribute.String("owner_api_version", pod.OwnerInfo.APIVersion) + ownerKind := attribute.String("owner_kind", pod.OwnerInfo.Kind) + ownerNamespace := attribute.String("owner_namespace", pod.OwnerInfo.Namespace) + ownerName := attribute.String("owner_name", pod.OwnerInfo.Name) + attributes = append(attributes, &namespace, &name, &ownerAPIVersion, &ownerKind, &ownerNamespace, &ownerName) } for _, stats := range statsList { - processStats(stats, observer, getObservable, attributeNamespace, attributeName, e.nodeName) + processStats(stats, observer, getObservable, attributes...) } return nil } diff --git a/pkg/rdmametrics/metrics_test.go b/pkg/rdmametrics/metrics_test.go index 9672ffafb..05350119c 100644 --- a/pkg/rdmametrics/metrics_test.go +++ b/pkg/rdmametrics/metrics_test.go @@ -29,16 +29,27 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/spidernet-io/spiderpool/pkg/logutils" + "github.com/spidernet-io/spiderpool/pkg/podownercache" ) // Label(K00002) +type FakeCache struct { + IPToPodMap map[string]podownercache.Pod +} + +func (f *FakeCache) GetPodByIP(ip string) *podownercache.Pod { + if val, ok := f.IPToPodMap[ip]; ok { + return &val + } + return nil +} + func TestRegister(t *testing.T) { ctx := context.Background() meter := noop.NewMeterProvider().Meter("test") - cli := fake.NewClientBuilder().Build() - err := Register(ctx, meter, cli) + err := Register(ctx, meter, &FakeCache{IPToPodMap: map[string]podownercache.Pod{}}) if err != nil { t.Errorf("Unexpected error: %v", err) } @@ -972,13 +983,13 @@ func TestProcessNetNS(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { meter := noop.NewMeterProvider().Meter("test") - cli = fake.NewClientBuilder().Build() fakeExec := &testexec.FakeExec{ CommandScript: tt.commandScript, } e := &exporter{ + cache: &FakeCache{}, meter: meter, netlinkImpl: NetlinkImpl{ RdmaLinkList: func() ([]*netlink.RdmaLink, error) { @@ -998,7 +1009,7 @@ func TestProcessNetNS(t *testing.T) { guidMapNetDeviceName := map[string]string{ "b6:65:05:0c:9c:5c:f6:00": "ib1", } - err := e.processNetNS(tt.netnsID, tt.ipPodMap, guidMapNetDeviceName, observer, tt.getObservable) + err := e.processNetNS(tt.netnsID, guidMapNetDeviceName, observer, tt.getObservable) if (err != nil) != tt.expectError { t.Errorf("Expected error: %v, but got: %v", tt.expectError, err) } @@ -1010,7 +1021,6 @@ func TestUpdateUnregisteredMetrics(t *testing.T) { var commandScript []testexec.FakeCommandAction meter := noop.NewMeterProvider().Meter("test") - cli = fake.NewClientBuilder().Build() fakeExec := &testexec.FakeExec{ CommandScript: commandScript, @@ -1066,7 +1076,6 @@ func TestCallback(t *testing.T) { } meter := noop.NewMeterProvider().Meter("test") - cli = fake.NewClientBuilder().Build() fakeExec := &testexec.FakeExec{ CommandScript: commandScript, @@ -1087,8 +1096,9 @@ func TestCallback(t *testing.T) { netns: func(netnsID string, toRun func() error) error { return toRun() }, - exec: fakeExec, - log: logutils.Logger.Named("rdma-metrics-exporter"), + exec: fakeExec, + log: logutils.Logger.Named("rdma-metrics-exporter"), + cache: &FakeCache{}, } err := e.registerMetrics(e.meter) if err != nil {