diff --git a/examples/grafana-agent-auto-instrumentation/ebpf/k8s/config.river b/examples/grafana-agent-auto-instrumentation/ebpf/k8s/config.river deleted file mode 100644 index 8a74b9fb75..0000000000 --- a/examples/grafana-agent-auto-instrumentation/ebpf/k8s/config.river +++ /dev/null @@ -1,70 +0,0 @@ -// This is an example grafana agent config to set up eBPF profiling in kubernetes. -// for more info see https://grafana.com/docs/pyroscope/latest/configure-client/grafana-agent/ebpf/setup-kubernetes/ - -discovery.kubernetes "local_pods" { - selectors { - field = "spec.nodeName=" + env("HOSTNAME") // Note: this assume HOSTNAME is set to the node name - role = "pod" - } - role = "pod" -} - -discovery.relabel "specific_pods" { - targets = discovery.kubernetes.local_pods.targets - rule { - action = "drop" - regex = "Succeeded|Failed|Completed" - source_labels = ["__meta_kubernetes_pod_phase"] - } - rule { - action = "replace" - source_labels = ["__meta_kubernetes_namespace"] - target_label = "namespace" - } - rule { - action = "replace" - source_labels = ["__meta_kubernetes_pod_name"] - target_label = "pod" - } - rule { - action = "replace" - source_labels = ["__meta_kubernetes_pod_node_name"] - target_label = "node" - } - rule { - action = "replace" - source_labels = ["__meta_kubernetes_pod_container_name"] - target_label = "container" - } - // provide arbitrary service_name label, otherwise it will be set to {__meta_kubernetes_namespace}/{__meta_kubernetes_pod_container_name} - rule { - action = "replace" - regex = "(.*)@(.*)" - replacement = "ebpf/k8s/${1}/${2}" - separator = "@" - source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] - target_label = "service_name" - } - // Filter specific targets to profile - rule { - source_labels = ["service_name"] - regex = "(ebpf/grafana-agent/agent|ebpf/pyroscope/pyroscope)" - action = "keep" - } -} - -pyroscope.ebpf "instance" { - forward_to = [pyroscope.write.endpoint.receiver] - targets = discovery.relabel.specific_pods.output -} - -pyroscope.write "endpoint" { - endpoint { - url = "http://pyroscope:4040" - // url = "" - // basic_auth { - // username = "" - // password = "" - // } - } -} diff --git a/examples/grafana-agent-auto-instrumentation/ebpf/kubernetes/README.md b/examples/grafana-agent-auto-instrumentation/ebpf/kubernetes/README.md new file mode 100644 index 0000000000..6b1fe74d14 --- /dev/null +++ b/examples/grafana-agent-auto-instrumentation/ebpf/kubernetes/README.md @@ -0,0 +1,40 @@ +# Grafana Alloy eBPF profiling via auto-instrumentation in Kubernetes + +This repository provides a practical demonstration of leveraging Grafana Alloy for continuous application profiling +using eBPF and Pyroscope in Kubernetes. It illustrates a seamless approach to profiling Golang and Python processes, +aiding in performance optimization. + +## Overview + +eBPF profiling via Grafana Alloy is based on a few components: +- `discovery.kubernetes` for discovering Kubernetes pods +- `discovery.relabel` for detecting and filtering target processes and setting up labels +- `pyroscope.ebpf` for enabling eBPF profiling for specific applications +- `pyroscope.write` for writing the profiles data to a remote endpoint + +Refer to the [official documentation](https://grafana.com/docs/alloy/latest/reference/components/pyroscope/pyroscope.ebpf/) for an in-depth understanding and additional configuration options for eBPF with Grafana Alloy. +Also, check the [Grafana Alloy Components reference](https://grafana.com/docs/alloy/latest/reference/components/) for more details on each used component. + + + +## Getting started + +To use this example: + +1. Set up a local kubernetes cluster using Kind or a similar tool. +2. Clone this repository and navigate to this example's directory. +3. Deploy the manifests: + ```shell + kubectl apply -f alloy.yaml -f grafana.yaml -f pyroscope.yaml -f python-fast-slow.yaml + ``` +4. Port-forward the Grafana service to access the Explore Profiles app: + ```shell + kubectl port-forward -n pyroscope-ebpf service/grafana 3000:3000 + ``` +5. Explore profiles http://localhost:3000/a/grafana-pyroscope-app/profiles-explore + +After the deployment is operational, the Grafana Alloy will profile the Go and Python applications using `pyroscope.ebpf` component. + +## Documentation + +Refer to the [official documentation](https://grafana.com/docs/alloy/latest/reference/components/pyroscope/pyroscope.ebpf/) for an in-depth understanding and additional configuration options for eBPF profiling with Grafana Alloy. diff --git a/examples/grafana-agent-auto-instrumentation/ebpf/kubernetes/alloy.yaml b/examples/grafana-agent-auto-instrumentation/ebpf/kubernetes/alloy.yaml new file mode 100644 index 0000000000..cb39a98c99 --- /dev/null +++ b/examples/grafana-agent-auto-instrumentation/ebpf/kubernetes/alloy.yaml @@ -0,0 +1,177 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: pyroscope-ebpf +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole # needed for the discovery.kubernetes alloy component +metadata: + name: grafana-alloy-role +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: grafana-alloy + namespace: pyroscope-ebpf +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: grafana-alloy-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: grafana-alloy-role +subjects: + - kind: ServiceAccount + name: grafana-alloy + namespace: pyroscope-ebpf + +--- + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: grafana-alloy + namespace: pyroscope-ebpf +spec: + selector: + matchLabels: + app: grafana-alloy + template: + metadata: + labels: + app: grafana-alloy + spec: + serviceAccountName: grafana-alloy + containers: + - name: grafana-alloy + image: grafana/alloy:latest + command: + - /bin/alloy + - run + - /etc/agent-config/config.river + - --server.http.listen-addr=0.0.0.0:12345 + env: + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - containerPort: 12345 + volumeMounts: + - name: agent-config + mountPath: /etc/agent-config + securityContext: + privileged: true + runAsGroup: 0 + runAsUser: 0 + volumes: + - name: agent-config + configMap: + name: agent-config + + hostPID: true + +--- + +apiVersion: v1 +kind: ConfigMap +metadata: + name: agent-config + namespace: pyroscope-ebpf +data: + config.river: | + // This is an example grafana agent config to set up eBPF profiling in kubernetes. + // for more info see https://grafana.com/docs/pyroscope/latest/configure-client/grafana-agent/ebpf/setup-kubernetes/ + logging { + level = "debug" + format = "logfmt" + } + + discovery.kubernetes "local_pods" { + selectors { + field = "spec.nodeName=" + env("HOSTNAME") // Note: this assume HOSTNAME is set to the node name + role = "pod" + } + role = "pod" + } + + discovery.relabel "specific_pods" { + targets = discovery.kubernetes.local_pods.targets + rule { + action = "drop" + regex = "Succeeded|Failed|Completed" + source_labels = ["__meta_kubernetes_pod_phase"] + } + + rule { + action = "replace" + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_node_name"] + target_label = "node" + } + + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + + // provide arbitrary service_name label, otherwise it will be set to {__meta_kubernetes_namespace}/{__meta_kubernetes_pod_container_name} + rule { + action = "replace" + regex = "(.*)@(.*)" + replacement = "${1}/${2}" + separator = "@" + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] + target_label = "service_name" + } + + // Filter specific targets to profile + rule { + source_labels = ["service_name"] + regex = "(.*alloy|.*pyroscope|.*fast-slow)" + action = "keep" + } + } + + pyroscope.ebpf "instance" { + forward_to = [pyroscope.write.endpoint.receiver] + targets = discovery.relabel.specific_pods.output + python_enabled = true + } + + pyroscope.write "endpoint" { + endpoint { + url = "http://pyroscope.pyroscope-ebpf.svc.cluster.local.:4040" + // url = "" + // basic_auth { + // username = "" + // password = "" + // } + } + } +--- diff --git a/examples/grafana-agent-auto-instrumentation/ebpf/kubernetes/grafana.yaml b/examples/grafana-agent-auto-instrumentation/ebpf/kubernetes/grafana.yaml new file mode 100644 index 0000000000..8bf8a7c3bd --- /dev/null +++ b/examples/grafana-agent-auto-instrumentation/ebpf/kubernetes/grafana.yaml @@ -0,0 +1,78 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + namespace: pyroscope-ebpf +spec: + replicas: 1 + selector: + matchLabels: + app: grafana + template: + metadata: + labels: + app: grafana + spec: + containers: + - name: grafana + image: grafana/grafana:latest + env: + - name: GF_INSTALL_PLUGINS + value: grafana-pyroscope-app + - name: GF_AUTH_ANONYMOUS_ENABLED + value: "true" + - name: GF_AUTH_ANONYMOUS_ORG_ROLE + value: Admin + - name: GF_AUTH_DISABLE_LOGIN_FORM + value: "true" + ports: + - containerPort: 3000 + volumeMounts: + - name: grafana-provisioning + mountPath: /etc/grafana/provisioning + volumes: + - name: grafana-provisioning + configMap: + name: grafana-provisioning + items: + - key: datasources + path: datasources/datasources.yaml + - key: plugins + path: plugins/plugins.yaml +--- +apiVersion: v1 +kind: Service +metadata: + name: grafana + namespace: pyroscope-ebpf +spec: + selector: + app: grafana + ports: + - protocol: TCP + port: 3000 + targetPort: 3000 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-provisioning + namespace: pyroscope-ebpf +data: + "datasources": | + apiVersion: 1 + datasources: + - uid: local-pyroscope + type: grafana-pyroscope-datasource + name: Pyroscope + url: http://pyroscope:4040 + jsonData: + keepCookies: [pyroscope_git_session] + + "plugins": | + apiVersion: 1 + apps: + - type: grafana-pyroscope-app + jsonData: + backendUrl: http://pyroscope:4040 + secureJsonData: diff --git a/examples/grafana-agent-auto-instrumentation/ebpf/kubernetes/pyroscope.yaml b/examples/grafana-agent-auto-instrumentation/ebpf/kubernetes/pyroscope.yaml new file mode 100644 index 0000000000..65a602a841 --- /dev/null +++ b/examples/grafana-agent-auto-instrumentation/ebpf/kubernetes/pyroscope.yaml @@ -0,0 +1,34 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pyroscope + namespace: pyroscope-ebpf +spec: + replicas: 1 + selector: + matchLabels: + app: pyroscope + template: + metadata: + labels: + app: pyroscope + spec: + containers: + - name: pyroscope + image: grafana/pyroscope:latest + ports: + - containerPort: 4040 +--- + +apiVersion: v1 +kind: Service +metadata: + name: pyroscope + namespace: pyroscope-ebpf +spec: + selector: + app: pyroscope + ports: + - protocol: TCP + port: 4040 + targetPort: 4040 diff --git a/examples/grafana-agent-auto-instrumentation/ebpf/kubernetes/python-fast-slow.yaml b/examples/grafana-agent-auto-instrumentation/ebpf/kubernetes/python-fast-slow.yaml new file mode 100644 index 0000000000..66e4a2732e --- /dev/null +++ b/examples/grafana-agent-auto-instrumentation/ebpf/kubernetes/python-fast-slow.yaml @@ -0,0 +1,60 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: python-app + namespace: pyroscope-ebpf +data: + main: | + #!/usr/bin/env python3 + + import logging + import os + + def work(n): + i = 0 + while i < n: + i += 1 + + def fast_function(): + work(20000) + + def slow_function(): + work(80000) + + if __name__ == "__main__": + while True: + fast_function() + slow_function() + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: python-fast-slow + namespace: pyroscope-ebpf +spec: + replicas: 1 + selector: + matchLabels: + app: python-fast-slow + template: + metadata: + labels: + app: python-fast-slow + spec: + containers: + - name: python-fast-slow + image: python:3.11 + imagePullPolicy: IfNotPresent + command: [ "python3" ] + args: [ "/app/main.py" ] + volumeMounts: + - name: app + mountPath: /app + volumes: + - name: app + configMap: + name: python-app + items: + - key: main + path: main.py