docs: update ebpf/k8s example (#3644)

* docs: update ebpf/k8s example

* Update README.md

Co-authored-by: Marc Sanmiquel <marcsanmiquel@gmail.com>

* Update README.md

Co-authored-by: Marc Sanmiquel <marcsanmiquel@gmail.com>

* Apply suggestions from code review

---------

Co-authored-by: Marc Sanmiquel <marcsanmiquel@gmail.com>
Co-authored-by: Kim Nylander <104772500+knylander-grafana@users.noreply.github.com>

examples/grafana-agent-auto-instrumentation/ebpf/kubernetes/README.md

@@ -0,0 +1,40 @@
+# Grafana Alloy eBPF profiling via auto-instrumentation in Kubernetes
+
+This repository provides a practical demonstration of leveraging Grafana Alloy for continuous application profiling 
+using eBPF and Pyroscope in Kubernetes. It illustrates a seamless approach to profiling Golang and Python processes, 
+aiding in performance optimization.
+
+## Overview
+
+eBPF profiling via Grafana Alloy is based on a few components:
+- `discovery.kubernetes` for discovering Kubernetes pods
+- `discovery.relabel` for detecting and filtering target processes and setting up labels
+- `pyroscope.ebpf` for enabling eBPF profiling for specific applications
+- `pyroscope.write` for writing the profiles data to a remote endpoint
+
+Refer to the [official documentation](https://grafana.com/docs/alloy/latest/reference/components/pyroscope/pyroscope.ebpf/) for an in-depth understanding and additional configuration options for eBPF with Grafana Alloy.
+Also, check the [Grafana Alloy Components reference](https://grafana.com/docs/alloy/latest/reference/components/) for more details on each used component.
+
+
+
+## Getting started
+
+To use this example:
+
+1. Set up a local kubernetes cluster using Kind or a similar tool.
+2. Clone this repository and navigate to this example's directory.
+3. Deploy the manifests:
+    ```shell
+       kubectl apply -f alloy.yaml -f grafana.yaml -f pyroscope.yaml -f python-fast-slow.yaml
+    ```
+4. Port-forward the Grafana service to access the Explore Profiles app:
+    ```shell
+       kubectl port-forward -n pyroscope-ebpf service/grafana  3000:3000
+    ```
+5. Explore profiles http://localhost:3000/a/grafana-pyroscope-app/profiles-explore
+
+After the deployment is operational, the Grafana Alloy will profile the Go and Python applications using `pyroscope.ebpf` component.
+
+## Documentation
+
+Refer to the [official documentation](https://grafana.com/docs/alloy/latest/reference/components/pyroscope/pyroscope.ebpf/) for an in-depth understanding and additional configuration options for eBPF profiling with Grafana Alloy.

examples/grafana-agent-auto-instrumentation/ebpf/kubernetes/alloy.yaml

@@ -0,0 +1,177 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: pyroscope-ebpf
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole # needed for the discovery.kubernetes alloy component
+metadata:
+  name: grafana-alloy-role
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: grafana-alloy
+  namespace: pyroscope-ebpf
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: grafana-alloy-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: grafana-alloy-role
+subjects:
+  - kind: ServiceAccount
+    name: grafana-alloy
+    namespace: pyroscope-ebpf
+
+---
+
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: grafana-alloy
+  namespace: pyroscope-ebpf
+spec:
+  selector:
+    matchLabels:
+      app: grafana-alloy
+  template:
+    metadata:
+      labels:
+        app: grafana-alloy
+    spec:
+      serviceAccountName: grafana-alloy
+      containers:
+        - name: grafana-alloy
+          image: grafana/alloy:latest
+          command:
+            - /bin/alloy
+            - run
+            - /etc/agent-config/config.river
+            - --server.http.listen-addr=0.0.0.0:12345
+          env:
+            - name: HOSTNAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          ports:
+            - containerPort: 12345
+          volumeMounts:
+            - name: agent-config
+              mountPath: /etc/agent-config
+          securityContext:
+            privileged: true
+            runAsGroup: 0
+            runAsUser: 0
+      volumes:
+        - name: agent-config
+          configMap:
+            name: agent-config
+
+      hostPID: true
+
+---
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: agent-config
+  namespace: pyroscope-ebpf
+data:
+  config.river: |
+    // This is an example grafana agent config to set up eBPF profiling in kubernetes.
+    // for more info see https://grafana.com/docs/pyroscope/latest/configure-client/grafana-agent/ebpf/setup-kubernetes/
+    logging {
+      level = "debug"
+      format = "logfmt"
+    }
+    
+    discovery.kubernetes "local_pods" {
+        selectors {
+          field = "spec.nodeName=" + env("HOSTNAME") // Note: this assume HOSTNAME is set to the node name
+          role = "pod"
+        }
+        role = "pod"
+    }
+    
+    discovery.relabel "specific_pods" {
+      targets = discovery.kubernetes.local_pods.targets
+      rule {
+        action = "drop"
+        regex = "Succeeded|Failed|Completed"
+        source_labels = ["__meta_kubernetes_pod_phase"]
+      }
+      
+      rule {
+        action = "replace"
+        source_labels = ["__meta_kubernetes_namespace"]
+        target_label = "namespace"
+      }
+      rule {
+        action = "replace"
+        source_labels = ["__meta_kubernetes_pod_name"]
+        target_label = "pod"
+      }
+      rule {
+        action = "replace"
+        source_labels = ["__meta_kubernetes_pod_node_name"]
+        target_label = "node"
+      }
+      
+      rule {
+        action = "replace"
+        source_labels = ["__meta_kubernetes_pod_container_name"]
+        target_label = "container"
+      }
+      
+      // provide arbitrary service_name label, otherwise it will be set to {__meta_kubernetes_namespace}/{__meta_kubernetes_pod_container_name}
+      rule {
+        action = "replace"
+        regex = "(.*)@(.*)"
+        replacement = "${1}/${2}"
+        separator = "@"
+        source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"]
+        target_label = "service_name"
+      }
+      
+      // Filter specific targets to profile
+      rule {
+        source_labels = ["service_name"]
+        regex = "(.*alloy|.*pyroscope|.*fast-slow)"
+        action = "keep"
+      }
+    }
+    
+    pyroscope.ebpf "instance" {
+        forward_to = [pyroscope.write.endpoint.receiver]
+        targets = discovery.relabel.specific_pods.output
+        python_enabled = true
+    }
+    
+    pyroscope.write "endpoint" {
+        endpoint {
+          url = "http://pyroscope.pyroscope-ebpf.svc.cluster.local.:4040"
+          // url = "<Grafana Cloud URL>"
+          // basic_auth {
+          //  username = "<Grafana Cloud User>"
+          //  password = "<Grafana Cloud Password>"
+          // }
+        }
+    }
+---

examples/grafana-agent-auto-instrumentation/ebpf/kubernetes/grafana.yaml

@@ -0,0 +1,78 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: grafana
+  namespace: pyroscope-ebpf
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: grafana
+  template:
+    metadata:
+      labels:
+        app: grafana
+    spec:
+      containers:
+        - name: grafana
+          image: grafana/grafana:latest
+          env:
+            - name: GF_INSTALL_PLUGINS
+              value: grafana-pyroscope-app
+            - name: GF_AUTH_ANONYMOUS_ENABLED
+              value: "true"
+            - name: GF_AUTH_ANONYMOUS_ORG_ROLE
+              value: Admin
+            - name: GF_AUTH_DISABLE_LOGIN_FORM
+              value: "true"
+          ports:
+            - containerPort: 3000
+          volumeMounts:
+            - name: grafana-provisioning
+              mountPath: /etc/grafana/provisioning
+      volumes:
+        - name: grafana-provisioning
+          configMap:
+            name: grafana-provisioning
+            items:
+              - key: datasources
+                path: datasources/datasources.yaml
+              - key: plugins
+                path: plugins/plugins.yaml
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: grafana
+  namespace: pyroscope-ebpf
+spec:
+  selector:
+    app: grafana
+  ports:
+    - protocol: TCP
+      port: 3000
+      targetPort: 3000
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-provisioning
+  namespace: pyroscope-ebpf
+data:
+  "datasources": |
+    apiVersion: 1
+    datasources:
+      - uid: local-pyroscope
+        type: grafana-pyroscope-datasource
+        name: Pyroscope
+        url: http://pyroscope:4040
+        jsonData:
+          keepCookies: [pyroscope_git_session]
+
+  "plugins": |
+    apiVersion: 1
+    apps:
+      - type: grafana-pyroscope-app
+        jsonData:
+          backendUrl: http://pyroscope:4040
+        secureJsonData: