Merge pull request #19 from scality/feature/COSI-34-debug-with-delve

COSI-34: remote-debugging-golang-on-Kubernetes

.dockerignore

@@ -0,0 +1,10 @@
+bin/
+test/
+coverage/
+docker/
+.vscode/
+helm/
+kustomize/
+.git/
+.github/
+.devcontainer/

Makefile

@@ -25,3 +25,7 @@ clean:
 container:
 	@echo "Building container image..."
 	docker build -t $(IMAGE_NAME) .
+
+delve:
+	@echo "Building Delve container image..."
+	docker build -f docker/delve/Dockerfile -t $(IMAGE_NAME)-delve .

docker/delve/Dockerfile

@@ -0,0 +1,33 @@
+# Stage 1: Build the Go application and install Delve
+FROM golang:1.23.3 AS builder
+
+# Install Delve for debugging
+RUN go install github.com/go-delve/delve/cmd/dlv@latest
+
+# Set the working directory
+WORKDIR /app
+
+# Copy Go modules and install dependencies
+COPY go.mod go.sum ./
+RUN go mod download
+
+# Copy the source code
+COPY . .
+
+# Build the Go app with debugging flags
+RUN CGO_ENABLED=0 GOOS=linux go build -gcflags "all=-N -l" -o scality-cosi-driver ./cmd/scality-cosi-driver
+
+# Stage 2: Create a minimal container with the built binary and Delve
+FROM golang:1.23.3
+
+WORKDIR /app
+
+# Copy the Go binary and Delve debugger from the builder stage
+COPY --from=builder /go/bin/dlv /dlv
+COPY --from=builder /app/scality-cosi-driver /app/scality-cosi-driver
+
+# Expose Delve debugger port
+EXPOSE 2345
+
+# Run the Go app with Delve in headless mode
+CMD ["/dlv", "exec", "/app/scality-cosi-driver", "--headless", "--listen=:2345", "--api-version=2", "--accept-multiclient", "--log"]

docs/development/remote-debugging-golang-on-kubernetes.md

@@ -0,0 +1,111 @@
+# Remote Debugging Golang-Based Services on Kubernetes with Delve and VS Code
+
+This guide walks you through setting up a remote debugging environment for Go applications deployed on a Kubernetes cluster using Delve and VS Code. The steps apply to any Go service on Kubernetes, using the Scality COSI driver as an example.
+
+---
+
+## Prerequisites
+
+Ensure the following are installed and configured before starting:
+
+1. **Kubernetes Cluster**: A running Kubernetes cluster, such as [Minikube](https://minikube.sigs.k8s.io/docs/start/), [Docker Desktop Kubernetes](https://docs.docker.com/desktop/features/kubernetes/), or a remote cluster.
+2. **kubectl**: [Installed](https://kubernetes.io/docs/tasks/tools/) and configured for your cluster.
+3. **VS Code**: [Installed](https://code.visualstudio.com/), with the [Go extension](https://marketplace.visualstudio.com/items?itemName=golang.Go).
+4. **COSI Driver**: Clone the repository and navigate to the directory:
+
+   ```bash
+   git clone git@github.com:scality/cosi-driver.git && cd cosi-driver
+   ```
+
+5. **Delve**: [Installed locally](https://github.com/go-delve/delve/tree/master/Documentation/installation).
+
+---
+
+## Step 1: Build the Container Image
+
+Build the Docker image with Delve by running:
+
+```bash
+make delve
+```
+
+---
+
+## Step 2: Deploy the COSI Driver
+
+Deploy the COSI driver to Kubernetes using Kustomize. This deployment is configured to run Delve in wait mode, meaning it won’t start the COSI service until a debugger attaches.
+
+```bash
+kubectl apply -k kustomize/overlays/debug
+```
+
+### Verify the Pod Status
+
+Wait until the pod is ready to ensure the deployment succeeded:
+
+```bash
+kubectl wait --namespace scality-object-storage --for=condition=ready pod --selector=app.kubernetes.io/name=scality-cosi-driver --timeout=120s
+```
+
+---
+
+## Step 3: Forward the Delve Debugger Port
+
+Identify the pod name for the COSI driver:
+
+```bash
+kubectl get pods -n scality-object-storage
+```
+
+Forward port `2345` from the Kubernetes pod to your local machine to connect VS Code to the Delve debugger:
+
+```bash
+kubectl port-forward -n scality-object-storage pod/<pod-name> 2345:2345
+```
+
+Replace `<pod-name>` with the actual name of the pod.
+
+---
+
+## Step 4: Configure VS Code for Remote Debugging
+
+1. Confirm **Delve** is installed locally.
+2. Open VS Code and create a `launch.json` file under the `.vscode` directory.
+3. Add the following configuration to `launch.json`:
+
+   ```json
+   {
+       "version": "0.2.0",
+       "configurations": [
+           {
+               "name": "Remote Debug Scality COSI Driver",
+               "type": "go",
+               "request": "attach",
+               "mode": "remote",
+               "remotePath": "/app",
+               "port": 2345,
+               "host": "127.0.0.1",
+               "apiVersion": 2,
+               "trace": "verbose"
+           }
+       ]
+   }
+   ```
+
+---
+
+## Step 6: Start Debugging
+
+1. **Run Port Forwarding**: Ensure port forwarding is active (Step 4).
+2. **Initiate Debugging in VS Code**:
+   - Open VS Code, set breakpoints in your Go code, and press **F5** to start debugging with the "Remote Debug Scality COSI Driver" configuration.
+3. **Inspect Variables and Stack**: You can now inspect variables, step through the code, and debug your Go application as it runs on Kubernetes.
+
+---
+
+## Troubleshooting
+
+- **Delve Not Found**: Verify Delve is installed in your Docker image and available at `/dlv`.
+- **Port Forwarding Issues**: Confirm the Kubernetes pod is running, and that port `2345` is open.
+- **Breakpoints Not Hit**: Ensure the code in Kubernetes matches the local code in VS Code.
+- **Connection Timeout**: Check firewall rules, network policies, and Kubernetes pod permissions if the debugger cannot connect.

kustomization.yaml

@@ -2,4 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
-- kustomize/overlays
+- kustomize/overlays/production

kustomize/overlays/debug/deployment.yaml

@@ -0,0 +1,53 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: scality-cosi-driver
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: scality-cosi-driver
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: scality-cosi-driver
+        app.kubernetes.io/part-of: container-object-storage-interface
+        app.kubernetes.io/component: driver
+        app.kubernetes.io/version: main
+        app.kubernetes.io/managed-by: kustomize
+    spec:
+      serviceAccountName: scality-object-storage-provisioner
+      containers:
+        - name: scality-cosi-driver
+          image: ghcr.io/scality/cosi-driver-delve:latest
+          imagePullPolicy: IfNotPresent
+          command: ["/dlv"]
+          args:
+            - "exec"
+            - "/app/scality-cosi-driver"
+            - "--headless"
+            - "--listen=:2345"
+            - "--api-version=2"
+            - "--accept-multiclient"
+            - "--log"
+            - "--"
+            - "--driver-prefix=cosi"
+            - "--v=$(COSI_DRIVER_LOG_LEVEL)"
+          ports:
+            - containerPort: 2345
+          env:
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+        - name: objectstorage-provisioner-sidecar
+          image: gcr.io/k8s-staging-sig-storage/objectstorage-sidecar:latest
+          imagePullPolicy: IfNotPresent
+          args:
+            - "--v=$(OBJECTSTORAGE_PROVISIONER_SIDECAR_LOG_LEVEL)"
+          volumeMounts:
+            - mountPath: /var/lib/cosi
+              name: socket
+      volumes:
+        - name: socket
+          emptyDir: {}

kustomize/overlays/debug/kustomization.yaml

@@ -0,0 +1,40 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+configMapGenerator:
+- name: scality-cosi-driver-properties
+  env: scality-cosi-driver.properties
+generatorOptions:         
+  disableNameSuffixHash: true
+  labels:                    
+    generated-by: "kustomize"
+
+resources:
+  - ../../base
+
+patchesStrategicMerge:
+  - deployment.yaml
+
+commonLabels:
+  app.kubernetes.io/version: debug
+  app.kubernetes.io/component: driver
+  app.kubernetes.io/name: scality-cosi-driver
+  app.kubernetes.io/part-of: container-object-storage-interface
+  app.kubernetes.io/managed-by: kustomize
+
+vars:
+- name: COSI_DRIVER_LOG_LEVEL
+  objref:
+    name: scality-cosi-driver-properties
+    kind: ConfigMap
+    apiVersion: v1
+  fieldref:
+    fieldpath: data.COSI_DRIVER_LOG_LEVEL
+
+- name: OBJECTSTORAGE_PROVISIONER_SIDECAR_LOG_LEVEL
+  objref:
+    name: scality-cosi-driver-properties
+    kind: ConfigMap
+    apiVersion: v1
+  fieldref:
+    fieldpath: data.OBJECTSTORAGE_PROVISIONER_SIDECAR_LOG_LEVEL

kustomize/overlays/kustomization.yaml → kustomize/overlays/production/kustomization.yaml

@@ -10,7 +10,7 @@ generatorOptions:
     generated-by: "kustomize"
 
 resources:
-  - ../base
+  - ../../base
 
 commonLabels:
   app.kubernetes.io/version: main