Collect core dumps (#229)

* Include debug symbols for golang and dqlite builds

We need the debug symbols to analyze core dumps or live processes
using debuggers such as gdb or delve.

The size increase is reasonable, so we'll include them by default.

* Get detailed inspection reports

* Collect core dumps from perf tests

We're experiencing some dqlite crashes. The collected core dumps
should allow us to pinpoint the issue.

* perf test: upload the snaps in case of failure

We'll upload the snaps in case of perf test failures so that we
can further analyse the snap or potential core dumps.

.github/workflows/performance.yaml

@@ -124,3 +124,15 @@ jobs:
         with:
           name: ${{ env.artifact_name }}
           path: ${{ github.workspace }}/inspection-reports.tar.gz
+      - name: Upload base-code.snap
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: base
+          path: ${{ github.workspace }}/base-code.snap
+      - name: Upload head.snap
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: base
+          path: ${{ github.workspace }}/head.snap

hack/dynamic-dqlite.sh

@@ -92,7 +92,7 @@ if [ ! -f "${BUILD_DIR}/dqlite/libdqlite.la" ]; then
     cd dqlite
     autoreconf -i > /dev/null
     ./configure --enable-build-raft \
-      CFLAGS="-I${BUILD_DIR}/sqlite -I${BUILD_DIR}/libuv/include -I${BUILD_DIR}/lz4/lib -Werror=implicit-function-declaration" \
+      CFLAGS="-g -I${BUILD_DIR}/sqlite -I${BUILD_DIR}/libuv/include -I${BUILD_DIR}/lz4/lib -Werror=implicit-function-declaration" \
       LDFLAGS=" -L${BUILD_DIR}/libuv/.libs -L${BUILD_DIR}/lz4/lib -L${BUILD_DIR}/libnsl/src" \
       UV_CFLAGS="-I${BUILD_DIR}/libuv/include" \
       UV_LIBS="-L${BUILD_DIR}/libuv/.libs" \

hack/dynamic-go-build.sh

@@ -6,5 +6,5 @@ DIR="$(realpath `dirname "${0}"`)"
 
 go build \
   -tags libsqlite3 \
-  -ldflags '-s -w -extldflags "-Wl,-rpath,$ORIGIN/lib -Wl,-rpath,$ORIGIN/../lib"' \
+  -ldflags '-extldflags "-Wl,-rpath,$ORIGIN/lib -Wl,-rpath,$ORIGIN/../lib"' \
   "${@}"

hack/static-dqlite.sh

@@ -123,7 +123,7 @@ if [ ! -f "${BUILD_DIR}/dqlite/libdqlite.la" ]; then
     cd dqlite
     autoreconf -i > /dev/null
     ./configure --disable-shared --enable-build-raft \
-      CFLAGS="${CFLAGS} -I${BUILD_DIR}/sqlite -I${BUILD_DIR}/libuv/include -I${BUILD_DIR}/lz4/lib -I${INSTALL_DIR}/musl/include -Werror=implicit-function-declaration" \
+      CFLAGS="-g ${CFLAGS} -I${BUILD_DIR}/sqlite -I${BUILD_DIR}/libuv/include -I${BUILD_DIR}/lz4/lib -I${INSTALL_DIR}/musl/include -Werror=implicit-function-declaration" \
       LDFLAGS="${LDFLAGS} -L${BUILD_DIR}/libuv/.libs -L${BUILD_DIR}/lz4/lib -L${BUILD_DIR}/libnsl/src" \
       UV_CFLAGS="-I${BUILD_DIR}/libuv/include" \
       UV_LIBS="-L${BUILD_DIR}/libuv/.libs" \

hack/static-go-build.sh

@@ -6,5 +6,5 @@ DIR="$(realpath `dirname "${0}"`)"
 
 go build \
   -tags libsqlite3 \
-  -ldflags '-s -w -linkmode "external" -extldflags "-static"' \
+  -ldflags '-linkmode "external" -extldflags "-static"' \
   "${@}"

test/performance/tests/conftest.py

@@ -31,7 +31,15 @@ def _generate_inspection_report(h: harness.Harness, instance_id: str):
     inspection_path = Path(config.INSPECTION_REPORTS_DIR)
     result = h.exec(
         instance_id,
-        ["/snap/k8s/current/k8s/scripts/inspect.sh", "/inspection-report.tar.gz"],
+        [
+            "/snap/k8s/current/k8s/scripts/inspect.sh",
+            "--all-namespaces",
+            "--num-snap-log-entries",
+            "1000000",
+            "--core-dump-dir",
+            config.CORE_DUMP_DIR,
+            "/inspection-report.tar.gz",
+        ],
         capture_output=True,
         text=True,
         check=False,
@@ -159,6 +167,7 @@ def instances(
         instance = h.new_instance(network_type=network_type)
         instances.append(instance)
         if not no_setup:
+            util.setup_core_dumps(instance)
             util.setup_k8s_snap(instance, tmp_path, snap)
 
     if not disable_k8s_bootstrapping and not no_setup:

test/performance/tests/test_util/config.py

@@ -24,6 +24,10 @@
 # after the tests complete.
 SKIP_CLEANUP = (os.getenv("TEST_SKIP_CLEANUP") or "") == "1"
 
+# Note that when using containers, this will override the host configuration.
+CORE_DUMP_PATTERN = (os.getenv("TEST_CORE_DUMP_PATTERN")) or r"core-%e.%p.%h"
+CORE_DUMP_DIR = (os.getenv("TEST_CORE_DUMP_DIR")) or "/var/crash"
+
 # SNAP is the path to the snap under test.
 SNAP = os.getenv("TEST_SNAP") or ""
 

test/performance/tests/test_util/util.py

@@ -4,6 +4,7 @@
 import ipaddress
 import json
 import logging
+import os
 import re
 import shlex
 import subprocess
@@ -146,6 +147,14 @@ def _as_int(value: Optional[str]) -> Optional[int]:
         return None
 
 
+def setup_core_dumps(instance: harness.Instance):
+    core_pattern = os.path.join(config.CORE_DUMP_DIR, config.CORE_DUMP_PATTERN)
+    LOG.info("Configuring core dumps. Pattern: %s", core_pattern)
+    instance.exec(["echo", core_pattern, ">", "/proc/sys/kernel/core_pattern"])
+    instance.exec(["echo", "1", ">", "/proc/sys/fs/suid_dumpable"])
+    instance.exec(["snap", "set", "system", "system.coredump.enable=true"])
+
+
 def configure_dqlite_logging(instance: harness.Instance):
     """Configure k8s-dqlite logging (requires restart)."""
     if config.DQLITE_TRACE_LEVEL: