ISSUE-1083: panic handler for router crashes (skupperproject#1084)

Very basic stack unwind + register dump to stderr on crash

.github/workflows/build.yaml

@@ -846,7 +846,7 @@ jobs:
           dnf config-manager --set-enabled powertools
           dnf install --setopt=tsflags=nodocs --setopt=install_weak_deps=False -y epel-release 'dnf-command(copr)' 'dnf-command(builddep)'
           dnf copr enable -y clime/rpkg-util
-          dnf install --setopt=tsflags=nodocs --setopt=install_weak_deps=False -y git rpkg
+          dnf install --setopt=tsflags=nodocs --setopt=install_weak_deps=False -y git rpkg libunwind-devel
 
       - uses: actions/checkout@v3
 

CMakeLists.txt

@@ -154,6 +154,7 @@ find_package(LibWebSockets 4.0.1 REQUIRED)
 ##
 
 find_library(dw_lib dw DOC "libdw used to symbolize QD_MEMORY_DEBUG backtraces")
+find_package(libunwind)
 
 # google benchmark tests are disabled by default
 OPTION(BUILD_BENCHMARKS "Enable building and running benchmarks with Google Benchmark" OFF)

Containerfile

@@ -19,12 +19,13 @@
 
 FROM registry.access.redhat.com/ubi9/ubi-minimal:latest as builder
 
-RUN microdnf -y --setopt=install_weak_deps=0 --setopt=tsflags=nodocs install \
+RUN rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm \
+ && microdnf -y --setopt=install_weak_deps=0 --setopt=tsflags=nodocs install \
     rpm-build \
     gcc gcc-c++ make cmake \
     cyrus-sasl-devel openssl-devel libuuid-devel \
     python3-devel python3-pip \
-    libnghttp2-devel \
+    libnghttp2-devel libunwind-devel \
     wget tar patch findutils git libasan libubsan libtsan \
  && microdnf clean all -y
 
@@ -43,11 +44,12 @@ RUN tar zxpf /qpid-proton-image.tar.gz --one-top-level=/image && tar zxpf /skupp
 FROM registry.access.redhat.com/ubi9/ubi-minimal:latest
 
 # gdb and sanitizers are part of final image as they can be used as debug options for Skupper
-RUN microdnf -y --setopt=install_weak_deps=0 --setopt=tsflags=nodocs install \
+RUN rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm \
+ && microdnf -y --setopt=install_weak_deps=0 --setopt=tsflags=nodocs install \
     glibc \
     cyrus-sasl-lib cyrus-sasl-plain cyrus-sasl-gssapi openssl \
     python3 \
-    libnghttp2 \
+    libnghttp2 libunwind \
     gdb libasan libubsan libtsan \
     gettext hostname iputils \
     shadow-utils \

cmake/Findlibunwind.cmake

@@ -0,0 +1,31 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Sets LIBUNWIND_LIBRARY to libunwind if it exists
+# Sets LIBUNWIND_INCLUDE_DIRS to directory containing libunwind.h header
+
+find_library(LIBUNWIND_LIBRARY unwind DOC "libunwind is used to dump the stack on crash")
+
+find_path(LIBUNWIND_INCLUDE_DIRS libunwind.h
+  HINTS "${CMAKE_INSTALL_PREFIX}/include"
+  PATHS "/usr/include")
+
+if (NOT (LIBUNWIND_INCLUDE_DIRS AND LIBUNWIND_LIBRARY))
+  message(STATUS "libunwind library not found: stack dump on crash disabled")
+endif()

include/qpid/dispatch/threading.h

@@ -65,5 +65,6 @@ sys_thread_t *sys_thread(const char *thread_name, void *(*run_function) (void *)
 void          sys_thread_free(sys_thread_t *thread);
 void          sys_thread_join(sys_thread_t *thread);
 sys_thread_t *sys_thread_self(void);
+const char   *sys_thread_name(sys_thread_t *thread);  // use self if thread == 0
 
 #endif

packaging/skupper-router.spec.rpkg

@@ -42,6 +42,7 @@
 %global proton_minimum_version 0.34.0
 %global libwebsockets_minimum_version 3.0.1
 %global libnghttp2_minimum_version 1.33.0
+%global libunwind_minimum_version 1.3.1
 
 Name:          skupper-router
 Version:       {{{ git_dir_version }}}
@@ -57,6 +58,7 @@ Requires: libwebsockets >= %{libwebsockets_minimum_version}
 Requires: libnghttp2 >= %{libnghttp2_minimum_version}
 Requires: cyrus-sasl-plain
 Requires: cyrus-sasl-gssapi
+Requires: libunwind >= %{libunwind_minimum_version}
 
 BuildRequires: gcc
 BuildRequires: gcc-c++
@@ -67,6 +69,7 @@ BuildRequires: python3-devel
 BuildRequires: python3-setuptools
 BuildRequires: libwebsockets-devel >= %{libwebsockets_minimum_version}
 BuildRequires: libnghttp2-devel >= %{libnghttp2_minimum_version}
+BuildRequires: libunwind-devel >= %{libunwind_minimum_version}
 # man pages --help
 BuildRequires: asciidoc
 BuildRequires: python3-qpid-proton >= %{proton_minimum_version}

router/CMakeLists.txt

@@ -30,9 +30,11 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR})
 ##
 set(router_SOURCES
     src/main.c
+    src/panic.c
     )
 
 add_executable(skrouterd ${router_SOURCES})
 target_link_libraries(skrouterd skupper-router)
+target_link_options(skrouterd PUBLIC LINKER:-Map=skrouterd.map)
 
 install(TARGETS skrouterd RUNTIME DESTINATION sbin)

router/src/main.c

@@ -39,6 +39,9 @@ static qd_dispatch_t *dispatch = 0;
 static qd_log_source_t *log_source = 0;
 static const char* argv0 = 0;
 
+// Install the panic handler for fatal signals. see panic.c
+extern void panic_handler_init(void);
+
 /**
  * Configures the handler function. Specify SIG_IGN to ignore incoming signals.
  */
@@ -91,6 +94,7 @@ static void check(int fd) {
 
 static void main_process(const char *config_path, const char *python_pkgdir, bool test_hooks, int fd)
 {
+    panic_handler_init();
     dispatch = qd_dispatch(python_pkgdir, test_hooks);
     check(fd);
     log_source = qd_log_source("MAIN"); /* Logging is initialized by qd_dispatch. */