Implement crash-tracking handler (#282)

build-profiling-ffi.sh

@@ -4,6 +4,11 @@
 # under the Apache License Version 2.0. This product includes software developed
 # at Datadog (https://www.datadoghq.com/). Copyright 2021-Present Datadog, Inc.
 
+get_abs_filename() {
+  # $1 : relative filename
+  echo "$(cd "$(dirname "$1")" && pwd)/$(basename "$1")"
+}
+
 # Location to place all artifacts
 if [ -z $CARGO_TARGET_DIR ] ; then
     export CARGO_TARGET_DIR=$PWD/target
@@ -162,4 +167,21 @@ cbindgen --crate "${datadog_profiling_ffi}" \
     --output "$destdir/include/datadog/profiling.h"
 "$CARGO_TARGET_DIR"/debug/dedup_headers "$destdir/include/datadog/common.h" "$destdir/include/datadog/profiling.h"
 
+# Don't build the crashtracker on windows
+if [[ "$target" != "x86_64-pc-windows-msvc" ]]; then
+    echo "Building binaries"
+    # $destdir might be relative. Get an absolute path that will work when we cd
+    export ABS_DESTDIR=$(get_abs_filename $destdir)
+    export CRASHTRACKER_BUILD_DIR=$CARGO_TARGET_DIR/build/crashtracker-receiver
+    export CRASHTRACKER_SRC_DIR=$PWD/profiling-crashtracking-receiver
+    # Always start with a clean directory
+    [ -d $CRASHTRACKER_BUILD_DIR ] && rm -r $CRASHTRACKER_BUILD_DIR
+    mkdir -p $CRASHTRACKER_BUILD_DIR
+    cd $CRASHTRACKER_BUILD_DIR
+    cmake -S $CRASHTRACKER_SRC_DIR -DDatadog_ROOT=$ABS_DESTDIR
+    cmake --build .
+    mkdir -p $ABS_DESTDIR/bin
+    cp libdatadog-crashtracking-receiver $ABS_DESTDIR/bin
+fi
+
 echo "Done."

profiling-crashtracking-receiver/CMakeLists.txt

@@ -0,0 +1,7 @@
+cmake_minimum_required(VERSION 3.19)
+project(datadog_profiling_crashtracking_reciever LANGUAGES C CXX)
+
+find_package(Datadog REQUIRED)
+
+add_executable(libdatadog-crashtracking-receiver libdatadog-crashtracking-receiver.c)
+target_link_libraries(libdatadog-crashtracking-receiver PRIVATE Datadog::Profiling)

profiling-crashtracking-receiver/libdatadog-crashtracking-receiver.c

@@ -0,0 +1,18 @@
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2024-Present Datadog, Inc.
+
+#include <datadog/common.h>
+#include <datadog/profiling.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+int main(void) {
+  ddog_prof_Profile_Result new_result = ddog_prof_crashtracker_receiver_entry_point();
+  if (new_result.tag != DDOG_PROF_PROFILE_NEW_RESULT_OK) {
+    ddog_CharSlice message = ddog_Error_message(&new_result.err);
+    fprintf(stderr, "%*s", (int)message.len, message.ptr);
+    ddog_Error_drop(&new_result.err);
+    exit(EXIT_FAILURE);
+  }
+  return 0;
+}

profiling-ffi/src/crashtracker.rs

@@ -0,0 +1,178 @@
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present Datadog, Inc.
+#![cfg(unix)]
+
+use crate::exporter::{self, Endpoint};
+use crate::profiles::ProfileResult;
+use datadog_profiling::crashtracker;
+use ddcommon::tag::Tag;
+use ddcommon_ffi::slice::{AsBytes, CharSlice};
+use ddcommon_ffi::Error;
+use std::ops::Not;
+
+pub use datadog_profiling::crashtracker::{CrashtrackerResolveFrames, ProfilingOpTypes};
+
+#[repr(C)]
+pub struct CrashtrackerConfiguration<'a> {
+    pub create_alt_stack: bool,
+    /// The endpoint to send the crash repor to (can be a file://)
+    pub endpoint: Endpoint<'a>,
+    /// Optional filename to forward stderr to (useful for logging/debugging)
+    pub optional_stderr_filename: CharSlice<'a>,
+    /// Optional filename to forward stdout to (useful for logging/debugging)
+    pub optional_stdout_filename: CharSlice<'a>,
+    pub path_to_receiver_binary: CharSlice<'a>,
+    /// Whether/when we should attempt to resolve frames
+    pub resolve_frames: CrashtrackerResolveFrames,
+}
+
+impl<'a> TryFrom<CrashtrackerConfiguration<'a>> for crashtracker::CrashtrackerConfiguration {
+    type Error = anyhow::Error;
+    fn try_from(value: CrashtrackerConfiguration<'a>) -> anyhow::Result<Self> {
+        fn option_from_char_slice(s: CharSlice) -> anyhow::Result<Option<String>> {
+            let s = unsafe { s.try_to_utf8()?.to_string() };
+            Ok(s.is_empty().not().then_some(s))
+        }
+
+        let create_alt_stack = value.create_alt_stack;
+        let endpoint = unsafe { Some(exporter::try_to_endpoint(value.endpoint)?) };
+        let path_to_receiver_binary =
+            unsafe { value.path_to_receiver_binary.try_to_utf8()?.to_string() };
+        let resolve_frames = value.resolve_frames;
+        let stderr_filename = option_from_char_slice(value.optional_stderr_filename)?;
+        let stdout_filename = option_from_char_slice(value.optional_stdout_filename)?;
+
+        crashtracker::CrashtrackerConfiguration::new(
+            create_alt_stack,
+            endpoint,
+            path_to_receiver_binary,
+            resolve_frames,
+            stderr_filename,
+            stdout_filename,
+        )
+    }
+}
+
+#[repr(C)]
+pub struct CrashtrackerMetadata<'a> {
+    pub profiling_library_name: CharSlice<'a>,
+    pub profiling_library_version: CharSlice<'a>,
+    pub family: CharSlice<'a>,
+    /// Should include "service", "environment", etc
+    pub tags: Option<&'a ddcommon_ffi::Vec<Tag>>,
+}
+
+impl<'a> TryFrom<CrashtrackerMetadata<'a>> for crashtracker::CrashtrackerMetadata {
+    type Error = anyhow::Error;
+    fn try_from(value: CrashtrackerMetadata<'a>) -> anyhow::Result<Self> {
+        let profiling_library_name =
+            unsafe { value.profiling_library_name.try_to_utf8()?.to_string() };
+        let profiling_library_version =
+            unsafe { value.profiling_library_version.try_to_utf8()?.to_string() };
+        let family = unsafe { value.family.try_to_utf8()?.to_string() };
+        let tags = value
+            .tags
+            .map(|tags| tags.iter().cloned().collect())
+            .unwrap_or_default();
+        Ok(crashtracker::CrashtrackerMetadata::new(
+            profiling_library_name,
+            profiling_library_version,
+            family,
+            tags,
+        ))
+    }
+}
+
+#[no_mangle]
+#[must_use]
+pub unsafe extern "C" fn ddog_prof_crashtracker_begin_profiling_op(
+    op: ProfilingOpTypes,
+) -> ProfileResult {
+    match crashtracker::begin_profiling_op(op) {
+        Ok(_) => ProfileResult::Ok(true),
+        Err(err) => ProfileResult::Err(Error::from(
+            err.context("ddog_prof_crashtracker_init failed"),
+        )),
+    }
+}
+
+#[no_mangle]
+#[must_use]
+pub unsafe extern "C" fn ddog_prof_crashtracker_end_profiling_op(
+    op: ProfilingOpTypes,
+) -> ProfileResult {
+    match crashtracker::end_profiling_op(op) {
+        Ok(_) => ProfileResult::Ok(true),
+        Err(err) => ProfileResult::Err(Error::from(
+            err.context("ddog_prof_crashtracker_init failed"),
+        )),
+    }
+}
+
+#[no_mangle]
+#[must_use]
+pub unsafe extern "C" fn ddog_prof_crashtracker_shutdown() -> ProfileResult {
+    match crashtracker::shutdown_crash_handler() {
+        Ok(_) => ProfileResult::Ok(true),
+        Err(err) => ProfileResult::Err(Error::from(
+            err.context("ddog_prof_crashtracker_init failed"),
+        )),
+    }
+}
+
+#[no_mangle]
+#[must_use]
+pub unsafe extern "C" fn ddog_prof_crashtracker_update_on_fork(
+    config: CrashtrackerConfiguration,
+    metadata: CrashtrackerMetadata,
+) -> ProfileResult {
+    match ddog_prof_crashtracker_update_on_fork_impl(config, metadata) {
+        Ok(_) => ProfileResult::Ok(true),
+        Err(err) => ProfileResult::Err(Error::from(
+            err.context("ddog_prof_crashtracker_init failed"),
+        )),
+    }
+}
+
+unsafe fn ddog_prof_crashtracker_update_on_fork_impl(
+    config: CrashtrackerConfiguration,
+    metadata: CrashtrackerMetadata,
+) -> anyhow::Result<()> {
+    let config = config.try_into()?;
+    let metadata = metadata.try_into()?;
+    crashtracker::on_fork(config, metadata)
+}
+
+#[no_mangle]
+#[must_use]
+pub unsafe extern "C" fn ddog_prof_crashtracker_receiver_entry_point() -> ProfileResult {
+    match crashtracker::receiver_entry_point() {
+        Ok(_) => ProfileResult::Ok(true),
+        Err(err) => ProfileResult::Err(Error::from(
+            err.context("ddog_prof_crashtracker_receiver_entry_point failed"),
+        )),
+    }
+}
+
+#[no_mangle]
+#[must_use]
+pub unsafe extern "C" fn ddog_prof_crashtracker_init(
+    config: CrashtrackerConfiguration,
+    metadata: CrashtrackerMetadata,
+) -> ProfileResult {
+    match ddog_prof_crashtracker_init_impl(config, metadata) {
+        Ok(_) => ProfileResult::Ok(true),
+        Err(err) => ProfileResult::Err(Error::from(
+            err.context("ddog_prof_crashtracker_init failed"),
+        )),
+    }
+}
+
+unsafe fn ddog_prof_crashtracker_init_impl(
+    config: CrashtrackerConfiguration,
+    metadata: CrashtrackerMetadata,
+) -> anyhow::Result<()> {
+    let config = config.try_into()?;
+    let metadata = metadata.try_into()?;
+    crashtracker::init(config, metadata)
+}

profiling-ffi/src/exporter.rs

@@ -92,7 +92,7 @@ unsafe fn try_to_url(slice: CharSlice) -> anyhow::Result<hyper::Uri> {
     Ok(hyper::Uri::from_str(str)?)
 }
 
-unsafe fn try_to_endpoint(endpoint: Endpoint) -> anyhow::Result<exporter::Endpoint> {
+pub unsafe fn try_to_endpoint(endpoint: Endpoint) -> anyhow::Result<exporter::Endpoint> {
     // convert to utf8 losslessly -- URLs and API keys should all be ASCII, so
     // a failed result is likely to be an error.
     match endpoint {

profiling-ffi/src/lib.rs

@@ -6,6 +6,7 @@ use std::time::SystemTime;
 
 use chrono::{DateTime, TimeZone, Utc};
 
+mod crashtracker;
 mod exporter;
 mod profiles;
 

profiling/Cargo.toml

@@ -13,14 +13,16 @@ crate-type = ["lib"]
 
 [dependencies]
 anyhow = "1.0"
+backtrace = "0.3.69"
 bitmaps = "3.2.0"
 bytes = "1.1"
-chrono = {version = "0.4", default-features = false, features = ["std", "clock"]}
+chrono = {version = "0.4", default-features = false, features = ["std", "clock", "serde"]}
 ddcommon = {path = "../ddcommon"}
 derivative = "2.2.0"
 futures = { version = "0.3", default-features = false }
 futures-core = {version = "0.3.0", default-features = false}
 futures-util = {version = "0.3.0", default-features = false}
+hex = { version = "0.4.3", features = ["serde"] }
 http = "0.2"
 http-body = "0.4"
 hyper = {version = "0.14", features = ["client"], default-features = false}
@@ -30,10 +32,14 @@ libc = "0.2"
 lz4_flex = { version = "0.9", default-features = false, features = ["std", "safe-encode", "frame"] }
 mime = "0.3.16"
 mime_guess = {version = "2.0", default-features = false}
+nix = { version = "0.27.1", features = ["signal"] }
+os_info = "3.7.0"
+page_size = "0.6.0"
 percent-encoding = "2.1"
 prost = "0.11"
 rustc-hash = { version = "1.1", default-features = false }
 serde = {version = "1.0", features = ["derive"]}
 serde_json = {version = "1.0"}
 tokio = {version = "1.23", features = ["rt", "macros"]}
 tokio-util = "0.7.1"
+uuid = { version = "1.4.1", features = ["v4", "serde"] }

profiling/src/crashtracker/README.md

@@ -0,0 +1,35 @@
+# Libdatadog Crashtracker
+This module implements a crash tracker that enables libdatadog using profilers to send crash-reports to the datadog backend.
+
+It has three related parts:
+1. A `CrashInfo` struct which stores information about a crash in a structured format, and has built-in functions to upload itself to the datadog backend.
+   This struct can be generated by the crash-handlers described below (e.g. for python or Ruby), or by language runtime tools from a crash dump (e.g. .Net).
+
+2. A UNIX crash-handler, which registers signal handlers to detect crashes.
+   The legal operations within a crash-handler are limited, so the handler does as much out of process as possible.
+   When a crash occurs, this handler collects relevant information, and puts it on a pipe to the receiver, which processes it out of process.
+
+3. A binary application `libdatadog-crashtracking-receiver`, which receives the crash-report over a pipe and formats it for transmission to the backend.
+
+## How to use the crashhandler
+
+1. Initilize it using `ddog_prof_crashtracker_init`
+2. After a fork, reset the crashtracker in the child using `ddog_prof_crashtracker_update_on_fork`.
+   This can be done in an `pthread_atfork` handler.  
+2. [Optional].  The crash-tracker can be shutdown, and the previous crash handler restored, using `ddog_prof_crashtracker_shutdown`.
+    Currently, there is a state machine that stops you from then restarting the crash-tracker.
+    Fixing this is a todo
+
+## Todos
+
+- [ ] Support windows
+- [ ] API for generating crash report externally.
+- [ ] Instrumentation Telemetry support
+- [ ] Enable multiple shutdown/restart cycles for crashtracker
+- [ ] Enable dynamic update of metadata/configuration
+
+
+
+## How to configure crashtracking
+
+#