Per thread logging (twitter#168)

Problem

Rust code should be able to log to ccommon's logging facility to reduce the complexity of configuring embedded features.

Solution

This PR contains code that allows rust to create per-thread logger instances for use with the standard log crate. This gives us lock-free logging on the rust side, and allows the use of any library without needing to be concerned that that library or its dependencies will log from other threads besides the calling thread. This is a fair tradeoff between speed, safety, and ease of use.

Result

Users will configure the rust logging extension cc_log_rs with a directory and a filename prefix. The extension will create new, unique log files under that directory based on that filename prefix per thread on first use.

ci/install-check.sh

@@ -23,12 +23,28 @@ CHECK_VERSION=0.12.0
 CHECK_TARBALL="check-${CHECK_VERSION}.tar.gz"
 CHECK_DIR="check-${CHECK_VERSION}"
 
+echo "building and installing check" >&2
+
 (
   cd "$TEMP" &&
     wget "https://github.com/libcheck/check/releases/download/${CHECK_VERSION}/${CHECK_TARBALL}" &&
-    tar xvfz "${CHECK_TARBALL}" &&
+    tar xfz "${CHECK_TARBALL}" &&
     cd "${CHECK_DIR}" &&
     ./configure --prefix="$CHECK_PREFIX" &&
     make &&
     make install
-) || die "check build failed"
+) >$TEMP/cmake-build.log 2>&1
+
+RESULT=$?
+if [[ $RESULT -ne 0 ]]; then
+  cat >&2 <<EOS
+check build failed! log below:
+
+EOS
+
+  cat $TEMP/cmake-build.log
+else
+  echo "Success!" >&2
+fi
+
+exit $RESULT

ci/local-run.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+## This file is potentially useful for doing a clean environment build on MacOS ##
+## This is my personal way of running the build, YMMV - jsimms                  ##
+
+set -euo pipefail
+IFS=$'\n\t'
+
+die() { echo "fatal: $*" >&2; exit 1; }
+
+TOPLEVEL=$(git -C "$(cd "$(dirname "$0")" >/dev/null || exit 1; pwd)" rev-parse --show-toplevel) || die "TOPLEVEL fail"
+
+cd "$TOPLEVEL"
+
+TEMP="$(mktemp -d -t TEMP.XXXXXXX)" || die "failed to make tmpdir"
+cleanup() { [[ -n "${TEMP:-}" ]] && rm -rf "${TEMP}"; }
+trap cleanup EXIT
+
+BUILD_PATH=(
+  "$HOME/.cargo/bin"
+  "/usr/local/bin"
+  "/usr/local/sbin"
+  "/usr/bin"
+  "/usr/sbin"
+  "/bin"
+  "/sbin"
+  "/opt/X11/bin"
+  "/usr/X11R6/bin"
+)
+
+PATH=$(echo "${BUILD_PATH[@]}"|tr ' ' ':')
+
+cat >&2 <<EOS
+CC=${CC:-}
+CFLAGS=${CFLAGS:-}
+LDFLAGS=${LDFLAGS:-}
+PATH=${PATH:-}
+EOS
+
+CMAKEFLAGS=(
+  -DHAVE_RUST=yes
+  -DRUST_VERBOSE_BUILD=yes
+  -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON
+)
+
+BUILD_DIR="${BUILD_DIR:-$TEMP}"
+
+mkdir -p "$BUILD_DIR" && (
+  cd "$BUILD_DIR" &&
+  cmake "${CMAKEFLAGS[@]}" "$TOPLEVEL" &&
+  make -j all &&
+  make check &&
+  cd rust &&
+  cargo test
+)

include/cc_log.h

@@ -61,6 +61,10 @@ typedef struct {
 void log_setup(log_metrics_st *metrics);
 void log_teardown(void);
 
+/* these two are for testing purposes only */
+log_metrics_st *log_metrics_create(void);
+void log_metrics_destroy(log_metrics_st **p);
+
 /**
  * Create a logger. If filename is NULL, created logger writes to stderr.
  * buf_cap is the size of the buffer used for pauseless logging. specify

include/rust/cc_log_rs.h

@@ -0,0 +1,75 @@
+/* ccommon - a cache common library.
+ * Copyright (C) 2013 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <cc_log.h>
+#include <cc_bstring.h>
+
+/* NOTE: for documentation see ccommon/rust/ccommon_rs/src/log.rs */
+
+typedef enum log_level_rs {
+    LOG_LEVEL_ERROR = 1,
+    LOG_LEVEL_WARN,
+    LOG_LEVEL_INFO,
+    LOG_LEVEL_DEBUG,
+    LOG_LEVEL_TRACE,
+} log_level_rs_e;
+
+
+typedef enum log_status_rs {
+    /* Good work! */
+    LOG_STATUS_OK = 0,
+    /* An action that requires log_rs_is_setup() to be true, but it isn't
+     * i.e. you need to call log_rs_setup() before whatever you just tried to do. */
+    LOG_STATUS_NOT_SETUP_ERROR,
+    /* We could not register as the backend for the log crate .
+     * This state is unrecoverable. */
+    LOG_STATUS_REGISTRATION_FAIL,
+    /* Returned when there is already a logger set up for rust. */
+    LOG_STATUS_ALREADY_SET_ERROR,
+    /* Data was expected to be valid UTF8 but was not */
+    LOG_STATUS_INVALID_UTF8,
+    /* Failed to create a logger instance */
+    LOG_STATUS_CREATION_ERROR,
+    /* An unexpected error occurred, check stderr */
+    LOG_STATUS_OTHER_FAILURE,
+    /* You suck at programming */
+    LOG_STATUS_NULL_POINTER_ERROR,
+} log_status_rs_e;
+
+struct log_config_rs {
+    uint32_t buf_size;
+    log_level_rs_e level;
+    struct bstring path;
+    struct bstring prefix;
+};
+
+struct log_handle_rs;
+
+struct log_handle_rs* log_create_handle_rs(struct log_config_rs *cfg);
+log_status_rs_e log_shutdown_rs(struct log_handle_rs *handle, uint32_t timeout_ms);
+void log_destroy_handle_rs(struct log_handle_rs **h);
+
+bool log_is_setup_rs(struct log_handle_rs *handle);
+
+#ifdef __cplusplus
+}
+#endif

include/rust/cc_util_rs.h

@@ -0,0 +1,33 @@
+/*
+ * ccommon - a cache common library.
+ * Copyright (C) 2018 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* recursively remove all content under path then unlinks path.
+ * returns 0 on success, -1 on failure and sets errno.
+*/
+int
+cc_util_rm_rf_rs(const char *path);
+
+
+#ifdef __cplusplus
+}
+#endif