Fault injection macros and functionality (plus example) (#264)

* Initial commit of fault injection macros and functionality

Signed-off-by: Stephen Brawner <brawner@gmail.com>

* Add can_return_with_failure to rcutils_vsnprintf

Signed-off-by: Stephen Brawner <brawner@gmail.com>

* Add getter for fault injection count

Signed-off-by: Stephen Brawner <brawner@gmail.com>

* Adjust headers

Signed-off-by: Stephen Brawner <brawner@gmail.com>

* Adding more fault injection locations

Signed-off-by: Stephen Brawner <brawner@gmail.com>

* Revert header strdup

Signed-off-by: Stephen Brawner <brawner@gmail.com>

* Adjust names

Signed-off-by: Stephen Brawner <brawner@gmail.com>

* Remove init/fini macros

Signed-off-by: Stephen Brawner <brawner@gmail.com>

* uncrustify fix

Signed-off-by: Stephen Brawner <brawner@gmail.com>

* Update definitions

Signed-off-by: Stephen Brawner <brawner@gmail.com>

* Change maybe_fail int int_least64

Signed-off-by: Stephen Brawner <brawner@gmail.com>

* ATOMIC_VAR_INIT

Signed-off-by: Stephen Brawner <brawner@gmail.com>

* Move get/set to public functions

Signed-off-by: Stephen Brawner <brawner@gmail.com>

* Add RCUTILS_CAN_FAIL_WITH

Signed-off-by: Stephen Brawner <brawner@gmail.com>

* Updating comments

Signed-off-by: Stephen Brawner <brawner@gmail.com>

CMakeLists.txt

@@ -69,6 +69,7 @@ set(rcutils_sources
   src/strerror.c
   src/string_array.c
   src/string_map.c
+  src/testing/fault_injection.c
   src/time.c
   ${time_impl_c}
   src/uint8_array.c
@@ -123,6 +124,10 @@ target_include_directories(${PROJECT_NAME} PUBLIC
 # which is appropriate when building the dll but not consuming it.
 target_compile_definitions(${PROJECT_NAME} PRIVATE "RCUTILS_BUILDING_DLL")
 
+if(BUILD_TESTING AND NOT RCUTILS_DISABLE_FAULT_INJECTION)
+  target_compile_definitions(${PROJECT_NAME} PUBLIC RCUTILS_ENABLE_FAULT_INJECTION)
+endif()
+
 target_link_libraries(${PROJECT_NAME} ${CMAKE_DL_LIBS})
 
 # Needed if pthread is used for thread local storage.

Doxyfile

@@ -24,6 +24,7 @@ EXPAND_ONLY_PREDEF     = YES
 PREDEFINED             += RCUTILS_PUBLIC=
 PREDEFINED             += RCUTILS_PUBLIC_TYPE=
 PREDEFINED             += RCUTILS_WARN_UNUSED=
+PREDEFINED             += RCUTILS_ENABLE_FAULT_INJECTION=
 
 # Tag files that do not exist will produce a warning and cross-project linking will not work.
 TAGFILES += "../../../doxygen_tag_files/cppreference-doxygen-web.tag.xml=http://en.cppreference.com/w/"

include/rcutils/error_handling.h

@@ -36,6 +36,7 @@ extern "C"
 #include "rcutils/allocator.h"
 #include "rcutils/macros.h"
 #include "rcutils/snprintf.h"
+#include "rcutils/testing/fault_injection.h"
 #include "rcutils/types/rcutils_ret.h"
 #include "rcutils/visibility_control.h"
 

include/rcutils/macros.h

@@ -134,6 +134,75 @@ extern "C"
 # define RCUTILS_UNLIKELY(x) (x)
 #endif  // _WIN32
 
+#if defined RCUTILS_ENABLE_FAULT_INJECTION
+#include "rcutils/testing/fault_injection.h"
+
+/**
+ * \def RCUTILS_CAN_RETURN_WITH_ERROR_OF
+ * Indicating macro that the function intends to return possible error value.
+ *
+ * Put this macro as the first line in the function. For example:
+ *
+ * int rcutils_function_that_can_fail() {
+ *   RCUTILS_CAN_RETURN_WITH_ERROR_OF(RCUTILS_RET_INVALID_ARGUMENT);
+ *   ...  // rest of function
+ * }
+ *
+ * For now, this macro just simply calls `RCUTILS_FAULT_INJECTION_MAYBE_RETURN_ERROR` if fault
+ * injection is enabled. However, for source code, the macro annotation
+ * `RCUTILS_CAN_RETURN_WITH_ERROR_OF` helps clarify that a function may return a value signifying
+ * an error and what those are.
+ *
+ * In general, you should only include a return value that originates in the function you're
+ * annotating instead of one that is merely passed on from a called function already annotated with
+ *`RCUTILS_CAN_RETURN_WITH_ERROR_OF`. If you are passing on return values from a called function,
+ * but that function is not annotated with `RCUTILS_CAN_RETURN_WITH_ERROR_OF`, then you might
+ * consider annotating that function first. If for some reason that is not desired or possible,
+ * then annotate your function as if the return values you are passing on originated from your
+ * function.
+ *
+ * If the function can return multiple return values indicating separate failure types, each one
+ * should go on a separate line.
+ *
+ * If in your function, there are expected effects on output parameters that occur during
+ * the failure case, then it will introduce a discrepancy between fault injection testing and
+ * production operation. This is because the fault injection will cause the function to return
+ * where this macro is used, not at the location the error values are typically returned. To help
+ * protect against this scenario you may consider adding unit tests that check your function does
+ * not modify output parameters when it actually returns a failing error code if it's possible for
+ * your code.
+ *
+ * If your function is void, this macro can be used without parameters. However, for the above
+ * reasoning, there should be no side effects on output parameters for all possible early returns.
+ *
+ * \param error_return_value the value returned as a result of an error. It does not need to be
+ * a rcutils_ret_t type. It could also be NULL, -1, a string error message, etc
+ */
+# define RCUTILS_CAN_RETURN_WITH_ERROR_OF(error_return_value) \
+  RCUTILS_FAULT_INJECTION_MAYBE_RETURN_ERROR(error_return_value);
+
+/**
+ * \def RCUTILS_CAN_FAIL_WITH
+ * Indicating macro similar to RCUTILS_CAN_RETURN_WITH_ERROR_OF but for use with more complicated
+ * statements.
+ *
+ * The `failure_code` will be executed inside a scoped if block, so any variables declared within
+ * will not be available outside of the macro.
+ *
+ * One example where you might need this version, is if a side-effect may occur within a function.
+ * For example, in snprintf, rcutils_snprintf needs to set both errno and return -1 on failure.
+ * This macro is used to capture both effects.
+ *
+ * \param failure_code Code that is representative of the failure case in this function.
+ */
+# define RCUTILS_CAN_FAIL_WITH(failure_code) \
+  RCUTILS_FAULT_INJECTION_MAYBE_FAIL(failure_code);
+
+#else
+# define RCUTILS_CAN_RETURN_WITH_ERROR_OF(error_return_value)
+# define RCUTILS_CAN_FAIL_WITH(failure_code)
+#endif  // defined RCUTILS_ENABLE_FAULT_INJECTION
+
 #ifdef __cplusplus
 }
 #endif

include/rcutils/testing/fault_injection.h

@@ -0,0 +1,174 @@
+// Copyright 2020 Open Source Robotics Foundation, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef RCUTILS__TESTING__FAULT_INJECTION_H_
+#define RCUTILS__TESTING__FAULT_INJECTION_H_
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdint.h>
+
+#include "rcutils/macros.h"
+#include "rcutils/visibility_control.h"
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+#define RCUTILS_FAULT_INJECTION_NEVER_FAIL -1
+
+#define RCUTILS_FAULT_INJECTION_FAIL_NOW 0
+
+RCUTILS_PUBLIC
+RCUTILS_WARN_UNUSED
+bool
+rcutils_fault_injection_is_test_complete(void);
+
+/**
+ * \brief Atomically set the fault injection counter.
+ *
+ * This is typically not the preferred method of interacting directly with the fault injection
+ * logic, instead use `RCUTILS_FAULT_INJECTION_TEST` instead.
+ *
+ * This function may also be used for pausing code inside of a `RCUTILS_FAULT_INJECTION_TEST` with
+ * something like the following:
+ *
+ * RCUTILS_FAULT_INJECTION_TEST({
+ *     ...  // code to run with fault injection
+ *     int64_t count = rcutils_fault_injection_get_count();
+ *     rcutils_fault_injection_set_count(RCUTILS_FAULT_INJECTION_NEVER_FAIL);
+ *     ...  // code to run without fault injection
+ *     rcutils_fault_injection_set_count(count);
+ *     ...  // code to run with fault injection
+ * });
+ *
+ * \param count The count to set the fault injection counter to. If count is negative, then fault
+ * injection errors will be disabled. The counter is globally initialized to
+ * RCUTILS_FAULT_INJECTION_NEVER_FAIL.
+ */
+RCUTILS_PUBLIC
+void
+rcutils_fault_injection_set_count(int count);
+
+/**
+ * \brief Atomically get the fault injection counter value
+ *
+ * This function is typically not used directly but instead indirectly inside an
+ * `RCUTILS_FAULT_INJECTION_TEST`
+ */
+RCUTILS_PUBLIC
+RCUTILS_WARN_UNUSED
+int_least64_t
+rcutils_fault_injection_get_count(void);
+
+/**
+ * \brief Implementation of fault injection decrementer
+ *
+ * This is included inside of macros, so it needs to be exported as a public function, but it
+ * should not be used directly.
+ */
+RCUTILS_PUBLIC
+RCUTILS_WARN_UNUSED
+int_least64_t
+_rcutils_fault_injection_maybe_fail(void);
+
+/**
+ * \def RCUTILS_FAULT_INJECTION_MAYBE_RETURN_ERROR
+ * \brief This macro checks and decrements a static global variable atomic counter and returns
+ * `return_value_on_error` if 0.
+ *
+ * This macro is not a function itself, so when this macro returns it will cause the calling
+ * function to return with the return value.
+ *
+ * Set the counter with `RCUTILS_FAULT_INJECTION_SET_COUNT`. If the count is less than 0, then
+ * `RCUTILS_FAULT_INJECTION_MAYBE_RETURN_ERROR` will not cause an early return.
+ *
+ * This macro is thread-safe, and ensures that at most one invocation results in a failure for each
+ * time the fault injection counter is set with `RCUTILS_FAULT_INJECTION_SET_COUNT`
+ *
+ * \param return_value_on_error the value to return in the case of fault injected failure.
+ */
+#define RCUTILS_FAULT_INJECTION_MAYBE_RETURN_ERROR(return_value_on_error) \
+  if (RCUTILS_FAULT_INJECTION_FAIL_NOW == _rcutils_fault_injection_maybe_fail()) { \
+    printf( \
+      "%s:%d Injecting fault and returning " #return_value_on_error "\n", __FILE__, __LINE__); \
+    return return_value_on_error; \
+  }
+
+/**
+ * \def RCUTILS_FAULT_INJECTION_MAYBE_FAIL
+ * \brief This macro checks and decrements a static global variable atomic counter and executes
+ * `failure_code` if the counter is 0 inside a scoped block (any variables declared in
+ * failure_code) will not be avaliable outside of this scoped block.
+ *
+ * This macro is not a function itself, so it will cause the calling function to execute the code
+ * from within an if loop.
+ *
+ * Set the counter with `RCUTILS_FAULT_INJECTION_SET_COUNT`. If the count is less than 0, then
+ * `RCUTILS_FAULT_INJECTION_MAYBE_FAIL` will not execute the failure code.
+ *
+ * This macro is thread-safe, and ensures that at most one invocation results in a failure for each
+ * time the fault injection counter is set with `RCUTILS_FAULT_INJECTION_SET_COUNT`
+ *
+ * \param failure_code the code to execute in the case of fault injected failure.
+ */
+#define RCUTILS_FAULT_INJECTION_MAYBE_FAIL(failure_code) \
+  if (RCUTILS_FAULT_INJECTION_FAIL_NOW == _rcutils_fault_injection_maybe_fail()) { \
+    printf( \
+      "%s:%d Injecting fault and executing " #failure_code "\n", __FILE__, __LINE__); \
+    failure_code; \
+  }
+
+/**
+ * \def RCUTILS_FAULT_INJECTION_TEST
+ *
+ * The fault injection macro for use with unit tests to check that `code` can tolerate injected
+ * failures at all points along the execution path where the indicating macros
+ * `RCUTILS_CAN_RETURN_WITH_ERROR_OF` and `RCUTILS_CAN_FAIL_WITH` are located.
+ *
+ * This macro is intended to be used within a gtest function macro like 'TEST', 'TEST_F', etc.
+ *
+ * `code` is executed within a do-while loop and therefore any variables declared within are in
+ * their own scope block.
+ *
+ * Here's a simple example:
+ *  RCUTILS_FAULT_INJECTION_TEST(
+ *    rcl_ret_t ret = rcl_init(argc, argv, options, context);
+ *    if (RCL_RET_OK == ret)
+ *    {
+ *        ret = rcl_shutdown(context);
+ *    }
+ * });
+ *
+ * In this example, you will need have conditional execution based on the return value of
+ * `rcl_init`. If it failed, then it wouldn't make sense to call rcl_shutdown. In your own test,
+ * there might be similar logic that requires conditional checks. The goal of writing this test
+ * is less about checking the behavior is consistent, but instead that failures do not cause
+ * program crashes, memory errors, or unnecessary memory leaks.
+ */
+#define RCUTILS_FAULT_INJECTION_TEST(code) \
+  do { \
+    int fault_injection_count = 0; \
+    do { \
+      rcutils_fault_injection_set_count(fault_injection_count++); \
+      code; \
+    } while (!rcutils_fault_injection_is_test_complete()); \
+    rcutils_fault_injection_set_count(RCUTILS_FAULT_INJECTION_NEVER_FAIL); \
+  } while (0)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // RCUTILS__TESTING__FAULT_INJECTION_H_

src/shared_library.c

@@ -27,6 +27,7 @@ C_ASSERT(sizeof(void *) == sizeof(HINSTANCE));
 #endif  // _WIN32
 
 #include "rcutils/error_handling.h"
+#include "rcutils/macros.h"
 #include "rcutils/shared_library.h"
 #include "rcutils/strdup.h"
 
@@ -46,6 +47,10 @@ rcutils_load_shared_library(
   const char * library_path,
   rcutils_allocator_t allocator)
 {
+  RCUTILS_CAN_RETURN_WITH_ERROR_OF(RCUTILS_RET_INVALID_ARGUMENT);
+  RCUTILS_CAN_RETURN_WITH_ERROR_OF(RCUTILS_RET_BAD_ALLOC);
+  RCUTILS_CAN_RETURN_WITH_ERROR_OF(RCUTILS_RET_ERROR);
+
   RCUTILS_CHECK_ARGUMENT_FOR_NULL(lib, RCUTILS_RET_INVALID_ARGUMENT);
   RCUTILS_CHECK_ARGUMENT_FOR_NULL(library_path, RCUTILS_RET_INVALID_ARGUMENT);
   RCUTILS_CHECK_ALLOCATOR(&allocator, return RCUTILS_RET_INVALID_ARGUMENT);

src/snprintf.c

@@ -39,6 +39,8 @@ rcutils_snprintf(char * buffer, size_t buffer_size, const char * format, ...)
 int
 rcutils_vsnprintf(char * buffer, size_t buffer_size, const char * format, va_list args)
 {
+  RCUTILS_CAN_FAIL_WITH({errno = EINVAL; return -1;});
+
   if (NULL == format) {
     errno = EINVAL;
     return -1;

src/strdup.c

@@ -23,10 +23,14 @@ extern "C"
 #include <string.h>
 
 #include "./common.h"
+#include "rcutils/macros.h"
+
 
 char *
 rcutils_strdup(const char * str, rcutils_allocator_t allocator)
 {
+  RCUTILS_CAN_RETURN_WITH_ERROR_OF(NULL);
+
   if (NULL == str) {
     return NULL;
   }
@@ -36,6 +40,8 @@ rcutils_strdup(const char * str, rcutils_allocator_t allocator)
 char *
 rcutils_strndup(const char * str, size_t string_length, rcutils_allocator_t allocator)
 {
+  RCUTILS_CAN_RETURN_WITH_ERROR_OF(NULL);
+
   if (NULL == str) {
     return NULL;
   }

src/string_array.c

@@ -42,6 +42,9 @@ rcutils_string_array_init(
   size_t size,
   const rcutils_allocator_t * allocator)
 {
+  RCUTILS_CAN_RETURN_WITH_ERROR_OF(RCUTILS_RET_INVALID_ARGUMENT);
+  RCUTILS_CAN_RETURN_WITH_ERROR_OF(RCUTILS_RET_BAD_ALLOC);
+
   if (NULL == allocator) {
     RCUTILS_SET_ERROR_MSG("allocator is null");
     return RCUTILS_RET_INVALID_ARGUMENT;
@@ -63,6 +66,8 @@ rcutils_string_array_init(
 rcutils_ret_t
 rcutils_string_array_fini(rcutils_string_array_t * string_array)
 {
+  RCUTILS_CAN_RETURN_WITH_ERROR_OF(RCUTILS_RET_INVALID_ARGUMENT);
+
   if (NULL == string_array) {
     RCUTILS_SET_ERROR_MSG("string_array is null");
     return RCUTILS_RET_INVALID_ARGUMENT;