Add AddressSanitizer and valgrind support

CMakeLists.txt

@@ -52,6 +52,7 @@ include(${BLT_SOURCE_DIR}/SetupBLT.cmake)
 
 include(${PROJECT_SOURCE_DIR}/cmake/SeracMacros.cmake)
 include(${PROJECT_SOURCE_DIR}/cmake/SeracBasics.cmake)
+include(${PROJECT_SOURCE_DIR}/cmake/SeracCompilerFlags.cmake)
 include(${PROJECT_SOURCE_DIR}/cmake/thirdparty/SetupSeracThirdParty.cmake)
 include(${PROJECT_SOURCE_DIR}/cmake/SeracConfigHeader.cmake)
 

cmake/SeracBasics.cmake

@@ -4,6 +4,16 @@
 #
 # SPDX-License-Identifier: (BSD-3-Clause)
 
+#------------------------------------------------------------------------------
+# Options
+#------------------------------------------------------------------------------
+option(ENABLE_ASAN "Enable AddressSanitizer for memory checking (Clang or GCC only)" OFF)
+if(ENABLE_ASAN)
+    if(NOT (C_COMPILER_FAMILY_IS_CLANG OR C_COMPILER_FAMILY_IS_GNU))
+        message(FATAL_ERROR "ENABLE_ASAN only supports Clang and GCC")
+    endif()
+endif()
+
 #------------------------------------------------------------------------------
 # Create symlink in installed bin
 #------------------------------------------------------------------------------
@@ -13,6 +23,7 @@ if(GLVIS_EXECUTABLE)
                       -E create_symlink ${GLVIS_EXECUTABLE} ${CMAKE_INSTALL_BINDIR}/glvis)
 endif()
 
+
 #------------------------------------------------------------------------------
 # Global includes (restrict these as much as possible)
 #------------------------------------------------------------------------------

cmake/SeracCompilerFlags.cmake

@@ -0,0 +1,12 @@
+# Copyright (c) 2019-2020, Lawrence Livermore National Security, LLC and
+# other Serac Project Developers. See the top-level LICENSE file for
+# details.
+#
+# SPDX-License-Identifier: (BSD-3-Clause)
+
+if(ENABLE_ASAN)
+    message(STATUS "AddressSanitizer is ON (ENABLE_ASAN)")
+    foreach(_flagvar CMAKE_C_FLAGS CMAKE_CXX_FLAGS CMAKE_LINKER_FLAGS)
+        string(APPEND ${_flagvar} " -fsanitize=address -fno-omit-frame-pointer")
+    endforeach()
+endif()

src/docs/sphinx/index.rst

@@ -4,6 +4,7 @@
 .. ## SPDX-License-Identifier: (BSD-3-Clause)
 
 .. toctree::
+   memory_checking
    :maxdepth: 2
 
 =======

src/docs/sphinx/memory_checking.rst

@@ -0,0 +1,87 @@
+.. ## Copyright (c) 2019-2020, Lawrence Livermore National Security, LLC and
+.. ## other Serac Project Developers. See the top-level COPYRIGHT file for details.
+.. ##
+.. ## SPDX-License-Identifier: (BSD-3-Clause)
+
+===============
+Memory Checking
+===============
+
+There are two commonly available memory checkers available to use with Serac on LC:
+`AddressSanitizer <https://github.com/google/sanitizers/wiki/AddressSanitizer>`_
+and `Valgrind <https://valgrind.org/>`_.
+
+AddressSanitizer
+----------------
+
+AddressSanitizer (aka Asan) is memory error detection tool that is a part of LLVM.  It
+very fast and easy to use but doesn't seem as robust as Valgrind.  It requires compile
+and link flags which are enabled via the CMake option ENABLE_ASAN.  Anything in our CMake
+system will get those flags after that is enabled but our third-party libraries (like MFEM)
+will not. After that just run your built executable and Asan will output a log to the screen
+after your program is done running.  Asan's behavior can be modified with a set of
+`environment variables <https://github.com/google/sanitizers/wiki/AddressSanitizerFlags>`_ .
+
+..note::
+    Asan only works with the Clang and GCC compiler chains.  Our build system will throw
+    and error if you try to build with anything else while ENABLE_ASAN is ON.
+
+Here is a recommended workflow:
+
+.. code-block:: bash
+
+    ./config-build.py -hc host-configs/rzgenie-toss_3_x86_64_ib-gcc@8.1.0.cmake -DENABLE_ASAN=ON
+    cd build-rzgenie-toss_3_x86_64_ib-gcc@8.1.0-debug
+    srun -N1 --exclusive --mpi-bind=off make -j
+    LSAN_OPTIONS=suppressions=../suppressions.asan ASAN_OPTIONS=log_path=asan.out:log_exe_name=true srun -n2 bin/serac
+
+This will output files in the current directory for each process that follow the pattern:
+``asan.out.<exe name>.<pid>``.  It also sets your return code to a non-zero value if there
+were any non-suppressed memory errors.
+
+LSAN_OPTIONS and ASAN_OPTIONS are delimited by ':'.
+
+Here is an explanation of the given options (all should be added to ASAN_OPTIONS unless noted):
+
+  * ``suppressions``: Location of memory leak suppression file (LSAN_OPTIONS)
+  * ``log_path``: Logs to the given file instead of to the screen. This is very helpful
+    to avoid intermingled lines on the screen from every process
+  * ``log_exe_name``: Adds executable name to log_path
+
+Helpful options:
+
+  * ``fast_unwind_on_malloc=0``: This improves Asan's stack tracing ability but also greatly slows
+    down the run
+  * ``exitcode=0``: This stops Asan from returning a a non-zero exit code from your executable
+    (defaults to 23) (LSAN_OPTIONS)
+
+Valgrind
+--------
+
+Valgrind is a very powerful set of tools that help with dynamic analysis tools.  We will
+focus on `memcheck <https://valgrind.org/docs/manual/mc-manual.html>`_ which is a memory
+error detection tool.
+
+Unlike Asan, valgrind does not need any special compiler flags.  Just build your executable
+and run your executable with ``valgrind``. Valgrind's suppression files are easily generated by
+valgrind with `--gen-suppressions=all` and are more customizable than Asan's.
+
+Here is a recommended workflow:
+
+.. code-block:: bash
+
+    ./config-build.py -hc host-configs/rzgenie-toss_3_x86_64_ib-gcc@8.1.0.cmake
+    cd build-rzgenie-toss_3_x86_64_ib-gcc@8.1.0-debug
+    srun -N1 --exclusive --mpi-bind=off make -j
+    srun -n2 valgrind --tool=memcheck --log-file=valgrind.out --leak-check=yes --show-leak-kinds=all --num-callers=20 --suppressions=../suppressions.valgrind bin/serac
+
+This will will produce a file called ``valgrind.out`` in the current directory with a valgrind report.
+
+Here is an explanation of the given options:
+
+ * ``--tool=memcheck``: valgrind is a tool-suite so this runs the memcheck tool
+ * ``--log-file=valgrind.out``: Logs report to the given file
+ * ``--leak-check=yes``: Enables memory leak checks
+ * ``--show-leak-kinds=all```: Enables showing all memory leak kinds
+ * ``--num-callers=20``: Limits the size of the stack traces to 20
+ * ``--suppressions=../suppressions.valgrind``: Location of memory leak suppression file

src/drivers/serac.cpp

@@ -20,6 +20,8 @@
 #include <iostream>
 #include <memory>
 
+#include "serac_config.hpp"
+
 #include "coefficients/loading_functions.hpp"
 #include "coefficients/traction_coefficient.hpp"
 #include "mfem.hpp"
@@ -34,7 +36,8 @@ int main(int argc, char *argv[])
   MPI_Comm_rank(MPI_COMM_WORLD, &myid);
 
   // mesh
-  const char *mesh_file = "../../data/beam-hex.mesh";
+  std::string base_mesh_file = std::string(SERAC_SRC_DIR) + "/data/beam-hex.mesh";
+  const char *mesh_file = base_mesh_file.c_str();
 
   // serial and parallel refinement levels
   int ser_ref_levels = 0;

suppressions.asan

@@ -0,0 +1,2 @@
+# Library that isn't built by us
+leak:libpsm2.so.2

suppressions.valgrind

@@ -0,0 +1,36 @@
+{
+   <mpi_init_leak>
+   Memcheck:Leak
+   ...
+   fun:PMPI_Init
+   fun:main
+}
+
+{
+   <mpi_init_leak2>
+   Memcheck:Leak
+   ...
+   fun:PMPI_Init
+}
+
+{
+   <mpi_init_param>
+   Memcheck:Param
+   rt_sigaction(act->sa_mask)
+   fun:__libc_sigaction
+   obj:/usr/lib64/libpsm2.so.2.1
+   obj:/usr/lib64/libpsm2.so.2.1
+   fun:psm2_ep_open
+   fun:psm_doinit
+   fun:MPID_Init
+   fun:MPIR_Init_thread
+   fun:PMPI_Init
+   fun:main
+}
+
+{
+   <psm2_lib>
+   Memcheck:Leak
+   ...
+   obj:/usr/lib64/libpsm2.so.2.1
+}