ci: rehaul sanitizer builds

- Based them on opensuse tumbleweed to get recent
  versions of their packages.
- Update clang/llvm to 17
- Add build-images-opensuse-tw script

.circleci/config.yml

@@ -25,29 +25,31 @@ defaults: &defaults
 version: 2
 
 jobs:
-  "fedora-ubsan":
+  "opensuse-tw-ubsan":
     <<: *defaults
     docker:
-      - image: ornladios/adios2:ci-fedora-ubsan
-  "fedora-asan":
+      - image: ghcr.io/ornladios/adios2:ci-opensuse-tw-ubsan
+    environment:
+      UBSAN_OPTIONS: print_stacktrace=1
+  "opensuse-tw-asan":
     <<: *defaults
     docker:
-      - image: ornladios/adios2:ci-fedora-asan
-  "fedora-msan":
+      - image: ghcr.io/ornladios/adios2:ci-opensuse-tw-asan
+  "opensuse-tw-msan":
     <<: *defaults
     docker:
-      - image: ornladios/adios2:ci-fedora-msan
+      - image: ghcr.io/ornladios/adios2:ci-opensuse-tw-msan
     resource_class: large
-  "fedora-tsan":
+  "opensuse-tw-tsan":
     <<: *defaults
     docker:
-      - image: ornladios/adios2:ci-fedora-tsan
+      - image: ghcr.io/ornladios/adios2:ci-opensuse-tw-tsan
 
 workflows:
   version: 2
   sanitizers:
     jobs:
-      - "fedora-ubsan"
-      - "fedora-asan"
-      - "fedora-msan"
-      - "fedora-tsan"
+      - "opensuse-tw-ubsan"
+      - "opensuse-tw-asan"
+      - "opensuse-tw-msan"
+      - "opensuse-tw-tsan"

scripts/ci/cmake/ci-fedora-asan.cmake → scripts/ci/cmake/ci-opensuse-tw-asan.cmake

@@ -20,8 +20,8 @@ ADIOS2_USE_ZFP:STRING=ON
 ")
 
 set(dashboard_track "Analysis")
-set(CTEST_CMAKE_GENERATOR "Unix Makefiles")
-set(CTEST_BUILD_FLAGS "-k -j4")
+set(CTEST_CMAKE_GENERATOR "Ninja")
+set(CTEST_BUILD_FLAGS "-k0 -j4")
 set(CTEST_MEMORYCHECK_TYPE "AddressSanitizer")
 
 set(ADIOS_TEST_REPEAT 0)

scripts/ci/cmake/ci-opensuse-tw-msan.cmake

@@ -0,0 +1,59 @@
+# Client maintainer: vicente.bolea@kitware.com
+
+set(ENV{CC}  clang)
+set(ENV{CXX} clang++)
+
+#CMAKE_CXX_FLAGS=-O1 -fsanitize=memory -fsanitize-memory-track-origins -fno-optimize-sibling-calls
+#CMAKE_C_FLAGS=-O1 -fsanitize=memory -fsanitize-memory-track-origins -fno-optimize-sibling-calls
+#CMAKE_LINKER_FLAGS=-fsanitize=memory -fsanitize-memory-track-origins -fno-optimize-sibling-calls -fPIE
+
+set(dashboard_cache "
+CMAKE_TOOLCHAIN_FILE:FILEPATH=/opt/msan/toolchain.cmake
+HDF5_C_COMPILER_EXECUTABLE:FILEPATH=/opt/msan/bin/h5cc
+HDF5_DIFF_EXECUTABLE:FILEPATH=/opt/msan/bin/h5diff
+
+BUILD_TESTING:BOOL=ON
+ADIOS2_BUILD_EXAMPLES:BOOL=ON
+ADIOS2_USE_Fortran:STRING=OFF
+ADIOS2_USE_HDF5:STRING=ON
+ADIOS2_USE_MPI:STRING=OFF
+ADIOS2_USE_Python:STRING=OFF
+")
+
+set(dashboard_track "Analysis")
+list(APPEND EXCLUDE_EXPRESSIONS
+  "Engine.BP.BPBufferSizeTest.SyncDeferredIdenticalUsage.BP3.Serial"
+  "Engine.BP.BPBufferSizeTest.SyncDeferredIdenticalUsage.BP4.Serial"
+  "Staging.1x1DefSync.BP3"
+  "Staging.1x1DefSync.BP5"
+  "Staging.1x1DataWrite.BP5"
+  "Staging.1x1.Attrs.CommMin.BP.SST"
+  "Staging.1x1.CommMin.BP5.SST"
+  "Staging.1x1.ForcePreload.CommMin.BP5.SST"
+  "Staging.1x1.ModAttrs.CommMin.BP.SST"
+  "Staging.1x1LockGeometry.CommMin.BP5.SST"
+  "Staging.1x1Struct.BP5"
+  "Staging.AllToAllDistribution.1x1x3.CommMin.BP5.SST"
+  "Staging.WriteMemorySelectionRead.1x1.CommMin.BP5.SST"
+  "Staging.1x1.LocalMultiblock.CommMin.BP.SST"
+  "Staging.1x1.NoPreload.CommMin.BP.SST"
+  "Staging.TimeoutReader.1x1.CommMin.BP5.SST"
+  "Staging.1x1.LocalMultiblock.CommMin.BP5.SST"
+  "Staging.1x1.SstRUDP.CommMin.BP5.SST"
+  "Staging.LatestReaderHold.1x1.CommMin.BP5.SST"
+  "Staging.RoundRobinDistribution.1x1x3.CommMin.BP5.SST"
+  "Staging.1x1.SstRUDP.CommMin.BP.SST"
+  "Staging.RoundRobinDistribution.1x1x3.CommMin.BP.SST"
+  "Staging.1x1.Attrs.CommMin.BP5.SST"
+  )
+list(JOIN EXCLUDE_EXPRESSIONS "|" TEST_EXCLUDE_STRING)
+set(CTEST_MEMCHECK_ARGS EXCLUDE "${TEST_EXCLUDE_STRING}")
+
+set(CTEST_CMAKE_GENERATOR "Ninja")
+set(CTEST_BUILD_FLAGS "-k0 -j8")
+set(CTEST_MEMORYCHECK_TYPE "MemorySanitizer")
+
+set(ADIOS_TEST_REPEAT 0)
+set(CTEST_TEST_ARGS PARALLEL_LEVEL 4)
+list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}")
+include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake)

scripts/ci/cmake/ci-fedora-tsan.cmake → scripts/ci/cmake/ci-opensuse-tw-tsan.cmake

@@ -15,8 +15,8 @@ HDF5_DIFF_EXECUTABLE:FILEPATH=/opt/tsan/bin/h5diff
 ")
 
 set(dashboard_track "Analysis")
-set(CTEST_CMAKE_GENERATOR "Unix Makefiles")
-set(CTEST_BUILD_FLAGS "-k -j4")
+set(CTEST_CMAKE_GENERATOR "Ninja")
+set(CTEST_BUILD_FLAGS "-k0 -j4")
 set(CTEST_MEMORYCHECK_TYPE "ThreadSanitizer")
 
 set(ADIOS_TEST_REPEAT 0)

scripts/ci/cmake/ci-fedora-ubsan.cmake → scripts/ci/cmake/ci-opensuse-tw-ubsan.cmake

@@ -7,7 +7,7 @@ set(UBSAN_FLAGS "-fsanitize=undefined -fno-sanitize-recover=all -pthread")
 set(ENV{CFLAGS}   "${UBSAN_FLAGS}")
 set(ENV{CXXFLAGS} "${UBSAN_FLAGS}")
 set(ENV{FFLAGS}   "${UBSAN_FLAGS}")
-set(ENV{UBSAN_OPTIONS} "print_stacktrace=1")
+set(CTEST_MEMORYCHECK_SANITIZER_OPTIONS "print_stacktrace=1")
 
 set(dashboard_cache "
 BUILD_TESTING:BOOL=ON
@@ -21,8 +21,8 @@ HDF5_DIFF_EXECUTABLE:FILEPATH=/usr/bin/h5diff
 ")
 
 set(dashboard_track "Analysis")
-set(CTEST_CMAKE_GENERATOR "Unix Makefiles")
-set(CTEST_BUILD_FLAGS "-k -j2")
+set(CTEST_CMAKE_GENERATOR "Ninja")
+set(CTEST_BUILD_FLAGS "-k0 -j2")
 set(CTEST_MEMORYCHECK_TYPE "UndefinedBehaviorSanitizer")
 
 set(ADIOS_TEST_REPEAT 0)

scripts/ci/images/opensuse-tw/build-images-opensuse-tw

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# This script is used to build the openSUSE Tumbleweed images
+
+set -xeuo pipefail
+
+declare -r DOCKER_REPOSITORY=ghcr.io/ornladios/adios2
+declare -u ENABLE_PUSH=
+
+#=============================================================================
+# Functions
+
+# Function to build a docker image
+function docker_build() {
+  local docker_file=$1
+  local docker_tag=$2
+  local args=(--progress=plain --rm)
+
+  if [[ -n "${ENABLE_PUSH}" ]]; then
+    args+=("--push")
+  fi
+
+  sudo docker build "${args[@]}"  \
+    --file "${docker_file}" \
+    --tag "${DOCKER_REPOSITORY}:${docker_tag}" \
+    .
+}
+
+function usage() {
+  echo "Usage: $0 [-p <PROJECT>] [-m <MILESTONE>]"
+  echo "    -p    Push to the docker registry"
+  exit 0
+}
+
+#=============================================================================
+# Main
+while getopts "p" o; do
+  case "${o}" in
+    p) ENABLE_PUSH=true ;;
+    *) usage ;;
+  esac
+done
+
+docker_build opensuse-tw-sanitizer-base.dockerfile ci-opensuse-tw-sanitizer-base
+docker_build opensuse-tw-full-stack-onbuild.dockerfile ci-opensuse-tw-full-stack-onbuild
+docker_build opensuse-tw-asan.dockerfile ci-opensuse-tw-asan
+docker_build opensuse-tw-msan.dockerfile ci-opensuse-tw-msan
+docker_build opensuse-tw-tsan.dockerfile ci-opensuse-tw-tsan
+docker_build opensuse-tw-ubsan.dockerfile ci-opensuse-tw-ubsan

scripts/ci/images/opensuse-tw/opensuse-tw-asan.dockerfile

@@ -0,0 +1,20 @@
+FROM ghcr.io/ornladios/adios2:ci-opensuse-tw-sanitizer-base
+
+# Install core dev packages
+RUN zypper install -y --no-recommends libasan8 hdf5-devel && \
+    zypper clean --all
+
+# Install ZFP
+WORKDIR /opt/zfp
+RUN curl -L https://github.com/LLNL/zfp/releases/download/0.5.5/zfp-0.5.5.tar.gz | tar -xvz && \
+    cmake -GNinja -S zfp-0.5.5 -B build \
+      -DBUILD_SHARED_LIBS=ON \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DCMAKE_INSTALL_PREFIX=/opt/zfp/0.5.5 \
+      && \
+    cmake --build build && \
+    cmake --install build && \
+    rm -rf zfp-0.5.5 build
+ENV PATH=/opt/zfp/0.5.5/bin:${PATH} \
+    LD_LIBRARY_PATH=/opt/zfp/0.5.5/lib64:${LD_LIBRARY_PATH} \
+    CMAKE_PREFIX_PATH=/opt/zfp/0.5.5:${CMAKE_PREFIX_PATH}