From 2fc2b29dc07b4c82fc49ee54754a4cf3e6f7383a Mon Sep 17 00:00:00 2001 From: Vicente Adolfo Bolea Sanchez Date: Tue, 26 Nov 2024 20:59:07 -0500 Subject: [PATCH] ci: rehaul sanitizer builds - Based them on opensuse tumbleweed to get recent versions of their packages. - Update clang/llvm to 17 - Add build-images-opensuse-tw script --- .circleci/config.yml | 26 +++-- scripts/ci/cmake/ci-fedora-msan.cmake | 32 ------ .../ci/cmake/ci-fedora-openmpi-ubsan.cmake | 35 ------ ...a-asan.cmake => ci-opensuse-tw-asan.cmake} | 4 +- scripts/ci/cmake/ci-opensuse-tw-msan.cmake | 59 ++++++++++ ...a-tsan.cmake => ci-opensuse-tw-tsan.cmake} | 4 +- ...ubsan.cmake => ci-opensuse-tw-ubsan.cmake} | 6 +- scripts/ci/images/fedora-asan/Dockerfile | 21 ---- .../images/fedora-sanitizers-base/Dockerfile | 22 ---- scripts/ci/images/fedora-tsan/Dockerfile | 10 -- scripts/ci/images/fedora-ubsan/Dockerfile | 8 -- .../opensuse-tw/build-images-opensuse-tw | 49 +++++++++ .../opensuse-tw/opensuse-tw-asan.dockerfile | 20 ++++ ...opensuse-tw-full-stack-onbuild.dockerfile} | 103 +++++++----------- .../opensuse-tw-msan.dockerfile} | 7 +- .../opensuse-tw-sanitizer-base.dockerfile | 32 ++++++ .../opensuse-tw/opensuse-tw-tsan.dockerfile | 9 ++ .../opensuse-tw/opensuse-tw-ubsan.dockerfile | 5 + .../toolchain-msan.cmake} | 0 .../toolchain-tsan.cmake} | 0 scripts/dashboard/common.cmake | 2 +- .../toolkit/format/bp/bp3/BP3Serializer.tcc | 2 + 22 files changed, 242 insertions(+), 214 deletions(-) delete mode 100644 scripts/ci/cmake/ci-fedora-msan.cmake delete mode 100644 scripts/ci/cmake/ci-fedora-openmpi-ubsan.cmake rename scripts/ci/cmake/{ci-fedora-asan.cmake => ci-opensuse-tw-asan.cmake} (91%) create mode 100644 scripts/ci/cmake/ci-opensuse-tw-msan.cmake rename scripts/ci/cmake/{ci-fedora-tsan.cmake => ci-opensuse-tw-tsan.cmake} (89%) rename scripts/ci/cmake/{ci-fedora-ubsan.cmake => ci-opensuse-tw-ubsan.cmake} (85%) delete mode 100644 scripts/ci/images/fedora-asan/Dockerfile delete mode 100644 scripts/ci/images/fedora-sanitizers-base/Dockerfile delete mode 100644 scripts/ci/images/fedora-tsan/Dockerfile delete mode 100644 scripts/ci/images/fedora-ubsan/Dockerfile create mode 100755 scripts/ci/images/opensuse-tw/build-images-opensuse-tw create mode 100644 scripts/ci/images/opensuse-tw/opensuse-tw-asan.dockerfile rename scripts/ci/images/{fedora-full-stack-onbuild/Dockerfile => opensuse-tw/opensuse-tw-full-stack-onbuild.dockerfile} (64%) rename scripts/ci/images/{fedora-msan/Dockerfile => opensuse-tw/opensuse-tw-msan.dockerfile} (57%) create mode 100644 scripts/ci/images/opensuse-tw/opensuse-tw-sanitizer-base.dockerfile create mode 100644 scripts/ci/images/opensuse-tw/opensuse-tw-tsan.dockerfile create mode 100644 scripts/ci/images/opensuse-tw/opensuse-tw-ubsan.dockerfile rename scripts/ci/images/{fedora-msan/toolchain.cmake => opensuse-tw/toolchain-msan.cmake} (100%) rename scripts/ci/images/{fedora-tsan/toolchain.cmake => opensuse-tw/toolchain-tsan.cmake} (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index ead73aac22..8f96878f6a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,29 +25,31 @@ defaults: &defaults version: 2 jobs: - "fedora-ubsan": + "opensuse-tw-ubsan": <<: *defaults docker: - - image: ornladios/adios2:ci-fedora-ubsan - "fedora-asan": + - image: ghcr.io/ornladios/adios2:ci-opensuse-tw-ubsan + environment: + UBSAN_OPTIONS: print_stacktrace=1 + "opensuse-tw-asan": <<: *defaults docker: - - image: ornladios/adios2:ci-fedora-asan - "fedora-msan": + - image: ghcr.io/ornladios/adios2:ci-opensuse-tw-asan + "opensuse-tw-msan": <<: *defaults docker: - - image: ornladios/adios2:ci-fedora-msan + - image: ghcr.io/ornladios/adios2:ci-opensuse-tw-msan resource_class: large - "fedora-tsan": + "opensuse-tw-tsan": <<: *defaults docker: - - image: ornladios/adios2:ci-fedora-tsan + - image: ghcr.io/ornladios/adios2:ci-opensuse-tw-tsan workflows: version: 2 sanitizers: jobs: - - "fedora-ubsan" - - "fedora-asan" - - "fedora-msan" - - "fedora-tsan" + - "opensuse-tw-ubsan" + - "opensuse-tw-asan" + - "opensuse-tw-msan" + - "opensuse-tw-tsan" diff --git a/scripts/ci/cmake/ci-fedora-msan.cmake b/scripts/ci/cmake/ci-fedora-msan.cmake deleted file mode 100644 index b1a8022fdd..0000000000 --- a/scripts/ci/cmake/ci-fedora-msan.cmake +++ /dev/null @@ -1,32 +0,0 @@ -# Client maintainer: chuck.atkins@kitware.com - -set(dashboard_cache " -BUILD_TESTING:BOOL=ON -ADIOS2_BUILD_EXAMPLES:BOOL=ON - -CMAKE_TOOLCHAIN_FILE:FILEPATH=/opt/msan/toolchain.cmake -ADIOS2_USE_Fortran:STRING=OFF -ADIOS2_USE_HDF5:STRING=ON -ADIOS2_USE_MPI:STRING=OFF -ADIOS2_USE_Python:STRING=OFF - -HDF5_C_COMPILER_EXECUTABLE:FILEPATH=/opt/msan/bin/h5cc -HDF5_DIFF_EXECUTABLE:FILEPATH=/opt/msan/bin/h5diff -") - -set(dashboard_track "Analysis") -set(CTEST_CMAKE_GENERATOR "Unix Makefiles") -set(CTEST_BUILD_FLAGS "-k -j8") -set(CTEST_MEMORYCHECK_TYPE "MemorySanitizer") - -list(APPEND EXCLUDE_EXPRESSIONS - "Engine.BP.BPBufferSizeTest.SyncDeferredIdenticalUsage.BP3.Serial" - "Engine.BP.BPBufferSizeTest.SyncDeferredIdenticalUsage.BP4.Serial" - ) -list(JOIN EXCLUDE_EXPRESSIONS "|" TEST_EXCLUDE_STRING) -set(CTEST_MEMCHECK_ARGS EXCLUDE "${TEST_EXCLUDE_STRING}") - -set(ADIOS_TEST_REPEAT 0) -set(CTEST_TEST_ARGS PARALLEL_LEVEL 4) -list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") -include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake) diff --git a/scripts/ci/cmake/ci-fedora-openmpi-ubsan.cmake b/scripts/ci/cmake/ci-fedora-openmpi-ubsan.cmake deleted file mode 100644 index 956b004137..0000000000 --- a/scripts/ci/cmake/ci-fedora-openmpi-ubsan.cmake +++ /dev/null @@ -1,35 +0,0 @@ -# Client maintainer: chuck.atkins@kitware.com - -set(ENV{CC} gcc) -set(ENV{CXX} g++) -set(ENV{FC} gfortran) -set(UBSAN_FLAGS "-fsanitize=undefined -fno-sanitize-recover=all -pthread") -set(ENV{CFLAGS} "${UBSAN_FLAGS}") -set(ENV{CXXFLAGS} "${UBSAN_FLAGS}") -set(ENV{FFLAGS} "${UBSAN_FLAGS}") -set(ENV{UBSAN_OPTIONS} "print_stacktrace=1") - -find_package(EnvModules REQUIRED) -env_module(load mpi) - -set(dashboard_cache " -BUILD_TESTING:BOOL=ON -ADIOS2_BUILD_EXAMPLES:BOOL=ON - -ADIOS2_USE_HDF5:STRING=ON -ADIOS2_USE_MPI:STRING=ON - -HDF5_C_COMPILER_EXECUTABLE:FILEPATH=/usr/lib64/openmpi/bin/h5pcc -HDF5_DIFF_EXECUTABLE:FILEPATH=/usr/lib64/openmpi/bin/h5diff - -MPIEXEC_EXTRA_FLAGS:STRING=--allow-run-as-root --oversubscribe -MPIEXEC_MAX_NUMPROCS:STRING=4 -") - -set(dashboard_track "Analysis") -set(CTEST_CMAKE_GENERATOR "Unix Makefiles") -set(CTEST_BUILD_FLAGS "-k -j2") -set(CTEST_MEMORYCHECK_TYPE "UndefinedBehaviorSanitizer") - -list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") -include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake) diff --git a/scripts/ci/cmake/ci-fedora-asan.cmake b/scripts/ci/cmake/ci-opensuse-tw-asan.cmake similarity index 91% rename from scripts/ci/cmake/ci-fedora-asan.cmake rename to scripts/ci/cmake/ci-opensuse-tw-asan.cmake index de819d49db..661ef59572 100644 --- a/scripts/ci/cmake/ci-fedora-asan.cmake +++ b/scripts/ci/cmake/ci-opensuse-tw-asan.cmake @@ -20,8 +20,8 @@ ADIOS2_USE_ZFP:STRING=ON ") set(dashboard_track "Analysis") -set(CTEST_CMAKE_GENERATOR "Unix Makefiles") -set(CTEST_BUILD_FLAGS "-k -j4") +set(CTEST_CMAKE_GENERATOR "Ninja") +set(CTEST_BUILD_FLAGS "-k0 -j4") set(CTEST_MEMORYCHECK_TYPE "AddressSanitizer") set(ADIOS_TEST_REPEAT 0) diff --git a/scripts/ci/cmake/ci-opensuse-tw-msan.cmake b/scripts/ci/cmake/ci-opensuse-tw-msan.cmake new file mode 100644 index 0000000000..6000cbf732 --- /dev/null +++ b/scripts/ci/cmake/ci-opensuse-tw-msan.cmake @@ -0,0 +1,59 @@ +# Client maintainer: vicente.bolea@kitware.com + +set(ENV{CC} clang) +set(ENV{CXX} clang++) + +#CMAKE_CXX_FLAGS=-O1 -fsanitize=memory -fsanitize-memory-track-origins -fno-optimize-sibling-calls +#CMAKE_C_FLAGS=-O1 -fsanitize=memory -fsanitize-memory-track-origins -fno-optimize-sibling-calls +#CMAKE_LINKER_FLAGS=-fsanitize=memory -fsanitize-memory-track-origins -fno-optimize-sibling-calls -fPIE + +set(dashboard_cache " +CMAKE_TOOLCHAIN_FILE:FILEPATH=/opt/msan/toolchain.cmake +HDF5_C_COMPILER_EXECUTABLE:FILEPATH=/opt/msan/bin/h5cc +HDF5_DIFF_EXECUTABLE:FILEPATH=/opt/msan/bin/h5diff + +BUILD_TESTING:BOOL=ON +ADIOS2_BUILD_EXAMPLES:BOOL=ON +ADIOS2_USE_Fortran:STRING=OFF +ADIOS2_USE_HDF5:STRING=ON +ADIOS2_USE_MPI:STRING=OFF +ADIOS2_USE_Python:STRING=OFF +") + +set(dashboard_track "Analysis") +list(APPEND EXCLUDE_EXPRESSIONS + "Engine.BP.BPBufferSizeTest.SyncDeferredIdenticalUsage.BP3.Serial" + "Engine.BP.BPBufferSizeTest.SyncDeferredIdenticalUsage.BP4.Serial" + "Staging.1x1DefSync.BP3" + "Staging.1x1DefSync.BP5" + "Staging.1x1DataWrite.BP5" + "Staging.1x1.Attrs.CommMin.BP.SST" + "Staging.1x1.CommMin.BP5.SST" + "Staging.1x1.ForcePreload.CommMin.BP5.SST" + "Staging.1x1.ModAttrs.CommMin.BP.SST" + "Staging.1x1LockGeometry.CommMin.BP5.SST" + "Staging.1x1Struct.BP5" + "Staging.AllToAllDistribution.1x1x3.CommMin.BP5.SST" + "Staging.WriteMemorySelectionRead.1x1.CommMin.BP5.SST" + "Staging.1x1.LocalMultiblock.CommMin.BP.SST" + "Staging.1x1.NoPreload.CommMin.BP.SST" + "Staging.TimeoutReader.1x1.CommMin.BP5.SST" + "Staging.1x1.LocalMultiblock.CommMin.BP5.SST" + "Staging.1x1.SstRUDP.CommMin.BP5.SST" + "Staging.LatestReaderHold.1x1.CommMin.BP5.SST" + "Staging.RoundRobinDistribution.1x1x3.CommMin.BP5.SST" + "Staging.1x1.SstRUDP.CommMin.BP.SST" + "Staging.RoundRobinDistribution.1x1x3.CommMin.BP.SST" + "Staging.1x1.Attrs.CommMin.BP5.SST" + ) +list(JOIN EXCLUDE_EXPRESSIONS "|" TEST_EXCLUDE_STRING) +set(CTEST_MEMCHECK_ARGS EXCLUDE "${TEST_EXCLUDE_STRING}") + +set(CTEST_CMAKE_GENERATOR "Ninja") +set(CTEST_BUILD_FLAGS "-k0 -j8") +set(CTEST_MEMORYCHECK_TYPE "MemorySanitizer") + +set(ADIOS_TEST_REPEAT 0) +set(CTEST_TEST_ARGS PARALLEL_LEVEL 4) +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake) diff --git a/scripts/ci/cmake/ci-fedora-tsan.cmake b/scripts/ci/cmake/ci-opensuse-tw-tsan.cmake similarity index 89% rename from scripts/ci/cmake/ci-fedora-tsan.cmake rename to scripts/ci/cmake/ci-opensuse-tw-tsan.cmake index d5d77dd038..8db9323642 100644 --- a/scripts/ci/cmake/ci-fedora-tsan.cmake +++ b/scripts/ci/cmake/ci-opensuse-tw-tsan.cmake @@ -15,8 +15,8 @@ HDF5_DIFF_EXECUTABLE:FILEPATH=/opt/tsan/bin/h5diff ") set(dashboard_track "Analysis") -set(CTEST_CMAKE_GENERATOR "Unix Makefiles") -set(CTEST_BUILD_FLAGS "-k -j4") +set(CTEST_CMAKE_GENERATOR "Ninja") +set(CTEST_BUILD_FLAGS "-k0 -j4") set(CTEST_MEMORYCHECK_TYPE "ThreadSanitizer") set(ADIOS_TEST_REPEAT 0) diff --git a/scripts/ci/cmake/ci-fedora-ubsan.cmake b/scripts/ci/cmake/ci-opensuse-tw-ubsan.cmake similarity index 85% rename from scripts/ci/cmake/ci-fedora-ubsan.cmake rename to scripts/ci/cmake/ci-opensuse-tw-ubsan.cmake index b7b852bc24..d97e2fbcc3 100644 --- a/scripts/ci/cmake/ci-fedora-ubsan.cmake +++ b/scripts/ci/cmake/ci-opensuse-tw-ubsan.cmake @@ -7,7 +7,7 @@ set(UBSAN_FLAGS "-fsanitize=undefined -fno-sanitize-recover=all -pthread") set(ENV{CFLAGS} "${UBSAN_FLAGS}") set(ENV{CXXFLAGS} "${UBSAN_FLAGS}") set(ENV{FFLAGS} "${UBSAN_FLAGS}") -set(ENV{UBSAN_OPTIONS} "print_stacktrace=1") +set(CTEST_MEMORYCHECK_SANITIZER_OPTIONS "print_stacktrace=1") set(dashboard_cache " BUILD_TESTING:BOOL=ON @@ -21,8 +21,8 @@ HDF5_DIFF_EXECUTABLE:FILEPATH=/usr/bin/h5diff ") set(dashboard_track "Analysis") -set(CTEST_CMAKE_GENERATOR "Unix Makefiles") -set(CTEST_BUILD_FLAGS "-k -j2") +set(CTEST_CMAKE_GENERATOR "Ninja") +set(CTEST_BUILD_FLAGS "-k0 -j2") set(CTEST_MEMORYCHECK_TYPE "UndefinedBehaviorSanitizer") set(ADIOS_TEST_REPEAT 0) diff --git a/scripts/ci/images/fedora-asan/Dockerfile b/scripts/ci/images/fedora-asan/Dockerfile deleted file mode 100644 index e8daae0ca0..0000000000 --- a/scripts/ci/images/fedora-asan/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM ornladios/adios2:ci-fedora-sanitizers-base - -# Install core dev packages -RUN dnf -y install libasan python3-devel python3-numpy hdf5-devel - -# Install ZFP -WORKDIR /opt/zfp -RUN curl -L https://github.com/LLNL/zfp/releases/download/0.5.5/zfp-0.5.5.tar.gz | tar -xvz && \ - mkdir build && \ - cd build && \ - cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/zfp/0.5.5 ../zfp-0.5.5 && \ - make -j$(grep -c '^processor' /proc/cpuinfo) install && \ - cd .. && \ - rm -rf zfp-0.5.5 build -ENV PATH=/opt/zfp/0.5.5/bin:${PATH} \ - LD_LIBRARY_PATH=/opt/zfp/0.5.5/lib64:${LD_LIBRARY_PATH} \ - CMAKE_PREFIX_PATH=/opt/zfp/0.5.5:${CMAKE_PREFIX_PATH} - -# Misc cleanup -RUN dnf clean all && \ - rm -rfv /tmp/* /var/cache/dnf diff --git a/scripts/ci/images/fedora-sanitizers-base/Dockerfile b/scripts/ci/images/fedora-sanitizers-base/Dockerfile deleted file mode 100644 index 7d4948facd..0000000000 --- a/scripts/ci/images/fedora-sanitizers-base/Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -FROM fedora:latest - -# Install core dev packages -RUN dnf -y install gcc gcc-c++ gcc-gfortran git make curl file patch \ - zlib-devel bzip2 bzip2-libs bzip2-devel blosc-devel vim valgrind \ - zeromq-devel libpng-devel libfabric-devel libffi-devel \ - python3-devel python3-numpy \ - clang llvm-devel libcxxabi-devel libcxx-devel - -# Install the most recent CMake nightly binary -WORKDIR /opt/cmake -RUN curl -L https://cmake.org/files/dev/$(curl https://cmake.org/files/dev/ | sed -n '/Linux-x86_64.tar.gz/s/.*>\(cmake[^<]*\)<.*/\1/p' | sort | tail -1) | tar --strip-components=1 -xzv -ENV PATH=/opt/cmake/bin:${PATH} - -# Installing Kitware Ninja -RUN mkdir -p /usr/local/bin && \ - curl -L https://github.com/Kitware/ninja/releases/download/v1.9.0.g99df1.kitware.dyndep-1.jobserver-1/ninja-1.9.0.g99df1.kitware.dyndep-1.jobserver-1_x86_64-linux-gnu.tar.gz | \ - tar -C /usr/local/bin --strip-components=1 -xzv - -# Misc cleanup -RUN dnf clean all && \ - rm -rfv /tmp/* /var/cache/dnf diff --git a/scripts/ci/images/fedora-tsan/Dockerfile b/scripts/ci/images/fedora-tsan/Dockerfile deleted file mode 100644 index 123daf1bab..0000000000 --- a/scripts/ci/images/fedora-tsan/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -ARG INSTALL_PREFIX=/opt/tsan -ARG TOOLCHAIN_FILE=toolchain.cmake -ARG CFLAGS="-fsanitize=thread" -ARG CXXFLAGS="-fsanitize=thread" -ARG LLVM_USE_SANITIZER=Thread - -FROM ornladios/adios2:ci-fedora-full-stack-onbuild AS tmp-stage - -FROM ornladios/adios2:ci-fedora-sanitizers-base -COPY --from=tmp-stage /opt/tsan/ /opt/tsan/ diff --git a/scripts/ci/images/fedora-ubsan/Dockerfile b/scripts/ci/images/fedora-ubsan/Dockerfile deleted file mode 100644 index 1af105610f..0000000000 --- a/scripts/ci/images/fedora-ubsan/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM ornladios/adios2:ci-fedora-sanitizers-base - -# Install core dev packages -RUN dnf -y install libubsan python3-devel python3-numpy hdf5-devel - -# Misc cleanup -RUN dnf clean all && \ - rm -rfv /tmp/* /var/cache/dnf diff --git a/scripts/ci/images/opensuse-tw/build-images-opensuse-tw b/scripts/ci/images/opensuse-tw/build-images-opensuse-tw new file mode 100755 index 0000000000..48d14f4687 --- /dev/null +++ b/scripts/ci/images/opensuse-tw/build-images-opensuse-tw @@ -0,0 +1,49 @@ +#!/bin/bash + +# This script is used to build the openSUSE Tumbleweed images + +set -xeuo pipefail + +declare -r DOCKER_REPOSITORY=ghcr.io/ornladios/adios2 +declare -u ENABLE_PUSH= + +#============================================================================= +# Functions + +# Function to build a docker image +function docker_build() { + local docker_file=$1 + local docker_tag=$2 + local args=(--progress=plain --rm) + + if [[ -n "${ENABLE_PUSH}" ]]; then + args+=("--push") + fi + + sudo docker build "${args[@]}" \ + --file "${docker_file}" \ + --tag "${DOCKER_REPOSITORY}:${docker_tag}" \ + . +} + +function usage() { + echo "Usage: $0 [-p ] [-m ]" + echo " -p Push to the docker registry" + exit 0 +} + +#============================================================================= +# Main +while getopts "p" o; do + case "${o}" in + p) ENABLE_PUSH=true ;; + *) usage ;; + esac +done + +docker_build opensuse-tw-sanitizer-base.dockerfile ci-opensuse-tw-sanitizer-base +docker_build opensuse-tw-full-stack-onbuild.dockerfile ci-opensuse-tw-full-stack-onbuild +docker_build opensuse-tw-asan.dockerfile ci-opensuse-tw-asan +docker_build opensuse-tw-msan.dockerfile ci-opensuse-tw-msan +docker_build opensuse-tw-tsan.dockerfile ci-opensuse-tw-tsan +docker_build opensuse-tw-ubsan.dockerfile ci-opensuse-tw-ubsan diff --git a/scripts/ci/images/opensuse-tw/opensuse-tw-asan.dockerfile b/scripts/ci/images/opensuse-tw/opensuse-tw-asan.dockerfile new file mode 100644 index 0000000000..8ceaf63b39 --- /dev/null +++ b/scripts/ci/images/opensuse-tw/opensuse-tw-asan.dockerfile @@ -0,0 +1,20 @@ +FROM ghcr.io/ornladios/adios2:ci-opensuse-tw-sanitizer-base + +# Install core dev packages +RUN zypper install -y --no-recommends libasan8 hdf5-devel && \ + zypper clean --all + +# Install ZFP +WORKDIR /opt/zfp +RUN curl -L https://github.com/LLNL/zfp/releases/download/0.5.5/zfp-0.5.5.tar.gz | tar -xvz && \ + cmake -GNinja -S zfp-0.5.5 -B build \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/opt/zfp/0.5.5 \ + && \ + cmake --build build && \ + cmake --install build && \ + rm -rf zfp-0.5.5 build +ENV PATH=/opt/zfp/0.5.5/bin:${PATH} \ + LD_LIBRARY_PATH=/opt/zfp/0.5.5/lib64:${LD_LIBRARY_PATH} \ + CMAKE_PREFIX_PATH=/opt/zfp/0.5.5:${CMAKE_PREFIX_PATH} diff --git a/scripts/ci/images/fedora-full-stack-onbuild/Dockerfile b/scripts/ci/images/opensuse-tw/opensuse-tw-full-stack-onbuild.dockerfile similarity index 64% rename from scripts/ci/images/fedora-full-stack-onbuild/Dockerfile rename to scripts/ci/images/opensuse-tw/opensuse-tw-full-stack-onbuild.dockerfile index 51edf7e6f9..5fa8d83152 100644 --- a/scripts/ci/images/fedora-full-stack-onbuild/Dockerfile +++ b/scripts/ci/images/opensuse-tw/opensuse-tw-full-stack-onbuild.dockerfile @@ -1,4 +1,4 @@ -FROM ornladios/adios2:ci-fedora-sanitizers-base +FROM ghcr.io/ornladios/adios2:ci-opensuse-tw-sanitizer-base # Set up some arguments ONBUILD ARG INSTALL_PREFIX @@ -7,24 +7,20 @@ ONBUILD ARG CFLAGS ONBUILD ARG CXXFLAGS ONBUILD ARG LLVM_USE_SANITIZER +ENV CC=clang +ENV CXX=clang++ + # Build and install libc++ -# Have to build in debug mode due to a bug in LLVM: -# https://bugs.llvm.org/show_bug.cgi?id=44586 ONBUILD WORKDIR /root/llvm -ONBUILD RUN git clone --branch llvmorg-9.0.0 --depth 1 \ - https://github.com/llvm/llvm-project.git source && \ - mkdir -p build && \ - cd build && \ - /opt/cmake/bin/cmake \ - -GNinja \ - -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ - -DCMAKE_C_COMPILER=/usr/bin/clang \ - -DCMAKE_CXX_COMPILER=/usr/bin/clang++ \ - -DLLVM_ENABLE_PROJECTS="libcxx;libcxxabi" \ - -DLLVM_USE_SANITIZER=${LLVM_USE_SANITIZER} \ - ../source/llvm && \ - ninja install-cxxabi install-cxx +ONBUILD RUN git clone --branch llvmorg-14.0.6 --depth 1 \ + https://github.com/llvm/llvm-project.git source +ONBUILD RUN cmake -GNinja -B build -S source/runtimes \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ + -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi" \ + -DLLVM_USE_SANITIZER=${LLVM_USE_SANITIZER} \ + && \ + cmake --build build --target install-cxxabi install-cxx # Copy in the toolchain ONBUILD COPY \ @@ -35,19 +31,17 @@ ONBUILD COPY \ ONBUILD WORKDIR /root/zlib ONBUILD RUN git clone --branch v1.2.11 --depth 1 \ https://github.com/madler/zlib.git source && \ - mkdir -p build && \ - cd build && \ - /opt/cmake/bin/cmake \ - -GNinja \ + cmake -GNinja -S source -B build \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_TOOLCHAIN_FILE=${INSTALL_PREFIX}/toolchain.cmake \ -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ - ../source && \ - ninja install + && \ + cmake --build build && \ + cmake --install build # Build and install bzip2 ONBUILD WORKDIR /root/bzip2 -ONBUILD RUN git clone --branch bzip2-1.0.8 \ +ONBUILD RUN git clone --branch bzip2-1.0.8 --depth 1 \ https://sourceware.org/git/bzip2.git source && \ cd source && \ sed -e "s_^CC=.*\$_CC=/usr/bin/clang_" \ @@ -65,69 +59,54 @@ ONBUILD RUN git clone --branch bzip2-1.0.8 \ # Build and install zeromq ONBUILD WORKDIR /root/zeromq -ONBUILD RUN git clone --branch v4.3.2 \ +ONBUILD RUN git clone --branch v4.3.2 --depth 1 \ https://github.com/zeromq/libzmq.git source && \ - mkdir -p build && \ - cd build && \ - /opt/cmake/bin/cmake \ - -GNinja \ + cmake -GNinja -S source -B build \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_TOOLCHAIN_FILE=${INSTALL_PREFIX}/toolchain.cmake \ -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ - ../source && \ - ninja install + && \ + cmake --build build && \ + cmake --install build + # Build and install libpng ONBUILD WORKDIR /root/libpng -ONBUILD RUN git clone --branch v1.6.9 \ +ONBUILD RUN git clone --branch v1.6.9 --depth 1 \ https://git.code.sf.net/p/libpng/code.git source && \ - mkdir -p build && \ - cd build && \ - /opt/cmake/bin/cmake \ - -GNinja \ + cmake -GNinja -S source -B build \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_TOOLCHAIN_FILE=${INSTALL_PREFIX}/toolchain.cmake \ -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ - ../source && \ - ninja install + && \ + cmake --build build && \ + cmake --install build # Build and install hdf5 ONBUILD WORKDIR /root/hdf5 -ONBUILD RUN git clone --branch hdf5-1_10_6 \ - https://bitbucket.hdfgroup.org/scm/hdffv/hdf5.git source && \ - mkdir -p build && \ - cd build && \ - /opt/cmake/bin/cmake \ - -GNinja \ +ONBUILD RUN git clone --branch hdf5-1_14_2 --depth=1 \ + https://github.com/HDFGroup/hdf5.git source && \ + cmake -GNinja -S source -B build \ + -DBUILD_TESTING=OFF \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_TOOLCHAIN_FILE=${INSTALL_PREFIX}/toolchain.cmake \ -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ - ../source && \ - ninja install - -# Build and install blosc -ONBUILD WORKDIR /root/blosc -ONBUILD RUN git clone --branch v1.17.1 \ - https://github.com/Blosc/c-blosc.git source && \ - mkdir -p build && \ - cd build && \ - /opt/cmake/bin/cmake \ - -GNinja \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_TOOLCHAIN_FILE=${INSTALL_PREFIX}/toolchain.cmake \ - -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ - ../source && \ - ninja install + && \ + cmake --build build && \ + cmake --install build # Build and install libfabric ONBUILD WORKDIR /root/libfabric ONBUILD RUN mkdir -p source build && \ curl -L \ - https://github.com/ofiwg/libfabric/releases/download/v1.9.0/libfabric-1.9.0.tar.bz2 | \ + https://github.com/ofiwg/libfabric/releases/download/v1.18.1/libfabric-1.18.1.tar.bz2 | \ tar -C source -xj --strip-components=1 && \ cd build && \ ../source/configure \ --prefix=${INSTALL_PREFIX} \ + --disable-efa \ + --disable-shm \ + --disable-verbs \ CC="/usr/bin/clang ${CFLAGS} -L${INSTALL_PREFIX}/lib -Wl,-rpath,${INSTALL_PREFIX}/lib -Wno-unused-command-line-argument" \ CXX="/usr/bin/clang++ ${CXXFLAGS} -L${INSTALL_PREFIX}/lib -Wl,-rpath,${INSTALL_PREFIX}/lib -Wno-unused-command-line-argument -nostdinc++ -isystem ${INSTALL_PREFIX}/include/c++/v1 -stdlib=libc++" && \ make -j4 install @@ -143,4 +122,4 @@ ONBUILD RUN mkdir -p source build && \ --prefix=${INSTALL_PREFIX} \ CC="/usr/bin/clang ${CFLAGS} -L${INSTALL_PREFIX}/lib -Wl,-rpath,${INSTALL_PREFIX}/lib -Wno-unused-command-line-argument" \ CXX="/usr/bin/clang++ ${CXXFLAGS} -L${INSTALL_PREFIX}/lib -Wl,-rpath,${INSTALL_PREFIX}/lib -Wno-unused-command-line-argument -nostdinc++ -isystem ${INSTALL_PREFIX}/include/c++/v1 -stdlib=libc++" && \ - make -j4 install + make -j4 install diff --git a/scripts/ci/images/fedora-msan/Dockerfile b/scripts/ci/images/opensuse-tw/opensuse-tw-msan.dockerfile similarity index 57% rename from scripts/ci/images/fedora-msan/Dockerfile rename to scripts/ci/images/opensuse-tw/opensuse-tw-msan.dockerfile index b289d1f4d6..e922d298f4 100644 --- a/scripts/ci/images/fedora-msan/Dockerfile +++ b/scripts/ci/images/opensuse-tw/opensuse-tw-msan.dockerfile @@ -1,10 +1,9 @@ ARG INSTALL_PREFIX=/opt/msan -ARG TOOLCHAIN_FILE=toolchain.cmake +ARG TOOLCHAIN_FILE=toolchain-msan.cmake ARG CFLAGS="-fsanitize=memory -fsanitize-memory-track-origins" ARG CXXFLAGS="-fsanitize=memory -fsanitize-memory-track-origins" ARG LLVM_USE_SANITIZER=MemoryWithOrigins -FROM ornladios/adios2:ci-fedora-full-stack-onbuild AS tmp-stage - -FROM ornladios/adios2:ci-fedora-sanitizers-base +FROM ghcr.io/ornladios/adios2:ci-opensuse-tw-full-stack-onbuild AS tmp-stage +FROM ghcr.io/ornladios/adios2:ci-opensuse-tw-sanitizer-base COPY --from=tmp-stage /opt/msan/ /opt/msan/ diff --git a/scripts/ci/images/opensuse-tw/opensuse-tw-sanitizer-base.dockerfile b/scripts/ci/images/opensuse-tw/opensuse-tw-sanitizer-base.dockerfile new file mode 100644 index 0000000000..6c329f4780 --- /dev/null +++ b/scripts/ci/images/opensuse-tw/opensuse-tw-sanitizer-base.dockerfile @@ -0,0 +1,32 @@ +FROM opensuse/tumbleweed +LABEL maintainer "Vicente Adolfo Bolea Sanchez" + +# Base dependencies for building ADIOS2 projects +RUN zypper refresh && \ + zypper update -y && \ + zypper install -y --no-recommends \ + awk \ + bzip2 \ + gzip \ + bzip3-devel \ + clang14 \ + cmake \ + curl \ + file \ + gcc \ + gcc-c++ \ + git \ + git-lfs \ + hdf5-devel \ + libffi-devel \ + libpng16-devel \ + llvm14 \ + ninja \ + patch \ + python3-devel \ + python3-numpy \ + tar \ + zeromq-devel \ + zlib-devel \ + && \ + zypper clean --all diff --git a/scripts/ci/images/opensuse-tw/opensuse-tw-tsan.dockerfile b/scripts/ci/images/opensuse-tw/opensuse-tw-tsan.dockerfile new file mode 100644 index 0000000000..295d1ac155 --- /dev/null +++ b/scripts/ci/images/opensuse-tw/opensuse-tw-tsan.dockerfile @@ -0,0 +1,9 @@ +ARG INSTALL_PREFIX=/opt/tsan +ARG TOOLCHAIN_FILE=toolchain-tsan.cmake +ARG CFLAGS="-fsanitize=thread" +ARG CXXFLAGS="-fsanitize=thread" +ARG LLVM_USE_SANITIZER=Thread + +FROM ghcr.io/ornladios/adios2:ci-opensuse-tw-full-stack-onbuild AS tmp-stage +FROM ghcr.io/ornladios/adios2:ci-opensuse-tw-sanitizer-base +COPY --from=tmp-stage /opt/tsan/ /opt/tsan/ diff --git a/scripts/ci/images/opensuse-tw/opensuse-tw-ubsan.dockerfile b/scripts/ci/images/opensuse-tw/opensuse-tw-ubsan.dockerfile new file mode 100644 index 0000000000..35be3e7f31 --- /dev/null +++ b/scripts/ci/images/opensuse-tw/opensuse-tw-ubsan.dockerfile @@ -0,0 +1,5 @@ +FROM ghcr.io/ornladios/adios2:ci-opensuse-tw-sanitizer-base + +# Install core dev packages +RUN zypper install -y --no-recommends libubsan1 libasan8 hdf5-devel zfp-devel && \ + zypper clean --all diff --git a/scripts/ci/images/fedora-msan/toolchain.cmake b/scripts/ci/images/opensuse-tw/toolchain-msan.cmake similarity index 100% rename from scripts/ci/images/fedora-msan/toolchain.cmake rename to scripts/ci/images/opensuse-tw/toolchain-msan.cmake diff --git a/scripts/ci/images/fedora-tsan/toolchain.cmake b/scripts/ci/images/opensuse-tw/toolchain-tsan.cmake similarity index 100% rename from scripts/ci/images/fedora-tsan/toolchain.cmake rename to scripts/ci/images/opensuse-tw/toolchain-tsan.cmake diff --git a/scripts/dashboard/common.cmake b/scripts/dashboard/common.cmake index 265b5c89e9..df71588478 100644 --- a/scripts/dashboard/common.cmake +++ b/scripts/dashboard/common.cmake @@ -65,7 +65,7 @@ # set(ENV{FC} /path/to/fc) # Fortran compiler (optional) # set(ENV{LD_LIBRARY_PATH} /path/to/vendor/lib) # (if necessary) -cmake_minimum_required(VERSION 2.8.12 FATAL_ERROR) +cmake_minimum_required(VERSION 3.14 FATAL_ERROR) if(NOT DEFINED dashboard_full) set(dashboard_full TRUE) diff --git a/source/adios2/toolkit/format/bp/bp3/BP3Serializer.tcc b/source/adios2/toolkit/format/bp/bp3/BP3Serializer.tcc index d666955bd1..2e315572a1 100644 --- a/source/adios2/toolkit/format/bp/bp3/BP3Serializer.tcc +++ b/source/adios2/toolkit/format/bp/bp3/BP3Serializer.tcc @@ -128,6 +128,8 @@ void BP3Serializer::PutSpanMetadata(const core::Variable &variable, const size_t minPosition = span.m_MinMaxMetadataPositions.first; const size_t maxPosition = span.m_MinMaxMetadataPositions.second; + + // Fix Ubsan issue here std::copy(&min, &min + 1, reinterpret_cast(buffer.data() + minPosition)); std::copy(&max, &max + 1, reinterpret_cast(buffer.data() + maxPosition)); }