diff --git a/.circleci/config.yml b/.circleci/config.yml index d95cac9fa1..75044102ac 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,6 +2,10 @@ defaults: &defaults steps: - checkout: path: source + - run: + # https://discuss.circleci.com/t/circle-working-directory-doesnt-expand/17007 + name: "Fix CIRCLE_WORKING_DIRECTORY" + command: echo 'CIRCLE_WORKING_DIRECTORY="${CIRCLE_WORKING_DIRECTORY/#\~/$HOME}"' >> $BASH_ENV - run: name: CDash command: bash source/scripts/ci/circle/postCDashStatus.sh diff --git a/.shellcheck_exclude_paths b/.shellcheck_exclude_paths index e84d523737..873e54f928 100644 --- a/.shellcheck_exclude_paths +++ b/.shellcheck_exclude_paths @@ -1,9 +1,3 @@ -scripts/ci/circle/postCDashStatus.sh -scripts/ci/circle/run.sh -scripts/ci/gh-actions/check-branch-name.sh -scripts/ci/gh-actions/get-changed-files.sh -scripts/ci/gh-actions/macos-setup.sh -scripts/ci/gh-actions/run.sh scripts/ci/scripts/github-prs-to-gitlab.sh scripts/ci/scripts/run-clang-format.sh scripts/ci/scripts/run-flake8.sh @@ -15,18 +9,3 @@ scripts/developer/setup.sh scripts/docker/setup-user.sh scripts/runconf/runconf.sh scripts/runconf/runconf_olcf.sh -testing/contract/lammps/build.sh -testing/contract/lammps/config.sh -testing/contract/lammps/install.sh -testing/contract/lammps/setup.sh -testing/contract/lammps/test.sh -testing/contract/scorpio/build.sh -testing/contract/scorpio/config.sh -testing/contract/scorpio/install.sh -testing/contract/scorpio/setup.sh -testing/contract/scorpio/test.sh -testing/contract/tau/build.sh -testing/contract/tau/config.sh -testing/contract/tau/install.sh -testing/contract/tau/setup.sh -testing/contract/tau/test.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index de866e8363..cd5d3e5fd3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -189,7 +189,7 @@ if(ADIOS2_HAVE_CUDA OR ADIOS2_HAVE_Kokkos_CUDA) endif() if(NOT DEFINED CMAKE_HIP_ARCHITECTURES AND DEFINED Kokkos_HIP_ARCHITECTURES) - set(CMAKE_HIP_ARCHITECTURES Kokkos_HIP_ARCHITECTURES) + set(CMAKE_HIP_ARCHITECTURES ${Kokkos_HIP_ARCHITECTURES}) endif() if(ADIOS2_HAVE_MPI) @@ -244,7 +244,7 @@ endif() set(ADIOS2_CONFIG_OPTS DataMan DataSpaces HDF5 HDF5_VOL MHS SST Fortran MPI Python Blosc2 BZip2 - LIBPRESSIO MGARD PNG SZ ZFP DAOS IME O_DIRECT Sodium Catalyst SysVShMem UCX + LIBPRESSIO MGARD MGARD_MDR PNG SZ ZFP DAOS IME O_DIRECT Sodium Catalyst SysVShMem UCX ZeroMQ Profiling Endian_Reverse Derived_Variable AWSSDK GPU_Support CUDA Kokkos Kokkos_CUDA Kokkos_HIP Kokkos_SYCL Campaign ) diff --git a/CTestConfig.cmake b/CTestConfig.cmake index 67ef8c43ee..49d6e3826c 100644 --- a/CTestConfig.cmake +++ b/CTestConfig.cmake @@ -11,3 +11,178 @@ set(CTEST_DROP_SITE "open.cdash.org") set(CTEST_DROP_LOCATION "/submit.php?project=ADIOS") set(CTEST_DROP_SITE_CDASH TRUE) set(MEMORYCHECK_SUPPRESSIONS_FILE ${CMAKE_SOURCE_DIR}/scripts/dashboard/nightly/valgrind-suppressions.txt) + +# Ignore tests that are currently failing, remove tests here as they are fixed +list(APPEND CTEST_CUSTOM_MEMCHECK_IGNORE + Bindings.C.ADIOS2_C_API.ADIOS2BPWriteTypes.Serial + Bindings.C.BPWriteReadMultiblockCC.ZeroSizeBlocks.Serial + Engine.BP.*/BPAppendAfterStepsP.Test/*.BP5.Serial + Engine.BP.*/BPChangingShapeWithinStep.MultiBlock/*.BP5.Serial + Engine.BP.*/BPParameterSelectStepsP.Read/*.BP5.Serial + Engine.BP.*/BPReadMultithreadedTestP.ReadFile/*.BP5.Serial + Engine.BP.*/BPStepsFileGlobalArrayParameters.EveryOtherStep/*.BP5.Serial + Engine.BP.*/BPStepsFileGlobalArrayReaders.EveryStep/*.BP5.Serial + Engine.BP.*/BPStepsFileGlobalArrayReaders.NewVarPerStep/*.BP5.Serial + Engine.BP.*/BPStepsFileLocalArrayParameters.EveryOtherStep/*.BP5.Serial + Engine.BP.*/BPStepsFileLocalArrayReaders.EveryStep/*.BP5.Serial + Engine.BP.*/BPStepsFileLocalArrayReaders.NewVarPerStep/*.BP5.Serial + Engine.BP.BPChangingShape.BPWriteReadShape2D.BP5.Serial + Engine.BP.BPLargeMetadata.ManyLongStrings.BP5.Serial + Engine.BP.BPWriteAppendReadTestADIOS2.ADIOS2BPWriteAppendRead2D2x4.BP5.Serial + Engine.BP.BPWriteAppendReadTestADIOS2.ADIOS2BPWriteAppendReadAggregate.BP5.Serial + Engine.BP.BPWriteAppendReadTestADIOS2.ADIOS2BPWriteAppendReadVaryingAggregation.BP5.Serial + Engine.BP.BPWriteMultiblockReadTest.ADIOS2BPWriteMultiblockRead1D8.BP5.Serial + Engine.BP.BPWriteMultiblockReadTest.ADIOS2BPWriteMultiblockRead2D2x4.BP5.Serial + Engine.BP.BPWriteMultiblockReadTest.ADIOS2BPWriteMultiblockRead2D4x2.BP5.Serial + Engine.BP.BPWriteMultiblockReadTest.ADIOS2BPWriteRead1D8ZeroBlock.BP5.Serial + Engine.BP.BPWriteReadAsStreamTestADIOS2.ReaderWriterDefineVariable.BP5.Serial + Engine.BP.BPWriteReadAttributes.BPWriteReadSingleTypesVar.BP5.Serial + Engine.BP.BPWriteReadAttributes.WriteReadArrayTypes.BP5.Serial + Engine.BP.BPWriteReadAttributes.WriteReadArrayTypesVar.BP5.Serial + Engine.BP.BPWriteReadAttributes.WriteReadSingleTypes.BP5.Serial + Engine.BP.BPWriteReadAttributes.WriteReadStreamVarp.BP5.Serial + Engine.BP.BPWriteReadAttributeTestMultirank.ADIOS2BPWriteReadArrayTypes.BP5.Serial + Engine.BP.BPWriteReadBlockInfo.BPWriteReadBlockInfo1D8_C.BP3.Serial + Engine.BP.BPWriteReadBlockInfo.BPWriteReadBlockInfo1D8_C.BP4.Serial + Engine.BP.BPWriteReadBlockInfo.BPWriteReadBlockInfo1D8_C.BP5.Serial + Engine.BP.BPWriteReadBlockInfo.BPWriteReadBlockInfo1D8.BP5.Serial + Engine.BP.BPWriteReadBlockInfo.BPWriteReadBlockInfo2D2x4.BP5.Serial + Engine.BP.BPWriteReadLocalVariables.ADIOS2BPWriteReadLocal1DAllSteps.BP5.Serial + Engine.BP.BPWriteReadLocalVariables.ADIOS2BPWriteReadLocal1DBlockInfo.BP5.Serial + Engine.BP.BPWriteReadLocalVariables.ADIOS2BPWriteReadLocal2DChangeCount.BP3.Serial + Engine.BP.BPWriteReadLocalVariables.ADIOS2BPWriteReadLocal2DChangeCount.BP4.Serial + Engine.BP.BPWriteReadLocalVariables.ADIOS2BPWriteReadLocal2DChangeCount.BP5.Serial + Engine.BP.BPWriteReadLocalVariablesSel.BPWriteReadLocal1DAllStepsSel.BP5.Serial + Engine.BP.BPWriteReadLocalVariablesSelHighLevel.BPWriteReadLocal1DAllStepsSel.BP5.Serial + Engine.BP.BPWriteReadMultiblockTest.ADIOS2BPWriteReadMultiblock1D8.BP5.Serial + Engine.BP.BPWriteReadMultiblockTest.ADIOS2BPWriteReadMultiblock2D2x4.BP5.Serial + Engine.BP.BPWriteReadMultiblockTest.ADIOS2BPWriteReadMultiblock2D4x2.BP5.Serial + Engine.BP.BPWriteReadMultiblockTest.MultiblockNullBlocks.BP5.Serial + Engine.BP.BPWriteReadMultiblockTest.MultiblockPerformDataWrite.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead10D2x2.Async.BP5.EWS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead10D2x2.Async.BP5.EWS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead10D2x2.Async.BP5.TLS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead10D2x2.Async.BP5.TLS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead10D2x2.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead1D8.Async.BP5.EWS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead1D8.Async.BP5.EWS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead1D8.Async.BP5.TLS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead1D8.Async.BP5.TLS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead1D8.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D2x4.Async.BP5.EWS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D2x4.Async.BP5.EWS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D2x4.Async.BP5.TLS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D2x4.Async.BP5.TLS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D2x4.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2_MultiStepsOverflow.Async.BP5.EWS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2_MultiStepsOverflow.Async.BP5.EWS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2_MultiStepsOverflow.Async.BP5.TLS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2_MultiStepsOverflow.Async.BP5.TLS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2_MultiStepsOverflow.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2_ReadMultiSteps.Async.BP5.EWS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2_ReadMultiSteps.Async.BP5.EWS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2_ReadMultiSteps.Async.BP5.TLS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2_ReadMultiSteps.Async.BP5.TLS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2_ReadMultiSteps.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2.Async.BP5.EWS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2.Async.BP5.EWS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2.Async.BP5.TLS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2.Async.BP5.TLS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.ADIOS2BPWriteRead2D4x2.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2.GetDeferredInClose.Async.BP5.EWS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.GetDeferredInClose.Async.BP5.EWS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.GetDeferredInClose.Async.BP5.TLS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.GetDeferredInClose.Async.BP5.TLS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.GetDeferredInClose.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2.ReadStartCount.Async.BP5.EWS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.ReadStartCount.Async.BP5.EWS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.ReadStartCount.Async.BP5.TLS.Guided.Serial + Engine.BP.BPWriteReadTestADIOS2.ReadStartCount.Async.BP5.TLS.Naive.Serial + Engine.BP.BPWriteReadTestADIOS2.ReadStartCount.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2fstream.ADIOS2BPWriteRead1D8.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2fstream.ADIOS2BPWriteRead2D2x4.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2fstream.ADIOS2BPWriteRead2D4x2_MultiStepsOverflow.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2fstream.ADIOS2BPWriteRead2D4x2_ReadMultiSteps.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2fstream.ADIOS2BPWriteRead2D4x2.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2stdio.ADIOS2BPWriteRead1D8.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2stdio.ADIOS2BPWriteRead2D2x4.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2stdio.ADIOS2BPWriteRead2D4x2_MultiStepsOverflow.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2stdio.ADIOS2BPWriteRead2D4x2_ReadMultiSteps.BP5.Serial + Engine.BP.BPWriteReadTestADIOS2stdio.ADIOS2BPWriteRead2D4x2.BP5.Serial + Engine.BP.BPWriteReadVector.ADIOS2BPWriteRead1D8.BP5.Serial + Engine.BP.BPWriteReadVector.ADIOS2BPWriteRead2D2x4.BP5.Serial + Engine.BP.BPWriteReadVector.ADIOS2BPWriteRead2D4x2.BP5.Serial + Engine.BP.BPWriteReadVector.ADIOS2BPWriteReadVector2D4x2_MultiSteps.BP5.Serial + Engine.BPEngineTest.ZfpComplex.Serial + Engine.SST.SstWriteFails.InvalidBeginStep.Serial + Engine.SST.SstWriteFails.InvalidPut.Serial + Engine.Staging.TestThreads.Basic.SST.BP.Serial + Engine.Staging.TestThreads.Basic.SST.FFS.Serial + Interface.ADIOS2_CXX11_API_Selection.MemorySelectionComplex.BPfile.Serial + Interface.ADIOS2_CXX11_API_Selection.MemorySelectionReadStart.BPfile.Serial + Remote.BPWriteMemorySelectionRead.FileRemote + Remote.BPWriteMemorySelectionRead.GetRemote + Remote.BPWriteReadADIOS2stdio.GetRemote + remoteServerCleanup + remoteServerSetup + Staging.1x1.Attrs.BP5 + Staging.1x1.Attrs.CommMin.BP.SST + Staging.1x1.Attrs.CommMin.BP5.SST + Staging.1x1.CommMin.BP.SST + Staging.1x1.CommMin.BP5.SST + Staging.1x1.ForcePreload.CommMin.BP.SST + Staging.1x1.ForcePreload.CommMin.BP5.SST + Staging.1x1.Local2.CommMin.BP.SST + Staging.1x1.Local2.CommMin.BP5.SST + Staging.1x1.LocalMultiblock.CommMin.BP.SST + Staging.1x1.LocalMultiblock.CommMin.BP5.SST + Staging.1x1.ModAttrs.BP5 + Staging.1x1.ModAttrs.CommMin.BP.SST + Staging.1x1.ModAttrs.CommMin.BP5.SST + Staging.1x1.NoPreload.CommMin.BP.SST + Staging.1x1.NoPreload.CommMin.BP5.SST + Staging.1x1.SstRUDP.CommMin.BP.SST + Staging.1x1.SstRUDP.CommMin.BP5.SST + Staging.1x1Joined.BP5 + Staging.1x1Joined.CommMin.BP5.SST + Staging.1x1LockGeometry.CommMin.BP.SST + Staging.1x1LockGeometry.CommMin.BP5.SST + Staging.1x1Struct.BP5 + Staging.1x1Struct.CommMin.BP5.SST + Staging.1x1VarDestruction.CommMin.BP.SST + Staging.1x1VarDestruction.CommMin.BP5.SST + Staging.AllToAllDistribution.1x1x3.CommMin.BP.SST + Staging.AllToAllDistribution.1x1x3.CommMin.BP5.SST + Staging.DiscardWriter.1x1.CommMin.BP.SST + Staging.DiscardWriter.1x1.CommMin.BP5.SST + Staging.LatestReader.1x1.CommMin.BP.SST + Staging.LatestReader.1x1.CommMin.BP5.SST + Staging.LatestReaderHold.1x1.CommMin.BP.SST + Staging.LatestReaderHold.1x1.CommMin.BP5.SST + Staging.OnDemandSingle.1x1.CommMin.BP.SST + Staging.OnDemandSingle.1x1.CommMin.BP5.SST + Staging.RoundRobinDistribution.1x1x3.CommMin.BP.SST + Staging.RoundRobinDistribution.1x1x3.CommMin.BP5.SST + Staging.TimeoutReader.1x1.CommMin.BP.SST + Staging.TimeoutReader.1x1.CommMin.BP5.SST + Staging.WriteMemorySelectionRead.1x1.CommMin.BP.SST + Staging.WriteMemorySelectionRead.1x1.CommMin.BP5.SST + Staging.ZFPCompression.1x1.CommMin.BP.SST + Staging.ZFPCompression.1x1.CommMin.BP5.SST + Staging.ZFPCompression.3x5.CommMin.BP.SST + Staging.ZFPCompression.3x5.CommMin.BP5.SST + Utils.ChangingShape.AlternatingStepsAndChangingShapeVar.Dump + Utils.ChangingShape.AlternatingStepsVarSelection.Dump + Utils.ChangingShape.ChangingShapeVarOneStep.Dump + Utils.ChangingShape.Dump + Utils.ChangingShape.FixedShapeVarTooManySteps.Dump + Utils.ChangingShape.Screen + Utils.CWriter + Utils.CWriter.Bpls.Al.Dump + Utils.CWriter.Bpls.h.Dump + Utils.CWriter.Bpls.la.Dump + Utils.CWriter.Bpls.la.Screen + Utils.CWriter.Bpls.ldDav.Dump + Utils.CWriter.Bpls.ldDavvv.Dump + Utils.CWriter.Bpls.ldvarI16.Dump +) diff --git a/bindings/C/adios2/c/adios2_c_adios.h b/bindings/C/adios2/c/adios2_c_adios.h index b5044b0447..147cb8ab6a 100644 --- a/bindings/C/adios2/c/adios2_c_adios.h +++ b/bindings/C/adios2/c/adios2_c_adios.h @@ -46,7 +46,7 @@ adios2_adios *adios2_init_config_mpi(const char *config_file, MPI_Comm comm); #else #define adios2_init() adios2_init_serial() -#define adios2_init_config(config_file) adios2_init_config_seria(config_file) +#define adios2_init_config(config_file) adios2_init_config_serial(config_file) #endif /** diff --git a/bindings/CXX11/adios2/cxx11/Variable.cpp b/bindings/CXX11/adios2/cxx11/Variable.cpp index cc1944fae9..d7d47c3428 100644 --- a/bindings/CXX11/adios2/cxx11/Variable.cpp +++ b/bindings/CXX11/adios2/cxx11/Variable.cpp @@ -78,6 +78,13 @@ namespace adios2 } \ \ template <> \ + void Variable::SetAccuracy(const adios2::Accuracy &a) \ + { \ + helper::CheckForNullptr(m_Variable, "in call to Variable::SetAccuracy"); \ + m_Variable->SetAccuracy(a); \ + } \ + \ + template <> \ size_t Variable::SelectionSize() const \ { \ helper::CheckForNullptr(m_Variable, "in call to Variable::SelectionSize"); \ @@ -219,6 +226,13 @@ namespace adios2 } \ \ template <> \ + adios2::Accuracy Variable::GetAccuracy() \ + { \ + helper::CheckForNullptr(m_Variable, "in call to Variable::GetAccuracy"); \ + return m_Variable->GetAccuracy(); \ + } \ + \ + template <> \ std::vector::Info>> Variable::AllStepsBlocksInfo() \ { \ return DoAllStepsBlocksInfo(); \ diff --git a/bindings/CXX11/adios2/cxx11/Variable.h b/bindings/CXX11/adios2/cxx11/Variable.h index 398d001667..4f8ea8f071 100644 --- a/bindings/CXX11/adios2/cxx11/Variable.h +++ b/bindings/CXX11/adios2/cxx11/Variable.h @@ -209,6 +209,12 @@ class Variable */ void SetStepSelection(const adios2::Box &stepSelection); + /** + * Sets the requested accuracy for the next read operation. + * The actual accuracy after the read is provided in GetAccuracy() + */ + void SetAccuracy(const adios2::Accuracy &a); + /** * Returns the number of elements required for pre-allocation based on * current count and stepsCount @@ -335,6 +341,13 @@ class Variable */ T Max(const size_t step = adios2::DefaultSizeT) const; + /** + * Get the provided accuracy for the last read operation. + * Most operations provide data as it was written, meaning that + * error is reported as 0.0 + */ + adios2::Accuracy GetAccuracy(); + /** Contains block information for a particular Variable */ struct Info { diff --git a/bindings/Matlab/Makefile b/bindings/Matlab/Makefile index 09978c32b8..aaaebf3a0e 100644 --- a/bindings/Matlab/Makefile +++ b/bindings/Matlab/Makefile @@ -18,6 +18,16 @@ MEXLIBS="LDFLAGS=${ADIOS_LIBS}" ADIOS_INC=-I${ADIOS_DIR}/include ADIOS_LIBS=`${ADIOS_DIR}/bin/adios2-config --c-libs` +### MacOS - example using homebrew installed ADIOS2 and Xcode 15 clang +### 1) Install homebrew (https://brew.sh/) and Xcode (App Store) +### 2) brew install adios2 +### OR +### 2) Compile Adios2 from scratch and update ADIOS_DIR below to match install directory +#ADIOS_DIR=/opt/homebrew/opt/adios2 +#ADIOS_INC=-I${ADIOS_DIR}/include +#ADIOS_LIBS=-Wl,-rpath,${ADIOS_DIR}/lib -shared -L${ADIOS_DIR}/lib -ladios2_c -ladios2_core +#MEXLIBS="LDFLAGS=${ADIOS_LIBS}" + MEXOPTS=-largeArrayDims -DDEBUG CFLAGS="-g -std=c99 -fPIC -O0" default: diff --git a/bindings/Matlab/adiosopenc.c b/bindings/Matlab/adiosopenc.c index 95dc6d41b3..07c4188d36 100644 --- a/bindings/Matlab/adiosopenc.c +++ b/bindings/Matlab/adiosopenc.c @@ -140,9 +140,9 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[]) /********************************************************/ /* Open ADIOS file now and get variables and attributes */ - adiosobj = adios2_init(false); + adiosobj = adios2_init_serial(); group = adios2_declare_io(adiosobj, "matlabiogroup"); // name is arbitrary - fp = adios2_open(group, fname, adios2_mode_read); + fp = adios2_open(group, fname, adios2_mode_readRandomAccess); if (fp == NULL) { mexErrMsgIdAndTxt("MATLAB:adiosopenc:open", "Opening the file failed\n"); diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index bbf253bb49..74a47f69e9 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -175,6 +175,7 @@ if(TARGET LibPressio::libpressio) endif() # MGARD +set(ADIOS2_HAVE_MGARD_MDR FALSE) if(ADIOS2_USE_MGARD STREQUAL AUTO) find_package(mgard CONFIG) elseif(ADIOS2_USE_MGARD) @@ -182,6 +183,9 @@ elseif(ADIOS2_USE_MGARD) endif() if(mgard_FOUND) set(ADIOS2_HAVE_MGARD TRUE) + if(MGARD_ENABLE_MDR) + set(ADIOS2_HAVE_MGARD_MDR TRUE) + endif() endif() # PNG @@ -450,6 +454,14 @@ if(ADIOS2_USE_SST AND NOT WIN32) if(CrayDRC_FOUND) set(ADIOS2_SST_HAVE_CRAY_DRC TRUE) endif() + + try_compile(ADIOS2_SST_HAVE_CRAY_CXI + ${ADIOS2_BINARY_DIR}/check_libfabric_cxi + ${ADIOS2_SOURCE_DIR}/cmake/check_libfabric_cxi.c + CMAKE_FLAGS + "-DINCLUDE_DIRECTORIES=${LIBFABRIC_INCLUDE_DIRS}" + "-DLINK_DIRECTORIES=${LIBFABRIC_LIBRARIES}") + message(STATUS "Libfabric support for the HPE CXI provider: ${ADIOS2_SST_HAVE_CRAY_CXI}") endif() if(ADIOS2_HAVE_MPI) set(CMAKE_REQUIRED_LIBRARIES "MPI::MPI_C;Threads::Threads") diff --git a/cmake/check_libfabric_cxi.c b/cmake/check_libfabric_cxi.c new file mode 100644 index 0000000000..c2f2c80878 --- /dev/null +++ b/cmake/check_libfabric_cxi.c @@ -0,0 +1,5 @@ +#include +#include +#include + +int main() {} diff --git a/docs/user_guide/source/components/anatomy.rst b/docs/user_guide/source/components/anatomy.rst new file mode 100644 index 0000000000..0832b6dc3c --- /dev/null +++ b/docs/user_guide/source/components/anatomy.rst @@ -0,0 +1,136 @@ +.. _sec:basics_interface_components_anatomy: + +*************************** +Anatomy of an ADIOS Program +*************************** + +Anatomy of an ADIOS Output +-------------------------- + +.. code:: C++ + + ADIOS adios("config.xml", MPI_COMM_WORLD); + | + | IO io = adios.DeclareIO(...); + | | + | | Variable<...> var = io.DefineVariable<...>(...) + | | Attribute<...> attr = io.DefineAttribute<...>(...) + | | Engine e = io.Open("OutputFileName.bp", adios2::Mode::Write); + | | | + | | | e.BeginStep() + | | | | + | | | | e.Put(var, datapointer); + | | | | + | | | e.EndStep() + | | | + | | e.Close(); + | | + | |--> IO goes out of scope + | + |--> ADIOS goes out of scope or adios2_finalize() + + +The pseudo code above depicts the basic structure of performing output. The ``ADIOS`` object is necessary to hold all +other objects. It is initialized with an MPI communicator in a parallel program or without in a serial program. +Additionally, a config file (XML or YAML format) can be specified here to load runtime configuration. Only one ADIOS +object is needed throughout the entire application but you can create as many as you want (e.g. if you need to separate +IO objects using the same name in a program that reads similar input from an ensemble of multiple applications). + +The ``IO`` object is required to hold the variable and attribute definitions, and runtime options for a particular input +or output stream. The IO object has a name, which is used only to refer to runtime options in the configuration file. +One IO object can only be used in one output or input stream. The only exception where an IO object can be used twice is +one input stream plus one output stream where the output is reusing the variable definitions loaded during input. + +``Variable`` and ``Attribute`` definitions belong to one IO object, which means, they can only be used in one output. +You need to define new ones for other outputs. Just because a Variable is defined, it will not appear in the output +unless an associated Put() call provides the content. + +A stream is opened and closed once. The ``Engine`` object implements the data movement for the stream. It depends on the +runtime options of the IO object that what type of an engine is created in the Open() call. One output step is denoted +by a pair of BeginStep..EndStep block. + +An output step consist of variables and attributes. Variables are just definitions without content, so one must call a +Put() function to provide the application data pointer that contains the data content one wants to write out. Attributes +have their content in their definitions so there is no need for an extra call. + +Some rules: + +* Variables can be defined any time, before the corresponding Put() call +* Attributes can be defined any time before EndStep +* The following functions must be treated as Collective operations + + * ADIOS + * Open + * BeginStep + * EndStep + * Close + +.. note:: + + If there is only one output step, and we only want to write it to a file on disk, never stream it to other + application, then BeginStep and EndStep are not required but it does not make any difference if they are called. + +Anatomy of an ADIOS Input +------------------------- + +.. code:: C++ + + ADIOS adios("config.xml", MPI_COMM_WORLD); + | + | IO io = adios.DeclareIO(...); + | | + | | Engine e = io.Open("InputFileName.bp", adios2::Mode::Read); + | | | + | | | e.BeginStep() + | | | | + | | | | varlist = io.AvailableVariables(...) + | | | | Variable var = io.InquireVariable(...) + | | | | Attribute attr = io.InquireAttribute(...) + | | | | | + | | | | | e.Get(var, datapointer); + | | | | | + | | | | + | | | e.EndStep() + | | | + | | e.Close(); + | | + | |--> IO goes out of scope + | + |--> ADIOS goes out of scope or adios2_finalize() + +The difference between input and output is that while we have to define the variables and attributes for an output, we +have to retrieve the available variables in an input first as definitions (Variable and Attribute objects). + +If we know the particular variable (name and type) in the input stream, we can get the definition using +InquireVariable(). Generic tools that process any input must use other functions to retrieve the list of variable names +and their types first and then get the individual Variable objects. The same is true for Attributes. + +Anatomy of an ADIOS File-only Input +----------------------------------- + +Previously we explored how to read using the input mode `adios2::Mode::Read`. Nonetheless, ADIOS has another input mode +named `adios2::Mode::ReadRandomAccess`. `adios2::Mode::Read` mode allows data access only timestep by timestep using +`BeginStep/EndStep`, but generally it is more memory efficient as ADIOS is only required to load metadata for the +current timestep. `ReadRandomAccess` can only be used with file engines and involves loading all the file metadata at +once. So it can be more memory intensive than `adios2::Mode::Read` mode, but allows reading data from any timestep using +`SetStepSelection()`. If you use `adios2::Mode::ReadRandomAccess` mode, be sure to allocate enough memory to hold +multiple steps of the variable content. + +.. code:: C++ + + ADIOS adios("config.xml", MPI_COMM_WORLD); + | + | IO io = adios.DeclareIO(...); + | | + | | Engine e = io.Open("InputFileName.bp", adios2::Mode::ReadRandomAccess); + | | | + | | | Variable var = io.InquireVariable(...) + | | | | var.SetStepSelection() + | | | | e.Get(var, datapointer); + | | | | + | | | + | | e.Close(); + | | + | |--> IO goes out of scope + | + |--> ADIOS goes out of scope or adios2_finalize() diff --git a/docs/user_guide/source/components/components.rst b/docs/user_guide/source/components/components.rst index 97b4f0b277..791a149a0f 100644 --- a/docs/user_guide/source/components/components.rst +++ b/docs/user_guide/source/components/components.rst @@ -10,3 +10,4 @@ Interface Components .. include:: engine.rst .. include:: operator.rst .. include:: runtime.rst +.. include:: anatomy.rst diff --git a/docs/user_guide/source/components/variable.rst b/docs/user_guide/source/components/variable.rst index baf071defb..69adbc884d 100644 --- a/docs/user_guide/source/components/variable.rst +++ b/docs/user_guide/source/components/variable.rst @@ -6,15 +6,17 @@ An ``adios2::Variable`` is the link between a piece of data coming from an appli This component handles all application variables classified by data type and shape. Each ``IO`` holds a set of Variables, and each ``Variable`` is identified with a unique name. -They are created using the reference from ``IO::DefineVariable`` or retrieved using the pointer from ``IO::InquireVariable`` functions in :ref:`IO`. +They are created using the reference from ``IO::DefineVariable`` or retrieved using the pointer from +``IO::InquireVariable`` functions in :ref:`IO`. Data Types --------------------- +---------- Only primitive types are supported in ADIOS2. -Fixed-width types from ` and `_ should be preferred when writing portable code. -ADIOS2 maps primitive types to equivalent fixed-width types (e.g. ``int`` -> ``int32_t``). -In C++, acceptable types ``T`` in ``Variable`` along with their preferred fix-width equivalent in 64-bit platforms are given below: +Fixed-width types from ` and `_ should be +preferred when writing portable code. ADIOS2 maps primitive types to equivalent fixed-width types +(e.g. ``int`` -> ``int32_t``). In C++, acceptable types ``T`` in ``Variable`` along with their preferred fix-width +equivalent in 64-bit platforms are given below: .. code-block:: c++ @@ -52,19 +54,19 @@ In C++, acceptable types ``T`` in ``Variable`` along with their preferred fix Python APIs: use the equivalent fixed-width types from numpy. If ``dtype`` is not specified, ADIOS2 handles numpy defaults just fine as long as primitive types are passed. - Shapes ---------------------- +------ ADIOS2 is designed for MPI applications. Thus different application data shapes must be supported depending on their scope within a particular MPI communicator. -The shape is defined at creation from the ``IO`` object by providing the dimensions: shape, start, count in the ``IO::DefineVariable``. -The supported shapes are described below. +The shape is defined at creation from the ``IO`` object by providing the dimensions: shape, start, count in the +``IO::DefineVariable``. The supported shapes are described below. 1. **Global Single Value**: Only a name is required for their definition. -These variables are helpful for storing global information, preferably managed by only one MPI process, that may or may not change over steps: *e.g.* total number of particles, collective norm, number of nodes/cells, etc. +These variables are helpful for storing global information, preferably managed by only one MPI process, that may or may +not change over steps: *e.g.* total number of particles, collective norm, number of nodes/cells, etc. .. code-block:: c++ @@ -157,8 +159,80 @@ be applicable to it. JoinedArrays are currently only supported by BP4 and BP5 engines, as well as the SST engine with BP5 marshalling. - +Global Array Capabilities and Limitations +----------------------------------------- + +ADIOS2 is focusing on writing and reading N-dimensional, distributed, global arrays of primitive types. The basic idea +is that, usually, a simulation has such a data structure in memory (distributed across multiple processes) and wants to +dump its content regularly as it progresses. ADIOS2 was designed to: + +1. to do this writing and reading as fast as possible +2. to enable reading any subsection of the array + +.. image:: https://imgur.com/6nX67yq.png + :width: 400 + +The figure above shows a parallel application of 12 processes producing a 2D array. Each process has a 2D array locally +and the output is created by placing them into a 4x3 pattern. A reading application's individual process then can read +any subsection of the entire global array. In the figure, a 6 process application decomposes the array in a 3x2 pattern +and each process reads a 2D array whose content comes from multiple producer processes. + +The figure hopefully helps to understand the basic concept but it can be also misleading if it suggests limitations that +are not there. Global Array is simply a boundary in N-dimensional space where processes can place their blocks of data. +In the global space: + +1. one process can place multiple blocks + + .. image:: https://imgur.com/Pb1s03h.png + :width: 400 + +2. does NOT need to be fully covered by the blocks + + .. image:: https://imgur.com/qJBXYcQ.png + :width: 400 + + * at reading, unfilled positions will not change the allocated memory + +3. blocks can overlap + + .. image:: https://imgur.com/GA59lZ2.png + :width: 300 + * the reader will get values in an overlapping position from one of the block but there is no control over from which + block +4. each process can put a different size of block, or put multiple blocks of different sizes + +5. some process may not contribute anything to the global array + +Over multiple output steps + +1. the processes CAN change the size (and number) of blocks in the array + + * E.g. atom table: global size is fixed but atoms wander around processes, so their block size is changing + + .. image:: https://imgur.com/DorjG2q.png + :width: 400 + +2. the global dimensions CAN change over output steps + + * but then you cannot read multiple steps at once + * E.g. particle table size changes due to particles disappearing or appearing + + .. image:: https://imgur.com/nkuHeVX.png + :width: 400 + + +Limitations of the ADIOS global array concept + +1. Indexing starts from 0 +2. Cyclic data patterns are not supported; only blocks can be written or read +3. If Some blocks may fully or partially fall outside of the global boundary, the reader will not be able to read those + parts + +.. note:: + Technically, the content of the individual blocks is kept in the BP format (but not in HDF5 format) and in staging. + If you really, really want to retrieve all the blocks, you need to handle this array as a Local Array and read the + blocks one by one. diff --git a/examples/basics/globalArrayND/globalArrayNDWrite.cpp b/examples/basics/globalArrayND/globalArrayNDWrite.cpp index b1509b7b54..7658a58843 100644 --- a/examples/basics/globalArrayND/globalArrayNDWrite.cpp +++ b/examples/basics/globalArrayND/globalArrayNDWrite.cpp @@ -49,7 +49,7 @@ int main(int argc, char *argv[]) MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nproc); #endif - const int NSTEPS = 5; + const int NSTEPS = 1; #if ADIOS2_USE_MPI adios2::ADIOS adios(MPI_COMM_WORLD); @@ -58,7 +58,7 @@ int main(int argc, char *argv[]) #endif // Application variables for output - const unsigned int Nx = 10; + const unsigned int Nx = 100000; // Global 2D array, size of nproc x Nx, with 1D decomposition // Each process writes one "row" of the 2D matrix. std::vector row(Nx); @@ -84,6 +84,9 @@ int main(int argc, char *argv[]) io.DefineAttribute("nsteps", NSTEPS); + adios2::Operator op = adios.DefineOperator("mdr", "mdr"); + varGlobalArray.AddOperation(op, {{"accuracy", std::to_string(0.1)}}); + // Open file. "w" means we overwrite any existing file on disk, // but Advance() will append steps to the same file. adios2::Engine writer = io.Open("globalArray.bp", adios2::Mode::Write); diff --git a/examples/hello/datamanKokkos/CMakeLists.txt b/examples/hello/datamanKokkos/CMakeLists.txt index e0d93d9fb8..9bbe1a2a3e 100644 --- a/examples/hello/datamanKokkos/CMakeLists.txt +++ b/examples/hello/datamanKokkos/CMakeLists.txt @@ -19,7 +19,16 @@ if(NOT TARGET adios2_core) find_package(ZeroMQ 4.1 QUIET) + find_package(Kokkos 3.7 QUIET) + if(Kokkos_FOUND AND DEFINED Kokkos_CXX_COMPILER) + set(CMAKE_CXX_COMPILER "${Kokkos_CXX_COMPILER}") + endif() + find_package(ADIOS2 REQUIRED COMPONENTS ${_components}) +else() + if(DEFINED Kokkos_CXX_COMPILER) + set(CMAKE_CXX_COMPILER "${Kokkos_CXX_COMPILER}") + endif() endif() if(ADIOS2_HAVE_MPI AND ADIOS2_HAVE_DataMan) diff --git a/examples/simulations/gray-scott-struct/CMakeLists.txt b/examples/simulations/gray-scott-struct/CMakeLists.txt index 8538e6bd07..5d67c49447 100644 --- a/examples/simulations/gray-scott-struct/CMakeLists.txt +++ b/examples/simulations/gray-scott-struct/CMakeLists.txt @@ -48,23 +48,4 @@ if(ADIOS2_HAVE_MPI) install(DIRECTORY "catalyst" DESTINATION ${CMAKE_INSTALL_PREFIX}/share/adios2/gray-scott-struct) install(PROGRAMS "cleanup.sh" DESTINATION ${CMAKE_INSTALL_PREFIX}/share/adios2/gray-scott-struct) - - find_package(VTK QUIET) - if(VTK_FOUND) - add_executable(adios2_simulations_gray-scott-struct_curvature analysis/curvature.cpp) - target_link_libraries(adios2_simulations_gray-scott-struct_curvature adios2::cxx11_mpi MPI::MPI_C ${VTK_LIBRARIES}) - install(TARGETS adios2_simulations_gray-scott-struct_curvature RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) - - add_executable(adios2_simulations_gray-scott-struct_find_blobs analysis/find_blobs.cpp) - target_link_libraries(adios2_simulations_gray-scott-struct_find_blobs adios2::cxx11_mpi MPI::MPI_C ${VTK_LIBRARIES}) - install(TARGETS adios2_simulations_gray-scott-struct_find_blobs RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) - - add_executable(adios2_simulations_gray-scott-struct_isosurface analysis/isosurface.cpp) - target_link_libraries(adios2_simulations_gray-scott-struct_isosurface adios2::cxx11_mpi MPI::MPI_C ${VTK_LIBRARIES}) - install(TARGETS adios2_simulations_gray-scott-struct_isosurface RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) - - # add_executable(adios2_simulations_gray-scott-struct_render_isosurface plot/render_isosurface.cpp) - # target_link_libraries(adios2_simulations_gray-scott_struct_render-isosurface adios2::cxx11_mpi MPI::MPI_C ${VTK_LIBRARIES}) - # install(TARGETS adios2_simulations_gray-scott-struct_render_isosurface RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) - endif() endif() diff --git a/examples/simulations/gray-scott-struct/analysis/curvature.cpp b/examples/simulations/gray-scott-struct/analysis/curvature.cpp deleted file mode 100644 index 586e2fe7d2..0000000000 --- a/examples/simulations/gray-scott-struct/analysis/curvature.cpp +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * Analysis code for the Gray-Scott simulation. - * Computes mean curvature at each point on an isosurface. - * - * Keichi Takahashi - * - */ - -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../common/timer.hpp" - -vtkSmartPointer read_mesh(const std::vector &bufPoints, - const std::vector &bufCells, - const std::vector &bufNormals) -{ - int nPoints = static_cast(bufPoints.size() / 3); - int nCells = static_cast(bufCells.size() / 3); - - auto points = vtkSmartPointer::New(); - points->SetNumberOfPoints(nPoints); - for (vtkIdType i = 0; i < nPoints; i++) - { - points->SetPoint(i, &bufPoints[i * 3]); - } - - auto polys = vtkSmartPointer::New(); - for (vtkIdType i = 0; i < nCells; i++) - { - vtkIdType a = bufCells[i * 3 + 0]; - vtkIdType b = bufCells[i * 3 + 1]; - vtkIdType c = bufCells[i * 3 + 2]; - - polys->InsertNextCell(3); - polys->InsertCellPoint(a); - polys->InsertCellPoint(b); - polys->InsertCellPoint(c); - } - - auto normals = vtkSmartPointer::New(); - normals->SetNumberOfComponents(3); - for (vtkIdType i = 0; i < nPoints; i++) - { - normals->InsertNextTuple(&bufNormals[i * 3]); - } - - auto polyData = vtkSmartPointer::New(); - polyData->SetPoints(points); - polyData->SetPolys(polys); - polyData->GetPointData()->SetNormals(normals); - - return polyData; -} - -void compute_curvature(const vtkSmartPointer polyData) -{ - vtkSmartPointer curvaturesFilter = vtkSmartPointer::New(); - curvaturesFilter->SetInputData(polyData); - // curvaturesFilter->SetCurvatureTypeToMinimum(); - // curvaturesFilter->SetCurvatureTypeToMaximum(); - // curvaturesFilter->SetCurvatureTypeToGaussian(); - curvaturesFilter->SetCurvatureTypeToMean(); - curvaturesFilter->Update(); -} - -int main(int argc, char *argv[]) -{ - int provided; - MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); - - int rank, procs, wrank; - - MPI_Comm_rank(MPI_COMM_WORLD, &wrank); - - const unsigned int color = 7; - MPI_Comm comm; - MPI_Comm_split(MPI_COMM_WORLD, color, wrank, &comm); - - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &procs); - - if (argc < 3) - { - if (rank == 0) - { - std::cerr << "Too few arguments" << std::endl; - std::cout << "Usage: curvature input output" << std::endl; - } - MPI_Abort(MPI_COMM_WORLD, -1); - } - - const std::string input_fname(argv[1]); - const std::string output_fname(argv[2]); - - adios2::ADIOS adios("adios2.xml", comm); - - adios2::IO inIO = adios.DeclareIO("IsosurfaceOutput"); - adios2::Engine reader = inIO.Open(input_fname, adios2::Mode::Read); - - adios2::IO outIO = adios.DeclareIO("CurvatureOutput"); - adios2::Engine writer = outIO.Open(output_fname, adios2::Mode::Write); - - std::vector points; - std::vector cells; - std::vector normals; - int step; - -#ifdef ENABLE_TIMERS - Timer timer_total; - Timer timer_read; - Timer timer_compute; - - std::ostringstream log_fname; - log_fname << "compute_curvature_pe_" << rank << ".log"; - - std::ofstream log(log_fname.str()); - log << "step\ttotal_curv\tread_curv\tcompute_curv" << std::endl; -#endif - - while (true) - { -#ifdef ENABLE_TIMERS - MPI_Barrier(comm); - timer_total.start(); - timer_read.start(); -#endif - - adios2::StepStatus status = reader.BeginStep(); - - if (status != adios2::StepStatus::OK) - { - break; - } - - auto varPoint = inIO.InquireVariable("point"); - auto varCell = inIO.InquireVariable("cell"); - auto varNormal = inIO.InquireVariable("normal"); - auto varStep = inIO.InquireVariable("step"); - - if (varPoint.Shape().size() > 0 || varCell.Shape().size() > 0) - { - varPoint.SetSelection({{0, 0}, {varPoint.Shape()[0], varPoint.Shape()[1]}}); - varCell.SetSelection({{0, 0}, {varCell.Shape()[0], varCell.Shape()[1]}}); - varNormal.SetSelection({{0, 0}, {varNormal.Shape()[0], varNormal.Shape()[1]}}); - - reader.Get(varPoint, points); - reader.Get(varCell, cells); - reader.Get(varNormal, normals); - } - - reader.Get(varStep, &step); - - reader.EndStep(); - -#ifdef ENABLE_TIMERS - double time_read = timer_read.stop(); - MPI_Barrier(comm); - timer_compute.start(); -#endif - - vtkSmartPointer polyData = read_mesh(points, cells, normals); - compute_curvature(polyData); - - if (!rank) - { - std::cout << "compute_curvature at step " << step << std::endl; - } - -#ifdef ENABLE_TIMERS - double time_compute = timer_compute.stop(); - double time_step = timer_total.stop(); - MPI_Barrier(comm); - - log << step << "\t" << time_step << "\t" << time_read << "\t" << time_compute << std::endl; -#endif - } - -#ifdef ENABLE_TIMERS - log << "total\t" << timer_total.elapsed() << "\t" << timer_read.elapsed() << "\t" - << timer_compute.elapsed() << "\t" << std::endl; - - log.close(); -#endif - - writer.Close(); - reader.Close(); -} diff --git a/examples/simulations/gray-scott-struct/analysis/find_blobs.cpp b/examples/simulations/gray-scott-struct/analysis/find_blobs.cpp deleted file mode 100644 index 8b5c8c54b2..0000000000 --- a/examples/simulations/gray-scott-struct/analysis/find_blobs.cpp +++ /dev/null @@ -1,245 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * Analysis code for the Gray-Scott simulation. - * Reads iso-surface mesh data and detects connected components. - * Counts the total number of connected components and measures the surface - * area of each component. - * - * Keichi Takahashi - * - */ - -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../common/timer.hpp" - -vtkSmartPointer read_mesh(const std::vector &bufPoints, - const std::vector &bufCells, - const std::vector &bufNormals) -{ - int nPoints = static_cast(bufPoints.size() / 3); - int nCells = static_cast(bufCells.size() / 3); - - auto points = vtkSmartPointer::New(); - points->SetNumberOfPoints(nPoints); - for (vtkIdType i = 0; i < nPoints; i++) - { - points->SetPoint(i, &bufPoints[i * 3]); - } - - auto polys = vtkSmartPointer::New(); - for (vtkIdType i = 0; i < nCells; i++) - { - vtkIdType a = bufCells[i * 3 + 0]; - vtkIdType b = bufCells[i * 3 + 1]; - vtkIdType c = bufCells[i * 3 + 2]; - - polys->InsertNextCell(3); - polys->InsertCellPoint(a); - polys->InsertCellPoint(b); - polys->InsertCellPoint(c); - } - - auto normals = vtkSmartPointer::New(); - normals->SetNumberOfComponents(3); - for (vtkIdType i = 0; i < nPoints; i++) - { - normals->InsertNextTuple(&bufNormals[i * 3]); - } - - auto polyData = vtkSmartPointer::New(); - polyData->SetPoints(points); - polyData->SetPolys(polys); - polyData->GetPointData()->SetNormals(normals); - - return polyData; -} - -void find_blobs(const vtkSmartPointer polyData) -{ - auto connectivityFilter = vtkSmartPointer::New(); - connectivityFilter->SetInputData(polyData); - connectivityFilter->SetExtractionModeToAllRegions(); - connectivityFilter->ColorRegionsOn(); - connectivityFilter->Update(); - - int nBlobs = connectivityFilter->GetNumberOfExtractedRegions(); - - std::cout << "Found " << nBlobs << " blobs" << std::endl; - - auto threshold = vtkSmartPointer::New(); - auto massProperties = vtkSmartPointer::New(); - auto surfaceFilter = vtkSmartPointer::New(); - - threshold->SetInputConnection(connectivityFilter->GetOutputPort()); - surfaceFilter->SetInputConnection(threshold->GetOutputPort()); - massProperties->SetInputConnection(surfaceFilter->GetOutputPort()); - - for (int i = 0; i < nBlobs; i++) - { - threshold->SetThresholdFunction(vtkThreshold::THRESHOLD_BETWEEN); - threshold->SetLowerThreshold(i); - threshold->SetUpperThreshold(i); - - std::cout << "Surface area of blob #" << i << " is " << massProperties->GetSurfaceArea() - << std::endl; - } -} - -void find_largest_blob(const vtkSmartPointer polyData) -{ - auto connectivityFilter = vtkSmartPointer::New(); - connectivityFilter->SetInputData(polyData); - connectivityFilter->SetExtractionModeToLargestRegion(); - connectivityFilter->Update(); - - auto massProperties = vtkSmartPointer::New(); - auto surfaceFilter = vtkSmartPointer::New(); - - surfaceFilter->SetInputConnection(connectivityFilter->GetOutputPort()); - massProperties->SetInputConnection(surfaceFilter->GetOutputPort()); - - std::cout << "Surface area of largest blob is " << massProperties->GetSurfaceArea() - << std::endl; -} - -int main(int argc, char *argv[]) -{ - int provided; - MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); - - int rank, procs, wrank; - - MPI_Comm_rank(MPI_COMM_WORLD, &wrank); - - const unsigned int color = 6; - MPI_Comm comm; - MPI_Comm_split(MPI_COMM_WORLD, color, wrank, &comm); - - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &procs); - - if (argc < 2) - { - if (rank == 0) - { - std::cerr << "Too few arguments" << std::endl; - std::cout << "Usage: find_blobs input" << std::endl; - } - MPI_Abort(MPI_COMM_WORLD, -1); - } - - if (procs != 1) - { - if (rank == 0) - { - std::cerr << "find_blobs only supports serial execution" << std::endl; - } - MPI_Abort(MPI_COMM_WORLD, -1); - } - - const std::string input_fname(argv[1]); - - adios2::ADIOS adios("adios2.xml", comm); - - adios2::IO inIO = adios.DeclareIO("IsosurfaceOutput"); - adios2::Engine reader = inIO.Open(input_fname, adios2::Mode::Read); - - std::vector points; - std::vector cells; - std::vector normals; - int step; - -#ifdef ENABLE_TIMERS - Timer timer_total; - Timer timer_compute; - Timer timer_read; - - std::ostringstream log_fname; - log_fname << "find_blobs_pe_" << rank << ".log"; - - std::ofstream log(log_fname.str()); - log << "step\ttotal_blobs\tread_blobs\tcompute_blobs" << std::endl; -#endif - - while (true) - { -#ifdef ENABLE_TIMERS - MPI_Barrier(comm); - timer_total.start(); - timer_read.start(); -#endif - - adios2::StepStatus status = reader.BeginStep(); - - if (status != adios2::StepStatus::OK) - { - break; - } - - auto varPoint = inIO.InquireVariable("point"); - auto varCell = inIO.InquireVariable("cell"); - auto varNormal = inIO.InquireVariable("normal"); - auto varStep = inIO.InquireVariable("step"); - - if (varPoint.Shape().size() > 0 || varCell.Shape().size() > 0) - { - varPoint.SetSelection({{0, 0}, {varPoint.Shape()[0], varPoint.Shape()[1]}}); - varCell.SetSelection({{0, 0}, {varCell.Shape()[0], varCell.Shape()[1]}}); - varNormal.SetSelection({{0, 0}, {varNormal.Shape()[0], varNormal.Shape()[1]}}); - - reader.Get(varPoint, points); - reader.Get(varCell, cells); - reader.Get(varNormal, normals); - } - - reader.Get(varStep, &step); - - reader.EndStep(); - -#ifdef ENABLE_TIMERS - double time_read = timer_read.stop(); - MPI_Barrier(comm); - timer_compute.start(); -#endif - - std::cout << "find_blobs at step " << step << std::endl; - - auto polyData = read_mesh(points, cells, normals); - // find_blobs(polyData); - find_largest_blob(polyData); - -#ifdef ENABLE_TIMERS - double time_compute = timer_compute.stop(); - double time_step = timer_total.stop(); - MPI_Barrier(comm); - - log << step << "\t" << time_step << "\t" << time_read << "\t" << time_compute << std::endl; -#endif - } - -#ifdef ENABLE_TIMERS - log << "total\t" << timer_total.elapsed() << "\t" << timer_read.elapsed() << "\t" - << timer_compute.elapsed() << std::endl; - - log.close(); -#endif - - reader.Close(); -} diff --git a/examples/simulations/gray-scott-struct/analysis/isosurface.cpp b/examples/simulations/gray-scott-struct/analysis/isosurface.cpp deleted file mode 100644 index e02a507aaa..0000000000 --- a/examples/simulations/gray-scott-struct/analysis/isosurface.cpp +++ /dev/null @@ -1,346 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * Analysis code for the Gray-Scott simulation. - * Reads variable U and and extracts the iso-surface using VTK. - * Writes the extracted iso-surface using ADIOS. - * - * Keichi Takahashi - * - */ - -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../common/timer.hpp" - -vtkSmartPointer compute_isosurface(const adios2::Variable &varField, - const std::vector &field, double isovalue) -{ - // Convert field values to vtkImageData - auto importer = vtkSmartPointer::New(); - importer->SetDataSpacing(1, 1, 1); - importer->SetDataOrigin(static_cast(varField.Start()[2]), - static_cast(varField.Start()[1]), - static_cast(varField.Start()[0])); - importer->SetWholeExtent(0, static_cast(varField.Count()[2] - 1), 0, - static_cast(varField.Count()[1] - 1), 0, - static_cast(varField.Count()[0] - 1)); - importer->SetDataExtentToWholeExtent(); - importer->SetDataScalarTypeToDouble(); - importer->SetNumberOfScalarComponents(1); - importer->SetImportVoidPointer(const_cast(field.data())); - - // Run the marching cubes algorithm - auto mcubes = vtkSmartPointer::New(); - mcubes->SetInputConnection(importer->GetOutputPort()); - mcubes->ComputeNormalsOn(); - mcubes->SetValue(0, isovalue); - mcubes->Update(); - - // Return the isosurface as vtkPolyData - return mcubes->GetOutput(); -} - -void write_vtk(const std::string &fname, const vtkSmartPointer polyData) -{ - auto writer = vtkSmartPointer::New(); - writer->SetFileName(fname.c_str()); - writer->SetInputData(polyData); - writer->Write(); -} - -void write_adios(adios2::Engine &writer, const vtkSmartPointer polyData, - adios2::Variable &varPoint, adios2::Variable &varCell, - adios2::Variable &varNormal, adios2::Variable &varOutStep, int step, - MPI_Comm comm) -{ - int numCells = static_cast(polyData->GetNumberOfPolys()); - int numPoints = static_cast(polyData->GetNumberOfPoints()); - int rank; - - MPI_Comm_rank(comm, &rank); - - std::vector points(static_cast(numPoints * 3)); - std::vector normals(static_cast(numPoints * 3)); - std::vector cells(static_cast(numCells * 3)); // Assumes that cells are triangles - - double coords[3]; - - auto cellArray = polyData->GetPolys(); - - cellArray->InitTraversal(); - - // Iterate through cells - for (vtkIdType i = 0; i < polyData->GetNumberOfPolys(); i++) - { - auto idList = vtkSmartPointer::New(); - - cellArray->GetNextCell(idList); - - // Iterate through points of a cell - for (vtkIdType j = 0; j < idList->GetNumberOfIds(); j++) - { - auto id = idList->GetId(j); - - cells[i * 3 + j] = static_cast(id); - - polyData->GetPoint(id, coords); - - points[id * 3 + 0] = coords[0]; - points[id * 3 + 1] = coords[1]; - points[id * 3 + 2] = coords[2]; - } - } - - auto normalArray = polyData->GetPointData()->GetNormals(); - - // Extract normals - for (int i = 0; i < normalArray->GetNumberOfTuples(); i++) - { - normalArray->GetTuple(i, coords); - - normals[i * 3 + 0] = coords[0]; - normals[i * 3 + 1] = coords[1]; - normals[i * 3 + 2] = coords[2]; - } - - int totalPoints, offsetPoints; - MPI_Allreduce(&numPoints, &totalPoints, 1, MPI_INT, MPI_SUM, comm); - MPI_Scan(&numPoints, &offsetPoints, 1, MPI_INT, MPI_SUM, comm); - - writer.BeginStep(); - - varPoint.SetShape( - {static_cast(totalPoints), static_cast(totalPoints > 0 ? 3 : 0)}); - varPoint.SetSelection( - {{static_cast(offsetPoints - numPoints), 0}, - {static_cast(numPoints), static_cast(numPoints > 0 ? 3 : 0)}}); - - varNormal.SetShape(varPoint.Shape()); - varNormal.SetSelection({varPoint.Start(), varPoint.Count()}); - - if (numPoints) - { - writer.Put(varPoint, points.data()); - writer.Put(varNormal, normals.data()); - } - - int totalCells, offsetCells; - MPI_Allreduce(&numCells, &totalCells, 1, MPI_INT, MPI_SUM, comm); - MPI_Scan(&numCells, &offsetCells, 1, MPI_INT, MPI_SUM, comm); - - for (int &cell : cells) - { - cell += (offsetPoints - numPoints); - } - - varCell.SetShape( - {static_cast(totalCells), static_cast(totalCells > 0 ? 3 : 0)}); - varCell.SetSelection( - {{static_cast(offsetCells - numCells), 0}, - {static_cast(numCells), static_cast(numCells > 0 ? 3 : 0)}}); - - if (numCells) - { - writer.Put(varCell, cells.data()); - } - - if (!rank) - { - std::cout << "isosurface at step " << step << " writing out " << totalCells << " cells and " - << totalPoints << " points" << std::endl; - } - - writer.Put(varOutStep, step); - - writer.EndStep(); -} - -int main(int argc, char *argv[]) -{ - int provided; - MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); - - int rank, procs, wrank; - - MPI_Comm_rank(MPI_COMM_WORLD, &wrank); - - const unsigned int color = 5; - MPI_Comm comm; - MPI_Comm_split(MPI_COMM_WORLD, color, wrank, &comm); - - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &procs); - - int dims[3] = {0}; - MPI_Dims_create(procs, 3, dims); - size_t npx = dims[0]; - size_t npy = dims[1]; - size_t npz = dims[2]; - - int coords[3] = {0}; - int periods[3] = {0}; - MPI_Comm cart_comm; - MPI_Cart_create(comm, 3, dims, periods, 0, &cart_comm); - MPI_Cart_coords(cart_comm, rank, 3, coords); - size_t px = coords[0]; - size_t py = coords[1]; - size_t pz = coords[2]; - - if (argc < 4) - { - if (rank == 0) - { - std::cerr << "Too few arguments" << std::endl; - std::cout << "Usage: isosurface input output isovalues..." << std::endl; - } - MPI_Abort(MPI_COMM_WORLD, -1); - } - - const std::string input_fname(argv[1]); - const std::string output_fname(argv[2]); - - std::vector isovalues; - for (int i = 3; i < argc; i++) - { - isovalues.push_back(std::stod(argv[i])); - } - - adios2::ADIOS adios("adios2.xml", comm); - - adios2::IO inIO = adios.DeclareIO("SimulationOutput"); - adios2::Engine reader = inIO.Open(input_fname, adios2::Mode::Read); - - adios2::IO outIO = adios.DeclareIO("IsosurfaceOutput"); - adios2::Engine writer = outIO.Open(output_fname, adios2::Mode::Write); - - auto varPoint = outIO.DefineVariable("point", {1, 3}, {0, 0}, {1, 3}); - auto varCell = outIO.DefineVariable("cell", {1, 3}, {0, 0}, {1, 3}); - auto varNormal = outIO.DefineVariable("normal", {1, 3}, {0, 0}, {1, 3}); - auto varOutStep = outIO.DefineVariable("step"); - - std::vector u; - int step; - -#ifdef ENABLE_TIMERS - Timer timer_total; - Timer timer_read; - Timer timer_compute; - Timer timer_write; - - std::ostringstream log_fname; - log_fname << "isosurface_pe_" << rank << ".log"; - - std::ofstream log(log_fname.str()); - log << "step\ttotal_iso\tread_iso\tcompute_iso\twrite_iso" << std::endl; -#endif - - while (true) - { -#ifdef ENABLE_TIMERS - MPI_Barrier(comm); - timer_total.start(); - timer_read.start(); -#endif - - adios2::StepStatus status = reader.BeginStep(); - - if (status != adios2::StepStatus::OK) - { - break; - } - - adios2::Variable varU = inIO.InquireVariable("U"); - const adios2::Variable varStep = inIO.InquireVariable("step"); - - adios2::Dims shape = varU.Shape(); - - size_t size_x = (shape[0] + npx - 1) / npx; - size_t size_y = (shape[1] + npy - 1) / npy; - size_t size_z = (shape[2] + npz - 1) / npz; - - size_t offset_x = size_x * px; - size_t offset_y = size_y * py; - size_t offset_z = size_z * pz; - - if (px == npx - 1) - { - size_x -= size_x * npx - shape[0]; - } - if (py == npy - 1) - { - size_y -= size_y * npy - shape[1]; - } - if (pz == npz - 1) - { - size_z -= size_z * npz - shape[2]; - } - - varU.SetSelection({{offset_x, offset_y, offset_z}, - {size_x + (px != npx - 1 ? 1 : 0), size_y + (py != npy - 1 ? 1 : 0), - size_z + (pz != npz - 1 ? 1 : 0)}}); - - reader.Get(varU, u); - reader.Get(varStep, step); - reader.EndStep(); - -#ifdef ENABLE_TIMERS - double time_read = timer_read.stop(); - MPI_Barrier(comm); - timer_compute.start(); -#endif - - auto appendFilter = vtkSmartPointer::New(); - - for (const auto isovalue : isovalues) - { - auto polyData = compute_isosurface(varU, u, isovalue); - appendFilter->AddInputData(polyData); - } - - appendFilter->Update(); - -#ifdef ENABLE_TIMERS - double time_compute = timer_compute.stop(); - MPI_Barrier(comm); - timer_write.start(); -#endif - - write_adios(writer, appendFilter->GetOutput(), varPoint, varCell, varNormal, varOutStep, - step, comm); - -#ifdef ENABLE_TIMERS - double time_write = timer_write.stop(); - double time_step = timer_total.stop(); - MPI_Barrier(comm); - - log << step << "\t" << time_step << "\t" << time_read << "\t" << time_compute << "\t" - << time_write << std::endl; -#endif - } - -#ifdef ENABLE_TIMERS - log << "total\t" << timer_total.elapsed() << "\t" << timer_read.elapsed() << "\t" - << timer_compute.elapsed() << "\t" << timer_write.elapsed() << std::endl; - - log.close(); -#endif - - writer.Close(); - reader.Close(); - - MPI_Finalize(); -} diff --git a/examples/simulations/gray-scott-struct/plot/render_isosurface.cpp b/examples/simulations/gray-scott-struct/plot/render_isosurface.cpp deleted file mode 100644 index 30a464b831..0000000000 --- a/examples/simulations/gray-scott-struct/plot/render_isosurface.cpp +++ /dev/null @@ -1,204 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * Visualization code for the Gray-Scott simulation. - * Reads and renders iso-surface mesh data. - * - * Keichi Takahashi - * - */ - -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -VTK_MODULE_INIT(vtkRenderingOpenGL2); - -typedef struct -{ - vtkRenderView *renderView; - vtkPolyDataMapper *mapper; - adios2::IO *inIO; - adios2::Engine *reader; -} Context; - -vtkSmartPointer read_mesh(const std::vector &bufPoints, - const std::vector &bufCells, - const std::vector &bufNormals) -{ - int nPoints = bufPoints.size() / 3; - int nCells = bufCells.size() / 3; - - auto points = vtkSmartPointer::New(); - points->SetNumberOfPoints(nPoints); - for (vtkIdType i = 0; i < nPoints; i++) - { - points->SetPoint(i, &bufPoints[i * 3]); - } - - auto polys = vtkSmartPointer::New(); - for (vtkIdType i = 0; i < nCells; i++) - { - vtkIdType a = bufCells[i * 3 + 0]; - vtkIdType b = bufCells[i * 3 + 1]; - vtkIdType c = bufCells[i * 3 + 2]; - - polys->InsertNextCell(3); - polys->InsertCellPoint(a); - polys->InsertCellPoint(b); - polys->InsertCellPoint(c); - } - - auto normals = vtkSmartPointer::New(); - normals->SetNumberOfComponents(3); - for (vtkIdType i = 0; i < nPoints; i++) - { - normals->InsertNextTuple(&bufNormals[i * 3]); - } - - auto polyData = vtkSmartPointer::New(); - polyData->SetPoints(points); - polyData->SetPolys(polys); - polyData->GetPointData()->SetNormals(normals); - - return polyData; -} - -void timer_func(vtkObject *object, unsigned long eid, void *clientdata, void *calldata) -{ - Context *context = static_cast(clientdata); - - std::vector points; - std::vector cells; - std::vector normals; - int step; - - adios2::StepStatus status = context->reader->BeginStep(); - - if (status != adios2::StepStatus::OK) - { - return; - } - - auto varPoint = context->inIO->InquireVariable("point"); - auto varCell = context->inIO->InquireVariable("cell"); - auto varNormal = context->inIO->InquireVariable("normal"); - auto varStep = context->inIO->InquireVariable("step"); - - if (varPoint.Shape().size() > 0 || varCell.Shape().size() > 0) - { - varPoint.SetSelection({{0, 0}, {varPoint.Shape()[0], varPoint.Shape()[1]}}); - varCell.SetSelection({{0, 0}, {varCell.Shape()[0], varCell.Shape()[1]}}); - varNormal.SetSelection({{0, 0}, {varNormal.Shape()[0], varNormal.Shape()[1]}}); - - context->reader->Get(varPoint, points); - context->reader->Get(varCell, cells); - context->reader->Get(varNormal, normals); - } - - context->reader->Get(varStep, &step); - - context->reader->EndStep(); - - std::cout << "render_isosurface at step " << step << std::endl; - - vtkSmartPointer polyData = read_mesh(points, cells, normals); - - context->mapper->SetInputData(polyData); - context->renderView->ResetCamera(); - context->renderView->Render(); -} - -int main(int argc, char *argv[]) -{ - MPI_Init(&argc, &argv); - - int rank, procs, wrank; - - MPI_Comm_rank(MPI_COMM_WORLD, &wrank); - - const unsigned int color = 7; - MPI_Comm comm; - MPI_Comm_split(MPI_COMM_WORLD, color, wrank, &comm); - - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &procs); - - if (argc < 2) - { - if (rank == 0) - { - std::cerr << "Too few arguments" << std::endl; - std::cout << "Usage: render_isosurface input" << std::endl; - } - MPI_Abort(MPI_COMM_WORLD, -1); - } - - if (procs != 1) - { - if (rank == 0) - { - std::cerr << "render_isosurface only supports serial execution" << std::endl; - } - MPI_Abort(MPI_COMM_WORLD, -1); - } - - const std::string input_fname(argv[1]); - - adios2::ADIOS adios("adios2.xml", comm); - - adios2::IO inIO = adios.DeclareIO("IsosurfaceOutput"); - adios2::Engine reader = inIO.Open(input_fname, adios2::Mode::Read); - - auto mapper = vtkSmartPointer::New(); - - auto actor = vtkSmartPointer::New(); - actor->SetMapper(mapper); - - auto renderView = vtkSmartPointer::New(); - renderView->GetRenderer()->AddActor(actor); - renderView->Update(); - - auto style = vtkSmartPointer::New(); - style->SetCurrentStyleToTrackballCamera(); - - auto interactor = renderView->GetInteractor(); - interactor->Initialize(); - interactor->SetInteractorStyle(style); - interactor->CreateRepeatingTimer(100); - - Context context = { - .renderView = renderView, - .mapper = mapper, - .inIO = &inIO, - .reader = &reader, - }; - - auto timerCallback = vtkSmartPointer::New(); - timerCallback->SetCallback(timer_func); - timerCallback->SetClientData(&context); - interactor->AddObserver(vtkCommand::TimerEvent, timerCallback); - - renderView->Render(); - interactor->Start(); - - reader.Close(); -} diff --git a/examples/simulations/gray-scott/CMakeLists.txt b/examples/simulations/gray-scott/CMakeLists.txt index e58acd4144..c39444698a 100644 --- a/examples/simulations/gray-scott/CMakeLists.txt +++ b/examples/simulations/gray-scott/CMakeLists.txt @@ -48,22 +48,4 @@ if(ADIOS2_HAVE_MPI) install(DIRECTORY "catalyst" DESTINATION ${CMAKE_INSTALL_PREFIX}/share/adios2/gray-scott) install(PROGRAMS "cleanup.sh" DESTINATION ${CMAKE_INSTALL_PREFIX}/share/adios2/gray-scott) - - find_package(VTK QUIET) - if(VTK_FOUND) - add_executable(adios2_simulations_gray-scott_curvature analysis/curvature.cpp) - target_link_libraries(adios2_simulations_gray-scott_curvature adios2::cxx11_mpi MPI::MPI_C ${VTK_LIBRARIES}) - install(TARGETS adios2_simulations_gray-scott_curvature RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) - - add_executable(adios2_simulations_gray-scott_find-blobs analysis/find_blobs.cpp) - target_link_libraries(adios2_simulations_gray-scott_find-blobs adios2::cxx11_mpi MPI::MPI_C ${VTK_LIBRARIES}) - install(TARGETS adios2_simulations_gray-scott_find-blobs RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) - - add_executable(adios2_simulations_gray-scott_isosurface analysis/isosurface.cpp) - target_link_libraries(adios2_simulations_gray-scott_isosurface adios2::cxx11_mpi MPI::MPI_C ${VTK_LIBRARIES}) - install(TARGETS adios2_simulations_gray-scott_isosurface RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) - - # add_executable(adios2_simulations_gray-scott_render-isosurface plot/render_isosurface.cpp) - # target_link_libraries(adios2_simulations_gray-scott_render-isosurface adios2::cxx11_mpi MPI::MPI_C ${VTK_LIBRARIES}) - endif() endif() diff --git a/examples/simulations/gray-scott/analysis/curvature.cpp b/examples/simulations/gray-scott/analysis/curvature.cpp deleted file mode 100644 index 586e2fe7d2..0000000000 --- a/examples/simulations/gray-scott/analysis/curvature.cpp +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * Analysis code for the Gray-Scott simulation. - * Computes mean curvature at each point on an isosurface. - * - * Keichi Takahashi - * - */ - -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../common/timer.hpp" - -vtkSmartPointer read_mesh(const std::vector &bufPoints, - const std::vector &bufCells, - const std::vector &bufNormals) -{ - int nPoints = static_cast(bufPoints.size() / 3); - int nCells = static_cast(bufCells.size() / 3); - - auto points = vtkSmartPointer::New(); - points->SetNumberOfPoints(nPoints); - for (vtkIdType i = 0; i < nPoints; i++) - { - points->SetPoint(i, &bufPoints[i * 3]); - } - - auto polys = vtkSmartPointer::New(); - for (vtkIdType i = 0; i < nCells; i++) - { - vtkIdType a = bufCells[i * 3 + 0]; - vtkIdType b = bufCells[i * 3 + 1]; - vtkIdType c = bufCells[i * 3 + 2]; - - polys->InsertNextCell(3); - polys->InsertCellPoint(a); - polys->InsertCellPoint(b); - polys->InsertCellPoint(c); - } - - auto normals = vtkSmartPointer::New(); - normals->SetNumberOfComponents(3); - for (vtkIdType i = 0; i < nPoints; i++) - { - normals->InsertNextTuple(&bufNormals[i * 3]); - } - - auto polyData = vtkSmartPointer::New(); - polyData->SetPoints(points); - polyData->SetPolys(polys); - polyData->GetPointData()->SetNormals(normals); - - return polyData; -} - -void compute_curvature(const vtkSmartPointer polyData) -{ - vtkSmartPointer curvaturesFilter = vtkSmartPointer::New(); - curvaturesFilter->SetInputData(polyData); - // curvaturesFilter->SetCurvatureTypeToMinimum(); - // curvaturesFilter->SetCurvatureTypeToMaximum(); - // curvaturesFilter->SetCurvatureTypeToGaussian(); - curvaturesFilter->SetCurvatureTypeToMean(); - curvaturesFilter->Update(); -} - -int main(int argc, char *argv[]) -{ - int provided; - MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); - - int rank, procs, wrank; - - MPI_Comm_rank(MPI_COMM_WORLD, &wrank); - - const unsigned int color = 7; - MPI_Comm comm; - MPI_Comm_split(MPI_COMM_WORLD, color, wrank, &comm); - - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &procs); - - if (argc < 3) - { - if (rank == 0) - { - std::cerr << "Too few arguments" << std::endl; - std::cout << "Usage: curvature input output" << std::endl; - } - MPI_Abort(MPI_COMM_WORLD, -1); - } - - const std::string input_fname(argv[1]); - const std::string output_fname(argv[2]); - - adios2::ADIOS adios("adios2.xml", comm); - - adios2::IO inIO = adios.DeclareIO("IsosurfaceOutput"); - adios2::Engine reader = inIO.Open(input_fname, adios2::Mode::Read); - - adios2::IO outIO = adios.DeclareIO("CurvatureOutput"); - adios2::Engine writer = outIO.Open(output_fname, adios2::Mode::Write); - - std::vector points; - std::vector cells; - std::vector normals; - int step; - -#ifdef ENABLE_TIMERS - Timer timer_total; - Timer timer_read; - Timer timer_compute; - - std::ostringstream log_fname; - log_fname << "compute_curvature_pe_" << rank << ".log"; - - std::ofstream log(log_fname.str()); - log << "step\ttotal_curv\tread_curv\tcompute_curv" << std::endl; -#endif - - while (true) - { -#ifdef ENABLE_TIMERS - MPI_Barrier(comm); - timer_total.start(); - timer_read.start(); -#endif - - adios2::StepStatus status = reader.BeginStep(); - - if (status != adios2::StepStatus::OK) - { - break; - } - - auto varPoint = inIO.InquireVariable("point"); - auto varCell = inIO.InquireVariable("cell"); - auto varNormal = inIO.InquireVariable("normal"); - auto varStep = inIO.InquireVariable("step"); - - if (varPoint.Shape().size() > 0 || varCell.Shape().size() > 0) - { - varPoint.SetSelection({{0, 0}, {varPoint.Shape()[0], varPoint.Shape()[1]}}); - varCell.SetSelection({{0, 0}, {varCell.Shape()[0], varCell.Shape()[1]}}); - varNormal.SetSelection({{0, 0}, {varNormal.Shape()[0], varNormal.Shape()[1]}}); - - reader.Get(varPoint, points); - reader.Get(varCell, cells); - reader.Get(varNormal, normals); - } - - reader.Get(varStep, &step); - - reader.EndStep(); - -#ifdef ENABLE_TIMERS - double time_read = timer_read.stop(); - MPI_Barrier(comm); - timer_compute.start(); -#endif - - vtkSmartPointer polyData = read_mesh(points, cells, normals); - compute_curvature(polyData); - - if (!rank) - { - std::cout << "compute_curvature at step " << step << std::endl; - } - -#ifdef ENABLE_TIMERS - double time_compute = timer_compute.stop(); - double time_step = timer_total.stop(); - MPI_Barrier(comm); - - log << step << "\t" << time_step << "\t" << time_read << "\t" << time_compute << std::endl; -#endif - } - -#ifdef ENABLE_TIMERS - log << "total\t" << timer_total.elapsed() << "\t" << timer_read.elapsed() << "\t" - << timer_compute.elapsed() << "\t" << std::endl; - - log.close(); -#endif - - writer.Close(); - reader.Close(); -} diff --git a/examples/simulations/gray-scott/analysis/find_blobs.cpp b/examples/simulations/gray-scott/analysis/find_blobs.cpp deleted file mode 100644 index 8b5c8c54b2..0000000000 --- a/examples/simulations/gray-scott/analysis/find_blobs.cpp +++ /dev/null @@ -1,245 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * Analysis code for the Gray-Scott simulation. - * Reads iso-surface mesh data and detects connected components. - * Counts the total number of connected components and measures the surface - * area of each component. - * - * Keichi Takahashi - * - */ - -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../common/timer.hpp" - -vtkSmartPointer read_mesh(const std::vector &bufPoints, - const std::vector &bufCells, - const std::vector &bufNormals) -{ - int nPoints = static_cast(bufPoints.size() / 3); - int nCells = static_cast(bufCells.size() / 3); - - auto points = vtkSmartPointer::New(); - points->SetNumberOfPoints(nPoints); - for (vtkIdType i = 0; i < nPoints; i++) - { - points->SetPoint(i, &bufPoints[i * 3]); - } - - auto polys = vtkSmartPointer::New(); - for (vtkIdType i = 0; i < nCells; i++) - { - vtkIdType a = bufCells[i * 3 + 0]; - vtkIdType b = bufCells[i * 3 + 1]; - vtkIdType c = bufCells[i * 3 + 2]; - - polys->InsertNextCell(3); - polys->InsertCellPoint(a); - polys->InsertCellPoint(b); - polys->InsertCellPoint(c); - } - - auto normals = vtkSmartPointer::New(); - normals->SetNumberOfComponents(3); - for (vtkIdType i = 0; i < nPoints; i++) - { - normals->InsertNextTuple(&bufNormals[i * 3]); - } - - auto polyData = vtkSmartPointer::New(); - polyData->SetPoints(points); - polyData->SetPolys(polys); - polyData->GetPointData()->SetNormals(normals); - - return polyData; -} - -void find_blobs(const vtkSmartPointer polyData) -{ - auto connectivityFilter = vtkSmartPointer::New(); - connectivityFilter->SetInputData(polyData); - connectivityFilter->SetExtractionModeToAllRegions(); - connectivityFilter->ColorRegionsOn(); - connectivityFilter->Update(); - - int nBlobs = connectivityFilter->GetNumberOfExtractedRegions(); - - std::cout << "Found " << nBlobs << " blobs" << std::endl; - - auto threshold = vtkSmartPointer::New(); - auto massProperties = vtkSmartPointer::New(); - auto surfaceFilter = vtkSmartPointer::New(); - - threshold->SetInputConnection(connectivityFilter->GetOutputPort()); - surfaceFilter->SetInputConnection(threshold->GetOutputPort()); - massProperties->SetInputConnection(surfaceFilter->GetOutputPort()); - - for (int i = 0; i < nBlobs; i++) - { - threshold->SetThresholdFunction(vtkThreshold::THRESHOLD_BETWEEN); - threshold->SetLowerThreshold(i); - threshold->SetUpperThreshold(i); - - std::cout << "Surface area of blob #" << i << " is " << massProperties->GetSurfaceArea() - << std::endl; - } -} - -void find_largest_blob(const vtkSmartPointer polyData) -{ - auto connectivityFilter = vtkSmartPointer::New(); - connectivityFilter->SetInputData(polyData); - connectivityFilter->SetExtractionModeToLargestRegion(); - connectivityFilter->Update(); - - auto massProperties = vtkSmartPointer::New(); - auto surfaceFilter = vtkSmartPointer::New(); - - surfaceFilter->SetInputConnection(connectivityFilter->GetOutputPort()); - massProperties->SetInputConnection(surfaceFilter->GetOutputPort()); - - std::cout << "Surface area of largest blob is " << massProperties->GetSurfaceArea() - << std::endl; -} - -int main(int argc, char *argv[]) -{ - int provided; - MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); - - int rank, procs, wrank; - - MPI_Comm_rank(MPI_COMM_WORLD, &wrank); - - const unsigned int color = 6; - MPI_Comm comm; - MPI_Comm_split(MPI_COMM_WORLD, color, wrank, &comm); - - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &procs); - - if (argc < 2) - { - if (rank == 0) - { - std::cerr << "Too few arguments" << std::endl; - std::cout << "Usage: find_blobs input" << std::endl; - } - MPI_Abort(MPI_COMM_WORLD, -1); - } - - if (procs != 1) - { - if (rank == 0) - { - std::cerr << "find_blobs only supports serial execution" << std::endl; - } - MPI_Abort(MPI_COMM_WORLD, -1); - } - - const std::string input_fname(argv[1]); - - adios2::ADIOS adios("adios2.xml", comm); - - adios2::IO inIO = adios.DeclareIO("IsosurfaceOutput"); - adios2::Engine reader = inIO.Open(input_fname, adios2::Mode::Read); - - std::vector points; - std::vector cells; - std::vector normals; - int step; - -#ifdef ENABLE_TIMERS - Timer timer_total; - Timer timer_compute; - Timer timer_read; - - std::ostringstream log_fname; - log_fname << "find_blobs_pe_" << rank << ".log"; - - std::ofstream log(log_fname.str()); - log << "step\ttotal_blobs\tread_blobs\tcompute_blobs" << std::endl; -#endif - - while (true) - { -#ifdef ENABLE_TIMERS - MPI_Barrier(comm); - timer_total.start(); - timer_read.start(); -#endif - - adios2::StepStatus status = reader.BeginStep(); - - if (status != adios2::StepStatus::OK) - { - break; - } - - auto varPoint = inIO.InquireVariable("point"); - auto varCell = inIO.InquireVariable("cell"); - auto varNormal = inIO.InquireVariable("normal"); - auto varStep = inIO.InquireVariable("step"); - - if (varPoint.Shape().size() > 0 || varCell.Shape().size() > 0) - { - varPoint.SetSelection({{0, 0}, {varPoint.Shape()[0], varPoint.Shape()[1]}}); - varCell.SetSelection({{0, 0}, {varCell.Shape()[0], varCell.Shape()[1]}}); - varNormal.SetSelection({{0, 0}, {varNormal.Shape()[0], varNormal.Shape()[1]}}); - - reader.Get(varPoint, points); - reader.Get(varCell, cells); - reader.Get(varNormal, normals); - } - - reader.Get(varStep, &step); - - reader.EndStep(); - -#ifdef ENABLE_TIMERS - double time_read = timer_read.stop(); - MPI_Barrier(comm); - timer_compute.start(); -#endif - - std::cout << "find_blobs at step " << step << std::endl; - - auto polyData = read_mesh(points, cells, normals); - // find_blobs(polyData); - find_largest_blob(polyData); - -#ifdef ENABLE_TIMERS - double time_compute = timer_compute.stop(); - double time_step = timer_total.stop(); - MPI_Barrier(comm); - - log << step << "\t" << time_step << "\t" << time_read << "\t" << time_compute << std::endl; -#endif - } - -#ifdef ENABLE_TIMERS - log << "total\t" << timer_total.elapsed() << "\t" << timer_read.elapsed() << "\t" - << timer_compute.elapsed() << std::endl; - - log.close(); -#endif - - reader.Close(); -} diff --git a/examples/simulations/gray-scott/analysis/isosurface.cpp b/examples/simulations/gray-scott/analysis/isosurface.cpp deleted file mode 100644 index e02a507aaa..0000000000 --- a/examples/simulations/gray-scott/analysis/isosurface.cpp +++ /dev/null @@ -1,346 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * Analysis code for the Gray-Scott simulation. - * Reads variable U and and extracts the iso-surface using VTK. - * Writes the extracted iso-surface using ADIOS. - * - * Keichi Takahashi - * - */ - -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../common/timer.hpp" - -vtkSmartPointer compute_isosurface(const adios2::Variable &varField, - const std::vector &field, double isovalue) -{ - // Convert field values to vtkImageData - auto importer = vtkSmartPointer::New(); - importer->SetDataSpacing(1, 1, 1); - importer->SetDataOrigin(static_cast(varField.Start()[2]), - static_cast(varField.Start()[1]), - static_cast(varField.Start()[0])); - importer->SetWholeExtent(0, static_cast(varField.Count()[2] - 1), 0, - static_cast(varField.Count()[1] - 1), 0, - static_cast(varField.Count()[0] - 1)); - importer->SetDataExtentToWholeExtent(); - importer->SetDataScalarTypeToDouble(); - importer->SetNumberOfScalarComponents(1); - importer->SetImportVoidPointer(const_cast(field.data())); - - // Run the marching cubes algorithm - auto mcubes = vtkSmartPointer::New(); - mcubes->SetInputConnection(importer->GetOutputPort()); - mcubes->ComputeNormalsOn(); - mcubes->SetValue(0, isovalue); - mcubes->Update(); - - // Return the isosurface as vtkPolyData - return mcubes->GetOutput(); -} - -void write_vtk(const std::string &fname, const vtkSmartPointer polyData) -{ - auto writer = vtkSmartPointer::New(); - writer->SetFileName(fname.c_str()); - writer->SetInputData(polyData); - writer->Write(); -} - -void write_adios(adios2::Engine &writer, const vtkSmartPointer polyData, - adios2::Variable &varPoint, adios2::Variable &varCell, - adios2::Variable &varNormal, adios2::Variable &varOutStep, int step, - MPI_Comm comm) -{ - int numCells = static_cast(polyData->GetNumberOfPolys()); - int numPoints = static_cast(polyData->GetNumberOfPoints()); - int rank; - - MPI_Comm_rank(comm, &rank); - - std::vector points(static_cast(numPoints * 3)); - std::vector normals(static_cast(numPoints * 3)); - std::vector cells(static_cast(numCells * 3)); // Assumes that cells are triangles - - double coords[3]; - - auto cellArray = polyData->GetPolys(); - - cellArray->InitTraversal(); - - // Iterate through cells - for (vtkIdType i = 0; i < polyData->GetNumberOfPolys(); i++) - { - auto idList = vtkSmartPointer::New(); - - cellArray->GetNextCell(idList); - - // Iterate through points of a cell - for (vtkIdType j = 0; j < idList->GetNumberOfIds(); j++) - { - auto id = idList->GetId(j); - - cells[i * 3 + j] = static_cast(id); - - polyData->GetPoint(id, coords); - - points[id * 3 + 0] = coords[0]; - points[id * 3 + 1] = coords[1]; - points[id * 3 + 2] = coords[2]; - } - } - - auto normalArray = polyData->GetPointData()->GetNormals(); - - // Extract normals - for (int i = 0; i < normalArray->GetNumberOfTuples(); i++) - { - normalArray->GetTuple(i, coords); - - normals[i * 3 + 0] = coords[0]; - normals[i * 3 + 1] = coords[1]; - normals[i * 3 + 2] = coords[2]; - } - - int totalPoints, offsetPoints; - MPI_Allreduce(&numPoints, &totalPoints, 1, MPI_INT, MPI_SUM, comm); - MPI_Scan(&numPoints, &offsetPoints, 1, MPI_INT, MPI_SUM, comm); - - writer.BeginStep(); - - varPoint.SetShape( - {static_cast(totalPoints), static_cast(totalPoints > 0 ? 3 : 0)}); - varPoint.SetSelection( - {{static_cast(offsetPoints - numPoints), 0}, - {static_cast(numPoints), static_cast(numPoints > 0 ? 3 : 0)}}); - - varNormal.SetShape(varPoint.Shape()); - varNormal.SetSelection({varPoint.Start(), varPoint.Count()}); - - if (numPoints) - { - writer.Put(varPoint, points.data()); - writer.Put(varNormal, normals.data()); - } - - int totalCells, offsetCells; - MPI_Allreduce(&numCells, &totalCells, 1, MPI_INT, MPI_SUM, comm); - MPI_Scan(&numCells, &offsetCells, 1, MPI_INT, MPI_SUM, comm); - - for (int &cell : cells) - { - cell += (offsetPoints - numPoints); - } - - varCell.SetShape( - {static_cast(totalCells), static_cast(totalCells > 0 ? 3 : 0)}); - varCell.SetSelection( - {{static_cast(offsetCells - numCells), 0}, - {static_cast(numCells), static_cast(numCells > 0 ? 3 : 0)}}); - - if (numCells) - { - writer.Put(varCell, cells.data()); - } - - if (!rank) - { - std::cout << "isosurface at step " << step << " writing out " << totalCells << " cells and " - << totalPoints << " points" << std::endl; - } - - writer.Put(varOutStep, step); - - writer.EndStep(); -} - -int main(int argc, char *argv[]) -{ - int provided; - MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); - - int rank, procs, wrank; - - MPI_Comm_rank(MPI_COMM_WORLD, &wrank); - - const unsigned int color = 5; - MPI_Comm comm; - MPI_Comm_split(MPI_COMM_WORLD, color, wrank, &comm); - - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &procs); - - int dims[3] = {0}; - MPI_Dims_create(procs, 3, dims); - size_t npx = dims[0]; - size_t npy = dims[1]; - size_t npz = dims[2]; - - int coords[3] = {0}; - int periods[3] = {0}; - MPI_Comm cart_comm; - MPI_Cart_create(comm, 3, dims, periods, 0, &cart_comm); - MPI_Cart_coords(cart_comm, rank, 3, coords); - size_t px = coords[0]; - size_t py = coords[1]; - size_t pz = coords[2]; - - if (argc < 4) - { - if (rank == 0) - { - std::cerr << "Too few arguments" << std::endl; - std::cout << "Usage: isosurface input output isovalues..." << std::endl; - } - MPI_Abort(MPI_COMM_WORLD, -1); - } - - const std::string input_fname(argv[1]); - const std::string output_fname(argv[2]); - - std::vector isovalues; - for (int i = 3; i < argc; i++) - { - isovalues.push_back(std::stod(argv[i])); - } - - adios2::ADIOS adios("adios2.xml", comm); - - adios2::IO inIO = adios.DeclareIO("SimulationOutput"); - adios2::Engine reader = inIO.Open(input_fname, adios2::Mode::Read); - - adios2::IO outIO = adios.DeclareIO("IsosurfaceOutput"); - adios2::Engine writer = outIO.Open(output_fname, adios2::Mode::Write); - - auto varPoint = outIO.DefineVariable("point", {1, 3}, {0, 0}, {1, 3}); - auto varCell = outIO.DefineVariable("cell", {1, 3}, {0, 0}, {1, 3}); - auto varNormal = outIO.DefineVariable("normal", {1, 3}, {0, 0}, {1, 3}); - auto varOutStep = outIO.DefineVariable("step"); - - std::vector u; - int step; - -#ifdef ENABLE_TIMERS - Timer timer_total; - Timer timer_read; - Timer timer_compute; - Timer timer_write; - - std::ostringstream log_fname; - log_fname << "isosurface_pe_" << rank << ".log"; - - std::ofstream log(log_fname.str()); - log << "step\ttotal_iso\tread_iso\tcompute_iso\twrite_iso" << std::endl; -#endif - - while (true) - { -#ifdef ENABLE_TIMERS - MPI_Barrier(comm); - timer_total.start(); - timer_read.start(); -#endif - - adios2::StepStatus status = reader.BeginStep(); - - if (status != adios2::StepStatus::OK) - { - break; - } - - adios2::Variable varU = inIO.InquireVariable("U"); - const adios2::Variable varStep = inIO.InquireVariable("step"); - - adios2::Dims shape = varU.Shape(); - - size_t size_x = (shape[0] + npx - 1) / npx; - size_t size_y = (shape[1] + npy - 1) / npy; - size_t size_z = (shape[2] + npz - 1) / npz; - - size_t offset_x = size_x * px; - size_t offset_y = size_y * py; - size_t offset_z = size_z * pz; - - if (px == npx - 1) - { - size_x -= size_x * npx - shape[0]; - } - if (py == npy - 1) - { - size_y -= size_y * npy - shape[1]; - } - if (pz == npz - 1) - { - size_z -= size_z * npz - shape[2]; - } - - varU.SetSelection({{offset_x, offset_y, offset_z}, - {size_x + (px != npx - 1 ? 1 : 0), size_y + (py != npy - 1 ? 1 : 0), - size_z + (pz != npz - 1 ? 1 : 0)}}); - - reader.Get(varU, u); - reader.Get(varStep, step); - reader.EndStep(); - -#ifdef ENABLE_TIMERS - double time_read = timer_read.stop(); - MPI_Barrier(comm); - timer_compute.start(); -#endif - - auto appendFilter = vtkSmartPointer::New(); - - for (const auto isovalue : isovalues) - { - auto polyData = compute_isosurface(varU, u, isovalue); - appendFilter->AddInputData(polyData); - } - - appendFilter->Update(); - -#ifdef ENABLE_TIMERS - double time_compute = timer_compute.stop(); - MPI_Barrier(comm); - timer_write.start(); -#endif - - write_adios(writer, appendFilter->GetOutput(), varPoint, varCell, varNormal, varOutStep, - step, comm); - -#ifdef ENABLE_TIMERS - double time_write = timer_write.stop(); - double time_step = timer_total.stop(); - MPI_Barrier(comm); - - log << step << "\t" << time_step << "\t" << time_read << "\t" << time_compute << "\t" - << time_write << std::endl; -#endif - } - -#ifdef ENABLE_TIMERS - log << "total\t" << timer_total.elapsed() << "\t" << timer_read.elapsed() << "\t" - << timer_compute.elapsed() << "\t" << timer_write.elapsed() << std::endl; - - log.close(); -#endif - - writer.Close(); - reader.Close(); - - MPI_Finalize(); -} diff --git a/examples/simulations/gray-scott/plot/render_isosurface.cpp b/examples/simulations/gray-scott/plot/render_isosurface.cpp deleted file mode 100644 index 30a464b831..0000000000 --- a/examples/simulations/gray-scott/plot/render_isosurface.cpp +++ /dev/null @@ -1,204 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * Visualization code for the Gray-Scott simulation. - * Reads and renders iso-surface mesh data. - * - * Keichi Takahashi - * - */ - -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -VTK_MODULE_INIT(vtkRenderingOpenGL2); - -typedef struct -{ - vtkRenderView *renderView; - vtkPolyDataMapper *mapper; - adios2::IO *inIO; - adios2::Engine *reader; -} Context; - -vtkSmartPointer read_mesh(const std::vector &bufPoints, - const std::vector &bufCells, - const std::vector &bufNormals) -{ - int nPoints = bufPoints.size() / 3; - int nCells = bufCells.size() / 3; - - auto points = vtkSmartPointer::New(); - points->SetNumberOfPoints(nPoints); - for (vtkIdType i = 0; i < nPoints; i++) - { - points->SetPoint(i, &bufPoints[i * 3]); - } - - auto polys = vtkSmartPointer::New(); - for (vtkIdType i = 0; i < nCells; i++) - { - vtkIdType a = bufCells[i * 3 + 0]; - vtkIdType b = bufCells[i * 3 + 1]; - vtkIdType c = bufCells[i * 3 + 2]; - - polys->InsertNextCell(3); - polys->InsertCellPoint(a); - polys->InsertCellPoint(b); - polys->InsertCellPoint(c); - } - - auto normals = vtkSmartPointer::New(); - normals->SetNumberOfComponents(3); - for (vtkIdType i = 0; i < nPoints; i++) - { - normals->InsertNextTuple(&bufNormals[i * 3]); - } - - auto polyData = vtkSmartPointer::New(); - polyData->SetPoints(points); - polyData->SetPolys(polys); - polyData->GetPointData()->SetNormals(normals); - - return polyData; -} - -void timer_func(vtkObject *object, unsigned long eid, void *clientdata, void *calldata) -{ - Context *context = static_cast(clientdata); - - std::vector points; - std::vector cells; - std::vector normals; - int step; - - adios2::StepStatus status = context->reader->BeginStep(); - - if (status != adios2::StepStatus::OK) - { - return; - } - - auto varPoint = context->inIO->InquireVariable("point"); - auto varCell = context->inIO->InquireVariable("cell"); - auto varNormal = context->inIO->InquireVariable("normal"); - auto varStep = context->inIO->InquireVariable("step"); - - if (varPoint.Shape().size() > 0 || varCell.Shape().size() > 0) - { - varPoint.SetSelection({{0, 0}, {varPoint.Shape()[0], varPoint.Shape()[1]}}); - varCell.SetSelection({{0, 0}, {varCell.Shape()[0], varCell.Shape()[1]}}); - varNormal.SetSelection({{0, 0}, {varNormal.Shape()[0], varNormal.Shape()[1]}}); - - context->reader->Get(varPoint, points); - context->reader->Get(varCell, cells); - context->reader->Get(varNormal, normals); - } - - context->reader->Get(varStep, &step); - - context->reader->EndStep(); - - std::cout << "render_isosurface at step " << step << std::endl; - - vtkSmartPointer polyData = read_mesh(points, cells, normals); - - context->mapper->SetInputData(polyData); - context->renderView->ResetCamera(); - context->renderView->Render(); -} - -int main(int argc, char *argv[]) -{ - MPI_Init(&argc, &argv); - - int rank, procs, wrank; - - MPI_Comm_rank(MPI_COMM_WORLD, &wrank); - - const unsigned int color = 7; - MPI_Comm comm; - MPI_Comm_split(MPI_COMM_WORLD, color, wrank, &comm); - - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &procs); - - if (argc < 2) - { - if (rank == 0) - { - std::cerr << "Too few arguments" << std::endl; - std::cout << "Usage: render_isosurface input" << std::endl; - } - MPI_Abort(MPI_COMM_WORLD, -1); - } - - if (procs != 1) - { - if (rank == 0) - { - std::cerr << "render_isosurface only supports serial execution" << std::endl; - } - MPI_Abort(MPI_COMM_WORLD, -1); - } - - const std::string input_fname(argv[1]); - - adios2::ADIOS adios("adios2.xml", comm); - - adios2::IO inIO = adios.DeclareIO("IsosurfaceOutput"); - adios2::Engine reader = inIO.Open(input_fname, adios2::Mode::Read); - - auto mapper = vtkSmartPointer::New(); - - auto actor = vtkSmartPointer::New(); - actor->SetMapper(mapper); - - auto renderView = vtkSmartPointer::New(); - renderView->GetRenderer()->AddActor(actor); - renderView->Update(); - - auto style = vtkSmartPointer::New(); - style->SetCurrentStyleToTrackballCamera(); - - auto interactor = renderView->GetInteractor(); - interactor->Initialize(); - interactor->SetInteractorStyle(style); - interactor->CreateRepeatingTimer(100); - - Context context = { - .renderView = renderView, - .mapper = mapper, - .inIO = &inIO, - .reader = &reader, - }; - - auto timerCallback = vtkSmartPointer::New(); - timerCallback->SetCallback(timer_func); - timerCallback->SetClientData(&context); - interactor->AddObserver(vtkCommand::TimerEvent, timerCallback); - - renderView->Render(); - interactor->Start(); - - reader.Close(); -} diff --git a/examples/simulations/heatTransfer/ReadMe.md b/examples/simulations/heatTransfer/ReadMe.md index 8693d00265..b1a45a24e7 100644 --- a/examples/simulations/heatTransfer/ReadMe.md +++ b/examples/simulations/heatTransfer/ReadMe.md @@ -22,6 +22,13 @@ an application to the ADIOS2 library for its IO. #### Example +##### 1. Build the example +$ mkdir build +$ cd build +$ cmake -DCMAKE_PREFIX_PATH= -DCMAKE_INSTALL_PREFIX=install .. +$ make -j 8 +$ cd .. + ##### 1. Produce an output ``` @@ -44,24 +51,23 @@ The adios1, ph5 and hdf5 versions of the example do not use XML config files, so argument. ``` -$ mpirun -np 12 ./bin/adios2_simulations_heatTransferWrite ../examples/simulations/heatTransfer/heat_file.xml heat.bp 4 -3 5 10 10 10 -$ mpirun -np 12 ./bin/adios2_simulations_heatTransferWrite ../examples/simulations/heatTransfer/heat_file.xml heat.h5 4 -3 5 10 10 10 +$ mpirun -np 12 ./build/write/adios2_simulations_heatTransferWrite heat_file.xml heat.bp 4 3 5 10 10 10 +$ mpirun -np 12 ./build/write/adios2_simulations_heatTransferWrite heat_hdf5.xml heat.h5 4 3 5 10 10 10 ``` ##### 2. Read the output step-by-step and print data into text files (data. per reader process) ``` -Reader Usage: heatRead config input N M - config: XML config file to use - input: name of input data file/stream - N: number of processes in X dimension - M: number of processes in Y dimension +Reader Usage: heatRead config input output N M + config: XML config file to use + input: name of input data file/stream + output: name of output data file/stream + N: number of processes in X dimension + M: number of processes in Y dimension ``` ``` -$ mpirun -np 2 ./bin/heatTransfer_read ../examples/simulations/heatTransfer/heat_file.xml heat 2 1 +$ mpirun -np 2 ./build/read/adios2_simulations_heatTransferRead heat_file.xml heat.bp read.bp 2 1 ``` ##### Notes: diff --git a/scripts/ci/circle/postCDashStatus.sh b/scripts/ci/circle/postCDashStatus.sh index 800b192d77..78ec1e2228 100755 --- a/scripts/ci/circle/postCDashStatus.sh +++ b/scripts/ci/circle/postCDashStatus.sh @@ -4,7 +4,7 @@ USER=${STATUS_ROBOT_NAME} TOKEN=${STATUS_ROBOT_KEY} COMMIT=${CIRCLE_SHA1} CDASH_STATUS_CONTEXT="cdash" -SOURCE_DIR="$(readlink -f ${CIRCLE_WORKING_DIRECTORY}/source)" +SOURCE_DIR="$(readlink -f "${CIRCLE_WORKING_DIRECTORY}"/source)" build_status_body() { cat < /dev/null)" - if [ $? -eq 0 ] + RESULT="$(curl -s "${APIURL}" | python3 -c "import sys, json; print(json.load(sys.stdin)['head']['ref'])" 2> /dev/null)" + exit_status=$? + if [ "$exit_status" -eq 0 ] then REALBRANCH="$(echo "${RESULT}" | tr '/' '-')" fi @@ -70,8 +72,9 @@ echo "**********Env End************" echo "**********CTest Begin**********" ${CTEST} --version -echo ${CTEST} -VV -S ${CTEST_SCRIPT} -Ddashboard_full=OFF ${CTEST_STEP_ARGS} -${CTEST} -VV -S ${CTEST_SCRIPT} -Ddashboard_full=OFF ${CTEST_STEP_ARGS} +echo ${CTEST} -VV -S "${CTEST_SCRIPT}" -Ddashboard_full=OFF "${CTEST_STEP_ARGS}" +# shellcheck disable=SC2086 +${CTEST} -VV -S "${CTEST_SCRIPT}" -Ddashboard_full=OFF ${CTEST_STEP_ARGS} RET=$? echo "**********CTest End************" diff --git a/scripts/ci/cmake/ci-fedora-asan.cmake b/scripts/ci/cmake/ci-fedora-asan.cmake index 8651193fd8..8b7e6dcd65 100644 --- a/scripts/ci/cmake/ci-fedora-asan.cmake +++ b/scripts/ci/cmake/ci-fedora-asan.cmake @@ -4,7 +4,7 @@ set(ENV{CC} clang) set(ENV{CXX} clang++) set(ASAN_FLAGS "-fsanitize=address -fno-omit-frame-pointer -pthread -mllvm -asan-use-private-alias=1 -Wno-unused-command-line-argument") set(ENV{ASAN_OPTIONS} "use_odr_indicator=1") -set(ENV{LSAN_OPTIONS} "suppressions=${CMAKE_SOURCE_DIR}/thirdparty/perfstubs/perfstubs.supp") +set(ENV{LSAN_OPTIONS} "suppressions=$ENV{CI_SOURCE_DIR}/thirdparty/perfstubs/perfstubs.supp") set(ENV{CFLAGS} "${ASAN_FLAGS}") set(ENV{CXXFLAGS} "${ASAN_FLAGS}") diff --git a/scripts/ci/gh-actions/check-branch-name.sh b/scripts/ci/gh-actions/check-branch-name.sh index 83bffd8e85..52d09e6cb0 100755 --- a/scripts/ci/gh-actions/check-branch-name.sh +++ b/scripts/ci/gh-actions/check-branch-name.sh @@ -4,7 +4,7 @@ if [ "${GITHUB_EVENT_NAME}" = "pull_request" ] then if [ -z "${BASE_REF}" ] then - BASE_REF="$(jq -r .pull_request.base.ref ${GITHUB_EVENT_PATH})" + BASE_REF="$(jq -r .pull_request.base.ref "${GITHUB_EVENT_PATH}")" fi echo "Base ref: ${BASE_REF}" echo "Head ref: ${GITHUB_HEAD_REF}" diff --git a/scripts/ci/gh-actions/get-changed-files.sh b/scripts/ci/gh-actions/get-changed-files.sh index 9e3ed8fa5b..5afcd19396 100755 --- a/scripts/ci/gh-actions/get-changed-files.sh +++ b/scripts/ci/gh-actions/get-changed-files.sh @@ -3,12 +3,12 @@ case "${GITHUB_EVENT_NAME}" in pull_request) - BASE_SHA=$(jq -r .pull_request.base.sha ${GITHUB_EVENT_PATH}) - HEAD_SHA=$(jq -r .pull_request.head.sha ${GITHUB_EVENT_PATH}) + BASE_SHA=$(jq -r .pull_request.base.sha "${GITHUB_EVENT_PATH}") + HEAD_SHA=$(jq -r .pull_request.head.sha "${GITHUB_EVENT_PATH}") ;; push) - BASE_SHA=$(jq -r .before ${GITHUB_EVENT_PATH}) - HEAD_SHA=$(jq -r .after ${GITHUB_EVENT_PATH}) + BASE_SHA=$(jq -r .before "${GITHUB_EVENT_PATH}") + HEAD_SHA=$(jq -r .after "${GITHUB_EVENT_PATH}") ;; *) echo "Unable to get changed files from '${GITHUB_EVENT_NAME}' event" @@ -20,11 +20,11 @@ echo "Base: ${BASE_SHA}" echo "Head: ${HEAD_SHA}" echo "" -git fetch origin ${BASE_SHA} +git fetch origin "${BASE_SHA}" echo "" echo "::group::All changed files" -git diff --name-only ${BASE_SHA}...${HEAD_SHA} | tee all-changed-files.txt +git diff --name-only "${BASE_SHA}"..."${HEAD_SHA}" | tee all-changed-files.txt echo "::group::Filtered changes" grep -v '^docs/' all-changed-files.txt | tee filtered-changed-files.txt diff --git a/scripts/ci/gh-actions/macos-setup.sh b/scripts/ci/gh-actions/macos-setup.sh index 8cae241e0e..0eb6850f20 100755 --- a/scripts/ci/gh-actions/macos-setup.sh +++ b/scripts/ci/gh-actions/macos-setup.sh @@ -8,13 +8,13 @@ then echo "Error: GH_YML_MATRIX_COMPILER variable is not defined" exit 1 fi -XCODE_VER="$(echo ${GH_YML_MATRIX_COMPILER} | sed -e 's|_|.|g' -e 's|xcode||')" -if [ ! -d /Applications/Xcode_${XCODE_VER}.app ] +XCODE_VER="$(echo "${GH_YML_MATRIX_COMPILER}" | sed -e 's|_|.|g' -e 's|xcode||')" +if [ ! -d "/Applications/Xcode_${XCODE_VER}.app" ] then echo "Error: XCode installation directory /Applications/Xcode_${XCODE_VER}.app does not exist" exit 2 fi -sudo xcode-select --switch /Applications/Xcode_${XCODE_VER}.app +sudo xcode-select --switch "/Applications/Xcode_${XCODE_VER}.app" echo "Installing CMake" @@ -36,7 +36,7 @@ brew install ninja echo "Installing GCC" brew install gcc -sudo ln -v -s $(which gfortran-11) /usr/local/bin/gfortran +sudo ln -v -s "$(which gfortran-11)" /usr/local/bin/gfortran echo "Installing blosc compression" brew install c-blosc diff --git a/scripts/ci/gh-actions/run.sh b/scripts/ci/gh-actions/run.sh index 5dcfcca7aa..c511730eea 100755 --- a/scripts/ci/gh-actions/run.sh +++ b/scripts/ci/gh-actions/run.sh @@ -100,19 +100,19 @@ export ADIOS2_IP=127.0.0.1 export OMP_NUM_THREADS=1 # Load any additional setup scripts -if [ -f gha/scripts/ci/setup-run/ci-${GH_YML_JOBNAME}.sh ] +if [ -f "gha/scripts/ci/setup-run/ci-${GH_YML_JOBNAME}.sh" ] then SETUP_RUN_SCRIPT=gha/scripts/ci/setup-run/ci-${GH_YML_JOBNAME}.sh -elif [ -f gha/scripts/ci/setup-run/ci-${GH_YML_MATRIX_OS}-${GH_YML_MATRIX_COMPILER}-${GH_YML_MATRIX_PARALLEL}.sh ] +elif [ -f "gha/scripts/ci/setup-run/ci-${GH_YML_MATRIX_OS}-${GH_YML_MATRIX_COMPILER}-${GH_YML_MATRIX_PARALLEL}.sh" ] then SETUP_RUN_SCRIPT=gha/scripts/ci/setup-run/ci-${GH_YML_MATRIX_OS}-${GH_YML_MATRIX_COMPILER}-${GH_YML_MATRIX_PARALLEL}.sh -elif [ -f gha/scripts/ci/setup-run/ci-${GH_YML_MATRIX_OS}-${GH_YML_MATRIX_COMPILER}.sh ] +elif [ -f "gha/scripts/ci/setup-run/ci-${GH_YML_MATRIX_OS}-${GH_YML_MATRIX_COMPILER}.sh" ] then SETUP_RUN_SCRIPT=gha/scripts/ci/setup-run/ci-${GH_YML_MATRIX_OS}-${GH_YML_MATRIX_COMPILER}.sh -elif [ -f gha/scripts/ci/setup-run/ci-${GH_YML_MATRIX_OS}.sh ] +elif [ -f "gha/scripts/ci/setup-run/ci-${GH_YML_MATRIX_OS}.sh" ] then SETUP_RUN_SCRIPT=gha/scripts/ci/setup-run/ci-${GH_YML_MATRIX_OS}.sh -elif [ -f gha/scripts/ci/setup-run/ci-${GH_YML_BASE_OS}.sh ] +elif [ -f "gha/scripts/ci/setup-run/ci-${GH_YML_BASE_OS}.sh" ] then SETUP_RUN_SCRIPT=gha/scripts/ci/setup-run/ci-${GH_YML_BASE_OS}.sh fi @@ -126,7 +126,8 @@ echo "::endgroup::" echo "::group::Job-run setup (if any)" if [ "${SETUP_RUN_SCRIPT:-UNSET}" != "UNSET" ] then - source ${SETUP_RUN_SCRIPT} + # shellcheck source=/dev/null + source "${SETUP_RUN_SCRIPT}" fi echo "::endgroup::" @@ -139,7 +140,8 @@ echo "::group::CTest version" echo "::endgroup::" echo "::group::Execute job step" -"${CTEST}" -VV -S ${CTEST_SCRIPT} -Ddashboard_full=OFF ${CTEST_STEP_ARGS} +# shellcheck disable=SC2086 +"${CTEST}" -VV -S "${CTEST_SCRIPT}" -Ddashboard_full=OFF ${CTEST_STEP_ARGS} RET=$? echo "::endgroup::" diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index 3646c59051..225f4105a5 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -191,6 +191,7 @@ target_compile_features(adios2_core PUBLIC "$; template struct TypeInfo; +/** Data accuracy **/ + +/* Error. Accuracy can be requested for reading data. + norm: 0.0 = L2, inf() = Linf + relative: true = relative error, false = absolute error + */ +struct Accuracy +{ + double error; + double norm; + bool relative; +}; + +constexpr double L2_norm = 0.0; +constexpr double Linf_norm = std::numeric_limits::infinity(); + /** * Return the actual size in bytes of elements of the given type. Returns -1 * for strings. @@ -408,6 +424,19 @@ constexpr char s[] = "s"; } #endif +#ifdef ADIOS2_HAVE_MGARD_MDR + +constexpr char MDR[] = "mdr"; + +namespace mdr +{ +constexpr double DOUBLE_ROUNDING_ERROR_LIMIT = 5.0e-16; +constexpr double FLOAT_ROUNDING_ERROR_LIMIT = 3.0e-7; +namespace key +{} +} +#endif + #ifdef ADIOS2_HAVE_LIBPRESSIO constexpr char LossyLIBPRESSIO[] = "libpressio"; namespace libpressio diff --git a/source/adios2/core/ADIOS.cpp b/source/adios2/core/ADIOS.cpp index b5006c99e7..6504474d81 100644 --- a/source/adios2/core/ADIOS.cpp +++ b/source/adios2/core/ADIOS.cpp @@ -118,7 +118,7 @@ ADIOS::ADIOS(const std::string configFile, helper::Comm comm, const std::string static bool perfstubsInit(false); if (!perfstubsInit) { - PERFSTUBS_INITIALIZE(m_Comm.Rank()); + PERFSTUBS_INITIALIZE(); perfstubsInit = true; atexit(ps_finalize_); } @@ -133,11 +133,11 @@ ADIOS::ADIOS(const std::string configFile, helper::Comm comm, const std::string } if (helper::EndsWith(configFile, ".xml")) { - XMLInit(configFile); + m_ConfigFileContents = XMLInit(configFile); } else if (helper::EndsWith(configFile, ".yaml") || helper::EndsWith(configFile, ".yml")) { - YAMLInit(configFile); + m_ConfigFileContents = YAMLInit(configFile); } } #ifdef ADIOS2_HAVE_KOKKOS @@ -195,6 +195,20 @@ IO &ADIOS::DeclareIO(const std::string name, const ArrayOrdering ArrayOrder) IO &io = ioPair.first->second; io.SetDeclared(); io.SetArrayOrder(ArrayOrder); + + // Configure new IO objects with config file (if present) + if (!m_ConfigFile.empty() && !m_ConfigFileContents.empty()) + { + if (helper::EndsWith(m_ConfigFile, ".xml")) + { + XMLIOInit(m_ConfigFile, m_ConfigFileContents, io); + } + else if (helper::EndsWith(m_ConfigFile, ".yaml") || helper::EndsWith(m_ConfigFile, ".yml")) + { + YAMLInitIO(m_ConfigFile, m_ConfigFileContents, io); + } + } + return io; } @@ -292,14 +306,26 @@ void ADIOS::CheckOperator(const std::string name) const } } -void ADIOS::XMLInit(const std::string &configFileXML) +std::string ADIOS::XMLInit(const std::string &configFileXML) +{ + return helper::ParseConfigXML(*this, configFileXML, m_IOs, m_Operators); +} + +void ADIOS::XMLIOInit(const std::string &configFileXML, const std::string &configFileContents, + core::IO &io) +{ + helper::ParseConfigXMLIO(*this, configFileXML, configFileContents, io, m_Operators); +} + +std::string ADIOS::YAMLInit(const std::string &configFileYAML) { - helper::ParseConfigXML(*this, configFileXML, m_IOs, m_Operators); + return helper::ParseConfigYAML(*this, configFileYAML, m_IOs); } -void ADIOS::YAMLInit(const std::string &configFileYAML) +void ADIOS::YAMLInitIO(const std::string &configFileYAML, const std::string &configFileContents, + core::IO &io) { - helper::ParseConfigYAML(*this, configFileYAML, m_IOs); + helper::ParseConfigYAMLIO(*this, configFileYAML, configFileContents, io); } void ADIOS::RecordOutputStep(const std::string &name, const size_t step, const double time) diff --git a/source/adios2/core/ADIOS.h b/source/adios2/core/ADIOS.h index cee4f65664..bf534a960e 100644 --- a/source/adios2/core/ADIOS.h +++ b/source/adios2/core/ADIOS.h @@ -170,6 +170,7 @@ class ADIOS /** XML File to be read containing configuration information */ const std::string m_ConfigFile; + std::string m_ConfigFileContents; /** * @brief List of IO class objects defined from either ADIOS @@ -193,9 +194,15 @@ class ADIOS void CheckOperator(const std::string name) const; - void XMLInit(const std::string &configFileXML); + std::string XMLInit(const std::string &configFileXML); - void YAMLInit(const std::string &configFileYAML); + void XMLIOInit(const std::string &configFileXML, const std::string &configFileContents, + core::IO &io); + + std::string YAMLInit(const std::string &configFileYAML); + + void YAMLInitIO(const std::string &configFileYAML, const std::string &configFileContents, + core::IO &io); private: /* Global services that we want to initialize at most once and shutdown diff --git a/source/adios2/core/Info.cpp b/source/adios2/core/Info.cpp index 39686b8e0f..ae5dca63b3 100644 --- a/source/adios2/core/Info.cpp +++ b/source/adios2/core/Info.cpp @@ -72,6 +72,9 @@ static const char *const operators[] = { "MGARD", "MGARDPlus", #endif +#ifdef ADIOS2_HAVE_MGARD_MDR + "MDR", +#endif #ifdef ADIOS2_HAVE_SZ "SZ", #endif diff --git a/source/adios2/core/Operator.cpp b/source/adios2/core/Operator.cpp index 946c6fbada..481749e57a 100644 --- a/source/adios2/core/Operator.cpp +++ b/source/adios2/core/Operator.cpp @@ -11,6 +11,8 @@ #include "Operator.h" #include "adios2/helper/adiosFunctions.h" +#include + namespace adios2 { namespace core @@ -30,6 +32,9 @@ void Operator::SetParameter(const std::string key, const std::string value) noex Params &Operator::GetParameters() noexcept { return m_Parameters; } +void Operator::SetAccuracy(const adios2::Accuracy &a) noexcept { m_AccuracyRequested = a; } +adios2::Accuracy Operator::GetAccuracy() const noexcept { return m_AccuracyProvided; } + #define declare_type(T) \ \ void Operator::RunCallback1(const T *arg0, const std::string &arg1, const std::string &arg2, \ @@ -96,6 +101,12 @@ Dims Operator::ConvertDims(const Dims &dimensions, const DataType type, const si size_t Operator::GetHeaderSize() const { return 0; } +size_t Operator::GetEstimatedSize(const size_t ElemCount, const size_t ElemSize, const size_t ndims, + const size_t *dims) const +{ + return ElemCount * ElemSize + 128; +}; + // PRIVATE void Operator::CheckCallbackType(const std::string type) const { diff --git a/source/adios2/core/Operator.h b/source/adios2/core/Operator.h index a13b596a56..d87e6788b4 100644 --- a/source/adios2/core/Operator.h +++ b/source/adios2/core/Operator.h @@ -37,6 +37,7 @@ class Operator COMPRESS_SZ = 6, COMPRESS_ZFP = 7, COMPRESS_MGARDPLUS = 8, + REFACTOR_MDR = 41, CALLBACK_SIGNATURE1 = 51, CALLBACK_SIGNATURE2 = 52, PLUGIN_INTERFACE = 53, @@ -56,6 +57,9 @@ class Operator Params &GetParameters() noexcept; + void SetAccuracy(const adios2::Accuracy &a) noexcept; + adios2::Accuracy GetAccuracy() const noexcept; + #define declare_type(T) \ virtual void RunCallback1(const T *, const std::string &, const std::string &, \ const std::string &, const size_t, const Dims &, const Dims &, \ @@ -68,6 +72,10 @@ class Operator virtual size_t GetHeaderSize() const; + /** Give an upper bound estimate how big the transformed data could be */ + virtual size_t GetEstimatedSize(const size_t ElemCount, const size_t ElemSize, + const size_t ndims, const size_t *dims) const; + /** * @param dataIn * @param blockStart @@ -94,6 +102,11 @@ class Operator /** Parameters associated with a particular Operator */ Params m_Parameters; + /** user requested accuracy */ + Accuracy m_AccuracyRequested = {0.0, 0.0, false}; + /** provided accuracy */ + Accuracy m_AccuracyProvided = {0.0, 0.0, false}; + /** * Used by lossy compressors with a limitation on complex data types or * dimentions Returns a adios2::Dims object that meets the requirement of a diff --git a/source/adios2/core/VariableBase.cpp b/source/adios2/core/VariableBase.cpp index a6430f67ba..3f76bb1f48 100644 --- a/source/adios2/core/VariableBase.cpp +++ b/source/adios2/core/VariableBase.cpp @@ -204,6 +204,14 @@ void VariableBase::SetMemorySelection(const Box &memorySelection) m_MemoryCount = memorySelection.second; } +void VariableBase::SetAccuracy(const adios2::Accuracy &a) noexcept +{ + m_AccuracyRequested = a; + m_AccuracyProvided = {0.0, a.norm, a.relative}; +} +adios2::Accuracy VariableBase::GetAccuracy() const noexcept { return m_AccuracyProvided; } +adios2::Accuracy VariableBase::GetAccuracyRequested() const noexcept { return m_AccuracyRequested; } + size_t VariableBase::GetAvailableStepsStart() const { return m_AvailableStepsStart; } size_t VariableBase::GetAvailableStepsCount() const { return m_AvailableStepsCount; } diff --git a/source/adios2/core/VariableBase.h b/source/adios2/core/VariableBase.h index bd888a3828..b777400f40 100644 --- a/source/adios2/core/VariableBase.h +++ b/source/adios2/core/VariableBase.h @@ -98,6 +98,11 @@ class VariableBase Engine *m_Engine = nullptr; + /** user requested accuracy */ + Accuracy m_AccuracyRequested = {0.0, 0.0, false}; + /** provided accuracy */ + Accuracy m_AccuracyProvided = {0.0, 0.0, false}; + /** Index to Step and blocks' (inside a step) characteristics position in a * serial metadata buffer * 
@@ -166,6 +171,22 @@ class VariableBase
      */
     void SetMemorySelection(const Box &memorySelection);
 
+    /**
+     * Sets the requested accuracy for the next read operation.
+     * The actual accuracy after the read is provided in GetAccuracy()
+     */
+    void SetAccuracy(const adios2::Accuracy &a) noexcept;
+
+    /**
+     * Get the provided accuracy for the last read operation.
+     * Most operations provide data as it was written, meaning that
+     * error is reported as 0.0
+     */
+    adios2::Accuracy GetAccuracy() const noexcept;
+
+    /** Return the requested accuracy set by user with SetAccuracy */
+    adios2::Accuracy GetAccuracyRequested() const noexcept;
+
     size_t GetAvailableStepsStart() const;
 
     size_t GetAvailableStepsCount() const;
diff --git a/source/adios2/helper/adiosXML.cpp b/source/adios2/helper/adiosXML.cpp
index a1595fcefb..a904c01ee5 100644
--- a/source/adios2/helper/adiosXML.cpp
+++ b/source/adios2/helper/adiosXML.cpp
@@ -13,14 +13,12 @@
 
 /// \cond EXCLUDE_FROM_DOXYGEN
 #include  //std::transform
-#include   // std::distance
 #include  //std::invalid_argument
 /// \endcond
 
 #include "adios2/common/ADIOSTypes.h"
 #include "adios2/core/IO.h"
 #include "adios2/helper/adiosLog.h"
-#include "adios2/helper/adiosString.h"
 #include "adios2/helper/adiosXMLUtil.h"
 
 #include 
@@ -30,162 +28,194 @@ namespace adios2
 namespace helper
 {
 
-void ParseConfigXML(core::ADIOS &adios, const std::string &configFileXML,
-                    std::map &ios,
-                    std::unordered_map> &operators)
+std::string FileContents(core::ADIOS &adios, const std::string &configXML)
 {
-    const std::string hint("for config file " + configFileXML + " in call to ADIOS constructor");
+    const std::string configFileContents(adios.GetComm().BroadcastFile(
+        configXML, "when parsing configXML file, in call to the ADIOS constructor"));
 
-    auto lf_FileContents = [&](const std::string &configXML) -> std::string {
-        const std::string fileContents(adios.GetComm().BroadcastFile(
-            configXML, "when parsing configXML file, in call to the ADIOS constructor"));
+    if (configFileContents.empty())
+    {
+        helper::Throw("Helper", "AdiosXML", "ParseConfigXML",
+                                             "empty config xml file");
+    }
+    return configFileContents;
+}
 
-        if (fileContents.empty())
-        {
-            helper::Throw("Helper", "AdiosXML", "ParseConfigXML",
-                                                 "empty config xml file");
-        }
-        return fileContents;
-    };
+void OperatorXML(core::ADIOS &adios, const pugi::xml_node &operatorNode, const std::string &hint)
+{
+    const std::unique_ptr name =
+        helper::XMLAttribute("name", operatorNode, hint);
 
-    auto lf_OperatorXML = [&](const pugi::xml_node &operatorNode) {
-        const std::unique_ptr name =
-            helper::XMLAttribute("name", operatorNode, hint);
+    const std::unique_ptr type =
+        helper::XMLAttribute("type", operatorNode, hint);
 
-        const std::unique_ptr type =
-            helper::XMLAttribute("type", operatorNode, hint);
+    std::string typeLowerCase = std::string(type->value());
+    std::transform(typeLowerCase.begin(), typeLowerCase.end(), typeLowerCase.begin(), ::tolower);
 
-        std::string typeLowerCase = std::string(type->value());
-        std::transform(typeLowerCase.begin(), typeLowerCase.end(), typeLowerCase.begin(),
-                       ::tolower);
+    const Params parameters = helper::XMLGetParameters(operatorNode, hint);
 
-        const Params parameters = helper::XMLGetParameters(operatorNode, hint);
+    adios.DefineOperator(name->value(), typeLowerCase, parameters);
+}
 
-        adios.DefineOperator(name->value(), typeLowerCase, parameters);
-    };
+void IOVariableXML(const pugi::xml_node &variableNode, core::IO &io, const std::string &hint,
+                   std::unordered_map> &operators)
+{
+    const std::string variableName =
+        std::string(helper::XMLAttribute("name", variableNode, hint)->value());
 
-    // node is the variable node
-    auto lf_IOVariableXML = [&](const pugi::xml_node &node, core::IO ¤tIO) {
-        const std::string variableName =
-            std::string(helper::XMLAttribute("name", node, hint)->value());
+    for (const pugi::xml_node &operation : variableNode.children("operation"))
+    {
+        const std::unique_ptr opName =
+            helper::XMLAttribute("operator", operation, hint, false);
 
-        for (const pugi::xml_node &operation : node.children("operation"))
+        const std::unique_ptr opType =
+            helper::XMLAttribute("type", operation, hint, false);
+
+        if (*opName && *opType)
         {
-            const std::unique_ptr opName =
-                helper::XMLAttribute("operator", operation, hint, false);
+            helper::Throw(
+                "Helper", "AdiosXML", "ParseConfigXML",
+                "operator (" + std::string(opName->value()) + ") and type (" +
+                    std::string(opType->value()) +
+                    ") attributes can't coexist in  element "
+                    "inside  element");
+        }
 
-            const std::unique_ptr opType =
-                helper::XMLAttribute("type", operation, hint, false);
+        if (!*opName && !*opType)
+        {
+            helper::Throw("Helper", "AdiosXML", "ParseConfigXML",
+                                                 " element "
+                                                 "inside  element requires either operator "
+                                                     "(existing) or type (supported) attribute");
+        }
 
-            if (*opName && *opType)
-            {
-                helper::Throw(
-                    "Helper", "AdiosXML", "ParseConfigXML",
-                    "operator (" + std::string(opName->value()) + ") and type (" +
-                        std::string(opType->value()) +
-                        ") attributes can't coexist in  element "
-                        "inside  element");
-            }
+        std::string type;
+        Params params;
 
-            if (!*opName && !*opType)
+        if (*opName)
+        {
+            auto itOperator = operators.find(std::string(opName->value()));
+            if (itOperator == operators.end())
             {
-                helper::Throw(
-                    "Helper", "AdiosXML", "ParseConfigXML",
-                    " element "
-                    "inside  element requires either operator "
-                        "(existing) or type (supported) attribute");
+                helper::Throw("Helper", "AdiosXML", "ParseConfigXML",
+                                                     "operator " + std::string(opName->value()) +
+                                                         " not previously defined, from variable " +
+                                                         variableName + " inside io " + io.m_Name);
             }
+            type = itOperator->second.first;
+            params = itOperator->second.second;
+        }
 
-            std::string type;
-            Params params;
+        if (*opType)
+        {
+            type = std::string(opType->value());
+        }
 
-            if (*opName)
-            {
-                auto itOperator = operators.find(std::string(opName->value()));
-                if (itOperator == operators.end())
-                {
-                    helper::Throw(
-                        "Helper", "AdiosXML", "ParseConfigXML",
-                        "operator " + std::string(opName->value()) +
-                            " not previously defined, from variable " + variableName +
-                            " inside io " + currentIO.m_Name);
-                }
-                type = itOperator->second.first;
-                params = itOperator->second.second;
-            }
+        for (const auto &p : helper::XMLGetParameters(operation, hint))
+        {
+            params[p.first] = p.second;
+        }
 
-            if (*opType)
-            {
-                type = std::string(opType->value());
-            }
+        io.m_VarOpsPlaceholder[variableName].emplace_back(type, params);
+    }
+}
 
-            for (const auto &p : helper::XMLGetParameters(operation, hint))
-            {
-                params[p.first] = p.second;
-            }
+void IOXML(core::ADIOS &adios, const pugi::xml_node &ioNode, core::IO &io, const std::string &hint,
+           std::unordered_map> &operators)
+{
+    // must be unique per io
+    const std::unique_ptr engine =
+        helper::XMLNode("engine", ioNode, hint, false, true);
 
-            currentIO.m_VarOpsPlaceholder[variableName].push_back({type, params});
-        }
-    };
+    if (*engine)
+    {
+        const std::unique_ptr type =
+            helper::XMLAttribute("type", *engine, hint);
+        io.SetEngine(type->value());
 
-    auto lf_IOXML = [&](const pugi::xml_node &io) {
-        const std::unique_ptr ioName = helper::XMLAttribute("name", io, hint);
+        const Params parameters = helper::XMLGetParameters(*engine, hint);
+        io.SetParameters(parameters);
+    }
 
-        // Build the IO object
-        auto itCurrentIO =
-            ios.emplace(std::piecewise_construct, std::forward_as_tuple(ioName->value()),
-                        std::forward_as_tuple(adios, ioName->value(), true, adios.m_HostLanguage));
-        core::IO ¤tIO = itCurrentIO.first->second;
+    for (const pugi::xml_node &transport : ioNode.children("transport"))
+    {
+        const std::unique_ptr type =
+            helper::XMLAttribute("type", transport, hint);
 
-        // must be unique per io
-        const std::unique_ptr engine =
-            helper::XMLNode("engine", io, hint, false, true);
+        const Params parameters = helper::XMLGetParameters(transport, hint);
+        io.AddTransport(type->value(), parameters);
+    }
 
-        if (*engine)
-        {
-            const std::unique_ptr type =
-                helper::XMLAttribute("type", *engine, hint);
-            currentIO.SetEngine(type->value());
+    for (const pugi::xml_node &variable : ioNode.children("variable"))
+    {
+        IOVariableXML(variable, io, hint, operators);
+    }
+}
 
-            const Params parameters = helper::XMLGetParameters(*engine, hint);
-            currentIO.SetParameters(parameters);
-        }
+void ParseConfigXMLIO(core::ADIOS &adios, const std::string &configFileXML,
+                      const std::string &configFileContents, core::IO &io,
+                      std::unordered_map> &operators)
+{
+    const std::string hint("for config file " + configFileXML + " in call to ADIOS constructor");
 
-        for (const pugi::xml_node &transport : io.children("transport"))
-        {
-            const std::unique_ptr type =
-                helper::XMLAttribute("type", transport, hint);
+    // the following copy is needed because pugi::xml_document modifies configFileContents
+    const std::string configFileContentsCopy = configFileContents;
+    const std::unique_ptr document =
+        helper::XMLDocument(configFileContentsCopy, hint);
 
-            const Params parameters = helper::XMLGetParameters(transport, hint);
-            currentIO.AddTransport(type->value(), parameters);
-        }
+    // must be unique
+    const std::unique_ptr config =
+        helper::XMLNode("adios-config", *document, hint, true);
 
-        for (const pugi::xml_node &variable : io.children("variable"))
+    for (const pugi::xml_node &ioNode : config->children("io"))
+    {
+        const std::unique_ptr ioName =
+            helper::XMLAttribute("name", ioNode, hint);
+        if (io.m_Name == ioName->value())
         {
-            lf_IOVariableXML(variable, currentIO);
+            IOXML(adios, ioNode, io, hint, operators);
+            return;
         }
-    };
+    }
+}
 
-    // BODY OF FUNCTION
-    const std::string fileContents = lf_FileContents(configFileXML);
-    const std::unique_ptr document = helper::XMLDocument(fileContents, hint);
+std::string
+ParseConfigXML(core::ADIOS &adios, const std::string &configFileXML,
+               std::map &ios,
+               std::unordered_map> &operators)
+{
+    const std::string hint("for config file " + configFileXML + " in call to ADIOS constructor");
+
+    const std::string configFileContents = FileContents(adios, configFileXML);
+    // the following copy is needed because pugi::xml_document modifies configFileContents
+    const std::string configFileContentsCopy = configFileContents;
+    const std::unique_ptr document =
+        helper::XMLDocument(configFileContentsCopy, hint);
 
     // must be unique
     const std::unique_ptr config =
         helper::XMLNode("adios-config", *document, hint, true);
 
-    for (const pugi::xml_node &op : config->children("operator"))
+    for (const pugi::xml_node &opNode : config->children("operator"))
     {
-        lf_OperatorXML(op);
+        OperatorXML(adios, opNode, hint);
     }
 
-    for (const pugi::xml_node &io : config->children("io"))
+    for (const pugi::xml_node &ioNode : config->children("io"))
     {
-        lf_IOXML(io);
+        const std::unique_ptr ioName =
+            helper::XMLAttribute("name", ioNode, hint);
+        // Build the IO object
+        auto itCurrentIO =
+            ios.emplace(std::piecewise_construct, std::forward_as_tuple(ioName->value()),
+                        std::forward_as_tuple(adios, ioName->value(), true, adios.m_HostLanguage));
+        core::IO ¤tIO = itCurrentIO.first->second;
+        IOXML(adios, ioNode, currentIO, hint, operators);
     }
+    return configFileContents;
 }
 
 } // end namespace helper
diff --git a/source/adios2/helper/adiosXML.h b/source/adios2/helper/adiosXML.h
index 2c7aff26fe..6385aecfa5 100644
--- a/source/adios2/helper/adiosXML.h
+++ b/source/adios2/helper/adiosXML.h
@@ -26,9 +26,14 @@ namespace adios2
 namespace helper
 {
 
-void ParseConfigXML(core::ADIOS &adios, const std::string &configFile,
-                    std::map &ios,
-                    std::unordered_map> &operators);
+void ParseConfigXMLIO(core::ADIOS &adios, const std::string &configFileXML,
+                      const std::string &configFileContents, core::IO &io,
+                      std::unordered_map> &operators);
+
+std::string
+ParseConfigXML(core::ADIOS &adios, const std::string &configFile,
+               std::map &ios,
+               std::unordered_map> &operators);
 
 } // end namespace helper
 } // end namespace adios2
diff --git a/source/adios2/helper/adiosYAML.cpp b/source/adios2/helper/adiosYAML.cpp
index a12d8f9d0c..ec835ce26c 100644
--- a/source/adios2/helper/adiosYAML.cpp
+++ b/source/adios2/helper/adiosYAML.cpp
@@ -63,102 +63,134 @@ Params YAMLNodeMapToParams(const YAML::Node &node, const std::string &hint)
     return parameters;
 }
 
+constexpr bool isMandatory = true;
+constexpr bool isNotMandatory = false;
 } // end empty  namespace
 
-void ParseConfigYAML(core::ADIOS &adios, const std::string &configFileYAML,
-                     std::map &ios)
+void IOVariableYAML(const YAML::Node &variableMap, core::IO ¤tIO, const std::string &hint)
 {
-    const std::string hint =
-        "when parsing config file " + configFileYAML + " in call to ADIOS constructor";
+    const YAML::Node &variableNameScalar =
+        YAMLNode("Variable", variableMap, hint, isMandatory, YAML::NodeType::Scalar);
+    const std::string variableName = variableNameScalar.as();
 
-    constexpr bool isMandatory = true;
-    constexpr bool isNotMandatory = false;
+    const YAML::Node operationsSequence =
+        YAMLNode("Operations", variableMap, hint, isNotMandatory, YAML::NodeType::Sequence);
 
-    auto lf_IOVariableYAML = [&](const YAML::Node &variableMap, core::IO ¤tIO) {
-        const YAML::Node &variableNameScalar =
-            YAMLNode("Variable", variableMap, hint, isMandatory, YAML::NodeType::Scalar);
-        const std::string variableName = variableNameScalar.as();
-
-        const YAML::Node operationsSequence =
-            YAMLNode("Operations", variableMap, hint, isNotMandatory, YAML::NodeType::Sequence);
+    if (operationsSequence)
+    {
+        // loop through each transport node
+        const std::string errorMessage =
+            " in operations node from variable " + variableName + ", " + hint;
 
-        if (operationsSequence)
+        for (auto it = operationsSequence.begin(); it != operationsSequence.end(); ++it)
         {
-            // loop through each transport node
-            const std::string errorMessage =
-                " in operations node from variable " + variableName + ", " + hint;
+            const YAML::Node typeScalar =
+                YAMLNode("Type", *it, errorMessage, isMandatory, YAML::NodeType::Scalar);
 
-            for (auto it = operationsSequence.begin(); it != operationsSequence.end(); ++it)
-            {
-                const YAML::Node typeScalar =
-                    YAMLNode("Type", *it, errorMessage, isMandatory, YAML::NodeType::Scalar);
-
-                Params parameters = YAMLNodeMapToParams(*it, hint);
-                const std::string operatorType = EraseKey("Type", parameters);
+            Params parameters = YAMLNodeMapToParams(*it, hint);
+            const std::string operatorType = EraseKey("Type", parameters);
 
-                currentIO.m_VarOpsPlaceholder[variableName].push_back({operatorType, parameters});
-            }
+            currentIO.m_VarOpsPlaceholder[variableName].emplace_back(operatorType, parameters);
         }
-    };
-
-    auto lf_IOYAML = [&](const std::string &ioName, const YAML::Node &ioMap) {
-        // Build the IO object
-        auto itCurrentIO =
-            ios.emplace(std::piecewise_construct, std::forward_as_tuple(ioName),
-                        std::forward_as_tuple(adios, ioName, true, adios.m_HostLanguage));
-        core::IO ¤tIO = itCurrentIO.first->second;
+    }
+}
 
-        // Engine parameters
-        const YAML::Node engineMap = YAMLNode("Engine", ioMap, hint, false, YAML::NodeType::Map);
+void IOYAML(core::ADIOS &adios, const YAML::Node &ioMap, core::IO &io, const std::string &hint)
+{
+    // Engine parameters
+    const YAML::Node engineMap = YAMLNode("Engine", ioMap, hint, false, YAML::NodeType::Map);
 
-        if (engineMap)
+    if (engineMap)
+    {
+        Params parameters = YAMLNodeMapToParams(engineMap, hint);
+        auto itType = parameters.find("Type");
+        if (itType != parameters.end())
         {
-            Params parameters = YAMLNodeMapToParams(engineMap, hint);
-            auto itType = parameters.find("Type");
-            if (itType != parameters.end())
-            {
-                const std::string type = EraseKey("Type", parameters);
-                currentIO.SetEngine(type);
-            }
-            currentIO.SetParameters(parameters);
+            const std::string type = EraseKey("Type", parameters);
+            io.SetEngine(type);
         }
+        io.SetParameters(parameters);
+    }
 
-        // Variables
-        const YAML::Node variablesSequence =
-            YAMLNode("Variables", ioMap, hint, false, YAML::NodeType::Sequence);
+    // Variables
+    const YAML::Node variablesSequence =
+        YAMLNode("Variables", ioMap, hint, false, YAML::NodeType::Sequence);
 
-        if (variablesSequence)
+    if (variablesSequence)
+    {
+        // loop through each variable node
+        for (const YAML::Node &variableMap : variablesSequence)
         {
-            // loop through each variable node
-            for (const YAML::Node &variableMap : variablesSequence)
-            {
-                lf_IOVariableYAML(variableMap, currentIO);
-            }
+            IOVariableYAML(variableMap, io, hint);
         }
+    }
 
-        // Transports
-        const YAML::Node transportsSequence =
-            YAMLNode("Transports", ioMap, hint, false, YAML::NodeType::Sequence);
+    // Transports
+    const YAML::Node transportsSequence =
+        YAMLNode("Transports", ioMap, hint, false, YAML::NodeType::Sequence);
 
-        if (transportsSequence)
+    if (transportsSequence)
+    {
+        // loop through each transport node
+        for (auto it = transportsSequence.begin(); it != transportsSequence.end(); ++it)
         {
-            // loop through each transport node
-            for (auto it = transportsSequence.begin(); it != transportsSequence.end(); ++it)
-            {
-                YAMLNode("Type", *it, " in transport node " + hint, isMandatory,
-                         YAML::NodeType::Scalar);
+            YAMLNode("Type", *it, " in transport node " + hint, isMandatory,
+                     YAML::NodeType::Scalar);
+
+            Params parameters = YAMLNodeMapToParams(*it, hint);
+            const std::string type = EraseKey("Type", parameters);
+
+            io.AddTransport(type, parameters);
+        }
+    }
+}
+
+void ParseConfigYAMLIO(core::ADIOS &adios, const std::string &configFileYAML,
+                       const std::string &configFileContents, core::IO &io)
+{
+    const std::string hint =
+        "when parsing config file " + configFileYAML + " in call to ADIOS constructor";
+
+    // the following copy is needed because YAML::Load modifies configFileContents
+    const std::string configFileContentsCopy = configFileContents;
+    const YAML::Node document = YAML::Load(configFileContentsCopy);
 
-                Params parameters = YAMLNodeMapToParams(*it, hint);
-                const std::string type = EraseKey("Type", parameters);
+    if (!document)
+    {
+        helper::Throw(
+            "Helper", "adiosYAML", "ParseConfigYAML",
+            "parser error in file " + configFileYAML +
+                " invalid format check with any YAML editor if format is "
+                "ill-formed, " +
+                hint);
+    }
 
-                currentIO.AddTransport(type, parameters);
+    for (auto itNode = document.begin(); itNode != document.end(); ++itNode)
+    {
+        const YAML::Node ioScalar =
+            YAMLNode("IO", *itNode, hint, isNotMandatory, YAML::NodeType::Scalar);
+        if (ioScalar)
+        {
+            const std::string ioName = ioScalar.as();
+            if (ioName == io.m_Name)
+            {
+                IOYAML(adios, *itNode, io, hint);
+                return;
             }
         }
-    };
+    }
+}
+
+std::string ParseConfigYAML(core::ADIOS &adios, const std::string &configFileYAML,
+                            std::map &ios)
+{
+    const std::string hint =
+        "when parsing config file " + configFileYAML + " in call to ADIOS constructor";
 
-    // BODY OF FUNCTION STARTS HERE
-    const std::string fileContents = adios.GetComm().BroadcastFile(configFileYAML, hint);
-    const YAML::Node document = YAML::Load(fileContents);
+    const std::string configFileContents = adios.GetComm().BroadcastFile(configFileYAML, hint);
+    // the following copy is needed because YAML::Load modifies configFileContents
+    const std::string configFileContentsCopy = configFileContents;
+    const YAML::Node document = YAML::Load(configFileContentsCopy);
 
     if (!document)
     {
@@ -177,9 +209,15 @@ void ParseConfigYAML(core::ADIOS &adios, const std::string &configFileYAML,
         if (ioScalar)
         {
             const std::string ioName = ioScalar.as();
-            lf_IOYAML(ioName, *itNode);
+            // Build the IO object
+            auto itCurrentIO =
+                ios.emplace(std::piecewise_construct, std::forward_as_tuple(ioName),
+                            std::forward_as_tuple(adios, ioName, true, adios.m_HostLanguage));
+            core::IO ¤tIO = itCurrentIO.first->second;
+            IOYAML(adios, *itNode, currentIO, hint);
         }
     }
+    return configFileContents;
 }
 
 } // end namespace helper
diff --git a/source/adios2/helper/adiosYAML.h b/source/adios2/helper/adiosYAML.h
index 6d8bfa12a1..8014d0cf06 100644
--- a/source/adios2/helper/adiosYAML.h
+++ b/source/adios2/helper/adiosYAML.h
@@ -25,9 +25,11 @@ namespace adios2
 {
 namespace helper
 {
+void ParseConfigYAMLIO(core::ADIOS &adios, const std::string &configFileYAML,
+                       const std::string &configFileContents, core::IO &io);
 
-void ParseConfigYAML(core::ADIOS &adios, const std::string &configFile,
-                     std::map &ios);
+std::string ParseConfigYAML(core::ADIOS &adios, const std::string &configFileYAML,
+                            std::map &ios);
 
 } // end namespace helper
 } // end namespace adios2
diff --git a/source/adios2/operator/OperatorFactory.cpp b/source/adios2/operator/OperatorFactory.cpp
index 3e9b0b419c..87139a92ff 100644
--- a/source/adios2/operator/OperatorFactory.cpp
+++ b/source/adios2/operator/OperatorFactory.cpp
@@ -31,6 +31,10 @@
 #include "adios2/operator/compress/CompressMGARDPlus.h"
 #endif
 
+#ifdef ADIOS2_HAVE_MGARD_MDR
+#include "adios2/operator/refactor/RefactorMDR.h"
+#endif
+
 #ifdef ADIOS2_HAVE_PNG
 #include "adios2/operator/compress/CompressPNG.h"
 #endif
@@ -74,6 +78,8 @@ std::string OperatorTypeToString(const Operator::OperatorType type)
         return "sz";
     case Operator::COMPRESS_ZFP:
         return "zfp";
+    case Operator::REFACTOR_MDR:
+        return "mdr";
     case Operator::PLUGIN_INTERFACE:
         return "plugin";
     default:
@@ -139,6 +145,12 @@ std::shared_ptr MakeOperator(const std::string &type, const Params &pa
     {
 #ifdef ADIOS2_HAVE_ZFP
         ret = std::make_shared(parameters);
+#endif
+    }
+    else if (typeLowerCase == "mdr")
+    {
+#ifdef ADIOS2_HAVE_MGARD_MDR
+        ret = std::make_shared(parameters);
 #endif
     }
     else if (typeLowerCase == "plugin")
diff --git a/source/adios2/operator/OperatorFactory.h b/source/adios2/operator/OperatorFactory.h
index 50519f93a4..0439a81bd1 100644
--- a/source/adios2/operator/OperatorFactory.h
+++ b/source/adios2/operator/OperatorFactory.h
@@ -17,6 +17,8 @@ namespace adios2
 namespace core
 {
 
+std::string OperatorTypeToString(const Operator::OperatorType type);
+
 std::shared_ptr MakeOperator(const std::string &type, const Params ¶meters);
 
 size_t Decompress(const char *bufferIn, const size_t sizeIn, char *dataOut, MemorySpace memSpace,
diff --git a/source/adios2/operator/refactor/RefactorMDR.cpp b/source/adios2/operator/refactor/RefactorMDR.cpp
new file mode 100644
index 0000000000..b2a5518bfa
--- /dev/null
+++ b/source/adios2/operator/refactor/RefactorMDR.cpp
@@ -0,0 +1,536 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ *
+ * RefactorMDR.cpp :
+ *
+ *  Created on: Sep 14, 2023
+ *      Author: Norbert Podhorszki 
+ */
+
+#include "RefactorMDR.h"
+#include "adios2/helper/adiosFunctions.h"
+#include "adios2/operator/compress/CompressNull.h"
+#include 
+#include 
+
+#include 
+
+#include 
+
+namespace adios2
+{
+namespace core
+{
+namespace refactor
+{
+
+RefactorMDR::RefactorMDR(const Params ¶meters)
+: Operator("mdr", REFACTOR_MDR, "refactor", parameters)
+{
+    config.normalize_coordinates = false;
+    config.log_level = 1;
+    config.decomposition = mgard_x::decomposition_type::MultiDim;
+    config.domain_decomposition = mgard_x::domain_decomposition_type::MaxDim;
+    // config.domain_decomposition = mgard_x::domain_decomposition_type::Block;
+    // config.block_size = 64;
+
+    config.dev_type = mgard_x::device_type::AUTO;
+    config.prefetch = false;
+    // config.max_memory_footprint = max_memory_footprint;
+
+    config.lossless = mgard_x::lossless_type::Huffman_Zstd;
+    // mgard_x::lossless_type::Huffman, mgard_x::lossless_type::Huffman_Zstd,
+    // mgard_x::lossless_type::CPU_Lossless
+
+    // This should be changed to 4 for float later
+    config.total_num_bitplanes = 64;
+}
+
+size_t RefactorMDR::GetEstimatedSize(const size_t ElemCount, const size_t ElemSize,
+                                     const size_t ndims, const size_t *dims) const
+{
+    std::cout << "RefactorMDR::GetEstimatedSize() called \n";
+    mgard_x::data_type mtype =
+        (ElemSize == 8 ? mgard_x::data_type::Double : mgard_x::data_type::Float);
+    DataType datatype = (ElemSize == 8 ? DataType::Double : DataType::Float);
+    Dims dimsV(ndims);
+    for (size_t i = 0; i < ndims; ++i)
+    {
+        dimsV[i] = dims[i];
+    }
+    Dims convertedDims = ConvertDims(dimsV, datatype, 3);
+    mgard_x::DIM mgardDim = ndims;
+    std::vector mgardCount;
+    for (const auto &c : convertedDims)
+    {
+        mgardCount.push_back(c);
+    }
+
+    mgard_x::Config cfg(config); // copy of const config
+    if (mtype == mgard_x::data_type::Float)
+    {
+        cfg.total_num_bitplanes = 32;
+    }
+
+    auto s = mgard_x::MDR::MDRMaxOutputDataSize(mgardDim, mtype, mgardCount, config);
+
+    size_t sizeIn = helper::GetTotalSize(convertedDims, ElemSize);
+    std::cout << "RefactorMDR Estimated Max output size = " << s << " for input size = " << sizeIn
+              << std::endl;
+    return (size_t)s + 128; // count in the header
+};
+
+struct RefactorMDRHeader
+{
+    uint64_t ndims;
+    std::vector dims; // ndims values
+    adios2::DataType type;
+    uint8_t mgard_version_major;
+    uint8_t mgard_version_minor;
+    uint8_t mgard_version_patch;
+    bool wasRefactored;
+    uint64_t metadataHeaderSize;
+    uint64_t metadataSize;
+    uint8_t nSubdomains;
+    uint8_t nLevels;
+    uint8_t nBitPlanes;
+};
+
+size_t RefactorMDR::Operate(const char *dataIn, const Dims &blockStart, const Dims &blockCount,
+                            const DataType type, char *bufferOut)
+{
+    const uint8_t bufferVersion = 1;
+    size_t bufferOutOffset = 0;
+
+    MakeCommonHeader(bufferOut, bufferOutOffset, bufferVersion);
+
+    Dims convertedDims = ConvertDims(blockCount, type, 3);
+
+    const size_t ndims = convertedDims.size();
+    if (ndims > 5)
+    {
+        helper::Throw("Operator", "RefactorMDR", "Operate",
+                                             "MGARD does not support data in " +
+                                                 std::to_string(ndims) + " dimensions");
+    }
+
+    // mgard V1 metadata
+    PutParameter(bufferOut, bufferOutOffset, ndims);
+    for (const auto &d : convertedDims)
+    {
+        PutParameter(bufferOut, bufferOutOffset, d);
+    }
+    PutParameter(bufferOut, bufferOutOffset, type);
+    PutParameter(bufferOut, bufferOutOffset, static_cast(MGARD_VERSION_MAJOR));
+    PutParameter(bufferOut, bufferOutOffset, static_cast(MGARD_VERSION_MINOR));
+    PutParameter(bufferOut, bufferOutOffset, static_cast(MGARD_VERSION_PATCH));
+    // mgard V1 metadata end
+
+    // set type
+    mgard_x::data_type mgardType;
+    if (type == helper::GetDataType())
+    {
+        mgardType = mgard_x::data_type::Float;
+        config.total_num_bitplanes = 32;
+    }
+    else if (type == helper::GetDataType())
+    {
+        mgardType = mgard_x::data_type::Double;
+        config.total_num_bitplanes = 64;
+    }
+    else if (type == helper::GetDataType>())
+    {
+        mgardType = mgard_x::data_type::Float;
+        config.total_num_bitplanes = 32;
+    }
+    else if (type == helper::GetDataType>())
+    {
+        mgardType = mgard_x::data_type::Double;
+        config.total_num_bitplanes = 64;
+    }
+    else
+    {
+        helper::Throw("Operator", "RefactorMDR", "Operate",
+                                             "MGARD only supports float and double types");
+    }
+    // set type end
+
+    // set mgard style dim info
+    mgard_x::DIM mgardDim = ndims;
+    std::vector mgardCount;
+    for (const auto &c : convertedDims)
+    {
+        mgardCount.push_back(c);
+    }
+    // set mgard style dim info end
+
+    // input under this size will not be refactored
+    const size_t thresholdSize = 100000;
+
+    size_t sizeIn = helper::GetTotalSize(blockCount, helper::GetDataTypeSize(type));
+
+    if (sizeIn < thresholdSize)
+    {
+        /* disable compression and add marker in the header*/
+        PutParameter(bufferOut, bufferOutOffset, false);
+        headerSize = bufferOutOffset;
+        return 0;
+    }
+    PutParameter(bufferOut, bufferOutOffset, true);
+
+    mgard_x::MDR::RefactoredMetadata refactored_metadata;
+    mgard_x::MDR::RefactoredData refactored_data;
+    char *dataptr = const_cast(dataIn);
+    mgard_x::pin_memory(dataptr, sizeIn, config);
+    mgard_x::MDR::MDRefactor(mgardDim, mgardType, mgardCount, dataIn, refactored_metadata,
+                             refactored_data, config, false);
+    mgard_x::unpin_memory(dataptr, config);
+
+    size_t nbytes = SerializeRefactoredData(refactored_metadata, refactored_data,
+                                            bufferOut + bufferOutOffset, SIZE_MAX);
+
+    bufferOutOffset += nbytes;
+
+    return bufferOutOffset;
+}
+
+// return number of bytes written
+size_t RefactorMDR::SerializeRefactoredData(mgard_x::MDR::RefactoredMetadata &refactored_metadata,
+                                            mgard_x::MDR::RefactoredData &refactored_data,
+                                            char *buffer, size_t maxsize)
+{
+    size_t offset = 0;
+    size_t MDRHeaderSize = 0;
+
+    /* Metadata header */
+    const uint64_t metadata_header_size = refactored_metadata.header.size();
+    {
+        PutParameter(buffer, offset, metadata_header_size);
+        std::memcpy(buffer + offset, refactored_metadata.header.data(), metadata_header_size);
+        offset += metadata_header_size;
+    }
+
+    /* Metadata */
+    std::vector serialized_metadata = refactored_metadata.Serialize();
+    const uint64_t metadata_size = (uint64_t)serialized_metadata.size();
+    {
+        PutParameter(buffer, offset, metadata_size);
+        std::memcpy(buffer + offset, serialized_metadata.data(), metadata_size);
+        offset += metadata_size;
+    }
+
+    std::cout << "MDR metadata seralized " << offset << " bytes. header = " << metadata_header_size
+              << " metadata = " << metadata_size << "\n";
+
+    /* 3D table of subdomain X level X bitplane offsets, not all of them has data */
+    uint8_t nSubdomains = refactored_metadata.metadata.size();
+    uint8_t nLevels = 0;
+    uint8_t nBitPlanes = 0;
+
+    for (size_t subdomain_id = 0; subdomain_id < refactored_metadata.metadata.size();
+         subdomain_id++)
+    {
+        nLevels = std::max(nLevels,
+                           (uint8_t)refactored_metadata.metadata[subdomain_id].level_sizes.size());
+        for (size_t level_idx = 0;
+             level_idx < refactored_metadata.metadata[subdomain_id].level_sizes.size(); level_idx++)
+        {
+            nBitPlanes = std::max(
+                nBitPlanes,
+                (uint8_t)refactored_metadata.metadata[subdomain_id].level_sizes[level_idx].size());
+        }
+    }
+
+    PutParameter(buffer, offset, nSubdomains);
+    PutParameter(buffer, offset, nLevels);
+    PutParameter(buffer, offset, nBitPlanes);
+    uint64_t tableSize = (nSubdomains * nLevels * nBitPlanes);
+    uint64_t *table = (uint64_t *)(buffer + offset);
+    std::fill(table, table + tableSize, 0ULL);
+    offset += tableSize * sizeof(uint64_t);
+    MDRHeaderSize = offset;
+
+    /* Individual components of refactored data */
+    size_t nBlocks = 0;
+    uint64_t tableIdx = 0;
+    for (size_t subdomain_id = 0; subdomain_id < refactored_metadata.metadata.size();
+         subdomain_id++)
+    {
+        for (size_t level_idx = 0;
+             level_idx < refactored_metadata.metadata[subdomain_id].level_sizes.size(); level_idx++)
+        {
+            tableIdx = subdomain_id * nLevels * nBitPlanes + level_idx * nBitPlanes;
+            for (size_t bitplane_idx = 0;
+                 bitplane_idx <
+                 refactored_metadata.metadata[subdomain_id].level_sizes[level_idx].size();
+                 bitplane_idx++)
+            {
+                std::memcpy(buffer + offset,
+                            refactored_data.data[subdomain_id][level_idx][bitplane_idx],
+                            refactored_metadata.metadata[subdomain_id]
+                                .level_sizes[level_idx][bitplane_idx]);
+                table[tableIdx] = offset - MDRHeaderSize;
+                offset +=
+                    refactored_metadata.metadata[subdomain_id].level_sizes[level_idx][bitplane_idx];
+                ++nBlocks;
+                ++tableIdx;
+            }
+        }
+    }
+
+    std::cout << "MDR serialized " << offset << " bytes, MDR header size = " << MDRHeaderSize
+              << " subdomains = " << (size_t)nSubdomains << " levels = " << (size_t)nLevels
+              << " bitplanes = " << (size_t)nBitPlanes << " blocks = " << nBlocks << "\n";
+    return offset;
+}
+
+size_t RefactorMDR::GetHeaderSize() const { return headerSize; }
+
+size_t RefactorMDR::ReconstructV1(const char *bufferIn, const size_t sizeIn, char *dataOut)
+{
+    // Do NOT remove even if the buffer version is updated. Data might be still
+    // in lagacy formats. This function must be kept for backward compatibility.
+    // If a newer buffer format is implemented, create another function, e.g.
+    // ReconstructV1 and keep this function for reconstructing legacy data.
+
+    config.log_level = 1;
+    // double s = std::numeric_limits::infinity();
+
+    size_t bufferInOffset = 0;
+
+    const size_t ndims = GetParameter(bufferIn, bufferInOffset);
+    Dims blockCount(ndims);
+    for (size_t i = 0; i < ndims; ++i)
+    {
+        blockCount[i] = GetParameter(bufferIn, bufferInOffset);
+    }
+    const DataType type = GetParameter(bufferIn, bufferInOffset);
+    m_VersionInfo = " Data is compressed using MGARD Version " +
+                    std::to_string(GetParameter(bufferIn, bufferInOffset)) + "." +
+                    std::to_string(GetParameter(bufferIn, bufferInOffset)) + "." +
+                    std::to_string(GetParameter(bufferIn, bufferInOffset)) +
+                    ". Please make sure a compatible version is used for decompression.";
+
+    const bool isRefactored = GetParameter(bufferIn, bufferInOffset);
+
+    if (!isRefactored)
+    {
+        // data was copied as is from this offset
+        headerSize += bufferInOffset;
+        m_AccuracyProvided = m_AccuracyRequested;
+        return 0;
+    }
+
+    size_t sizeOut = helper::GetTotalSize(blockCount, helper::GetDataTypeSize(type));
+
+    mgard_x::MDR::RefactoredMetadata refactored_metadata;
+
+    /* Metadata header */
+    {
+        const uint64_t metadata_header_size = GetParameter(bufferIn, bufferInOffset);
+        refactored_metadata.header.resize(metadata_header_size);
+        std::memcpy(refactored_metadata.header.data(), bufferIn + bufferInOffset,
+                    metadata_header_size);
+        bufferInOffset += metadata_header_size;
+    }
+
+    /* Metadata */
+    {
+        const uint64_t metadata_size = GetParameter(bufferIn, bufferInOffset);
+        std::vector serialized_metadata;
+        serialized_metadata.resize(metadata_size);
+        std::memcpy(serialized_metadata.data(), bufferIn + bufferInOffset, metadata_size);
+        bufferInOffset += metadata_size;
+        refactored_metadata.Deserialize(serialized_metadata);
+        refactored_metadata.InitializeForReconstruction();
+    }
+
+    mgard_x::MDR::RefactoredData refactored_data;
+    refactored_data.InitializeForReconstruction(refactored_metadata);
+
+    const uint8_t nSubdomains = GetParameter(bufferIn, bufferInOffset);
+    const uint8_t nLevels = GetParameter(bufferIn, bufferInOffset);
+    const uint8_t nBitPlanes = GetParameter(bufferIn, bufferInOffset);
+
+    const uint64_t tableSize = (nSubdomains * nLevels * nBitPlanes);
+    const uint64_t *table = (uint64_t *)(bufferIn + bufferInOffset);
+    bufferInOffset += tableSize * sizeof(uint64_t);
+
+    /*
+        Reconstruction
+    */
+    const char *componentData = bufferIn + bufferInOffset; // data pieces are here in memory
+
+    mgard_x::MDR::ReconstructedData reconstructed_data;
+    for (auto &metadata : refactored_metadata.metadata)
+    {
+        metadata.requested_tol = m_AccuracyRequested.error;
+        metadata.requested_s = m_AccuracyRequested.norm;
+    }
+    mgard_x::MDR::MDRequest(refactored_metadata, config);
+    /*for (auto &metadata : refactored_metadata.metadata)
+    {
+        metadata.PrintStatus();
+    }*/
+
+    bool first_reconstruction = true; // only will be needed with progressive reconstruction
+
+    // Assemble data pieces from buffer
+    {
+        uint64_t tableIdx;
+        int num_subdomains = refactored_metadata.metadata.size();
+        assert(nSubdomains == refactored_metadata.metadata.size());
+        for (int subdomain_id = 0; subdomain_id < num_subdomains; subdomain_id++)
+        {
+            mgard_x::MDR::MDRMetadata metadata = refactored_metadata.metadata[subdomain_id];
+            int num_levels = metadata.level_sizes.size();
+            for (int level_idx = 0; level_idx < num_levels; level_idx++)
+            {
+                assert(nLevels >= metadata.level_sizes.size());
+                tableIdx = subdomain_id * nLevels * nBitPlanes + level_idx * nBitPlanes;
+                int num_bitplanes = metadata.level_sizes[level_idx].size();
+                int loaded_bitplanes = metadata.loaded_level_num_bitplanes[level_idx];
+                int reqested_bitplanes = metadata.requested_level_num_bitplanes[level_idx];
+                assert(nBitPlanes >= metadata.requested_level_num_bitplanes[level_idx]);
+                for (int bitplane_idx = loaded_bitplanes; bitplane_idx < reqested_bitplanes;
+                     bitplane_idx++)
+                {
+
+                    uint64_t componentSize = refactored_metadata.metadata[subdomain_id]
+                                                 .level_sizes[level_idx][bitplane_idx];
+                    const mgard_x::Byte *cdata =
+                        reinterpret_cast(componentData + table[tableIdx]);
+
+                    refactored_data.data[subdomain_id][level_idx][bitplane_idx] =
+                        const_cast(cdata);
+
+                    ++tableIdx;
+                }
+                if (first_reconstruction)
+                {
+                    // initialize level signs
+                    refactored_data.level_signs[subdomain_id][level_idx] =
+                        (bool *)malloc(sizeof(bool) * metadata.level_num_elems[level_idx]);
+                    std::memset(refactored_data.level_signs[subdomain_id][level_idx], 0,
+                                sizeof(bool) * metadata.level_num_elems[level_idx]);
+                }
+            }
+        }
+    }
+
+    /* Initialize reconstructed data manually here to force using
+        user allocated memory for the final result
+    */
+    if (false)
+    {
+        reconstructed_data.Initialize(1);
+        reconstructed_data.data[0] = reinterpret_cast(dataOut);
+        std::memset(reconstructed_data.data[0], 0, sizeOut);
+        std::vector offsets = std::vector(ndims, 0);
+        std::vector shape = std::vector();
+        for (const auto &c : blockCount)
+        {
+            shape.push_back(static_cast(c));
+        }
+        reconstructed_data.offset[0] = offsets;
+        reconstructed_data.shape[0] = shape;
+    }
+
+    mgard_x::MDR::MDReconstruct(refactored_metadata, refactored_data, reconstructed_data, config,
+                                false);
+
+    size_t refactoredSize =
+        helper::GetTotalSize(reconstructed_data.shape[0], helper::GetDataTypeSize(type));
+    assert(sizeOut == refactoredSize);
+    std::memcpy(dataOut, reconstructed_data.data[0], sizeOut);
+
+    first_reconstruction = false;
+
+    /*for (int subdomain_id = 0; subdomain_id < reconstructed_data.data.size(); subdomain_id++)
+    {
+        std::cout << "reconstructed_data " << subdomain_id << " : offset(";
+        for (auto n : reconstructed_data.offset[subdomain_id])
+        {
+            std::cout << n << " ";
+        }
+        std::cout << ") shape(";
+        for (auto n : reconstructed_data.shape[subdomain_id])
+        {
+            std::cout << n << " ";
+        }
+        std::cout << ")\n";
+    }*/
+
+    if (type == DataType::FloatComplex || type == DataType::Float)
+    {
+        m_AccuracyProvided.error =
+            std::max(m_AccuracyRequested.error, adios2::ops::mdr::FLOAT_ROUNDING_ERROR_LIMIT);
+    }
+    else
+    {
+        m_AccuracyProvided.error =
+            std::max(m_AccuracyRequested.error, adios2::ops::mdr::DOUBLE_ROUNDING_ERROR_LIMIT);
+    }
+    m_AccuracyProvided.norm = m_AccuracyRequested.norm;
+    m_AccuracyProvided.relative = false; // should be m_AccuracyRequested.relative
+
+    if (type == DataType::FloatComplex || type == DataType::DoubleComplex)
+    {
+        sizeOut /= 2;
+    }
+    return sizeOut;
+}
+
+size_t RefactorMDR::InverseOperate(const char *bufferIn, const size_t sizeIn, char *dataOut)
+{
+
+    for (auto &p : m_Parameters)
+    {
+        std::cout << "User parameter " << p.first << " = " << p.second << std::endl;
+        const std::string key = helper::LowerCase(p.first);
+        // const std::string value = helper::LowerCase(p.second);
+        if (key == "accuracy")
+        {
+            m_AccuracyRequested.error =
+                helper::StringTo(p.second, " in Parameter key=" + key);
+            std::cout << "Accuracy error set from Parameter to " << m_AccuracyRequested.error;
+        }
+    }
+
+    size_t bufferInOffset = 1; // skip operator type
+    const uint8_t bufferVersion = GetParameter(bufferIn, bufferInOffset);
+    bufferInOffset += 2; // skip two reserved bytes
+    headerSize = bufferInOffset;
+
+    if (bufferVersion == 1)
+    {
+        return ReconstructV1(bufferIn + bufferInOffset, sizeIn - bufferInOffset, dataOut);
+    }
+    /*else if (bufferVersion == 2)
+    {
+        // TODO: if a Version 2 mgard buffer is being implemented, put it here
+        // and keep the ReconstructV1 routine for backward compatibility
+    }*/
+    else
+    {
+        helper::Throw("Operator", "RefactorMDR", "InverseOperate",
+                                          "invalid mgard buffer version" + bufferVersion);
+    }
+
+    return 0;
+}
+
+bool RefactorMDR::IsDataTypeValid(const DataType type) const
+{
+    if (type == DataType::Double || type == DataType::Float || type == DataType::DoubleComplex ||
+        type == DataType::FloatComplex)
+    {
+        return true;
+    }
+    return false;
+}
+
+} // end namespace compress
+} // end namespace core
+} // end namespace adios2
diff --git a/source/adios2/operator/refactor/RefactorMDR.h b/source/adios2/operator/refactor/RefactorMDR.h
new file mode 100644
index 0000000000..99a39a82a2
--- /dev/null
+++ b/source/adios2/operator/refactor/RefactorMDR.h
@@ -0,0 +1,85 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ *
+ * RefactorMDR.h :
+ *
+ *  Created on: Sep 14, 2023
+ *      Author: Norbert Podhorszki 
+ */
+
+#ifndef ADIOS2_OPERATOR_COMPRESS_REFACTORMDR_H_
+#define ADIOS2_OPERATOR_COMPRESS_REFACTORMDR_H_
+
+#include "adios2/core/Operator.h"
+
+#include 
+#include 
+
+namespace adios2
+{
+namespace core
+{
+namespace refactor
+{
+
+class RefactorMDR : public Operator
+{
+
+public:
+    RefactorMDR(const Params ¶meters);
+
+    ~RefactorMDR() = default;
+
+    /**
+     * @param dataIn
+     * @param blockStart
+     * @param blockCount
+     * @param type
+     * @param bufferOut
+     * @return size of compressed buffer
+     */
+    size_t Operate(const char *dataIn, const Dims &blockStart, const Dims &blockCount,
+                   const DataType type, char *bufferOut) final;
+
+    /**
+     * @param bufferIn
+     * @param sizeIn
+     * @param dataOut
+     * @return size of decompressed buffer
+     */
+    size_t InverseOperate(const char *bufferIn, const size_t sizeIn, char *dataOut) final;
+
+    bool IsDataTypeValid(const DataType type) const final;
+
+    size_t GetHeaderSize() const;
+
+    size_t GetEstimatedSize(const size_t ElemCount, const size_t ElemSize, const size_t ndims,
+                            const size_t *dims) const;
+
+private:
+    size_t headerSize = 0;
+    /**
+     * Decompress function for V1 buffer. Do NOT remove even if the buffer
+     * version is updated. Data might be still in lagacy formats. This function
+     * must be kept for backward compatibility
+     * @param bufferIn : compressed data buffer (V1 only)
+     * @param sizeIn : number of bytes in bufferIn
+     * @param dataOut : decompressed data buffer
+     * @return : number of bytes in dataOut
+     */
+    size_t ReconstructV1(const char *bufferIn, const size_t sizeIn, char *dataOut);
+    size_t SerializeRefactoredData(mgard_x::MDR::RefactoredMetadata &refactored_metadata,
+                                   mgard_x::MDR::RefactoredData &refactored_data, char *buffer,
+                                   size_t maxsize);
+
+    std::string m_VersionInfo;
+
+    mgard_x::Config config;
+};
+
+} // end namespace compress
+} // end namespace core
+} // end namespace adios2
+
+#endif /* ADIOS2_OPERATOR_COMPRESS_REFACTORMDR_H_ */
diff --git a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp
index 6e6ea7e806..84fd6d9531 100644
--- a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp
+++ b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp
@@ -761,14 +761,7 @@ void BP5Deserializer::InstallMetaData(void *MetadataBlock, size_t BlockLen, size
             m_FreeableMBA = nullptr;
         }
 
-        JoinedDimArray.resize(Step + 1);
-        if (JoinedDimArray[Step] == nullptr)
-        {
-            m_JoinedDimenOffsetArrays = new std::vector();
-            m_JoinedDimenOffsetArrays->resize(writerCohortSize);
-            JoinedDimArray[Step] = m_JoinedDimenOffsetArrays;
-            m_FreeableJDOA = nullptr;
-        }
+        JDAIdx = Step;
     }
     else
     {
@@ -782,16 +775,11 @@ void BP5Deserializer::InstallMetaData(void *MetadataBlock, size_t BlockLen, size
             m_MetadataBaseAddrs->resize(writerCohortSize);
         }
 
-        if (!m_JoinedDimenOffsetArrays)
-        {
-            m_JoinedDimenOffsetArrays = new std::vector();
-            m_FreeableJDOA = m_JoinedDimenOffsetArrays;
-        }
-        if (writerCohortSize > m_JoinedDimenOffsetArrays->size())
-        {
-            m_JoinedDimenOffsetArrays->resize(writerCohortSize);
-        }
+        JDAIdx = 0;
     }
+    JoinedDimArray.resize(JDAIdx + 1);
+    JoinedDimArray[JDAIdx].resize(writerCohortSize);
+
     (*m_MetadataBaseAddrs)[WriterRank] = BaseData;
 
     size_t JoinedDimenTotal = 0;
@@ -823,13 +811,15 @@ void BP5Deserializer::InstallMetaData(void *MetadataBlock, size_t BlockLen, size
 
     //  Allocate memory to hold new offset values for Joined Arrays
     size_t CurJoinedDimenOffset = 0;
-    size_t *JoinedDimenOffsetArray = NULL;
     if (JoinedDimenTotal)
-        JoinedDimenOffsetArray =
-            (size_t *)malloc(JoinedDimenTotal * writerCohortSize * sizeof(size_t));
+    {
+        JoinedDimArray[JDAIdx][WriterRank] =
+            (size_t *)realloc(JoinedDimArray[JDAIdx][WriterRank],
+                              JoinedDimenTotal * writerCohortSize * sizeof(size_t));
+    }
 
-    // store this away so it can be deallocated later
-    (*m_JoinedDimenOffsetArrays)[WriterRank] = JoinedDimenOffsetArray;
+    // shortcut name. should be const
+    size_t *JoinedDimenOffsetArray = JoinedDimArray[JDAIdx][WriterRank];
 
     for (int i = 0; i < Control->ControlCount; i++)
     {
@@ -1778,12 +1768,29 @@ void BP5Deserializer::FinalizeGet(const ReadRequest &Read, const bool freeAddr)
             DestSize *= writer_meta_base->Count[dim + Read.BlockID * writer_meta_base->Dims];
         }
         decompressBuffer.resize(DestSize);
+
+        // Get the operator of the variable if exists or create one
+        std::shared_ptr op = nullptr;
+        VariableBase *VB = static_cast(((struct BP5VarRec *)Req.VarRec)->Variable);
+        if (!VB->m_Operations.empty())
+        {
+            op = VB->m_Operations[0];
+        }
+        else
+        {
+            Operator::OperatorType compressorType =
+                static_cast(IncomingData[0]);
+            op = MakeOperator(OperatorTypeToString(compressorType), {});
+        }
+        op->SetAccuracy(VB->GetAccuracyRequested());
+
         {
             std::lock_guard lockGuard(mutexDecompress);
             core::Decompress(
                 IncomingData,
                 ((MetaArrayRecOperator *)writer_meta_base)->DataBlockSize[Read.BlockID],
-                decompressBuffer.data(), Req.MemSpace);
+                decompressBuffer.data(), Req.MemSpace, op);
+            VB->m_AccuracyProvided = op->GetAccuracy();
         }
         IncomingData = decompressBuffer.data();
         VirtualIncomingData = IncomingData;
@@ -1985,14 +1992,20 @@ BP5Deserializer::~BP5Deserializer()
     {
         delete m_FreeableMBA;
     }
-    if (m_FreeableJDOA)
-    {
-        delete m_FreeableJDOA;
-    }
     for (auto &step : MetadataBaseArray)
     {
         delete step;
     }
+    for (auto &pvec : JoinedDimArray)
+    {
+        for (auto &p : pvec)
+        {
+            if (p)
+            {
+                free(p);
+            }
+        }
+    }
 }
 
 void *BP5Deserializer::GetMetadataBase(BP5VarRec *VarRec, size_t Step, size_t WriterRank) const
diff --git a/source/adios2/toolkit/format/bp5/BP5Deserializer.h b/source/adios2/toolkit/format/bp5/BP5Deserializer.h
index c2d6f39568..ad6c53fb13 100644
--- a/source/adios2/toolkit/format/bp5/BP5Deserializer.h
+++ b/source/adios2/toolkit/format/bp5/BP5Deserializer.h
@@ -190,9 +190,6 @@ class BP5Deserializer : virtual public BP5Base
         nullptr; // may be a pointer into MetadataBaseArray or m_FreeableMBA
     std::vector *m_FreeableMBA = nullptr;
 
-    std::vector *m_JoinedDimenOffsetArrays = nullptr;
-    std::vector *m_FreeableJDOA = nullptr;
-
     // for random access mode, for each timestep, for each writerrank, what
     // metameta info applies to the metadata
     std::vector> m_ControlArray;
@@ -200,8 +197,9 @@ class BP5Deserializer : virtual public BP5Base
     // address of the metadata
     std::vector *> MetadataBaseArray;
     // for random access mode, for each timestep, for each writerrank, base
-    // address of the joined dim arrays
-    std::vector *> JoinedDimArray;
+    // address of the joined dim arrays, for streaming use 0 index
+    std::vector> JoinedDimArray;
+    size_t JDAIdx = 0;
 
     ControlInfo *ControlBlocks = nullptr;
     ControlInfo *GetPriorControl(FMFormat Format);
diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
index d7145bd7fb..84fcd33fc3 100644
--- a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
+++ b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
@@ -837,7 +837,8 @@ void BP5Serializer::Marshal(void *Variable, const char *Name, const DataType Typ
                 if (Offsets)
                     tmpOffsets.push_back(Offsets[i]);
             }
-            size_t AllocSize = ElemCount * ElemSize + 100;
+            size_t AllocSize =
+                VB->m_Operations[0]->GetEstimatedSize(ElemCount, ElemSize, DimCount, Count);
             BufferV::BufferPos pos = CurDataBuffer->Allocate(AllocSize, ElemSize);
             char *CompressedData = (char *)GetPtr(pos.bufferIdx, pos.posInBuffer);
             DataOffset = m_PriorDataBufferSizeTotal + pos.globalPos;
diff --git a/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp b/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp
index e96fc2bdc0..1b8c8c409c 100644
--- a/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp
+++ b/source/adios2/toolkit/interop/hdf5/HDF5Common.cpp
@@ -1709,6 +1709,16 @@ void HDF5Common::StaticGetAdiosStepString(std::string &stepName, size_t ts)
     stepName = "/Step" + std::to_string(ts);
 }
 
+void HDF5Common::CheckVariableOperations(const core::VariableBase &variable) const
+{
+    if (!variable.m_Operations.empty())
+    {
+        helper::Throw("Toolkit", "interop::hdf5::HDF5Common",
+                                          "CheckVariableOperations",
+                                          "ADIOS2 Operators are not supported for HDF5 engine");
+    }
+}
+
 #define declare_template_instantiation(T)                                                          \
     template void HDF5Common::Write(core::Variable &, const T *);
 
diff --git a/source/adios2/toolkit/interop/hdf5/HDF5Common.h b/source/adios2/toolkit/interop/hdf5/HDF5Common.h
index 1fe2ffa5ca..2b9b95c6bb 100644
--- a/source/adios2/toolkit/interop/hdf5/HDF5Common.h
+++ b/source/adios2/toolkit/interop/hdf5/HDF5Common.h
@@ -245,6 +245,8 @@ class HDF5Common
     void GetHDF5SpaceSpec(const core::Variable &variable, std::vector &,
                           std::vector &, std::vector &);
 
+    void CheckVariableOperations(const core::VariableBase &variable) const;
+
     bool m_WriteMode = false;
 
     size_t m_NumAdiosSteps = 0;
diff --git a/source/adios2/toolkit/interop/hdf5/HDF5Common.tcc b/source/adios2/toolkit/interop/hdf5/HDF5Common.tcc
index 77f946f799..9ef93b3498 100644
--- a/source/adios2/toolkit/interop/hdf5/HDF5Common.tcc
+++ b/source/adios2/toolkit/interop/hdf5/HDF5Common.tcc
@@ -25,6 +25,7 @@ namespace interop
 template 
 void HDF5Common::DefineDataset(core::Variable &variable)
 {
+    CheckVariableOperations(variable);
     size_t dimSize = std::max(variable.m_Shape.size(), variable.m_Count.size());
     hid_t h5Type = GetHDF5Type();
 
@@ -118,6 +119,7 @@ template 
 void HDF5Common::Write(core::Variable &variable, const T *values)
 {
     CheckWriteGroup();
+    CheckVariableOperations(variable);
     size_t dimSize = std::max(variable.m_Shape.size(), variable.m_Count.size());
     hid_t h5Type = GetHDF5Type();
 
diff --git a/source/adios2/toolkit/sst/CMakeLists.txt b/source/adios2/toolkit/sst/CMakeLists.txt
index 070c8f41a1..c77fc0459b 100644
--- a/source/adios2/toolkit/sst/CMakeLists.txt
+++ b/source/adios2/toolkit/sst/CMakeLists.txt
@@ -65,6 +65,7 @@ set(SST_CONFIG_OPTS
   UCX
   FI_GNI
   CRAY_DRC
+  CRAY_CXI
   NVStream
   MPI
 )
diff --git a/source/adios2/toolkit/sst/dp/rdma_dp.c b/source/adios2/toolkit/sst/dp/rdma_dp.c
index e61c6f5f3f..e3800f4ff0 100644
--- a/source/adios2/toolkit/sst/dp/rdma_dp.c
+++ b/source/adios2/toolkit/sst/dp/rdma_dp.c
@@ -19,6 +19,13 @@
 #include 
 #include 
 
+#ifdef SST_HAVE_CRAY_CXI
+#include 
+// This comment prevents clang-format from reordering these includes.
+// The CXI extension header requires the bool header, but does not include it on its own.
+#include 
+#endif
+
 #if defined(__has_feature)
 #if __has_feature(thread_sanitizer)
 #define NO_SANITIZE_THREAD __attribute__((no_sanitize("thread")))
@@ -53,6 +60,66 @@ static pthread_mutex_t fabric_mutex = PTHREAD_MUTEX_INITIALIZER;
 pthread_mutex_t wsr_mutex = PTHREAD_MUTEX_INITIALIZER;
 pthread_mutex_t ts_mutex = PTHREAD_MUTEX_INITIALIZER;
 
+/*
+ * Wrapper for fi_mr_reg() with additional parameters endpoint and mr_mode.
+ * If mr_mode includes FI_MR_ENDPOINT, the memory region must be bound to the
+ * endpoint and enabled before use.
+ */
+int sst_fi_mr_reg(
+    /* first two parameters for verbose logging */
+    CP_Services Svcs, void *CP_Stream,
+    /* regular fi_mir_reg() parameters*/
+    struct fid_domain *domain, const void *buf, size_t len, uint64_t acs, uint64_t offset,
+    uint64_t requested_key, uint64_t flags, struct fid_mr **mr, void *context,
+    /* additional parameters for binding the mr to the endpoint*/
+    struct fid_ep *endpoint, int mr_mode)
+{
+    int res = fi_mr_reg(domain, buf, len, acs, offset, requested_key, flags, mr, context);
+    int is_mr_endpoint = (mr_mode & FI_MR_ENDPOINT) != 0;
+    if (!is_mr_endpoint)
+    {
+        return res;
+    }
+    if (res != FI_SUCCESS)
+    {
+        Svcs->verbose(CP_Stream, DPCriticalVerbose, "fi_mr_reg failed with %ul (%s)\n", res,
+                      fi_strerror(res));
+        return res;
+    }
+
+    /*
+     * When the domain_attr->mr_mode includes FI_MR_ENDPOINT, the memory region
+     * needs to be bound to the endpoint and explicitly enabled after that.
+     */
+    res = fi_mr_bind(*mr, &endpoint->fid, 0);
+    if (res != FI_SUCCESS)
+    {
+        Svcs->verbose(CP_Stream, DPCriticalVerbose, "fi_mr_bind failed with %ul (%s)\n", res,
+                      fi_strerror(res));
+        return res;
+    }
+    res = fi_mr_enable(*mr);
+    if (res != FI_SUCCESS)
+    {
+        Svcs->verbose(CP_Stream, DPCriticalVerbose, "fi_mr_enable failed with %ul (%s)\n", res,
+                      fi_strerror(res));
+        return res;
+    }
+    return res;
+}
+
+/*
+ * Simple wrapper to create a log entry upon failing fi_*() function calls.
+ */
+int guard_fi_return(int code, CP_Services Svcs, CManager cm, char const *msg)
+{
+    if (code != FI_SUCCESS)
+    {
+        Svcs->verbose(cm, DPCriticalVerbose, "%s: %s (%lu)\n", msg, fi_strerror(code), code);
+    }
+    return code;
+}
+
 struct fabric_state
 {
     struct fi_context *ctx;
@@ -60,6 +127,7 @@ struct fabric_state
     struct fi_info *info;
     // struct fi_info *linfo;
     int local_mr_req;
+    int mr_virt_addr; /* Stores if the mr_mode includes FI_MR_VIRT_ADDR */
     int rx_cq_data;
     size_t addr_len;
     size_t msg_prefix_size;
@@ -69,6 +137,9 @@ struct fabric_state
     struct fid_cq *cq_signal;
     struct fid_av *av;
     pthread_t listener;
+#ifdef SST_HAVE_CRAY_CXI
+    struct cxi_auth_key *cxi_auth_key;
+#endif
 #ifdef SST_HAVE_CRAY_DRC
     drc_info_handle_t drc_info;
     uint32_t credential;
@@ -112,13 +183,24 @@ struct fabric_state
  *   plane would replace one or both of these with RDMA functionality.
  */
 
+static char const *get_preferred_domain(struct _SstParams *Params)
+{
+    if (Params->DataInterface)
+    {
+        return Params->DataInterface;
+    }
+    else
+    {
+        return getenv("FABRIC_IFACE");
+    }
+}
+
 static void init_fabric(struct fabric_state *fabric, struct _SstParams *Params, CP_Services Svcs,
-                        void *CP_Stream)
+                        void *CP_Stream, char const *ifname)
 {
     struct fi_info *hints, *info, *originfo, *useinfo;
     struct fi_av_attr av_attr = {FI_AV_UNSPEC};
     struct fi_cq_attr cq_attr = {0};
-    char *ifname;
     int result;
 
     hints = fi_allocinfo();
@@ -126,24 +208,76 @@ static void init_fabric(struct fabric_state *fabric, struct _SstParams *Params,
         FI_MSG | FI_SEND | FI_RECV | FI_REMOTE_READ | FI_REMOTE_WRITE | FI_RMA | FI_READ | FI_WRITE;
     hints->mode =
         FI_CONTEXT | FI_LOCAL_MR | FI_CONTEXT2 | FI_MSG_PREFIX | FI_ASYNC_IOV | FI_RX_CQ_DATA;
-    hints->domain_attr->mr_mode = FI_MR_BASIC;
-    hints->domain_attr->control_progress = FI_PROGRESS_AUTO;
-    hints->domain_attr->data_progress = FI_PROGRESS_AUTO;
     hints->ep_attr->type = FI_EP_RDM;
 
-    if (Params->DataInterface)
+    uint32_t fi_version;
+#ifdef SST_HAVE_CRAY_CXI
+    if (fabric->cxi_auth_key)
     {
-        ifname = Params->DataInterface;
+        fi_version = FI_VERSION(1, 11);
+
+        hints->domain_attr->mr_mode = FI_MR_ENDPOINT;
+        hints->domain_attr->control_progress = FI_PROGRESS_MANUAL;
+        hints->domain_attr->data_progress = FI_PROGRESS_MANUAL;
+
+        // Authentication is needed
+        // TODO: the first ID in SLINGSHOT_SVC_IDS is chosen, but we should
+        // rather choose the one corresponding with the FABRIC_IFACE
+        // example:
+        // SLINGSHOT_SVC_IDS=5,5,5,5
+        // SLINGSHOT_VNIS=1310,1271
+        // SLINGSHOT_DEVICES=cxi0,cxi1,cxi2,cxi3
+        // FABRIC_IFACE=cxi2 (user specified)
+
+        hints->ep_attr->auth_key = malloc(sizeof(struct cxi_auth_key));
+        memcpy(hints->ep_attr->auth_key, fabric->cxi_auth_key, sizeof(struct cxi_auth_key));
+        hints->ep_attr->auth_key_size = sizeof(struct cxi_auth_key);
+
+        hints->domain_attr->auth_key = malloc(sizeof(struct cxi_auth_key));
+        memcpy(hints->domain_attr->auth_key, fabric->cxi_auth_key, sizeof(struct cxi_auth_key));
+        hints->domain_attr->auth_key_size = sizeof(struct cxi_auth_key);
     }
     else
     {
-        ifname = getenv("FABRIC_IFACE");
+        fi_version = FI_VERSION(1, 5);
+
+        hints->domain_attr->mr_mode = FI_MR_BASIC;
+        hints->domain_attr->control_progress = FI_PROGRESS_AUTO;
+        hints->domain_attr->data_progress = FI_PROGRESS_AUTO;
+    }
+#else
+    fi_version = FI_VERSION(1, 5);
+
+    // Alternatively, one could set mr_mode to
+    // FI_MR_VIRT_ADDR | FI_MR_ALLOCATED | FI_MR_PROV_KEY | FI_MR_LOCAL
+    // here. These flags are equivalent to FI_MR_BASIC, but unlike basic
+    // registration, providers are not forced to keep those flags when they
+    // think that not using the flags is better.
+    // The RDMA DP is able to deal with this appropriately, and does so right
+    // before calling fi_fabric() further below in this function.
+    // The main reason for keeping FI_MR_BASIC here is backward compatibility.
+    hints->domain_attr->mr_mode = FI_MR_BASIC;
+    hints->domain_attr->control_progress = FI_PROGRESS_AUTO;
+    hints->domain_attr->data_progress = FI_PROGRESS_AUTO;
+#endif
+
+    /*
+     * ifname is passed as a function parameter of init_fabric() if
+     * a provider-specific key was configured and sent to the reader.
+     * Since the key is generally domain-specific, we must use that one in this
+     * case.
+     * The preferred domain is already considered upon key configuration,
+     * so this is fine.
+     */
+    if (!ifname)
+    {
+        ifname = get_preferred_domain(Params);
     }
 
     fabric->info = NULL;
 
     pthread_mutex_lock(&fabric_mutex);
-    fi_getinfo(FI_VERSION(1, 5), NULL, NULL, 0, hints, &info);
+    fi_getinfo(fi_version, NULL, NULL, 0, hints, &info);
     pthread_mutex_unlock(&fabric_mutex);
     if (!info)
     {
@@ -167,7 +301,8 @@ static void init_fabric(struct fabric_state *fabric, struct _SstParams *Params,
             break;
         }
         if ((((strcmp(prov_name, "verbs") == 0) && info->src_addr) ||
-             (strcmp(prov_name, "gni") == 0) || (strcmp(prov_name, "psm2") == 0)) &&
+             (strcmp(prov_name, "gni") == 0) || (strcmp(prov_name, "psm2") == 0) ||
+             (strcmp(prov_name, "cxi") == 0)) &&
             (!useinfo || !ifname || (strcmp(useinfo->domain_attr->name, ifname) != 0)))
         {
             Svcs->verbose(CP_Stream, DPTraceVerbose,
@@ -177,7 +312,7 @@ static void init_fabric(struct fabric_state *fabric, struct _SstParams *Params,
             useinfo = info;
         }
         else if (((strstr(prov_name, "verbs") && info->src_addr) || strstr(prov_name, "gni") ||
-                  strstr(prov_name, "psm2")) &&
+                  strstr(prov_name, "psm2") || strstr(prov_name, "cxi")) &&
                  !useinfo)
         {
             Svcs->verbose(CP_Stream, DPTraceVerbose,
@@ -253,7 +388,30 @@ static void init_fabric(struct fabric_state *fabric, struct _SstParams *Params,
 
     fabric->addr_len = info->src_addrlen;
 
-    info->domain_attr->mr_mode = FI_MR_BASIC;
+    /*
+     * The libfabric data-plane of SST was originally programmed to use
+     * FI_MR_BASIC as mr_mode, which is equivalent to
+     * FI_MR_VIRT_ADDR | FI_MR_ALLOCATED | FI_MR_PROV_KEY | FI_MR_LOCAL.
+     *
+     * However, HPE's CXI provider requires two changes to that:
+     * (1) It does not support FI_MR_VIRT_ADDR.
+     * (2) It requires use of FI_MR_ENDPOINT.
+     *
+     * So we propagate the bit value currently contained in the mr_mode
+     * for these flags.
+     */
+    if (info->domain_attr->mr_mode != FI_MR_BASIC)
+    {
+        info->domain_attr->mr_mode = FI_MR_ALLOCATED | FI_MR_PROV_KEY | FI_MR_LOCAL |
+                                     (FI_MR_ENDPOINT & info->domain_attr->mr_mode) |
+                                     (FI_MR_VIRT_ADDR & info->domain_attr->mr_mode);
+        fabric->mr_virt_addr = info->domain_attr->mr_mode & FI_MR_VIRT_ADDR ? 1 : 0;
+    }
+    else
+    {
+        fabric->mr_virt_addr = 1;
+    }
+
 #ifdef SST_HAVE_CRAY_DRC
     if (strstr(info->fabric_attr->prov_name, "gni") && fabric->auth_key)
     {
@@ -411,6 +569,12 @@ static void fini_fabric(struct fabric_state *fabric, CP_Services Svcs, void *CP_
         free(fabric->ctx);
     }
 
+#ifdef SST_HAVE_CRAY_CXI
+    if (fabric->cxi_auth_key)
+    {
+        free(fabric->cxi_auth_key);
+    }
+#endif
 #ifdef SST_HAVE_CRAY_DRC
     if (Fabric->auth_key)
     {
@@ -599,6 +763,191 @@ static TimestepList GetStep(Rdma_WS_Stream Stream, long Timestep)
     return (Step);
 }
 
+#ifdef SST_HAVE_CRAY_CXI
+static int get_cxi_auth_key_from_env(CP_Services Svcs, void *CP_Stream, struct _SstParams *Params,
+                                     struct cxi_auth_key *key, char **used_device)
+{
+    int vni, first_vni, second_vni, svc_id;
+
+    // Just some safety against faulty strings in string processing.
+    size_t const no_infinite_loops = 10000;
+
+    // struct cxi_auth_key {
+    //     /* The CXI service assigned to the Domain and Endpoints. A CXI
+    //     service
+    //     * is associated with a set of local resource limits, VNIs, and
+    //     Traffic
+    //     * Classes.
+    //     *
+    //     * The svc_id used by an OFI Domain must match all Endpoints belonging
+    //     * to the Domain.
+    //     */
+    //     uint32_t svc_id;
+
+    //     /* The Virtual Network ID (VNI) assigned to the Endpoint. Two
+    //     Endpoints
+    //     * must use the same VNI in order to communicate.
+    //     *
+    //     * Note that while the CXI service may define one or more VNIs which a
+    //     * process can access, an Endpoint is assigned to only one.
+    //     */
+    //     uint16_t vni;
+    // };
+
+    // typical value SLINGSHOT_DEVICES=cxi0,cxi1,cxi2,cxi3
+    char const *slingshot_devices = getenv("SLINGSHOT_DEVICES");
+    char const *preferred_device = get_preferred_domain(Params);
+
+    /*
+     * In the following loop, find out if the preferred_device is found within
+     * the slingshot_devices.
+     * If the preferred_device is NULL, just pick the first.
+     * Upon success, modifies the output parameter used_device and stores
+     * the retrieved device index.
+     */
+    size_t device_index = 0;
+    for (size_t no_infinite_loop_counter = 0;; ++device_index, ++no_infinite_loop_counter)
+    {
+        if (no_infinite_loop_counter > no_infinite_loops)
+        {
+            return EXIT_FAILURE;
+        }
+
+        // Are we at the end of the environment variable?
+        int found_end = 0;
+
+        // Find out the length of the current item in slingshot_devices.
+        size_t find_end_of_current_string = 0;
+        for (size_t no_infinite_loop_inner_counter = 0;;
+             ++find_end_of_current_string, ++no_infinite_loop_inner_counter)
+        {
+            if (no_infinite_loop_inner_counter > no_infinite_loops)
+            {
+                return EXIT_FAILURE;
+            }
+
+            switch (slingshot_devices[find_end_of_current_string])
+            {
+            case '\0':
+                found_end = 1;
+                goto break_first_loop;
+            case ',':
+                goto break_first_loop;
+            default:
+                break;
+            }
+        }
+    break_first_loop:;
+        int use_this_device = !preferred_device || (strncmp(preferred_device, slingshot_devices,
+                                                            find_end_of_current_string) == 0);
+        if (use_this_device)
+        {
+            char *construct_used_device = malloc(find_end_of_current_string + 1);
+            memcpy(construct_used_device, slingshot_devices, find_end_of_current_string);
+            construct_used_device[find_end_of_current_string] = '\0';
+            *used_device = construct_used_device;
+            break;
+        }
+        else if (found_end)
+        {
+            return EXIT_FAILURE;
+        }
+        else
+        {
+            // go to next iteration
+            slingshot_devices += find_end_of_current_string + 1;
+        }
+    }
+
+    Svcs->verbose(CP_Stream, DPTraceVerbose, "Found device %s at index %zu\n", *used_device,
+                  device_index);
+
+    // typical value SLINGSHOT_VNIS=4576,4530
+    char const *vni_env_str = getenv("SLINGSHOT_VNIS");
+    if (!vni_env_str)
+    {
+        return EXIT_FAILURE;
+    }
+
+    // typical value SLINGSHOT_SVC_IDS=5,5,5,5
+    char const *svc_ids_env_str = getenv("SLINGSHOT_SVC_IDS");
+    if (!svc_ids_env_str)
+    {
+        return EXIT_FAILURE;
+    }
+
+    {
+        int num_vnis = sscanf(vni_env_str, "%d,%d", &first_vni, &second_vni);
+        switch (num_vnis)
+        {
+        // first VNI is the subjob's VNI
+        case 1:
+            Svcs->verbose(CP_Stream, DPTraceVerbose, "Using first vni.\n");
+            vni = first_vni;
+            break;
+        // if present, the second VNI is the containing job's VNI
+        // the first VNI belongs to the subjob
+        case 2:
+            Svcs->verbose(CP_Stream, DPTraceVerbose, "Using second vni.\n");
+            vni = second_vni;
+            break;
+        default:
+            return EXIT_FAILURE;
+        }
+    }
+
+    {
+        // Pick the service ID according to the device_index found above.
+        for (size_t svc_id_index = 0; svc_id_index < device_index; ++svc_id_index)
+        {
+            for (size_t no_infinite_loop_counter = 0;; ++no_infinite_loop_counter)
+            {
+                if (no_infinite_loop_counter > no_infinite_loops)
+                {
+                    return EXIT_FAILURE;
+                }
+
+                switch (*(svc_ids_env_str++))
+                {
+                case ',':
+                    goto break_second_loop;
+                case '\0':
+                    return EXIT_FAILURE;
+                default:
+                    continue;
+                }
+            }
+        break_second_loop:;
+        }
+
+        int num_svc_ids = sscanf(svc_ids_env_str, "%d", &svc_id);
+        switch (num_svc_ids)
+        {
+        case 1:
+            break;
+        default:
+            return EXIT_FAILURE;
+        }
+    }
+
+    key->vni = vni;
+    key->svc_id = svc_id;
+
+    return EXIT_SUCCESS;
+}
+
+static int get_cxi_auth_key_from_writer(struct cxi_auth_key *key, attr_list WriterContact)
+{
+    long vni;
+    if (!get_long_attr(WriterContact, attr_atom_from_string("vni"), &vni))
+    {
+        return EXIT_FAILURE;
+    }
+    key->vni = (uint16_t)vni;
+    return EXIT_SUCCESS;
+}
+#endif
+
 static DP_RS_Stream RdmaInitReader(CP_Services Svcs, void *CP_Stream, void **ReaderContactInfoPtr,
                                    struct _SstParams *Params, attr_list WriterContact,
                                    SstStats Stats)
@@ -645,6 +994,38 @@ static DP_RS_Stream RdmaInitReader(CP_Services Svcs, void *CP_Stream, void **Rea
         Stream->PreloadAvail = 0;
     }
 
+    char *required_device = NULL;
+#ifdef SST_HAVE_CRAY_CXI
+    struct
+    {
+        struct cxi_auth_key key;
+        int valid;
+    } tagged_key;
+
+    /*
+     * The svc_id of the key must match the device that this particular reader
+     * connects with.
+     * The vni (virtual network ID) must be the same across all communicating
+     * instances (get this from the writer).
+     */
+
+    tagged_key.valid =
+        get_cxi_auth_key_from_env(Svcs, CP_Stream, Params, &tagged_key.key, &required_device);
+
+    if (tagged_key.valid == EXIT_SUCCESS &&
+        get_cxi_auth_key_from_writer(&tagged_key.key, WriterContact) == EXIT_SUCCESS)
+    {
+        Svcs->verbose(CP_Stream, DPSummaryVerbose, "Reader found CXI auth key: %d %d\n",
+                      tagged_key.key.vni, tagged_key.key.svc_id);
+        Stream->Fabric->cxi_auth_key = calloc(1, sizeof(struct cxi_auth_key));
+        memcpy(Stream->Fabric->cxi_auth_key, &tagged_key.key, sizeof(struct cxi_auth_key));
+    }
+    else
+    {
+        Svcs->verbose(CP_Stream, DPSummaryVerbose, "Reader found no CXI auth key\n");
+    }
+#endif
+
 #ifdef SST_HAVE_CRAY_DRC
     int attr_cred, try_left, rc;
     if (!get_int_attr(WriterContact, attr_atom_from_string("RDMA_DRC_KEY"), &attr_cred))
@@ -675,7 +1056,11 @@ static DP_RS_Stream RdmaInitReader(CP_Services Svcs, void *CP_Stream, void **Rea
 
 #endif /* SST_HAVE_CRAY_DRC */
 
-    init_fabric(Stream->Fabric, Stream->Params, Svcs, CP_Stream);
+    init_fabric(Stream->Fabric, Stream->Params, Svcs, CP_Stream, required_device);
+    if (required_device)
+    {
+        free(required_device);
+    }
     if (!Fabric->info)
     {
         Svcs->verbose(CP_Stream, DPCriticalVerbose, "Could not find a valid transport fabric.\n");
@@ -684,7 +1069,12 @@ static DP_RS_Stream RdmaInitReader(CP_Services Svcs, void *CP_Stream, void **Rea
 
     ContactInfo->Length = Fabric->info->src_addrlen;
     ContactInfo->Address = malloc(ContactInfo->Length);
-    fi_getname((fid_t)Fabric->signal, ContactInfo->Address, &ContactInfo->Length);
+    if (guard_fi_return(
+            fi_getname((fid_t)Fabric->signal, ContactInfo->Address, &ContactInfo->Length), Svcs,
+            CP_Stream, "[RdmaInitReader] fi_getname() failed with:") != FI_SUCCESS)
+    {
+        return NULL;
+    }
 
     Stream->PreloadStep = -1;
     Stream->ContactInfo = ContactInfo;
@@ -775,6 +1165,42 @@ static DP_WS_Stream RdmaInitWriter(CP_Services Svcs, void *CP_Stream, struct _Ss
 
     Stream->Fabric = calloc(1, sizeof(struct fabric_state));
     Fabric = Stream->Fabric;
+
+    char *required_device = NULL;
+#ifdef SST_HAVE_CRAY_CXI
+    struct
+    {
+        struct cxi_auth_key key;
+        int valid;
+    } tagged_key;
+
+    /*
+     * The svc_id of the key must match the device that this particular writer
+     * connects with.
+     * The vni (virtual network ID) must be the same across all communicating
+     * instances (use the one seen by rank 0).
+     */
+    tagged_key.valid =
+        get_cxi_auth_key_from_env(Svcs, CP_Stream, Params, &tagged_key.key, &required_device);
+
+    // Ensure that all writers use the same virtual network ID
+    SMPI_Bcast(&tagged_key.key.vni, sizeof(tagged_key.key.vni), SMPI_BYTE, 0, comm);
+
+    if (tagged_key.valid == EXIT_SUCCESS)
+    {
+        Svcs->verbose(CP_Stream, DPSummaryVerbose, "Writer found CXI auth key: %d %d\n",
+                      tagged_key.key.vni, tagged_key.key.svc_id);
+
+        set_long_attr(DPAttrs, attr_atom_from_string("vni"), tagged_key.key.vni);
+        Stream->Fabric->cxi_auth_key = calloc(1, sizeof(struct cxi_auth_key));
+        memcpy(Stream->Fabric->cxi_auth_key, &tagged_key.key, sizeof(struct cxi_auth_key));
+    }
+    else
+    {
+        Svcs->verbose(CP_Stream, DPSummaryVerbose, "Writer found no CXI auth key");
+    }
+#endif
+
 #ifdef SST_HAVE_CRAY_DRC
     int try_left, rc;
     if (Stream->Rank == 0)
@@ -818,7 +1244,11 @@ static DP_WS_Stream RdmaInitWriter(CP_Services Svcs, void *CP_Stream, struct _Ss
     set_long_attr(DPAttrs, attr_atom_from_string("RDMA_DRC_CRED"), attr_cred);
 #endif /* SST_HAVE_CRAY_DRC */
 
-    init_fabric(Stream->Fabric, Params, Svcs, CP_Stream);
+    init_fabric(Stream->Fabric, Params, Svcs, CP_Stream, required_device);
+    if (required_device)
+    {
+        free(required_device);
+    }
     Fabric = Stream->Fabric;
     if (!Fabric->info)
     {
@@ -872,8 +1302,15 @@ static DP_WSR_Stream RdmaInitWriterPerReader(CP_Services Svcs, DP_WS_Stream WS_S
 
     for (i = 0; i < readerCohortSize; i++)
     {
-        fi_av_insert(Fabric->av, providedReaderInfo[i]->Address, 1, &WSR_Stream->ReaderAddr[i], 0,
-                     NULL);
+        if (fi_av_insert(Fabric->av, providedReaderInfo[i]->Address, 1, &WSR_Stream->ReaderAddr[i],
+                         0, NULL) < 1)
+        {
+
+            Svcs->verbose(WS_Stream->CP_Stream, DPCriticalVerbose,
+                          "[RdmaInitWRiterPerReader] Failed inserting address "
+                          "into vector\n");
+            return NULL;
+        }
         Svcs->verbose(WS_Stream->CP_Stream, DPTraceVerbose,
                       "Received contact info for RS_Stream %p, WSR Rank %d\n",
                       providedReaderInfo[i]->RS_Stream, i);
@@ -895,13 +1332,20 @@ static DP_WSR_Stream RdmaInitWriterPerReader(CP_Services Svcs, DP_WS_Stream WS_S
 
     ContactInfo->Length = Fabric->info->src_addrlen;
     ContactInfo->Address = malloc(ContactInfo->Length);
-    fi_getname((fid_t)Fabric->signal, ContactInfo->Address, &ContactInfo->Length);
+    if (guard_fi_return(
+            fi_getname((fid_t)Fabric->signal, ContactInfo->Address, &ContactInfo->Length), Svcs,
+            WS_Stream->CP_Stream,
+            "[RdmaInitWriterPerReader] fi_getname() failed with") != FI_SUCCESS)
+    {
+        return NULL;
+    }
 
     ReaderRollHandle = &ContactInfo->ReaderRollHandle;
     ReaderRollHandle->Block = calloc(readerCohortSize, sizeof(struct _RdmaBuffer));
-    fi_mr_reg(Fabric->domain, ReaderRollHandle->Block,
-              readerCohortSize * sizeof(struct _RdmaBuffer), FI_REMOTE_WRITE, 0, 0, 0,
-              &WSR_Stream->rrmr, Fabric->ctx);
+    sst_fi_mr_reg(Svcs, WS_Stream->CP_Stream, Fabric->domain, ReaderRollHandle->Block,
+                  readerCohortSize * sizeof(struct _RdmaBuffer), FI_REMOTE_WRITE, 0, 0, 0,
+                  &WSR_Stream->rrmr, Fabric->ctx, Fabric->signal,
+                  Fabric->info->domain_attr->mr_mode);
     ReaderRollHandle->Key = fi_mr_key(WSR_Stream->rrmr);
 
     WSR_Stream->WriterContactInfo = ContactInfo;
@@ -943,8 +1387,15 @@ static void RdmaProvideWriterDataToReader(CP_Services Svcs, DP_RS_Stream RS_Stre
     for (int i = 0; i < writerCohortSize; i++)
     {
         RS_Stream->WriterContactInfo[i].WS_Stream = providedWriterInfo[i]->WS_Stream;
-        fi_av_insert(Fabric->av, providedWriterInfo[i]->Address, 1, &RS_Stream->WriterAddr[i], 0,
-                     NULL);
+        if (fi_av_insert(Fabric->av, providedWriterInfo[i]->Address, 1, &RS_Stream->WriterAddr[i],
+                         0, NULL) < 1)
+        {
+            Svcs->verbose(RS_Stream->CP_Stream, DPCriticalVerbose,
+                          "[RdmaProvideWriterDataToReader] "
+                          "Failed inserting address "
+                          "into vector\n");
+            return;
+        }
         RS_Stream->WriterRoll[i] = providedWriterInfo[i]->ReaderRollHandle;
         Svcs->verbose(RS_Stream->CP_Stream, DPTraceVerbose,
                       "Received contact info for WS_stream %p, WSR Rank %d\n",
@@ -1032,11 +1483,21 @@ static ssize_t PostRead(CP_Services Svcs, Rdma_RS_Stream RS_Stream, int Rank, lo
     if (Fabric->local_mr_req)
     {
         // register dest buffer
-        fi_mr_reg(Fabric->domain, Buffer, Length, FI_READ, 0, 0, 0, &ret->LocalMR, Fabric->ctx);
+        sst_fi_mr_reg(Svcs, RS_Stream->CP_Stream, Fabric->domain, Buffer, Length, FI_READ, 0, 0, 0,
+                      &ret->LocalMR, Fabric->ctx, Fabric->signal,
+                      Fabric->info->domain_attr->mr_mode);
         LocalDesc = fi_mr_desc(ret->LocalMR);
     }
 
-    Addr = Info->Block + Offset;
+    if (Fabric->mr_virt_addr)
+    {
+        Addr = Info->Block + Offset;
+    }
+    else
+    {
+        Addr = NULL;
+        Addr += Offset;
+    }
 
     Svcs->verbose(RS_Stream->CP_Stream, DPTraceVerbose,
                   "Remote read target is Rank %d (Offset = %zi, Length = %zi)\n", Rank, Offset,
@@ -1203,8 +1664,14 @@ static int DoPushWait(CP_Services Svcs, Rdma_RS_Stream Stream, RdmaCompletionHan
         rc = fi_cq_sread(Fabric->cq_signal, (void *)(&CQEntry), 1, NULL, -1);
         if (rc < 1)
         {
-            Svcs->verbose(Stream->CP_Stream, DPCriticalVerbose,
-                          "failure while waiting for completions (%d).\n", rc);
+            struct fi_cq_err_entry error;
+            fi_cq_readerr(Fabric->cq_signal, &error, 0);
+            Svcs->verbose(
+                Stream->CP_Stream, DPCriticalVerbose,
+                "failure while waiting for completions inside "
+                "DoPushWait() (%d (%s - %s)).\n",
+                rc, fi_strerror(error.err),
+                fi_cq_strerror(Fabric->cq_signal, error.err, error.err_data, NULL, error.len));
             return 0;
         }
         else if (CQEntry.flags & FI_REMOTE_CQ_DATA)
@@ -1276,8 +1743,14 @@ static int WaitForAnyPull(CP_Services Svcs, Rdma_RS_Stream Stream)
     rc = fi_cq_sread(Fabric->cq_signal, (void *)(&CQEntry), 1, NULL, -1);
     if (rc < 1)
     {
-        Svcs->verbose(Stream->CP_Stream, DPCriticalVerbose,
-                      "failure while waiting for completions (%d).\n", rc);
+        struct fi_cq_err_entry error;
+        fi_cq_readerr(Fabric->cq_signal, &error, 0);
+        Svcs->verbose(
+            Stream->CP_Stream, DPCriticalVerbose,
+            "failure while waiting for completions inside "
+            "WaitForAnyPull() (%d (%s - %s)).\n",
+            rc, fi_strerror(error.err),
+            fi_cq_strerror(Fabric->cq_signal, error.err, error.err_data, NULL, error.len));
         return 0;
     }
     else
@@ -1345,8 +1818,9 @@ static void RdmaProvideTimestep(CP_Services Svcs, DP_WS_Stream Stream_v, struct
     Entry->BufferSlot = -1;
     Entry->Desc = NULL;
 
-    fi_mr_reg(Fabric->domain, Data->block, Data->DataSize, FI_WRITE | FI_REMOTE_READ, 0, 0, 0,
-              &Entry->mr, Fabric->ctx);
+    sst_fi_mr_reg(Svcs, Stream->CP_Stream, Fabric->domain, Data->block, Data->DataSize,
+                  FI_WRITE | FI_REMOTE_READ, 0, 0, 0, &Entry->mr, Fabric->ctx, Fabric->signal,
+                  Fabric->info->domain_attr->mr_mode);
     Entry->Key = fi_mr_key(Entry->mr);
     if (Fabric->local_mr_req)
     {
@@ -1608,7 +2082,7 @@ static struct _CP_DP_Interface RdmaDPInterface = {0};
 static int RdmaGetPriority(CP_Services Svcs, void *CP_Stream, struct _SstParams *Params)
 {
     struct fi_info *hints, *info, *originfo;
-    char *ifname;
+    char const *ifname;
     char *forkunsafe;
     int Ret = -1;
 
@@ -1617,20 +2091,41 @@ static int RdmaGetPriority(CP_Services Svcs, void *CP_Stream, struct _SstParams
         FI_MSG | FI_SEND | FI_RECV | FI_REMOTE_READ | FI_REMOTE_WRITE | FI_RMA | FI_READ | FI_WRITE;
     hints->mode =
         FI_CONTEXT | FI_LOCAL_MR | FI_CONTEXT2 | FI_MSG_PREFIX | FI_ASYNC_IOV | FI_RX_CQ_DATA;
-    hints->domain_attr->mr_mode = FI_MR_BASIC;
-    hints->domain_attr->control_progress = FI_PROGRESS_AUTO;
-    hints->domain_attr->data_progress = FI_PROGRESS_AUTO;
     hints->ep_attr->type = FI_EP_RDM;
 
-    if (Params->DataInterface)
+    char const *vni_env_str = getenv("SLINGSHOT_VNIS");
+
+    uint32_t fi_version;
+    if (vni_env_str)
     {
-        ifname = Params->DataInterface;
+        // try fishing for the CXI provider
+        Svcs->verbose(CP_Stream, DPSummaryVerbose,
+                      "RDMA Dataplane trying to check for an available CXI "
+                      "provider since environment variable SLINGSHOT_VNIS is "
+                      "defined (value: '%s').\n",
+                      vni_env_str);
+        fi_version = FI_VERSION(1, 11);
+
+        hints->domain_attr->mr_mode = FI_MR_ENDPOINT;
+        hints->domain_attr->control_progress = FI_PROGRESS_MANUAL;
+        hints->domain_attr->data_progress = FI_PROGRESS_MANUAL;
     }
     else
     {
-        ifname = getenv("FABRIC_IFACE");
+        Svcs->verbose(CP_Stream, DPSummaryVerbose,
+                      "RDMA Dataplane trying to check for an available non-CXI "
+                      "provider since environment variable SLINGSHOT_VNIS is "
+                      "not defined.\n");
+
+        fi_version = FI_VERSION(1, 5);
+
+        hints->domain_attr->mr_mode = FI_MR_BASIC;
+        hints->domain_attr->control_progress = FI_PROGRESS_AUTO;
+        hints->domain_attr->data_progress = FI_PROGRESS_AUTO;
     }
 
+    ifname = get_preferred_domain(Params);
+
     forkunsafe = getenv("FI_FORK_UNSAFE");
     if (!forkunsafe)
     {
@@ -1638,7 +2133,7 @@ static int RdmaGetPriority(CP_Services Svcs, void *CP_Stream, struct _SstParams
     }
 
     pthread_mutex_lock(&fabric_mutex);
-    fi_getinfo(FI_VERSION(1, 5), NULL, NULL, 0, hints, &info);
+    fi_getinfo(fi_version, NULL, NULL, 0, hints, &info);
     pthread_mutex_unlock(&fabric_mutex);
     fi_freeinfo(hints);
 
@@ -1656,6 +2151,10 @@ static int RdmaGetPriority(CP_Services Svcs, void *CP_Stream, struct _SstParams
 
         prov_name = info->fabric_attr->prov_name;
         domain_name = info->domain_attr->name;
+        Svcs->verbose(CP_Stream, DPPerStepVerbose,
+                      "[RdmaGetPriority] Seeing and evaluating fabric with "
+                      "provider: '%s', domain: '%s'\n",
+                      prov_name, domain_name);
         if (ifname && strcmp(ifname, domain_name) == 0)
         {
             Svcs->verbose(CP_Stream, DPPerStepVerbose,
@@ -1666,7 +2165,7 @@ static int RdmaGetPriority(CP_Services Svcs, void *CP_Stream, struct _SstParams
             break;
         }
         if ((strstr(prov_name, "verbs") && info->src_addr) || strstr(prov_name, "gni") ||
-            strstr(prov_name, "psm2"))
+            strstr(prov_name, "psm2") || strstr(prov_name, "cxi"))
         {
 
             Svcs->verbose(CP_Stream, DPPerStepVerbose,
@@ -1733,7 +2232,12 @@ static void PushData(CP_Services Svcs, Rdma_WSR_Stream Stream, TimestepList Step
             {
                 rc = fi_writedata(Fabric->signal, StepBuffer + Req->Offset, Req->BufferLen,
                                   Step->Desc, Data, Stream->ReaderAddr[RankReq->Rank],
-                                  (uint64_t)Req->Handle.Block +
+                                  /*
+                                   * If mr_virt_addr is zero, we need just the offset,
+                                   * otherwise we need the remote virtual address composed by
+                                   * base pointer + offset.
+                                   */
+                                  Fabric->mr_virt_addr * (uint64_t)Req->Handle.Block +
                                       (BufferSlot * RankReq->PreloadBufferSize),
                                   RollBuffer->Offset, (void *)(Step->Timestep));
             } while (rc == -EAGAIN);
@@ -1822,15 +2326,17 @@ static void PostPreload(CP_Services Svcs, Rdma_RS_Stream Stream, long Timestep)
 
     PreloadBuffer->BufferLen = 2 * StepLog->BufferSize;
     PreloadBuffer->Handle.Block = malloc(PreloadBuffer->BufferLen);
-    fi_mr_reg(Fabric->domain, PreloadBuffer->Handle.Block, PreloadBuffer->BufferLen,
-              FI_REMOTE_WRITE, 0, 0, 0, &Stream->pbmr, Fabric->ctx);
+    sst_fi_mr_reg(Svcs, Stream->CP_Stream, Fabric->domain, PreloadBuffer->Handle.Block,
+                  PreloadBuffer->BufferLen, FI_REMOTE_WRITE, 0, 0, 0, &Stream->pbmr, Fabric->ctx,
+                  Fabric->signal, Fabric->info->domain_attr->mr_mode);
     PreloadKey = fi_mr_key(Stream->pbmr);
 
     SBSize = sizeof(*SendBuffer) * StepLog->WRanks;
     SendBuffer = malloc(SBSize);
     if (Fabric->local_mr_req)
     {
-        fi_mr_reg(Fabric->domain, SendBuffer, SBSize, FI_WRITE, 0, 0, 0, &sbmr, Fabric->ctx);
+        sst_fi_mr_reg(Svcs, Stream->CP_Stream, Fabric->domain, SendBuffer, SBSize, FI_WRITE, 0, 0,
+                      0, &sbmr, Fabric->ctx, Fabric->signal, Fabric->info->domain_attr->mr_mode);
         sbdesc = fi_mr_desc(sbmr);
     }
 
@@ -1838,8 +2344,9 @@ static void PostPreload(CP_Services Svcs, Rdma_RS_Stream Stream, long Timestep)
     {
         RBLen = 2 * StepLog->Entries * DP_DATA_RECV_SIZE;
         Stream->RecvDataBuffer = malloc(RBLen);
-        fi_mr_reg(Fabric->domain, Stream->RecvDataBuffer, RBLen, FI_RECV, 0, 0, 0, &Stream->rbmr,
-                  Fabric->ctx);
+        sst_fi_mr_reg(Svcs, Stream->CP_Stream, Fabric->domain, Stream->RecvDataBuffer, RBLen,
+                      FI_RECV, 0, 0, 0, &Stream->rbmr, Fabric->ctx, Fabric->signal,
+                      Fabric->info->domain_attr->mr_mode);
         Stream->rbdesc = fi_mr_desc(Stream->rbmr);
         RecvBuffer = (uint8_t *)Stream->RecvDataBuffer;
         for (i = 0; i < 2 * StepLog->Entries; i++)
@@ -1862,9 +2369,10 @@ static void PostPreload(CP_Services Svcs, Rdma_RS_Stream Stream, long Timestep)
         if (RankLog->Entries > 0)
         {
             RankLog->Buffer = (void *)RawPLBuffer;
-            fi_mr_reg(Fabric->domain, RankLog->ReqLog,
-                      (sizeof(struct _RdmaBuffer) * RankLog->Entries) + sizeof(uint64_t),
-                      FI_REMOTE_READ, 0, 0, 0, &RankLog->preqbmr, Fabric->ctx);
+            sst_fi_mr_reg(Svcs, Stream->CP_Stream, Fabric->domain, RankLog->ReqLog,
+                          (sizeof(struct _RdmaBuffer) * RankLog->Entries) + sizeof(uint64_t),
+                          FI_REMOTE_READ, 0, 0, 0, &RankLog->preqbmr, Fabric->ctx, Fabric->signal,
+                          Fabric->info->domain_attr->mr_mode);
             for (j = 0; j < RankLog->Entries; j++)
             {
                 ReqLog = &RankLog->ReqLog[j];
@@ -1883,11 +2391,17 @@ static void PostPreload(CP_Services Svcs, Rdma_RS_Stream Stream, long Timestep)
             SendBuffer[WRidx].Offset = (uint64_t)PreloadKey;
             SendBuffer[WRidx].Handle.Block = (void *)RankLog->ReqLog;
             SendBuffer[WRidx].Handle.Key = fi_mr_key(RankLog->preqbmr);
-            RollDest =
-                (uint64_t)Stream->WriterRoll[i].Block + (sizeof(struct _RdmaBuffer) * Stream->Rank);
-            fi_write(Fabric->signal, &SendBuffer[WRidx], sizeof(struct _RdmaBuffer), sbdesc,
-                     Stream->WriterAddr[i], RollDest, Stream->WriterRoll[i].Key,
-                     &SendBuffer[WRidx]);
+            /*
+             * If mr_virt_addr is zero, we need just the offset,
+             * otherwise we need the remote virtual address composed by
+             * base pointer + offset.
+             */
+            RollDest = Fabric->mr_virt_addr * (uint64_t)Stream->WriterRoll[i].Block +
+                       (sizeof(struct _RdmaBuffer) * Stream->Rank);
+            guard_fi_return((int)fi_write(Fabric->signal, &SendBuffer[WRidx],
+                                          sizeof(struct _RdmaBuffer), sbdesc, Stream->WriterAddr[i],
+                                          RollDest, Stream->WriterRoll[i].Key, &SendBuffer[WRidx]),
+                            Svcs, Stream->CP_Stream, "[PostPreload] fi_write failed with:");
             RankLog->PreloadHandles = malloc(sizeof(void *) * 2);
             RankLog->PreloadHandles[0] =
                 calloc(sizeof(struct _RdmaCompletionHandle), RankLog->Entries);
@@ -1899,7 +2413,19 @@ static void PostPreload(CP_Services Svcs, Rdma_RS_Stream Stream, long Timestep)
 
     while (WRidx > 0)
     {
-        fi_cq_sread(Fabric->cq_signal, (void *)(&CQEntry), 1, NULL, -1);
+        ssize_t rc = fi_cq_sread(Fabric->cq_signal, (void *)(&CQEntry), 1, NULL, -1);
+        if (rc < 1)
+        {
+            struct fi_cq_err_entry error;
+            fi_cq_readerr(Fabric->cq_signal, &error, 0);
+            Svcs->verbose(
+                Stream->CP_Stream, DPCriticalVerbose,
+                "[PostPreload] failure while waiting for completions "
+                "(%d (%s - %s)).\n",
+                rc, fi_strerror(error.err),
+                fi_cq_strerror(Fabric->cq_signal, error.err, error.err_data, NULL, error.len));
+            return;
+        }
         CQBuffer = CQEntry.op_context;
         if (CQBuffer >= SendBuffer && CQBuffer < (SendBuffer + StepLog->WRanks))
         {
@@ -2000,24 +2526,45 @@ static void PullSelection(CP_Services Svcs, Rdma_WSR_Stream Stream)
     ReqBuffer.Handle.Block = ReadBuffer = malloc(ReqBuffer.BufferLen);
     if (Fabric->local_mr_req)
     {
-        fi_mr_reg(Fabric->domain, ReqBuffer.Handle.Block, ReqBuffer.BufferLen, FI_READ, 0, 0, 0,
-                  &rrmr, Fabric->ctx);
+        sst_fi_mr_reg(Svcs, WS_Stream->CP_Stream, Fabric->domain, ReqBuffer.Handle.Block,
+                      ReqBuffer.BufferLen, FI_READ, 0, 0, 0, &rrmr, Fabric->ctx, Fabric->signal,
+                      Fabric->info->domain_attr->mr_mode);
         rrdesc = fi_mr_desc(rrmr);
     }
 
     for (RankReq = Stream->PreloadReq; RankReq; RankReq = RankReq->next)
     {
         RankReq->ReqLog = (RdmaBuffer)ReadBuffer;
-        fi_read(Fabric->signal, RankReq->ReqLog, RankReq->BufferSize, rrdesc,
-                Stream->ReaderAddr[RankReq->Rank], (uint64_t)ReaderRoll[RankReq->Rank].Handle.Block,
-                ReaderRoll[RankReq->Rank].Handle.Key, RankReq);
+        guard_fi_return(
+            (int)fi_read(Fabric->signal, RankReq->ReqLog, RankReq->BufferSize, rrdesc,
+                         Stream->ReaderAddr[RankReq->Rank],
+                         /*
+                          * If mr_virt_addr is 0, then this is a simple
+                          * null-pointer, indicating no offset. Otherwise, we
+                          * need the remote virtual memory read address.
+                          */
+                         Fabric->mr_virt_addr * (uint64_t)ReaderRoll[RankReq->Rank].Handle.Block,
+                         ReaderRoll[RankReq->Rank].Handle.Key, RankReq),
+            Svcs, WS_Stream->CP_Stream, "[PullSelection] fi_read() failed with:");
         ReadBuffer += RankReq->BufferSize;
     }
 
     RankReq = Stream->PreloadReq;
     while (RankReq)
     {
-        fi_cq_sread(Fabric->cq_signal, (void *)(&CQEntry), 1, NULL, -1);
+        ssize_t rc = fi_cq_sread(Fabric->cq_signal, (void *)(&CQEntry), 1, NULL, -1);
+        if (rc < 1)
+        {
+            struct fi_cq_err_entry error;
+            fi_cq_readerr(Fabric->cq_signal, &error, 0);
+            Svcs->verbose(
+                WS_Stream->CP_Stream, DPCriticalVerbose,
+                "[PullSelection] failure while waiting for completions "
+                "(%d (%s - %s)).\n",
+                rc, fi_strerror(error.err),
+                fi_cq_strerror(Fabric->cq_signal, error.err, error.err_data, NULL, error.len));
+            return;
+        }
         CQRankReq = CQEntry.op_context;
         if (CQEntry.flags & FI_READ)
         {
@@ -2049,7 +2596,19 @@ static void CompletePush(CP_Services Svcs, Rdma_WSR_Stream Stream, TimestepList
 
     while (Step->OutstandingWrites > 0)
     {
-        fi_cq_sread(Fabric->cq_signal, (void *)(&CQEntry), 1, NULL, -1);
+        ssize_t rc = fi_cq_sread(Fabric->cq_signal, (void *)(&CQEntry), 1, NULL, -1);
+        if (rc < 1)
+        {
+            struct fi_cq_err_entry error;
+            fi_cq_readerr(Fabric->cq_signal, &error, 0);
+            Svcs->verbose(
+                WS_Stream->CP_Stream, DPCriticalVerbose,
+                "[CompletePush] failure while waiting for completions "
+                "(%d (%s - %s)).\n",
+                rc, fi_strerror(error.err),
+                fi_cq_strerror(Fabric->cq_signal, error.err, error.err_data, NULL, error.len));
+            return;
+        }
         if (CQEntry.flags & FI_WRITE)
         {
             CQTimestep = (long)CQEntry.op_context;
diff --git a/source/adios2/toolkit/transport/file/FileHTTP.cpp b/source/adios2/toolkit/transport/file/FileHTTP.cpp
new file mode 100644
index 0000000000..59d06d81cc
--- /dev/null
+++ b/source/adios2/toolkit/transport/file/FileHTTP.cpp
@@ -0,0 +1,335 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ *
+ * FileDescriptor.h wrapper of POSIX library functions for file I/O
+ *
+ *  Created on: Jul 7, 2023
+ *      Author: Dmitry Ganyushin  ganyushin@gmail.com
+ */
+#include "FileHTTP.h"
+#include 
+#include 
+#include 
+#include 
+namespace adios2
+{
+namespace transport
+{
+
+FileHTTP::FileHTTP(helper::Comm const &comm) : Transport("File", "HTTP", comm) {}
+
+FileHTTP::~FileHTTP()
+{
+    if (m_IsOpen)
+    {
+    }
+}
+
+void FileHTTP::WaitForOpen()
+{
+    if (m_IsOpening)
+    {
+        m_IsOpening = false;
+        CheckFile("couldn't open file " + m_Name + ", in call to POSIX open");
+        m_IsOpen = true;
+    }
+}
+
+void FileHTTP::Open(const std::string &name, const Mode openMode, const bool async,
+                    const bool directio)
+{
+    struct protoent *protoent;
+    struct hostent *hostent;
+    in_addr_t in_addr;
+
+    m_Name = name;
+    /* Build the socket. */
+    protoent = getprotobyname("tcp");
+    if (protoent == NULL)
+    {
+        helper::Throw("Toolkit", "transport::file::FileHTTP", "Open",
+                                              "cannot make getprotobyname");
+    }
+    m_p_proto = protoent->p_proto;
+
+    /* get from parameters. Where the proxy should run*/
+
+    /* Build the address. */
+    hostent = gethostbyname(m_hostname.c_str());
+    if (hostent == NULL)
+    {
+        helper::Throw("Toolkit", "transport::file::FileHTTP", "Open",
+                                              "error: gethostbyname " + m_hostname);
+    }
+    in_addr = inet_addr(inet_ntoa(*(struct in_addr *)*(hostent->h_addr_list)));
+    if (in_addr == (in_addr_t)-1)
+    {
+        helper::Throw("Toolkit", "transport::file::FileHTTP", "Open",
+                                              "error: inet_addr " +
+                                                  std::string(*(hostent->h_addr_list)));
+    }
+    sockaddr_in.sin_addr.s_addr = in_addr;
+    sockaddr_in.sin_family = AF_INET;
+    sockaddr_in.sin_port = htons(m_server_port);
+
+    return;
+}
+
+void FileHTTP::OpenChain(const std::string &name, Mode openMode, const helper::Comm &chainComm,
+                         const bool async, const bool directio)
+{
+    return;
+}
+
+void FileHTTP::Write(const char *buffer, size_t size, size_t start) { return; }
+
+#ifdef REALLY_WANT_WRITEV
+void FilePOSIX::WriteV(const core::iovec *iov, const int iovcnt, size_t start)
+{
+    auto lf_Write = [&](const core::iovec *iov, const int iovcnt) {
+        ProfilerStart("write");
+        errno = 0;
+        size_t nBytesExpected = 0;
+        for (int i = 0; i < iovcnt; ++i)
+        {
+            nBytesExpected += iov[i].iov_len;
+        }
+        const iovec *v = reinterpret_cast(iov);
+        const auto ret = writev(m_FileDescriptor, v, iovcnt);
+        m_Errno = errno;
+        ProfilerStop("write");
+
+        size_t written;
+        if (ret == -1)
+        {
+            if (errno != EINTR)
+            {
+                helper::Throw(
+                    "Toolkit", "transport::file::FilePOSIX", "WriteV",
+                    "couldn't write to file " + m_Name + " " + SysErrMsg());
+            }
+            written = 0;
+        }
+        else
+        {
+            written = static_cast(ret);
+        }
+
+        ProfilerWriteBytes(written);
+
+        if (written < nBytesExpected)
+        {
+            /* Fall back to write calls with individual buffers */
+            // find where the writing has ended
+            int c = 0;
+            size_t n = 0;
+            size_t pos = 0;
+            while (n < written)
+            {
+                if (n + iov[c].iov_len <= written)
+                {
+                    n += iov[c].iov_len;
+                    ++c;
+                }
+                else
+                {
+                    pos = written - n;
+                    n = written;
+                }
+            }
+
+            // write the rest one by one
+            Write(static_cast(iov[c].iov_base) + pos, iov[c].iov_len - pos);
+            for (; c < iovcnt; ++c)
+            {
+                Write(static_cast(iov[c].iov_base), iov[c].iov_len);
+            }
+        }
+    };
+
+    WaitForOpen();
+    if (start != MaxSizeT)
+    {
+        errno = 0;
+        const auto newPosition = lseek(m_FileDescriptor, start, SEEK_SET);
+        m_Errno = errno;
+
+        if (static_cast(newPosition) != start)
+        {
+            helper::Throw("Toolkit", "transport::file::FilePOSIX", "WriteV",
+                                                  "couldn't move to start position " +
+                                                      std::to_string(start) + " in file " + m_Name +
+                                                      " " + SysErrMsg());
+        }
+    }
+
+    int cntTotal = 0;
+    while (cntTotal < iovcnt)
+    {
+        int cnt = iovcnt - cntTotal;
+        if (cnt > 8)
+        {
+            cnt = 8;
+        }
+        lf_Write(iov + cntTotal, cnt);
+        cntTotal += cnt;
+    }
+}
+#endif
+
+void FileHTTP::Read(char *buffer, size_t size, size_t start)
+{
+    /* not using BUFSIZ, the server might use another value for that */
+    const size_t BUF_SIZE = 8192;
+    enum CONSTEXPR
+    {
+        MAX_REQUEST_LEN = 1024
+    };
+    char request[MAX_REQUEST_LEN] = {'\0'};
+    int request_len = snprintf(request, MAX_REQUEST_LEN, request_template, m_Name.c_str(),
+                               m_hostname.c_str(), start, start + size - 1);
+    if (request_len >= MAX_REQUEST_LEN)
+    {
+        helper::Throw("Toolkit", "transport::file::FileHTTP", "Read",
+                                              "request length too long:  " +
+                                                  std::to_string(request_len));
+    }
+    m_socketFileDescriptor = socket(AF_INET, SOCK_STREAM, m_p_proto);
+    if (m_socketFileDescriptor == -1)
+    {
+        helper::Throw("Toolkit", "transport::file::FileHTTP", "Read",
+                                              "cannot open socket");
+    }
+    /* Actually connect. */
+    if (connect(m_socketFileDescriptor, (struct sockaddr *)&sockaddr_in, sizeof(sockaddr_in)) == -1)
+    {
+        helper::Throw("Toolkit", "transport::file::FileHTTP", "Read",
+                                              "cannot connect");
+    }
+    /* Send HTTP request. */
+    int nbytes_total = 0;
+    while (nbytes_total < request_len)
+    {
+        int nbytes_last =
+            write(m_socketFileDescriptor, request + nbytes_total, request_len - nbytes_total);
+        if (nbytes_last == -1)
+        {
+            helper::Throw("Toolkit", "transport::file::FileHTTP", "Read",
+                                                  "cannot send request");
+        }
+        nbytes_total += nbytes_last;
+    }
+
+    /* Read the response. */
+    size_t bytes_recd = 0;
+    while (bytes_recd < size)
+    {
+        nbytes_total = read(m_socketFileDescriptor, buffer + bytes_recd,
+                            std::min(size - bytes_recd, BUF_SIZE));
+        if (nbytes_total == -1)
+        {
+            helper::Throw("Toolkit", "transport::file::FileHTTP", "Read",
+                                                  "cannot get response");
+        }
+        bytes_recd += nbytes_total;
+    }
+
+    close(m_socketFileDescriptor);
+    return;
+}
+
+size_t FileHTTP::GetSize()
+{
+    char request_template[] = "GET %s HTTP/1.1\r\nHost: %s\r\nContent-Length: bytes\r\n\r\n";
+    enum CONSTEXPR
+    {
+        MAX_REQUEST_LEN = 1024,
+        BUF_SIZE = 128
+    };
+    char request[MAX_REQUEST_LEN];
+    char buffer[BUF_SIZE] = {'\0'};
+    int request_len =
+        snprintf(request, MAX_REQUEST_LEN, request_template, m_Name.c_str(), m_hostname.c_str());
+    if (request_len >= MAX_REQUEST_LEN)
+    {
+        helper::Throw("Toolkit", "transport::file::FileHTTP", "GetSize",
+                                              "request length too long:  " +
+                                                  std::to_string(request_len));
+    }
+    m_socketFileDescriptor = socket(AF_INET, SOCK_STREAM, m_p_proto);
+    if (m_socketFileDescriptor == -1)
+    {
+        helper::Throw("Toolkit", "transport::file::FileHTTP", "GetSize",
+                                              "cannot bind socket");
+    }
+    /* Actually connect. */
+    if (connect(m_socketFileDescriptor, (struct sockaddr *)&sockaddr_in, sizeof(sockaddr_in)) == -1)
+    {
+        helper::Throw("Toolkit", "transport::file::FileHTTP", "GetSize",
+                                              "cannot connect");
+    }
+    /* Send HTTP request. */
+    int nbytes_total = 0;
+    while (nbytes_total < request_len)
+    {
+        int nbytes_last =
+            write(m_socketFileDescriptor, request + nbytes_total, request_len - nbytes_total);
+        if (nbytes_last == -1)
+        {
+            helper::Throw("Toolkit", "transport::file::FileHTTP", "GetSize",
+                                                  "sending request failed");
+        }
+        nbytes_total += nbytes_last;
+    }
+
+    /* Read the response. */
+    while ((nbytes_total = read(m_socketFileDescriptor, buffer, BUF_SIZE)) > 0)
+        ;
+    if (nbytes_total == -1)
+    {
+        helper::Throw("Toolkit", "transport::file::FileHTTP", "GetSize",
+                                              "receiving response failed");
+    }
+    close(m_socketFileDescriptor);
+    size_t result = atoi(buffer);
+
+    return result;
+}
+
+void FileHTTP::Flush()
+{
+    /* Turn this off now because BP3/BP4 calls manager Flush and this syncing
+     * slows down IO performance */
+}
+
+void FileHTTP::Close() { return; }
+
+void FileHTTP::Delete() { return; }
+
+void FileHTTP::CheckFile(const std::string hint) const
+{
+    if (m_socketFileDescriptor == -1)
+    {
+        helper::Throw("Toolkit", "transport::file::FilePOSIX", "CheckFile",
+                                              hint + SysErrMsg());
+    }
+}
+
+std::string FileHTTP::SysErrMsg() const
+{
+    return std::string(": errno = " + std::to_string(m_Errno) + ": " + strerror(m_Errno));
+}
+
+void FileHTTP::SeekToEnd() { return; }
+
+void FileHTTP::SeekToBegin() { return; }
+
+void FileHTTP::Seek(const size_t start) { return; }
+
+void FileHTTP::Truncate(const size_t length) { return; }
+
+void FileHTTP::MkDir(const std::string &fileName) { return; }
+
+} // end namespace transport
+} // end namespace adios2
diff --git a/source/adios2/toolkit/transport/file/FileHTTP.h b/source/adios2/toolkit/transport/file/FileHTTP.h
new file mode 100644
index 0000000000..cf32da3a76
--- /dev/null
+++ b/source/adios2/toolkit/transport/file/FileHTTP.h
@@ -0,0 +1,94 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ *
+ * FileDescriptor.h wrapper of POSIX library functions for file I/O
+ *
+ *  Created on: Jul 7, 2023
+ *      Author: Dmitry Ganyushin  ganyushin@gmail.com
+ */
+#ifndef ADIOS2_FILEHTTP_H
+#define ADIOS2_FILEHTTP_H
+
+#include "../Transport.h"
+#include "adios2/common/ADIOSConfig.h"
+#include 
+
+namespace adios2
+{
+namespace helper
+{
+class Comm;
+}
+namespace transport
+{
+
+/** File descriptor transport using the POSIX IO library */
+class FileHTTP : public Transport
+{
+
+public:
+    FileHTTP(helper::Comm const &comm);
+
+    ~FileHTTP();
+
+    void Open(const std::string &name, const Mode openMode, const bool async = false,
+              const bool directio = false) final;
+
+    void OpenChain(const std::string &name, Mode openMode, const helper::Comm &chainComm,
+                   const bool async = false, const bool directio = false) final;
+
+    void Write(const char *buffer, size_t size, size_t start = MaxSizeT) final;
+
+#ifdef REALLY_WANT_WRITEV
+    /* Actual writev() function, inactive for now */
+    void WriteV(const core::iovec *iov, const int iovcnt, size_t start = MaxSizeT) final;
+#endif
+
+    void Read(char *buffer, size_t size, size_t start = MaxSizeT) final;
+
+    size_t GetSize() final;
+
+    /** Does nothing, each write is supposed to flush */
+    void Flush() final;
+
+    void Close() final;
+
+    void Delete() final;
+
+    void SeekToEnd() final;
+
+    void SeekToBegin() final;
+
+    void Seek(const size_t start = MaxSizeT) final;
+
+    void Truncate(const size_t length) final;
+
+    void MkDir(const std::string &fileName) final;
+
+private:
+    /** POSIX file handle returned by Open */
+    int m_socketFileDescriptor = -1;
+    int m_Errno = 0;
+    bool m_IsOpening = false;
+    /* if filename is very lomg, we can get lout from array boundaries */
+    char request_template[128] = "GET %s HTTP/1.1\r\nHost: %s\r\nRange: bytes=%d-%d\r\n\r\n";
+    std::string m_hostname = "localhost";
+    int m_server_port = 9999;
+    struct sockaddr_in sockaddr_in;
+    /* protocol number */
+    int m_p_proto;
+
+    /**
+     * Check if m_FileDescriptor is -1 after an operation
+     * @param hint exception message
+     */
+    void CheckFile(const std::string hint) const;
+    void WaitForOpen();
+    std::string SysErrMsg() const;
+};
+
+} // end namespace transport
+} // end namespace adios2
+
+#endif // ADIOS2_FILEHTTP_H
diff --git a/source/adios2/toolkit/transportman/TransportMan.cpp b/source/adios2/toolkit/transportman/TransportMan.cpp
index 4757495ea7..bb7ff8666d 100644
--- a/source/adios2/toolkit/transportman/TransportMan.cpp
+++ b/source/adios2/toolkit/transportman/TransportMan.cpp
@@ -38,6 +38,9 @@
 #endif
 
 #include "adios2/toolkit/transport/file/FileFStream.h"
+#ifndef _WIN32
+#include "adios2/toolkit/transport/file/FileHTTP.h"
+#endif
 #include "adios2/toolkit/transport/file/FileStdio.h"
 #include "adios2/toolkit/transport/null/NullTransport.h"
 
@@ -600,6 +603,18 @@ std::shared_ptr TransportMan::OpenFileTransport(const std::string &fi
         {
             transport = std::make_shared(m_Comm);
         }
+#endif
+#ifndef _WIN32
+        else if (library == "http")
+        {
+            transport = std::make_shared(m_Comm);
+            if (lf_GetBuffered("false"))
+            {
+                helper::Throw(
+                    "Toolkit", "TransportMan", "OpenFileTransport",
+                    library + " transport does not support buffered I/O.");
+            }
+        }
 #endif
         else if (library == "null")
         {
diff --git a/source/utils/bpls/bpls.cpp b/source/utils/bpls/bpls.cpp
index 5c4180b85b..bbd320cdd0 100644
--- a/source/utils/bpls/bpls.cpp
+++ b/source/utils/bpls/bpls.cpp
@@ -83,6 +83,7 @@ std::string format;           // format string for one data element (e.g. %6.2f)
 std::string transport_params; // Transport parameters (e.g. "Library=stdio,verbose=3")
 std::string engine_name;      // Engine name (e.g. "BP5")
 std::string engine_params;    // Engine parameters (e.g. "SelectSteps=0:5:2")
+std::string accuracy_def;     // Accuracy definition (e.g. "accuracy="0.0,0.0,rel")
 
 // Flags from arguments or defaults
 bool dump; // dump data not just list info(flag == 1)
@@ -102,6 +103,8 @@ bool hidden_attrs;       // show hidden attrs in BP file
 int hidden_attrs_flag;   // to be passed on in option struct
 bool show_decomp;        // show decomposition of arrays
 bool show_version;       // print binary version info of file before work
+adios2::Accuracy accuracy;
+bool accuracyWasSet = false;
 
 // other global variables
 char *prgname; /* argv[0] */
@@ -169,6 +172,10 @@ void display_help()
            "file\n"
            "  --decomp    | -D           Show decomposition of variables as layed "
            "out in file\n"
+           "  --error     | -X string    Specify read accuracy (error,norm,rel|abs)\n"
+           "                             e.g. error=\"0.0,0.0,abs\"\n"
+           "                             L2 norm = 0.0, Linf = inf\n"
+
            "  --transport-parameters | -T         Specify File transport "
            "parameters\n"
            "                                      e.g. \"Library=stdio\"\n"
@@ -636,6 +643,9 @@ int bplsMain(int argc, char *argv[])
                            "Print version information (add -verbose for additional"
                            " information)");
     arg.AddBooleanArgument("-V", &show_version, "");
+    arg.AddArgument("--error", argT::SPACE_ARGUMENT, &accuracy_def,
+                    "| -X string    Specify read accuracy (error,norm,rel|abs)");
+    arg.AddArgument("-X", argT::SPACE_ARGUMENT, &accuracy_def, "");
     arg.AddArgument("--transport-parameters", argT::SPACE_ARGUMENT, &transport_params,
                     "| -T string    Specify File transport parameters manually");
     arg.AddArgument("-T", argT::SPACE_ARGUMENT, &transport_params, "");
@@ -705,6 +715,10 @@ int bplsMain(int argc, char *argv[])
     if (attrsonly)
         listattrs = true;
 
+    retval = parseAccuracy();
+    if (retval)
+        return retval;
+
     if (verbose > 1)
         printSettings();
 
@@ -1273,6 +1287,7 @@ int printVariableInfo(core::Engine *fp, core::IO *io, core::Variable *variabl
 
     if (dump && !show_decomp)
     {
+        variable->SetAccuracy(accuracy);
         // print variable content
         if (variable->m_ShapeID == ShapeID::LocalArray)
         {
@@ -2210,6 +2225,14 @@ int readVar(core::Engine *fp, core::IO *io, core::Variable *variable)
         }
     } // end while sum < nelems
     print_endline();
+
+    if (accuracyWasSet)
+    {
+        adios2::Accuracy a = variable->GetAccuracy();
+        std::cout << "Read accuracy was (error = " << a.error << ", norm = " << a.norm << ", "
+                  << (a.relative ? "rel" : "abs") << ")\n";
+    }
+
     return 0;
 }
 
@@ -3804,6 +3827,61 @@ void print_decomp_singlestep(core::Engine *fp, core::IO *io, core::Variable *
     }
 }
 
+int parseAccuracy()
+{
+    if (accuracy_def.empty())
+        return 0;
+
+    // Vector of string to save tokens
+    std::vector tokens;
+
+    // stringstream class check1
+    std::stringstream ss(accuracy_def);
+
+    std::string intermediate;
+
+    // Tokenizing w.r.t. space ','
+    while (getline(ss, intermediate, ','))
+    {
+        tokens.push_back(intermediate);
+    }
+
+    if (tokens.size() != 3)
+    {
+        std::cout
+            << "ERROR: --error definition needs 3 values: error, norm, and relative|absolute\n"
+            << " error >= 0.0, norm >= 0.0 or inf, third arg is \"rel\" or \"abs\"\n";
+        return 1;
+    }
+
+    accuracy.error = adios2::helper::StringTo(tokens[0], "error value for accuracy");
+    std::string normval = adios2::helper::LowerCase(tokens[2]);
+    if (normval == "inf")
+        accuracy.norm = std::numeric_limits::infinity();
+    else
+        accuracy.norm = adios2::helper::StringTo(tokens[1], "norm value for accuracy");
+    std::string relval = adios2::helper::LowerCase(tokens[2]);
+    if (relval == "rel")
+        accuracy.relative = true;
+    else if (relval == "abs")
+        accuracy.relative = false;
+    else
+    {
+        std::cout << "ERROR: --error third value must be \"rel\" or \"abs\" but it was \""
+                  << tokens[2] << "\"\n";
+        return 1;
+    }
+
+    accuracyWasSet = true;
+    if (verbose > 0)
+    {
+        std::cout << "Read accuracy is set to (error = " << accuracy.error
+                  << ", norm = " << accuracy.norm << ", " << (accuracy.relative ? "rel" : "abs")
+                  << ")\n";
+    }
+    return 0;
+}
+
 // parse a string "0, 3; 027" into an integer array
 // of [0,3,27]
 // exits if parsing failes
diff --git a/source/utils/bpls/bpls.h b/source/utils/bpls/bpls.h
index f0a715a94f..683c582b4d 100644
--- a/source/utils/bpls/bpls.h
+++ b/source/utils/bpls/bpls.h
@@ -57,6 +57,7 @@ char *mystrndup(const char *s, size_t n);
 void init_globals();
 void processDimSpecs();
 void parseDimSpec(const std::string &str, int64_t *dims);
+int parseAccuracy();
 int compile_regexp_masks(void);
 void printSettings(void);
 int doList(const char *path);
diff --git a/testing/adios2/engine/bp/CMakeLists.txt b/testing/adios2/engine/bp/CMakeLists.txt
index 03479c7edc..8c5b90606b 100644
--- a/testing/adios2/engine/bp/CMakeLists.txt
+++ b/testing/adios2/engine/bp/CMakeLists.txt
@@ -184,6 +184,11 @@ gtest_add_tests_helper(ReadMultithreaded MPI_NONE BP Engine.BP. .BP5
   WORKING_DIRECTORY ${BP5_DIR} EXTRA_ARGS "BP5"
 )
 
+# Only a single test is enough, pick the latest engine
+gtest_add_tests_helper(AccuracyDefaults MPI_NONE BP Engine.BP. .BP5
+  WORKING_DIRECTORY ${BP5_DIR} EXTRA_ARGS "BP5"
+)
+
 # BP3 only for now
 gtest_add_tests_helper(WriteNull MPI_ALLOW BP Engine.BP. .BP3
   WORKING_DIRECTORY ${BP3_DIR} EXTRA_ARGS "BP3"
@@ -191,6 +196,7 @@ gtest_add_tests_helper(WriteNull MPI_ALLOW BP Engine.BP. .BP3
 
 # BP4 and BP5 but NOT BP3
 bp4_bp5_gtest_add_tests_helper(WriteAppendReadADIOS2 MPI_ALLOW)
+bp4_bp5_gtest_add_tests_helper(JoinedArray MPI_ALLOW)
 
 # BP4 only for now
 # gtest_add_tests_helper(WriteAppendReadADIOS2 MPI_ALLOW BP Engine.BP. .BP4
@@ -202,9 +208,9 @@ gtest_add_tests_helper(StepsInSituGlobalArray MPI_ALLOW BP Engine.BP. .BP4
 gtest_add_tests_helper(StepsInSituLocalArray MPI_ALLOW BP Engine.BP. .BP4
   WORKING_DIRECTORY ${BP4_DIR} EXTRA_ARGS "BP4"
 )
-gtest_add_tests_helper(JoinedArray MPI_ALLOW BP Engine.BP. .BP4
-  WORKING_DIRECTORY ${BP4_DIR} EXTRA_ARGS "BP4"
-)
+#gtest_add_tests_helper(JoinedArray MPI_ALLOW BP Engine.BP. .BP4
+#  WORKING_DIRECTORY ${BP4_DIR} EXTRA_ARGS "BP4"
+#)
 
 # InquireVaribleException only for BP4 because BP5 works differently
 gtest_add_tests_helper(InquireVariableException MPI_ALLOW BP Engine.BP. .BP4
diff --git a/testing/adios2/engine/bp/TestBPAccuracyDefaults.cpp b/testing/adios2/engine/bp/TestBPAccuracyDefaults.cpp
new file mode 100644
index 0000000000..d0493cdc22
--- /dev/null
+++ b/testing/adios2/engine/bp/TestBPAccuracyDefaults.cpp
@@ -0,0 +1,153 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ *
+ * TestBPAccuracyDefaults.cpp :
+ *
+ *  Created on: Dec 5, 2023
+ *      Author: Norbert Podhorszki
+ */
+
+#include 
+#include 
+
+#include 
+#include  //std::iota
+#include 
+
+#include 
+
+#include 
+
+std::string engineName;       // comes from command line
+std::string engineParameters; // comes from command line
+
+class AccuracyTests : public ::testing::Test
+{
+public:
+    AccuracyTests() = default;
+};
+
+// Check if SetAccuracy/GetAccuracy default behavior works
+TEST_F(AccuracyTests, DefaultAccuracy)
+{
+    const std::string fname("DefaultAccuracy.bp");
+
+    int mpiRank = 0, mpiSize = 1;
+
+    const std::size_t Nx = 10;
+
+#if ADIOS2_USE_MPI
+    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
+#endif
+
+    std::vector localData(Nx);
+    std::iota(localData.begin(), localData.end(), mpiRank * Nx);
+
+#if ADIOS2_USE_MPI
+    adios2::ADIOS adios(MPI_COMM_WORLD);
+#else
+    adios2::ADIOS adios;
+#endif
+    {
+        adios2::IO io = adios.DeclareIO("DefaultAccuracyWrite");
+        if (!engineName.empty())
+        {
+            io.SetEngine(engineName);
+        }
+        if (!engineParameters.empty())
+        {
+            io.SetParameters(engineParameters);
+        }
+
+        adios2::Variable var =
+            io.DefineVariable("range", {static_cast(Nx * mpiSize)},
+                                       {static_cast(Nx * mpiRank)}, {Nx});
+
+        adios2::Accuracy accuracyRequested = {0.001, std::numeric_limits::infinity(), true};
+        var.SetAccuracy(accuracyRequested);
+        adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write);
+
+        bpWriter.Put("range", localData.data());
+        bpWriter.Close();
+
+        auto accuracyGot = var.GetAccuracy();
+        assert(accuracyGot.error == 0.0); // no error whatsoever
+        assert(accuracyGot.norm == accuracyRequested.norm);
+        assert(accuracyGot.relative == accuracyRequested.relative);
+    }
+    // Reader
+    {
+        adios2::IO io = adios.DeclareIO("DefaultAccuracyRead");
+        if (!engineName.empty())
+        {
+            io.SetEngine(engineName);
+        }
+        if (!engineParameters.empty())
+        {
+            io.SetParameters(engineParameters);
+        }
+
+        adios2::Engine bpReader = io.Open(fname, adios2::Mode::ReadRandomAccess);
+        adios2::Variable varRange = io.InquireVariable("range");
+        adios2::Accuracy accuracyRequested = {0.001, 1.17, false};
+        varRange.SetAccuracy(accuracyRequested);
+
+        const std::size_t gNx = static_cast(Nx * mpiSize);
+        std::vector globalData(gNx);
+        bpReader.Get(varRange, globalData);
+        bpReader.PerformGets();
+
+        auto accuracyGot = varRange.GetAccuracy();
+        assert(accuracyGot.error == 0.0); // no error whatsoever
+        assert(accuracyGot.norm == accuracyRequested.norm);
+        assert(accuracyGot.relative == accuracyRequested.relative);
+
+        std::vector iStartEndData;
+        iStartEndData.reserve(gNx); // maximum possible
+
+        for (size_t i = 1; i < gNx; ++i)
+        {
+            varRange.SetSelection({{i}, {gNx - i}});
+
+            bpReader.Get("range", iStartEndData);
+            bpReader.PerformGets();
+
+            for (size_t j = i; j < gNx; ++j)
+            {
+                EXPECT_EQ(globalData[j], iStartEndData[j - i]);
+            }
+        }
+        bpReader.Close();
+    }
+}
+
+int main(int argc, char **argv)
+{
+#if ADIOS2_USE_MPI
+    int provided;
+
+    // MPI_THREAD_MULTIPLE is only required if you enable the SST MPI_DP
+    MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided);
+#endif
+
+    int result;
+    ::testing::InitGoogleTest(&argc, argv);
+
+    if (argc > 1)
+    {
+        engineName = std::string(argv[1]);
+    }
+    if (argc > 2)
+    {
+        engineParameters = std::string(argv[2]);
+    }
+    result = RUN_ALL_TESTS();
+
+#if ADIOS2_USE_MPI
+    MPI_Finalize();
+#endif
+
+    return result;
+}
diff --git a/testing/adios2/engine/bp/operations/CMakeLists.txt b/testing/adios2/engine/bp/operations/CMakeLists.txt
index 61a4544b01..b8c2a04362 100644
--- a/testing/adios2/engine/bp/operations/CMakeLists.txt
+++ b/testing/adios2/engine/bp/operations/CMakeLists.txt
@@ -60,6 +60,12 @@ if(ADIOS2_HAVE_MGARD)
       target_link_libraries(${tgt} CUDA::cudart)
     endforeach()
   endif()
+
+  if(ADIOS2_HAVE_MGARD_MDR)
+    gtest_add_tests_helper(WriteReadMGARDMDR MPI_ALLOW BP Engine.BP. .BP5
+    WORKING_DIRECTORY ${BP5_DIR} EXTRA_ARGS "BP5"
+    )
+  endif()
 endif()
 
 if(ADIOS2_HAVE_BZip2)
diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadMGARDMDR.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadMGARDMDR.cpp
new file mode 100644
index 0000000000..084e21631b
--- /dev/null
+++ b/testing/adios2/engine/bp/operations/TestBPWriteReadMGARDMDR.cpp
@@ -0,0 +1,227 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ */
+#include 
+#include 
+
+#include 
+#include  //std::cout
+#include   //std::iota
+#include 
+
+#include 
+
+#include 
+
+std::string engineName; // comes from command line
+
+class BPWriteReadMGARDMDR : public ::testing::TestWithParam
+{
+public:
+    BPWriteReadMGARDMDR() = default;
+    virtual void SetUp(){};
+    virtual void TearDown(){};
+};
+
+TEST_F(BPWriteReadMGARDMDR, BPWRMGARD1D)
+{
+    // Refactor a dataset with MDR, then
+    // read back with various accuracies
+    const std::string fname("BPWRMGARDMDR1D.bp");
+
+    int mpiRank = 0, mpiSize = 1;
+    const size_t Nx = 30000; // 100k minimum data size for MDR
+    const size_t NSteps = 1;
+
+    std::vector r32s(Nx);
+    std::vector r64s(Nx);
+
+    const double pi = 3.141592653589793238462643383279;
+    const double twopi = 2 * pi;
+    const double value = (2 * pi) / double(Nx);
+    for (size_t x = 0; x < Nx; ++x)
+    {
+        r64s[x] = std::cos(value * x);
+        r32s[x] = (float)r64s[x];
+    }
+
+#if ADIOS2_USE_MPI
+    MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
+#endif
+
+#if ADIOS2_USE_MPI
+    adios2::ADIOS adios(MPI_COMM_WORLD);
+#else
+    adios2::ADIOS adios;
+#endif
+    {
+        adios2::IO io = adios.DeclareIO("WriteIO");
+
+        if (!engineName.empty())
+        {
+            io.SetEngine(engineName);
+        }
+
+        const adios2::Dims shape{static_cast(Nx * mpiSize)};
+        const adios2::Dims start{static_cast(Nx * mpiRank)};
+        const adios2::Dims count{Nx};
+
+        auto var_r32 = io.DefineVariable("r32", shape, start, count, adios2::ConstantDims);
+        auto var_r64 = io.DefineVariable("r64", shape, start, count, adios2::ConstantDims);
+        auto var_raw64 =
+            io.DefineVariable("raw64", shape, start, count, adios2::ConstantDims);
+
+        // add operations
+        adios2::Operator mgardOp = adios.DefineOperator("mgardCompressor", adios2::ops::MDR);
+
+        var_r32.AddOperation(mgardOp, {});
+        var_r64.AddOperation(mgardOp, {});
+
+        adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write);
+
+        for (size_t step = 0; step < NSteps; ++step)
+        {
+            bpWriter.BeginStep();
+            bpWriter.Put("r32", r32s.data());
+            bpWriter.Put("r64", r64s.data());
+            bpWriter.Put("raw64", r64s.data());
+            bpWriter.EndStep();
+        }
+
+        bpWriter.Close();
+    }
+
+    {
+        adios2::IO io = adios.DeclareIO("ReadIO");
+
+        if (!engineName.empty())
+        {
+            io.SetEngine(engineName);
+        }
+
+        adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read);
+        size_t t = 0;
+        while (bpReader.BeginStep() == adios2::StepStatus::OK)
+        {
+            auto var_r32 = io.InquireVariable("r32");
+            EXPECT_TRUE(var_r32);
+            ASSERT_EQ(var_r32.ShapeID(), adios2::ShapeID::GlobalArray);
+            ASSERT_EQ(var_r32.Steps(), NSteps);
+            ASSERT_EQ(var_r32.Shape()[0], mpiSize * Nx);
+
+            auto var_r64 = io.InquireVariable("r64");
+            EXPECT_TRUE(var_r64);
+            ASSERT_EQ(var_r64.ShapeID(), adios2::ShapeID::GlobalArray);
+            ASSERT_EQ(var_r64.Steps(), NSteps);
+            ASSERT_EQ(var_r64.Shape()[0], mpiSize * Nx);
+
+            const adios2::Dims start{mpiRank * Nx};
+            const adios2::Dims count{Nx};
+            const adios2::Box sel(start, count);
+            var_r32.SetSelection(sel);
+            var_r64.SetSelection(sel);
+
+            std::vector errors = {0.0, 0.0000001, 0.0001, 0.1};
+            for (auto error : errors)
+            {
+                std::cout << "===== Read with error = " << error << std::endl;
+
+                std::vector read32s;
+                std::vector read64s;
+
+                adios2::Accuracy accuracyRequested = {
+                    error, std::numeric_limits::infinity(), false};
+                var_r32.SetAccuracy(accuracyRequested);
+                var_r64.SetAccuracy(accuracyRequested);
+                bpReader.Get(var_r32, read32s, adios2::Mode::Sync);
+                bpReader.Get(var_r64, read64s, adios2::Mode::Sync);
+
+                auto accuracyGot32 = var_r32.GetAccuracy();
+                assert(accuracyGot32.error <=
+                       std::max(accuracyRequested.error,
+                                adios2::ops::mdr::FLOAT_ROUNDING_ERROR_LIMIT));
+                assert(accuracyGot32.norm == accuracyRequested.norm);
+                assert(accuracyGot32.relative == accuracyRequested.relative);
+
+                auto accuracyGot64 = var_r64.GetAccuracy();
+                assert(accuracyGot64.error <=
+                       std::max(accuracyRequested.error,
+                                adios2::ops::mdr::DOUBLE_ROUNDING_ERROR_LIMIT));
+                assert(accuracyGot64.norm == accuracyRequested.norm);
+                assert(accuracyGot64.relative == accuracyRequested.relative);
+
+                double maxDiff = 0.0, relativeMaxDiff = 0.0;
+                size_t maxDiffPos = 0;
+
+                for (size_t i = 0; i < Nx; ++i)
+                {
+                    double diff = std::abs(r64s[i] - read64s[i]);
+                    if (diff > maxDiff)
+                    {
+                        maxDiff = diff;
+                        maxDiffPos = i;
+                    }
+                }
+
+                auto r64s_Max = std::max_element(r64s.begin(), r64s.end());
+                relativeMaxDiff = maxDiff / *r64s_Max;
+                std::cout << "double array: Relative Max Diff " << relativeMaxDiff << " Max Diff "
+                          << maxDiff << " requested error " << error << " result error "
+                          << accuracyGot64.error << " max diff pos " << maxDiffPos << " orig value "
+                          << r64s[maxDiffPos] << " read value " << read64s[maxDiffPos] << "\n";
+                ASSERT_LE(maxDiff, accuracyGot64.error);
+
+                for (size_t i = 0; i < Nx; ++i)
+                {
+                    double diff = std::abs(r32s[i] - read32s[i]);
+                    if (diff > maxDiff)
+                    {
+                        maxDiff = diff;
+                        maxDiffPos = i;
+                    }
+                }
+
+                auto r32s_Max = std::max_element(r32s.begin(), r32s.end());
+                relativeMaxDiff = maxDiff / *r32s_Max;
+
+                std::cout << "float array: Relative Max Diff " << relativeMaxDiff << " Max Diff "
+                          << maxDiff << " requested error " << error << " result error "
+                          << accuracyGot32.error << " max diff pos " << maxDiffPos << " orig value "
+                          << r32s[maxDiffPos] << " read value " << read32s[maxDiffPos] << "\n";
+                ASSERT_LE(maxDiff, accuracyGot32.error);
+            }
+            bpReader.EndStep();
+            ++t;
+        }
+
+        EXPECT_EQ(t, NSteps);
+
+        bpReader.Close();
+    }
+}
+
+int main(int argc, char **argv)
+{
+#if ADIOS2_USE_MPI
+    int provided;
+
+    // MPI_THREAD_MULTIPLE is only required if you enable the SST MPI_DP
+    MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided);
+#endif
+
+    int result;
+    ::testing::InitGoogleTest(&argc, argv);
+    if (argc > 1)
+    {
+        engineName = std::string(argv[1]);
+    }
+    result = RUN_ALL_TESTS();
+
+#if ADIOS2_USE_MPI
+    MPI_Finalize();
+#endif
+
+    return result;
+}
diff --git a/testing/adios2/xml/TestXMLConfig.cpp b/testing/adios2/xml/TestXMLConfig.cpp
index 93852393d4..1fd890ab5d 100644
--- a/testing/adios2/xml/TestXMLConfig.cpp
+++ b/testing/adios2/xml/TestXMLConfig.cpp
@@ -117,6 +117,32 @@ TEST_F(XMLConfigTest, OpNoneException)
 #endif
 }
 
+TEST_F(XMLConfigTest, RemoveIO)
+{
+    const std::string configFile(configDir + std::string(&adios2::PathSeparator, 1) +
+                                 "configRemoveIO.xml");
+
+#if ADIOS2_USE_MPI
+    adios2::ADIOS adios(configFile, MPI_COMM_WORLD);
+#else
+    adios2::ADIOS adios(configFile);
+#endif
+
+    adios2::IO io;
+    adios2::Engine engine;
+
+    std::string io_name_ = "checkpoint";
+    for (int c = 0; c < 3; c++)
+    {
+        io = adios.DeclareIO(io_name_);
+        std::string filename = "test.bp";
+        engine = io.Open(filename, adios2::Mode::Write);
+        EXPECT_TRUE(io.EngineType() == "BP4");
+        engine.Close();
+        adios.RemoveIO(io_name_);
+    }
+}
+
 int main(int argc, char **argv)
 {
 #if ADIOS2_USE_MPI
diff --git a/testing/adios2/xml/configRemoveIO.xml b/testing/adios2/xml/configRemoveIO.xml
new file mode 100644
index 0000000000..3c9a8ef22a
--- /dev/null
+++ b/testing/adios2/xml/configRemoveIO.xml
@@ -0,0 +1,8 @@
+
+
+    
+        
+            
+        
+    
+
diff --git a/testing/contract/lammps/build.sh b/testing/contract/lammps/build.sh
index 0290ec8f3d..436861e621 100755
--- a/testing/contract/lammps/build.sh
+++ b/testing/contract/lammps/build.sh
@@ -3,6 +3,10 @@
 set -x
 set -e
 
-source $(dirname $(readlink -f ${BASH_SOURCE}))/setup.sh
+# shellcheck disable=SC1091
+source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/setup.sh"
 
-cmake --build ${build_dir} -j8
+# Fail if is not set
+build_dir="${build_dir:?}"
+
+cmake --build "${build_dir}" -j8
diff --git a/testing/contract/lammps/config.sh b/testing/contract/lammps/config.sh
index 14837d843a..256306608d 100755
--- a/testing/contract/lammps/config.sh
+++ b/testing/contract/lammps/config.sh
@@ -3,17 +3,23 @@
 set -x
 set -e
 
-source $(dirname $(readlink -f ${BASH_SOURCE}))/setup.sh
+# shellcheck disable=SC1091
+source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/setup.sh"
 
-mkdir -p ${build_dir}
-cd ${build_dir}
+# Fail if is not set
+source_dir="${source_dir:?}"
+build_dir="${build_dir:?}"
+install_dir="${install_dir:?}"
+
+mkdir -p "${build_dir}"
+cd "${build_dir}"
 
 cmake \
-  -DCMAKE_INSTALL_PREFIX=${install_dir} \
+  -DCMAKE_INSTALL_PREFIX="${install_dir}" \
   -DBUILD_MPI=yes \
   -DBUILD_EXE=yes \
   -DBUILD_LIB=no \
   -DBUILD_DOC=no \
   -DLAMMPS_SIZES=smallbig \
   -DPKG_ADIOS=yes \
-  ${source_dir}/cmake
+  "${source_dir}/cmake"
diff --git a/testing/contract/lammps/install.sh b/testing/contract/lammps/install.sh
index f319d25356..26b65c1592 100755
--- a/testing/contract/lammps/install.sh
+++ b/testing/contract/lammps/install.sh
@@ -3,6 +3,10 @@
 set -x
 set -e
 
-source $(dirname $(readlink -f ${BASH_SOURCE}))/setup.sh
+# shellcheck disable=SC1091
+source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/setup.sh"
 
-cmake --install ${build_dir}
+# Fail if is not set
+build_dir="${build_dir:?}"
+
+cmake --install "${build_dir}"
diff --git a/testing/contract/lammps/setup.sh b/testing/contract/lammps/setup.sh
index ca74ccf68a..00a4a7f1d5 100755
--- a/testing/contract/lammps/setup.sh
+++ b/testing/contract/lammps/setup.sh
@@ -1,7 +1,14 @@
-source_dir=$(readlink -f ${PWD})/source
-build_dir=$(readlink -f ${PWD})/build
-install_dir=$(readlink -f ${PWD})/install
-test_dir=$(readlink -f ${PWD})/test
+#!/bin/bash
+
+source_dir=$(readlink -f "${PWD}")/source
+build_dir=$(readlink -f "${PWD}")/build
+install_dir=$(readlink -f "${PWD}")/install
+test_dir=$(readlink -f "${PWD}")/test
+
+export source_dir
+export build_dir
+export install_dir
+export test_dir
 
 echo "source_dir  = \"${source_dir}\""
 echo "build_dir   = \"${build_dir}\""
diff --git a/testing/contract/lammps/test.sh b/testing/contract/lammps/test.sh
index 2c378c1fda..28a0486b34 100755
--- a/testing/contract/lammps/test.sh
+++ b/testing/contract/lammps/test.sh
@@ -3,13 +3,18 @@
 set -x
 set -e
 
-source $(dirname $(readlink -f ${BASH_SOURCE}))/setup.sh
+# shellcheck disable=SC1091
+source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/setup.sh"
 
-mkdir -p ${test_dir}
-cd ${test_dir}
+# Fail if is not set
+install_dir="${install_dir:?}"
+test_dir="${test_dir:?}"
+
+mkdir -p "${test_dir}"
+cd "${test_dir}"
 cp -v /opt/adios2/source/testing/contract/lammps/{adios2_config.xml,check_results.sh,in.test} .
 
 
-mpiexec -np 4 --oversubscribe ${install_dir}/bin/lmp -in in.test
+mpiexec -np 4 --oversubscribe "${install_dir}/bin/lmp" -in in.test
 
 ./check_results.sh
diff --git a/testing/contract/scorpio/build.sh b/testing/contract/scorpio/build.sh
index 0290ec8f3d..436861e621 100755
--- a/testing/contract/scorpio/build.sh
+++ b/testing/contract/scorpio/build.sh
@@ -3,6 +3,10 @@
 set -x
 set -e
 
-source $(dirname $(readlink -f ${BASH_SOURCE}))/setup.sh
+# shellcheck disable=SC1091
+source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/setup.sh"
 
-cmake --build ${build_dir} -j8
+# Fail if is not set
+build_dir="${build_dir:?}"
+
+cmake --build "${build_dir}" -j8
diff --git a/testing/contract/scorpio/config.sh b/testing/contract/scorpio/config.sh
index bb88cc834c..9ff170bd49 100755
--- a/testing/contract/scorpio/config.sh
+++ b/testing/contract/scorpio/config.sh
@@ -3,22 +3,28 @@
 set -x
 set -e
 
-source $(dirname $(readlink -f ${BASH_SOURCE}))/setup.sh
+# shellcheck disable=SC1091
+source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/setup.sh"
 
-mkdir -p ${build_dir}
-cd ${build_dir}
+# Fail if is not set
+source_dir="${source_dir:?}"
+build_dir="${build_dir:?}"
+install_dir="${install_dir:?}"
+
+mkdir -p "${build_dir}"
+cd "${build_dir}"
 
 export CC=mpicc
 export CXX=mpic++
 export FC=mpifort
 
 cmake \
-  -DCMAKE_INSTALL_PREFIX=${install_dir} \
+  -DCMAKE_INSTALL_PREFIX="${install_dir}" \
   -DFPHSA_NAME_MISMATCHED=ON \
   -DPIO_ENABLE_TESTS=ON \
   -DPIO_ENABLE_EXAMPLES=ON \
   -DWITH_NETCDF=OFF \
   -DWITH_PNETCDF=ON \
-  -DPnetCDF_PATH=$(spack location -i parallel-netcdf) \
+  -DPnetCDF_PATH="$(spack location -i parallel-netcdf)" \
   -DWITH_ADIOS2=ON \
-  ${source_dir}
+  "${source_dir}"
diff --git a/testing/contract/scorpio/install.sh b/testing/contract/scorpio/install.sh
index f319d25356..26b65c1592 100755
--- a/testing/contract/scorpio/install.sh
+++ b/testing/contract/scorpio/install.sh
@@ -3,6 +3,10 @@
 set -x
 set -e
 
-source $(dirname $(readlink -f ${BASH_SOURCE}))/setup.sh
+# shellcheck disable=SC1091
+source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/setup.sh"
 
-cmake --install ${build_dir}
+# Fail if is not set
+build_dir="${build_dir:?}"
+
+cmake --install "${build_dir}"
diff --git a/testing/contract/scorpio/setup.sh b/testing/contract/scorpio/setup.sh
index 08fbcaa5e2..1e32491be1 100755
--- a/testing/contract/scorpio/setup.sh
+++ b/testing/contract/scorpio/setup.sh
@@ -1,7 +1,14 @@
-source_dir=$(readlink -f ${PWD})/source
-build_dir=$(readlink -f ${PWD})/build
-install_dir=$(readlink -f ${PWD})/install
-test_dir=$(readlink -f ${PWD})/test
+#!/bin/bash
+
+source_dir=$(readlink -f "${PWD}")/source
+build_dir=$(readlink -f "${PWD}")/build
+install_dir=$(readlink -f "${PWD}")/install
+test_dir=$(readlink -f "${PWD}")/test
+
+export source_dir
+export build_dir
+export install_dir
+export test_dir
 
 echo "source_dir  = \"${source_dir}\""
 echo "build_dir   = \"${build_dir}\""
diff --git a/testing/contract/scorpio/test.sh b/testing/contract/scorpio/test.sh
index 0249fa7d5f..94b4fca85d 100755
--- a/testing/contract/scorpio/test.sh
+++ b/testing/contract/scorpio/test.sh
@@ -3,12 +3,17 @@
 set -x
 set -e
 
-source $(dirname $(readlink -f ${BASH_SOURCE}))/setup.sh
+# shellcheck disable=SC1091
+source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/setup.sh"
 
-mkdir -p ${test_dir}
-cd ${test_dir}
+# Fail if is not set
+build_dir="${build_dir:?}"
+test_dir="${test_dir:?}"
 
-mpiexec --oversubscribe -np 4 ${build_dir}/examples/adios/example3 -v
+mkdir -p "${test_dir}"
+cd "${test_dir}"
+
+mpiexec --oversubscribe -np 4 "${build_dir}/examples/adios/example3" -v
 
 bpls -d example3_1.nc.bp.dir/example3_1.nc.bp.0 > 0.dump
 diff -u 0.dump /opt/adios2/source/testing/contract/scorpio/0.dump
diff --git a/testing/contract/tau/build.sh b/testing/contract/tau/build.sh
index 0290ec8f3d..436861e621 100755
--- a/testing/contract/tau/build.sh
+++ b/testing/contract/tau/build.sh
@@ -3,6 +3,10 @@
 set -x
 set -e
 
-source $(dirname $(readlink -f ${BASH_SOURCE}))/setup.sh
+# shellcheck disable=SC1091
+source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/setup.sh"
 
-cmake --build ${build_dir} -j8
+# Fail if is not set
+build_dir="${build_dir:?}"
+
+cmake --build "${build_dir}" -j8
diff --git a/testing/contract/tau/config.sh b/testing/contract/tau/config.sh
index a89b2deece..0fd803a1a6 100755
--- a/testing/contract/tau/config.sh
+++ b/testing/contract/tau/config.sh
@@ -3,11 +3,15 @@
 set -x
 set -e
 
-source $(dirname $(readlink -f ${BASH_SOURCE}))/setup.sh
+# shellcheck disable=SC1091
+source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/setup.sh"
 
-mkdir -p ${build_dir}
-cd ${build_dir}
+# Fail if is not set
+source_dir="${source_dir:?}"
+build_dir="${build_dir:?}"
+install_dir="${install_dir:?}"
 
-cmake \
-  -DCMAKE_INSTALL_PREFIX=${install_dir} \
-  ${source_dir}
+mkdir -p "${build_dir}"
+cd "${build_dir}"
+
+cmake -DCMAKE_INSTALL_PREFIX="${install_dir}" "${source_dir}"
diff --git a/testing/contract/tau/install.sh b/testing/contract/tau/install.sh
index f319d25356..26b65c1592 100755
--- a/testing/contract/tau/install.sh
+++ b/testing/contract/tau/install.sh
@@ -3,6 +3,10 @@
 set -x
 set -e
 
-source $(dirname $(readlink -f ${BASH_SOURCE}))/setup.sh
+# shellcheck disable=SC1091
+source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/setup.sh"
 
-cmake --install ${build_dir}
+# Fail if is not set
+build_dir="${build_dir:?}"
+
+cmake --install "${build_dir}"
diff --git a/testing/contract/tau/setup.sh b/testing/contract/tau/setup.sh
index d79a4d9ee9..a9e1aff92d 100755
--- a/testing/contract/tau/setup.sh
+++ b/testing/contract/tau/setup.sh
@@ -1,7 +1,14 @@
+#!/bin/bash
+
 source_dir="/opt/adios2/source/examples/basics/variablesShapes"
-build_dir=$(readlink -f ${PWD})/build
-install_dir=$(readlink -f ${PWD})/install
-test_dir=$(readlink -f ${PWD})/test
+build_dir=$(readlink -f "${PWD}")/build
+install_dir=$(readlink -f "${PWD}")/install
+test_dir=$(readlink -f "${PWD}")/test
+
+export source_dir
+export build_dir
+export install_dir
+export test_dir
 
 echo "source_dir  = \"${source_dir}\""
 echo "build_dir   = \"${build_dir}\""
diff --git a/testing/contract/tau/test.sh b/testing/contract/tau/test.sh
index a696145de3..47ac7b1769 100755
--- a/testing/contract/tau/test.sh
+++ b/testing/contract/tau/test.sh
@@ -3,14 +3,19 @@
 set -x
 set -e
 
-source $(dirname $(readlink -f ${BASH_SOURCE}))/setup.sh
+# shellcheck disable=SC1091
+source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/setup.sh"
 
-mkdir -p ${test_dir}
-cd ${test_dir}
+# Fail if is not set
+install_dir="${install_dir:?}"
+test_dir="${test_dir:?}"
+
+mkdir -p "${test_dir}"
+cd "${test_dir}"
 
 TAU=$(spack location -i tau)/bin/tau_exec
 
-mpiexec -np 2 ${TAU} ${install_dir}/bin/adios2_basics_variablesShapes
+mpiexec -np 2 "${TAU}" "${install_dir}/bin/adios2_basics_variablesShapes"
 
 [ ! -f profile.0.0.0 ] || [ ! -s profile.0.0.0 ] && { echo "Error: file profile.0.0.0 not found or empty"; exit 1; }
 [ ! -f profile.1.0.0 ] || [ ! -s profile.1.0.0 ] && { echo "Error: file profile.1.0.0 not found or empty"; exit 1; }
diff --git a/testing/utils/cwriter/TestUtilsCWriter.bplsh.expected.txt b/testing/utils/cwriter/TestUtilsCWriter.bplsh.expected.txt
index d9461a11fa..24bd5cd4c4 100644
--- a/testing/utils/cwriter/TestUtilsCWriter.bplsh.expected.txt
+++ b/testing/utils/cwriter/TestUtilsCWriter.bplsh.expected.txt
@@ -28,6 +28,9 @@ The time dimension is the first dimension then.
                                instead of the default. E.g. "%6.3f"
   --hidden_attrs             Show hidden ADIOS attributes in the file
   --decomp    | -D           Show decomposition of variables as layed out in file
+  --error     | -X string    Specify read accuracy (error,norm,rel|abs)
+                             e.g. error="0.0,0.0,abs"
+                             L2 norm = 0.0, Linf = inf
   --transport-parameters | -T         Specify File transport parameters
                                       e.g. "Library=stdio"
   --engine               | -E   Specify ADIOS Engine
diff --git a/thirdparty/perfstubs/adios2-perfstubs-interface.h.in b/thirdparty/perfstubs/adios2-perfstubs-interface.h.in
index d108329c5e..021c3118b0 100644
--- a/thirdparty/perfstubs/adios2-perfstubs-interface.h.in
+++ b/thirdparty/perfstubs/adios2-perfstubs-interface.h.in
@@ -6,7 +6,7 @@
 
 #else
 
-#define PERFSTUBS_INITIALIZE(rank)
+#define PERFSTUBS_INITIALIZE()
 #define PERFSTUBS_FINALIZE()
 #define PERFSTUBS_PAUSE_MEASUREMENT()
 #define PERFSTUBS_RESUME_MEASUREMENT()
diff --git a/thirdparty/perfstubs/perfstubs/LICENSE b/thirdparty/perfstubs/perfstubs/LICENSE
index b10754dbe6..667733e1f6 100644
--- a/thirdparty/perfstubs/perfstubs/LICENSE
+++ b/thirdparty/perfstubs/perfstubs/LICENSE
@@ -1,6 +1,6 @@
 BSD 3-Clause License
 
-Copyright (c) 2019-2020, Kevin Huck
+Copyright (c) 2019-2022, Kevin Huck and University of Oregon
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/thirdparty/perfstubs/perfstubs/README.md b/thirdparty/perfstubs/perfstubs/README.md
index 75f77ec1bb..60bcf5565f 100644
--- a/thirdparty/perfstubs/perfstubs/README.md
+++ b/thirdparty/perfstubs/perfstubs/README.md
@@ -1,6 +1,6 @@
 # PerfStubs
 
-Copyright (c) 2019-2020 University of Oregon
+Copyright (c) 2019-2022 University of Oregon
 Distributed under the BSD Software License
 (See accompanying file LICENSE.txt)
 
diff --git a/thirdparty/perfstubs/perfstubs/perfstubs_api/README.md b/thirdparty/perfstubs/perfstubs/perfstubs_api/README.md
index b0a86dfb3e..cb1579599c 100644
--- a/thirdparty/perfstubs/perfstubs/perfstubs_api/README.md
+++ b/thirdparty/perfstubs/perfstubs/perfstubs_api/README.md
@@ -1,6 +1,6 @@
-# Profiling Interface for ADIOS2
+# Profiling Interface for Libraries and Applications
 
-Copyright (c) 2019-2020 University of Oregon
+Copyright (c) 2019-2022 University of Oregon
 Distributed under the BSD Software License
 (See accompanying file LICENSE.txt)
 
@@ -10,7 +10,7 @@ This is a generic design and implementation for other libraries and tools.
 ## Todo Items
 - [x] Make the interface generic.
     - [x] Replace ADIOST-specific symbols with generic versions that will be
-      implemented by interested measurement libraries (i.e. Score-P). 
+      implemented by interested measurement libraries (i.e. Score-P).
     - ~~[ ] New environment variable specifying location of library containing
       function implementations.~~
     - [x] Remove dynamic-linking specific approach (checking ```LD_PRELOAD```)
diff --git a/thirdparty/perfstubs/perfstubs/perfstubs_api/config.h.in b/thirdparty/perfstubs/perfstubs/perfstubs_api/config.h.in
index 222b53cc62..afc84f1418 100644
--- a/thirdparty/perfstubs/perfstubs/perfstubs_api/config.h.in
+++ b/thirdparty/perfstubs/perfstubs/perfstubs_api/config.h.in
@@ -1,5 +1,5 @@
 // the configured options and settings for Tutorial
-// Copyright (c) 2019-2020 University of Oregon
+// Copyright (c) 2019-2022 University of Oregon
 // Distributed under the BSD Software License
 // (See accompanying file LICENSE.txt)
 #define PerfStubs_VERSION_MAJOR @PerfStubs_VERSION_MAJOR@
diff --git a/thirdparty/perfstubs/perfstubs/perfstubs_api/timer.c b/thirdparty/perfstubs/perfstubs/perfstubs_api/timer.c
index 8b62fa0c32..d219ea5e5b 100644
--- a/thirdparty/perfstubs/perfstubs/perfstubs_api/timer.c
+++ b/thirdparty/perfstubs/perfstubs/perfstubs_api/timer.c
@@ -1,22 +1,20 @@
-// Copyright (c) 2019-2020 University of Oregon
+// Copyright (c) 2019-2022 University of Oregon
 // Distributed under the BSD Software License
 // (See accompanying file LICENSE.txt)
 
 #ifndef _GNU_SOURCE
 #define _GNU_SOURCE // needed to define RTLD_DEFAULT
 #endif
+#include 
 #include 
 #include 
 #include 
-#include 
 #include "pthread.h"
 #ifndef PERFSTUBS_USE_TIMERS
 #define PERFSTUBS_USE_TIMERS
 #endif
 #include "perfstubs_api/timer.h"
 
-#define MAX_TOOLS 1
-
 /* Make sure that the Timer singleton is constructed when the
  * library is loaded.  This will ensure (on linux, anyway) that
  * we can assert that we have m_Initialized on the main thread. */
@@ -39,30 +37,30 @@ static void make_key(void) {
 
 /* Function pointers */
 
-ps_initialize_t initialize_functions[MAX_TOOLS];
-ps_finalize_t finalize_functions[MAX_TOOLS];
-ps_pause_measurement_t pause_measurement_functions[MAX_TOOLS];
-ps_resume_measurement_t resume_measurement_functions[MAX_TOOLS];
-ps_register_thread_t register_thread_functions[MAX_TOOLS];
-ps_dump_data_t dump_data_functions[MAX_TOOLS];
-ps_timer_create_t timer_create_functions[MAX_TOOLS];
-ps_timer_start_t timer_start_functions[MAX_TOOLS];
-ps_timer_stop_t timer_stop_functions[MAX_TOOLS];
-ps_start_string_t start_string_functions[MAX_TOOLS];
-ps_stop_string_t stop_string_functions[MAX_TOOLS];
-ps_stop_current_t stop_current_functions[MAX_TOOLS];
-ps_set_parameter_t set_parameter_functions[MAX_TOOLS];
-ps_dynamic_phase_start_t dynamic_phase_start_functions[MAX_TOOLS];
-ps_dynamic_phase_stop_t dynamic_phase_stop_functions[MAX_TOOLS];
-ps_create_counter_t create_counter_functions[MAX_TOOLS];
-ps_sample_counter_t sample_counter_functions[MAX_TOOLS];
-ps_set_metadata_t set_metadata_functions[MAX_TOOLS];
-ps_get_timer_data_t get_timer_data_functions[MAX_TOOLS];
-ps_get_counter_data_t get_counter_data_functions[MAX_TOOLS];
-ps_get_metadata_t get_metadata_functions[MAX_TOOLS];
-ps_free_timer_data_t free_timer_data_functions[MAX_TOOLS];
-ps_free_counter_data_t free_counter_data_functions[MAX_TOOLS];
-ps_free_metadata_t free_metadata_functions[MAX_TOOLS];
+ps_initialize_t initialize_function;
+ps_finalize_t finalize_function;
+ps_pause_measurement_t pause_measurement_function;
+ps_resume_measurement_t resume_measurement_function;
+ps_register_thread_t register_thread_function;
+ps_dump_data_t dump_data_function;
+ps_timer_create_t timer_create_function;
+ps_timer_start_t timer_start_function;
+ps_timer_stop_t timer_stop_function;
+ps_start_string_t start_string_function;
+ps_stop_string_t stop_string_function;
+ps_stop_current_t stop_current_function;
+ps_set_parameter_t set_parameter_function;
+ps_dynamic_phase_start_t dynamic_phase_start_function;
+ps_dynamic_phase_stop_t dynamic_phase_stop_function;
+ps_create_counter_t create_counter_function;
+ps_sample_counter_t sample_counter_function;
+ps_set_metadata_t set_metadata_function;
+ps_get_timer_data_t get_timer_data_function;
+ps_get_counter_data_t get_counter_data_function;
+ps_get_metadata_t get_metadata_function;
+ps_free_timer_data_t free_timer_data_function;
+ps_free_counter_data_t free_counter_data_function;
+ps_free_metadata_t free_metadata_function;
 
 #ifdef PERFSTUBS_USE_STATIC
 
@@ -102,94 +100,94 @@ PS_WEAK_PRE void ps_tool_free_counter_data(ps_tool_counter_data_t *) PS_WEAK_POS
 PS_WEAK_PRE void ps_tool_free_metadata(ps_tool_metadata_t *) PS_WEAK_POST;
 #endif
 
-void initialize_library(const int rank) {
+void initialize_library(void) {
 #ifdef PERFSTUBS_USE_STATIC
     /* The initialization function is the only required one */
-    initialize_functions[0] = &ps_tool_initialize;
-    if (initialize_functions[0] == NULL) {
+    initialize_function = &ps_tool_initialize;
+    if (initialize_function == NULL) {
         perfstubs_initialized = PERFSTUBS_FAILURE;
         return;
     }
-    if (rank == 0)
-        printf("Found ps_tool_initialize(), registering tool\n");
-    finalize_functions[0] = &ps_tool_finalize;
-    pause_measurement_functions[0] = &ps_tool_pause_measurement;
-    resume_measurement_functions[0] = &ps_tool_resume_measurement;
-    register_thread_functions[0] = &ps_tool_register_thread;
-    dump_data_functions[0] = &ps_tool_dump_data;
-    timer_create_functions[0] = &ps_tool_timer_create;
-    timer_start_functions[0] = &ps_tool_timer_start;
-    timer_stop_functions[0] = &ps_tool_timer_stop;
-    start_string_functions[0] = &ps_tool_start_string;
-    stop_string_functions[0] = &ps_tool_stop_string;
-    stop_current_functions[0] = &ps_tool_stop_current;
-    set_parameter_functions[0] = &ps_tool_set_parameter;
-    dynamic_phase_start_functions[0] = &ps_tool_dynamic_phase_start;
-    dynamic_phase_stop_functions[0] = &ps_tool_dynamic_phase_stop;
-    create_counter_functions[0] = &ps_tool_create_counter;
-    sample_counter_functions[0] = &ps_tool_sample_counter;
-    set_metadata_functions[0] = &ps_tool_set_metadata;
-    get_timer_data_functions[0] = &ps_tool_get_timer_data;
-    get_counter_data_functions[0] = &ps_tool_get_counter_data;
-    get_metadata_functions[0] = &ps_tool_get_metadata;
-    free_timer_data_functions[0] = &ps_tool_free_timer_data;
-    free_counter_data_functions[0] = &ps_tool_free_counter_data;
-    free_metadata_functions[0] = &ps_tool_free_metadata;
+    // removing printf statement for now, it's too noisy.
+    //printf("Found ps_tool_initialize(), registering tool\n");
+    finalize_function = &ps_tool_finalize;
+    pause_measurement_function = &ps_tool_pause_measurement;
+    resume_measurement_function = &ps_tool_resume_measurement;
+    register_thread_function = &ps_tool_register_thread;
+    dump_data_function = &ps_tool_dump_data;
+    timer_create_function = &ps_tool_timer_create;
+    timer_start_function = &ps_tool_timer_start;
+    timer_stop_function = &ps_tool_timer_stop;
+    start_string_function = &ps_tool_start_string;
+    stop_string_function = &ps_tool_stop_string;
+    stop_current_function = &ps_tool_stop_current;
+    set_parameter_function = &ps_tool_set_parameter;
+    dynamic_phase_start_function = &ps_tool_dynamic_phase_start;
+    dynamic_phase_stop_function = &ps_tool_dynamic_phase_stop;
+    create_counter_function = &ps_tool_create_counter;
+    sample_counter_function = &ps_tool_sample_counter;
+    set_metadata_function = &ps_tool_set_metadata;
+    get_timer_data_function = &ps_tool_get_timer_data;
+    get_counter_data_function = &ps_tool_get_counter_data;
+    get_metadata_function = &ps_tool_get_metadata;
+    free_timer_data_function = &ps_tool_free_timer_data;
+    free_counter_data_function = &ps_tool_free_counter_data;
+    free_metadata_function = &ps_tool_free_metadata;
 #else
-    initialize_functions[0] =
+    initialize_function =
         (ps_initialize_t)dlsym(RTLD_DEFAULT, "ps_tool_initialize");
-    if (initialize_functions[0] == NULL) {
+    if (initialize_function == NULL) {
         perfstubs_initialized = PERFSTUBS_FAILURE;
         return;
     }
     printf("Found ps_tool_initialize(), registering tool\n");
-    finalize_functions[0] =
+    finalize_function =
         (ps_finalize_t)dlsym(RTLD_DEFAULT, "ps_tool_finalize");
-    pause_measurement_functions[0] =
+    pause_measurement_function =
         (ps_pause_measurement_t)dlsym(RTLD_DEFAULT, "ps_tool_pause_measurement");
-    resume_measurement_functions[0] =
+    resume_measurement_function =
         (ps_resume_measurement_t)dlsym(RTLD_DEFAULT, "ps_tool_resume_measurement");
-    register_thread_functions[0] =
+    register_thread_function =
         (ps_register_thread_t)dlsym(RTLD_DEFAULT, "ps_tool_register_thread");
-    dump_data_functions[0] =
+    dump_data_function =
         (ps_dump_data_t)dlsym(RTLD_DEFAULT, "ps_tool_dump_data");
-    timer_create_functions[0] =
+    timer_create_function =
         (ps_timer_create_t)dlsym(RTLD_DEFAULT,
-        "ps_tool_timer_create");
-    timer_start_functions[0] =
+                "ps_tool_timer_create");
+    timer_start_function =
         (ps_timer_start_t)dlsym(RTLD_DEFAULT, "ps_tool_timer_start");
-    timer_stop_functions[0] =
+    timer_stop_function =
         (ps_timer_stop_t)dlsym(RTLD_DEFAULT, "ps_tool_timer_stop");
-    start_string_functions[0] =
+    start_string_function =
         (ps_start_string_t)dlsym(RTLD_DEFAULT, "ps_tool_start_string");
-    stop_string_functions[0] =
+    stop_string_function =
         (ps_stop_string_t)dlsym(RTLD_DEFAULT, "ps_tool_stop_string");
-    stop_current_functions[0] =
+    stop_current_function =
         (ps_stop_current_t)dlsym(RTLD_DEFAULT, "ps_tool_stop_current");
-    set_parameter_functions[0] =
+    set_parameter_function =
         (ps_set_parameter_t)dlsym(RTLD_DEFAULT, "ps_tool_set_parameter");
-    dynamic_phase_start_functions[0] = (ps_dynamic_phase_start_t)dlsym(
-        RTLD_DEFAULT, "ps_tool_dynamic_phase_start");
-    dynamic_phase_stop_functions[0] = (ps_dynamic_phase_stop_t)dlsym(
-        RTLD_DEFAULT, "ps_tool_dynamic_phase_stop");
-    create_counter_functions[0] = (ps_create_counter_t)dlsym(
-        RTLD_DEFAULT, "ps_tool_create_counter");
-    sample_counter_functions[0] = (ps_sample_counter_t)dlsym(
-        RTLD_DEFAULT, "ps_tool_sample_counter");
-    set_metadata_functions[0] =
+    dynamic_phase_start_function = (ps_dynamic_phase_start_t)dlsym(
+            RTLD_DEFAULT, "ps_tool_dynamic_phase_start");
+    dynamic_phase_stop_function = (ps_dynamic_phase_stop_t)dlsym(
+            RTLD_DEFAULT, "ps_tool_dynamic_phase_stop");
+    create_counter_function = (ps_create_counter_t)dlsym(
+            RTLD_DEFAULT, "ps_tool_create_counter");
+    sample_counter_function = (ps_sample_counter_t)dlsym(
+            RTLD_DEFAULT, "ps_tool_sample_counter");
+    set_metadata_function =
         (ps_set_metadata_t)dlsym(RTLD_DEFAULT, "ps_tool_set_metadata");
-    get_timer_data_functions[0] = (ps_get_timer_data_t)dlsym(
-        RTLD_DEFAULT, "ps_tool_get_timer_data");
-    get_counter_data_functions[0] = (ps_get_counter_data_t)dlsym(
-        RTLD_DEFAULT, "ps_tool_get_counter_data");
-    get_metadata_functions[0] = (ps_get_metadata_t)dlsym(
-        RTLD_DEFAULT, "ps_tool_get_metadata");
-    free_timer_data_functions[0] = (ps_free_timer_data_t)dlsym(
-        RTLD_DEFAULT, "ps_tool_free_timer_data");
-    free_counter_data_functions[0] = (ps_free_counter_data_t)dlsym(
-        RTLD_DEFAULT, "ps_tool_free_counter_data");
-    free_metadata_functions[0] = (ps_free_metadata_t)dlsym(
-        RTLD_DEFAULT, "ps_tool_free_metadata");
+    get_timer_data_function = (ps_get_timer_data_t)dlsym(
+            RTLD_DEFAULT, "ps_tool_get_timer_data");
+    get_counter_data_function = (ps_get_counter_data_t)dlsym(
+            RTLD_DEFAULT, "ps_tool_get_counter_data");
+    get_metadata_function = (ps_get_metadata_t)dlsym(
+            RTLD_DEFAULT, "ps_tool_get_metadata");
+    free_timer_data_function = (ps_free_timer_data_t)dlsym(
+            RTLD_DEFAULT, "ps_tool_free_timer_data");
+    free_counter_data_function = (ps_free_counter_data_t)dlsym(
+            RTLD_DEFAULT, "ps_tool_free_counter_data");
+    free_metadata_function = (ps_free_metadata_t)dlsym(
+            RTLD_DEFAULT, "ps_tool_free_metadata");
 #endif
     perfstubs_initialized = PERFSTUBS_SUCCESS;
     /* Increment the number of tools */
@@ -197,7 +195,7 @@ void initialize_library(const int rank) {
 }
 
 char * ps_make_timer_name_(const char * file,
-    const char * func, int line) {
+        const char * func, int line) {
     /* The length of the line number as a string is floor(log10(abs(num))) */
     int string_length = (strlen(file) + strlen(func) + floor(log10(abs(line))) + 12);
     char * name = calloc(string_length, sizeof(char));
@@ -207,59 +205,41 @@ char * ps_make_timer_name_(const char * file,
 
 // used internally to the class
 static inline void ps_register_thread_internal(void) {
-    //if (thread_seen == 0) {
     if (pthread_getspecific(key) == NULL) {
-    	int i;
-    	for (i = 0 ; i < num_tools_registered ; i++) {
-        	register_thread_functions[i]();
-    	}
-    	//thread_seen = 1;
-    	pthread_setspecific(key, (void*)1UL);
+        if (register_thread_function != NULL) {
+            register_thread_function();
+            pthread_setspecific(key, (void*)1UL);
+        }
     }
 }
 
 /* Initialization */
-void ps_initialize_(const int rank) {
-    int i;
+void ps_initialize_(void) {
     /* Only do this once */
     if (perfstubs_initialized != PERFSTUBS_UNKNOWN) {
         return;
     }
-    initialize_library(rank);
-    for (i = 0 ; i < num_tools_registered ; i++) {
-        initialize_functions[i]();
-    }
-    /* No need to register the main thread */
-    //thread_seen = 1;
-    (void) pthread_once(&key_once, make_key);
-    if (pthread_getspecific(key) == NULL) {
-        // set the key to 1, indicating we have seen this thread
+    initialize_library();
+    if (initialize_function != NULL) {
+        initialize_function();
+        (void) pthread_once(&key_once, make_key);
         pthread_setspecific(key, (void*)1UL);
     }
 }
 
 void ps_finalize_(void) {
-    int i;
-    for (i = 0 ; i < num_tools_registered ; i++) {
-        if (finalize_functions[i] != NULL)
-            finalize_functions[i]();
-    }
+    if (finalize_function != NULL)
+        finalize_function();
 }
 
 void ps_pause_measurement_(void) {
-    int i;
-    for (i = 0 ; i < num_tools_registered ; i++) {
-        if (pause_measurement_functions[i] != NULL)
-            pause_measurement_functions[i]();
-    }
+    if (pause_measurement_function != NULL)
+        pause_measurement_function();
 }
 
 void ps_resume_measurement_(void) {
-    int i;
-    for (i = 0 ; i < num_tools_registered ; i++) {
-        if (resume_measurement_functions[i] != NULL)
-            resume_measurement_functions[i]();
-    }
+    if (resume_measurement_function != NULL)
+        resume_measurement_function();
 }
 
 void ps_register_thread_(void) {
@@ -267,13 +247,10 @@ void ps_register_thread_(void) {
 }
 
 void* ps_timer_create_(const char *timer_name) {
-	ps_register_thread_internal();
+    ps_register_thread_internal();
     void ** objects = (void **)calloc(num_tools_registered, sizeof(void*));
-    int i;
-    for (i = 0 ; i < num_tools_registered ; i++) {
-        if (timer_create_functions[i] != NULL)
-            objects[i] = (void *)timer_create_functions[i](timer_name);
-    }
+    if (timer_create_function != NULL)
+        objects = (void *)timer_create_function(timer_name);
     return (void*)(objects);
 }
 
@@ -282,14 +259,10 @@ void ps_timer_create_fortran_(void ** object, const char *timer_name) {
 }
 
 void ps_timer_start_(void *timer) {
-	ps_register_thread_internal();
+    ps_register_thread_internal();
     void ** objects = (void **)timer;
-    int i;
-    for (i = 0; i < num_tools_registered ; i++) {
-        if (timer_start_functions[i] != NULL &&
-            objects[i] != NULL)
-            timer_start_functions[i](objects[i]);
-    }
+    if (timer_start_function != NULL && objects != NULL)
+        timer_start_function(objects);
 }
 
 void ps_timer_start_fortran_(void **timer) {
@@ -298,12 +271,9 @@ void ps_timer_start_fortran_(void **timer) {
 
 void ps_timer_stop_(void *timer) {
     void ** objects = (void **)timer;
-    int i;
-    for (i = 0; i < num_tools_registered ; i++) {
-        if (timer_stop_functions[i] != NULL &&
-            objects[i] != NULL)
-            timer_stop_functions[i](objects[i]);
-    }
+    if (timer_stop_function != NULL &&
+            objects != NULL)
+        timer_stop_function(objects);
 }
 
 void ps_timer_stop_fortran_(void **timer) {
@@ -311,62 +281,41 @@ void ps_timer_stop_fortran_(void **timer) {
 }
 
 void ps_start_string_(const char *timer_name) {
-	ps_register_thread_internal();
-    int i;
-    for (i = 0 ; i < num_tools_registered ; i++) {
-        if (start_string_functions[i] != NULL)
-            start_string_functions[i](timer_name);
-    }
+    ps_register_thread_internal();
+    if (start_string_function != NULL)
+        start_string_function(timer_name);
 }
 
 void ps_stop_string_(const char *timer_name) {
-    int i;
-    for (i = 0 ; i < num_tools_registered ; i++) {
-        if (stop_string_functions[i] != NULL)
-            stop_string_functions[i](timer_name);
-    }
+    if (stop_string_function != NULL)
+        stop_string_function(timer_name);
 }
 
 void ps_stop_current_(void) {
-    int i;
-    for (i = 0 ; i < num_tools_registered ; i++) {
-        if (stop_current_functions[i] != NULL)
-            stop_current_functions[i]();
-    }
+    if (stop_current_function != NULL)
+        stop_current_function();
 }
 
 void ps_set_parameter_(const char * parameter_name, int64_t parameter_value) {
-    int i;
-    for (i = 0; i < num_tools_registered ; i++) {
-        if (set_parameter_functions[i] != NULL)
-            set_parameter_functions[i](parameter_name, parameter_value);
-    }
+    if (set_parameter_function != NULL)
+        set_parameter_function(parameter_name, parameter_value);
 }
 
 void ps_dynamic_phase_start_(const char *phase_prefix, int iteration_index) {
-    int i;
-    for (i = 0; i < num_tools_registered ; i++) {
-        if (dynamic_phase_start_functions[i] != NULL)
-            dynamic_phase_start_functions[i](phase_prefix, iteration_index);
-    }
+    if (dynamic_phase_start_function != NULL)
+        dynamic_phase_start_function(phase_prefix, iteration_index);
 }
 
 void ps_dynamic_phase_stop_(const char *phase_prefix, int iteration_index) {
-    int i;
-    for (i = 0; i < num_tools_registered ; i++) {
-        if (dynamic_phase_stop_functions[i] != NULL)
-            dynamic_phase_stop_functions[i](phase_prefix, iteration_index);
-    }
+    if (dynamic_phase_stop_function != NULL)
+        dynamic_phase_stop_function(phase_prefix, iteration_index);
 }
 
 void* ps_create_counter_(const char *name) {
-	ps_register_thread_internal();
+    ps_register_thread_internal();
     void ** objects = (void **)calloc(num_tools_registered, sizeof(void*));
-    int i;
-    for (i = 0 ; i < num_tools_registered ; i++) {
-        if (create_counter_functions[i] != NULL)
-            objects[i] = (void*)create_counter_functions[i](name);
-    }
+    if (create_counter_function != NULL)
+        objects = (void*)create_counter_function(name);
     return (void*)(objects);
 }
 
@@ -376,12 +325,9 @@ void ps_create_counter_fortran_(void ** object, const char *name) {
 
 void ps_sample_counter_(void *counter, const double value) {
     void ** objects = (void **)counter;
-    int i;
-    for (i = 0; i < num_tools_registered ; i++) {
-        if (sample_counter_functions[i] != NULL &&
-            objects[i] != NULL)
-            sample_counter_functions[i](objects[i], value);
-    }
+    if (sample_counter_function != NULL &&
+            objects != NULL)
+        sample_counter_function(objects, value);
 }
 
 void ps_sample_counter_fortran_(void **counter, const double value) {
@@ -389,61 +335,43 @@ void ps_sample_counter_fortran_(void **counter, const double value) {
 }
 
 void ps_set_metadata_(const char *name, const char *value) {
-	ps_register_thread_internal();
-    int i;
-    for (i = 0; i < num_tools_registered ; i++) {
-        if (set_metadata_functions[i] != NULL)
-            set_metadata_functions[i](name, value);
-    }
+    ps_register_thread_internal();
+    if (set_metadata_function != NULL)
+        set_metadata_function(name, value);
 }
 
 void ps_dump_data_(void) {
-    int i;
-    for (i = 0; i < num_tools_registered ; i++) {
-        if (dump_data_functions[i] != NULL)
-            dump_data_functions[i]();
-    }
+    if (dump_data_function != NULL)
+        dump_data_function();
 }
 
-void ps_get_timer_data_(ps_tool_timer_data_t *timer_data, int tool_id) {
-    if (tool_id < num_tools_registered) {
-        if (get_timer_data_functions[tool_id] != NULL)
-            get_timer_data_functions[tool_id](timer_data);
-    }
+void ps_get_timer_data_(ps_tool_timer_data_t *timer_data) {
+    if (get_timer_data_function != NULL)
+        get_timer_data_function(timer_data);
 }
 
-void ps_get_counter_data_(ps_tool_counter_data_t *counter_data, int tool_id) {
-    if (tool_id < num_tools_registered) {
-        if (get_counter_data_functions[tool_id] != NULL)
-            get_counter_data_functions[tool_id](counter_data);
-    }
+void ps_get_counter_data_(ps_tool_counter_data_t *counter_data) {
+    if (get_counter_data_function != NULL)
+        get_counter_data_function(counter_data);
 }
 
-void ps_get_metadata_(ps_tool_metadata_t *metadata, int tool_id) {
-    if (tool_id < num_tools_registered) {
-        if (get_metadata_functions[tool_id] != NULL)
-            get_metadata_functions[tool_id](metadata);
-    }
+void ps_get_metadata_(ps_tool_metadata_t *metadata) {
+    if (get_metadata_function != NULL)
+        get_metadata_function(metadata);
 }
 
-void ps_free_timer_data_(ps_tool_timer_data_t *timer_data, int tool_id) {
-    if (tool_id < num_tools_registered) {
-        if (free_timer_data_functions[tool_id] != NULL)
-            free_timer_data_functions[tool_id](timer_data);
-    }
+void ps_free_timer_data_(ps_tool_timer_data_t *timer_data) {
+    if (free_timer_data_function != NULL)
+        free_timer_data_function(timer_data);
 }
 
-void ps_free_counter_data_(ps_tool_counter_data_t *counter_data, int tool_id) {
-    if (tool_id < num_tools_registered) {
-        if (free_counter_data_functions[tool_id] != NULL)
-            free_counter_data_functions[tool_id](counter_data);
-    }
+void ps_free_counter_data_(ps_tool_counter_data_t *counter_data) {
+    if (free_counter_data_function != NULL)
+        free_counter_data_function(counter_data);
 }
 
-void ps_free_metadata_(ps_tool_metadata_t *metadata, int tool_id) {
-    if (tool_id < num_tools_registered) {
-        if (free_metadata_functions[tool_id] != NULL)
-            free_metadata_functions[tool_id](metadata);
-    }
+void ps_free_metadata_(ps_tool_metadata_t *metadata) {
+    if (free_metadata_function != NULL)
+        free_metadata_function(metadata);
 }
 
diff --git a/thirdparty/perfstubs/perfstubs/perfstubs_api/timer.h b/thirdparty/perfstubs/perfstubs/perfstubs_api/timer.h
index 1e5af3d2e2..6bc36978f5 100644
--- a/thirdparty/perfstubs/perfstubs/perfstubs_api/timer.h
+++ b/thirdparty/perfstubs/perfstubs/perfstubs_api/timer.h
@@ -1,4 +1,4 @@
-// Copyright (c) 2019-2020 University of Oregon
+// Copyright (c) 2019-2022 University of Oregon
 // Distributed under the BSD Software License
 // (See accompanying file LICENSE.txt)
 
@@ -24,7 +24,8 @@
  * not just the function name.  If the compiler doesn't support it,
  * just use the function name. */
 
-#if defined(__GNUC__)
+/* ISO C doesn't allow __PRETTY_FUNCTION__, so only do it with C++ */
+#if defined(__GNUC__) && defined(__cplusplus)
 #define __PERFSTUBS_FUNCTION__ __PRETTY_FUNCTION__
 #else
 #define __PERFSTUBS_FUNCTION__ __func__
@@ -49,7 +50,7 @@ extern int perfstubs_initialized;
 extern "C" {
 #endif
 
-void  ps_initialize_(const int rank);
+void  ps_initialize_(void);
 void  ps_finalize_(void);
 void  ps_pause_measurement_(void);
 void  ps_resume_measurement_(void);
@@ -75,12 +76,12 @@ void  ps_set_metadata_(const char *name, const char *value);
 
 /* data query API */
 
-void  ps_get_timer_data_(ps_tool_timer_data_t *timer_data, int tool_id);
-void  ps_get_counter_data_(ps_tool_counter_data_t *counter_data, int tool_id);
-void  ps_get_metadata_(ps_tool_metadata_t *metadata, int tool_id);
-void  ps_free_timer_data_(ps_tool_timer_data_t *timer_data, int tool_id);
-void  ps_free_counter_data_(ps_tool_counter_data_t *counter_data, int tool_id);
-void  ps_free_metadata_(ps_tool_metadata_t *metadata, int tool_id);
+void  ps_get_timer_data_(ps_tool_timer_data_t *timer_data);
+void  ps_get_counter_data_(ps_tool_counter_data_t *counter_data);
+void  ps_get_metadata_(ps_tool_metadata_t *metadata);
+void  ps_free_timer_data_(ps_tool_timer_data_t *timer_data);
+void  ps_free_counter_data_(ps_tool_counter_data_t *counter_data);
+void  ps_free_metadata_(ps_tool_metadata_t *metadata);
 
 char* ps_make_timer_name_(const char * file, const char * func, int line);
 
@@ -93,7 +94,7 @@ char* ps_make_timer_name_(const char * file, const char * func, int line);
  * line or in a config.h file, however your project does it
  */
 
-#define PERFSTUBS_INITIALIZE(rank) ps_initialize_(rank);
+#define PERFSTUBS_INITIALIZE() ps_initialize_();
 
 #define PERFSTUBS_FINALIZE() ps_finalize_();
 
@@ -170,7 +171,7 @@ char* ps_make_timer_name_(const char * file, const char * func, int line);
 
 #else // defined(PERFSTUBS_USE_TIMERS)
 
-#define PERFSTUBS_INITIALIZE(rank)
+#define PERFSTUBS_INITIALIZE()
 #define PERFSTUBS_FINALIZE()
 #define PERFSTUBS_PAUSE_MEASUREMENT()
 #define PERFSTUBS_RESUME_MEASUREMENT()
diff --git a/thirdparty/perfstubs/perfstubs/perfstubs_api/tool.h b/thirdparty/perfstubs/perfstubs/perfstubs_api/tool.h
index d00f9ede3d..8668ffa9d8 100644
--- a/thirdparty/perfstubs/perfstubs/perfstubs_api/tool.h
+++ b/thirdparty/perfstubs/perfstubs/perfstubs_api/tool.h
@@ -1,4 +1,4 @@
-// Copyright (c) 2019-2020 University of Oregon
+// Copyright (c) 2019-2022 University of Oregon
 // Distributed under the BSD Software License
 // (See accompanying file LICENSE.txt)