Sync develop to stable (v.0.6)

.docker/dev_base.Dockerfile

@@ -4,13 +4,55 @@
 ARG ARCH
 FROM ${ARCH}ubuntu:jammy
 
-RUN echo "ARCH=${ARCH}" && sleep 3
-
 ARG ARCH_CODE
 
-RUN echo "ARCH_CODE=${ARCH_CODE}" && sleep 3
+ENV JULIA_HOME=/opt/julia
+ENV WORK_SPACE=/home/project
+ENV LIBFABRIC_SRC_DIR=$WORK_SPACE/source/libfabric
+ENV LIBFABRIC_DIR=$WORK_SPACE/install/libfabric
+ENV MERCURY_SRC_DIR=$WORK_SPACE/source/mercury
+ENV MERCURY_DIR=$WORK_SPACE/install/mercury
+
+ENV PDC_SRC_DIR=$WORK_SPACE/source/pdc
+ENV PDC_DIR=$WORK_SPACE/install/pdc
+
+ENV LD_LIBRARY_PATH="$LIBFABRIC_DIR/lib:$LD_LIBRARY_PATH"
+ENV PATH="$LIBFABRIC_DIR/include:$LIBFABRIC_DIR/lib:$PATH"
+ENV LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH"
+ENV PATH="$MERCURY_DIR/include:$MERCURY_DIR/lib:$PATH"
+
+
+ENV MERCURY_CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=$MERCURY_DIR -DCMAKE_C_COMPILER=mpicc -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DNA_USE_OFI=ON -DNA_USE_SM=OFF -DNA_OFI_TESTING_PROTOCOL=tcp "
+ENV CLANG_FORMAT_PATH=$WORK_SPACE/software/clang-format-lint-action/clang-format/clang-format10
+
 # Install necessary tools, MPICH, UUID library and developer files
-RUN apt-get update && apt-get install -y \
+RUN echo "ARCH=${ARCH}" && echo "ARCH_CODE=${ARCH_CODE}" && sleep 3 && mkdir -p $WORK_SPACE && \
+    mkdir -p $WORK_SPACE/software && \
+    mkdir -p $WORK_SPACE/source && \
+    mkdir -p $WORK_SPACE/install && \
+    mkdir -p $LIBFABRIC_SRC_DIR && \
+    mkdir -p $MERCURY_SRC_DIR && \
+    mkdir -p $PDC_SRC_DIR && \
+    mkdir -p $LIBFABRIC_DIR && \
+    mkdir -p $MERCURY_DIR && \
+    mkdir -p $PDC_DIR && \
+    rm -rf $LIBFABRIC_SRC_DIR/* && \
+    rm -rf $MERCURY_SRC_DIR/* && \
+    rm -rf $PDC_SRC_DIR/* && \
+    rm -rf $LIBFABRIC_DIR/* && \
+    rm -rf $MERCURY_DIR/* && \
+    rm -rf $PDC_DIR/* && \
+    echo "export LIBFABRIC_SRC_DIR=$WORK_SPACE/source/libfabric" > $WORK_SPACE/pdc_env.sh && \
+    echo "export LIBFABRIC_DIR=$WORK_SPACE/install/libfabric" >> $WORK_SPACE/pdc_env.sh && \
+    echo "export MERCURY_SRC_DIR=$WORK_SPACE/source/mercury" >> $WORK_SPACE/pdc_env.sh && \
+    echo "export MERCURY_DIR=$WORK_SPACE/install/mercury" >> $WORK_SPACE/pdc_env.sh && \
+    echo "export PDC_SRC_DIR=$WORK_SPACE/source/pdc" >> $WORK_SPACE/pdc_env.sh && \
+    echo "export PDC_DIR=$WORK_SPACE/install/pdc" >> $WORK_SPACE/pdc_env.sh && \
+    echo 'export LD_LIBRARY_PATH=$LIBFABRIC_DIR/lib:$LD_LIBRARY_PATH' >> $WORK_SPACE/pdc_env.sh  && \
+    echo 'export PATH=$LIBFABRIC_DIR/include:$LIBFABRIC_DIR/lib:$PATH' >> $WORK_SPACE/pdc_env.sh && \
+    echo 'export LD_LIBRARY_PATH=$MERCURY_DIR/lib:$LD_LIBRARY_PATH' >> $WORK_SPACE/pdc_env.sh \
+    echo 'export PATH=$MERCURY_DIR/include:$MERCURY_DIR/lib:$PATH' >> $WORK_SPACE/pdc_env.sh && \
+    apt-get update && apt-get install -y \
     build-essential \
     git \
     mpich \
@@ -28,107 +70,47 @@ RUN apt-get update && apt-get install -y \
     wget \
     axel \
     curl \
+    bc \
     vim \
     nano \
     gdb \
     cgdb \
     curl \
     valgrind \
-    python3 
-
-# Install Oh My Bash
-RUN bash -c "$(curl -fsSL https://raw.githubusercontent.com/ohmybash/oh-my-bash/master/tools/install.sh)" && \
-    sed -i 's/OSH_THEME="font"/OSH_THEME="powerline-multiline"/g' ~/.bashrc
-
-# Install Julia
-
-RUN echo "https://julialang-s3.julialang.org/bin/linux/aarch64/1.6/julia-1.6.7-linux-aarch64.tar.gz" > /julia_url_arm64v8.txt && \
-    echo "https://julialang-s3.julialang.org/bin/linux/x64/1.6/julia-1.6.7-linux-x86_64.tar.gz" > /julia_url_amd64.txt
-
-RUN echo $(cat /julia_url_${ARCH_CODE}.txt) && sleep 3
-
-RUN mkdir -p /opt/julia && wget -O - $(cat /julia_url_${ARCH_CODE}.txt) | tar -xz -C /opt/julia --strip-components=1 && \
-    ln -s /opt/julia/bin/julia /usr/local/bin/julia
-
-RUN rm -rf /tmp/julia_url_*.txt
-
-ENV JULIA_HOME=/opt/julia
-
-# Install Rust
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
-RUN echo 'source $HOME/.cargo/env' >> ~/.bashrc
-
-
-# Set WORK_SPACE environment variable and create necessary directories
-ENV WORK_SPACE=/home/project
-RUN mkdir -p $WORK_SPACE
-
-# Install clang-format repo
-RUN mkdir -p $WORK_SPACE/software
-RUN cd $WORK_SPACE/software && git clone https://github.com/DoozyX/clang-format-lint-action.git
-ENV CLANG_FORMAT_PATH=$WORK_SPACE/software/clang-format-lint-action/clang-format/clang-format10
-
-# Clone the repositories
-WORKDIR $WORK_SPACE/source
-RUN git clone https://github.com/ofiwg/libfabric.git && \
-    git clone https://github.com/mercury-hpc/mercury.git --recursive
-
-ENV LIBFABRIC_SRC_DIR=$WORK_SPACE/source/libfabric
-ENV LIBFABRIC_DIR=$WORK_SPACE/install/libfabric
-ENV MERCURY_SRC_DIR=$WORK_SPACE/source/mercury
-ENV MERCURY_DIR=$WORK_SPACE/install/mercury
-
-ENV PDC_SRC_DIR=$WORK_SPACE/source/pdc
-ENV PDC_DIR=$WORK_SPACE/install/pdc
-
-RUN mkdir -p $LIBFABRIC_SRC_DIR && \
-    mkdir -p $MERCURY_SRC_DIR && \
-    mkdir -p $PDC_SRC_DIR && \
-    mkdir -p $LIBFABRIC_DIR && \
-    mkdir -p $MERCURY_DIR && \
-    mkdir -p $PDC_DIR
-
-
-# Save the environment variables to a file
-RUN echo "export LIBFABRIC_SRC_DIR=$WORK_SPACE/source/libfabric" > $WORK_SPACE/pdc_env.sh && \
-    echo "export LIBFABRIC_DIR=$WORK_SPACE/install/libfabric" >> $WORK_SPACE/pdc_env.sh && \
-    echo "export MERCURY_SRC_DIR=$WORK_SPACE/source/mercury" >> $WORK_SPACE/pdc_env.sh && \
-    echo "export MERCURY_DIR=$WORK_SPACE/install/mercury" >> $WORK_SPACE/pdc_env.sh && \
-    echo "export PDC_SRC_DIR=$WORK_SPACE/source/pdc" >> $WORK_SPACE/pdc_env.sh && \
-    echo "export PDC_DIR=$WORK_SPACE/install/pdc" >> $WORK_SPACE/pdc_env.sh
-
-
-# Build and install libfabric
-WORKDIR $LIBFABRIC_SRC_DIR
-RUN git checkout v1.18.0 && \
+    python3 && \
+    cd $WORK_SPACE/software && \
+    bash -c "$(curl -fsSL https://raw.githubusercontent.com/ohmybash/oh-my-bash/master/tools/install.sh)" && \
+    sed -i 's/OSH_THEME="font"/OSH_THEME="powerline-multiline"/g' ~/.bashrc && \
+    echo "https://julialang-s3.julialang.org/bin/linux/aarch64/1.6/julia-1.6.7-linux-aarch64.tar.gz" > $WORK_SPACE/software/julia_url_arm64v8.txt && \
+    echo "https://julialang-s3.julialang.org/bin/linux/x64/1.6/julia-1.6.7-linux-x86_64.tar.gz" > $WORK_SPACE/software/julia_url_amd64.txt && \
+    echo $(cat $WORK_SPACE/software/julia_url_${ARCH_CODE}.txt) && sleep 3 && \
+    mkdir -p /opt/julia && wget -O - $(cat $WORK_SPACE/software/julia_url_${ARCH_CODE}.txt) | tar -xz -C /opt/julia --strip-components=1 && \
+    ln -s /opt/julia/bin/julia /usr/local/bin/julia && \
+    rm -rf $WORK_SPACE/software/julia_url_*.txt && \
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \
+    echo 'source $HOME/.cargo/env' >> ~/.bashrc && \
+    git clone https://github.com/DoozyX/clang-format-lint-action.git && \
+    git clone https://github.com/ofiwg/libfabric.git ${LIBFABRIC_SRC_DIR} && \
+    git clone https://github.com/mercury-hpc/mercury.git --recursive ${MERCURY_SRC_DIR} && \
+    cd $LIBFABRIC_SRC_DIR && \
+    git checkout v1.18.0 && \
     ./autogen.sh && \
     ./configure --prefix=$LIBFABRIC_DIR CC=mpicc CFLAG="-O2" && \
     make clean && \
     make -j 8 && make install && \
-    make check
-
-ENV LD_LIBRARY_PATH="$LIBFABRIC_DIR/lib:$LD_LIBRARY_PATH"
-ENV PATH="$LIBFABRIC_DIR/include:$LIBFABRIC_DIR/lib:$PATH"
-RUN echo 'export LD_LIBRARY_PATH=$LIBFABRIC_DIR/lib:$LD_LIBRARY_PATH' >> $WORK_SPACE/pdc_env.sh  && \
-    echo 'export PATH=$LIBFABRIC_DIR/include:$LIBFABRIC_DIR/lib:$PATH' >> $WORK_SPACE/pdc_env.sh
-
-
-# Build and install Mercury
-WORKDIR $MERCURY_SRC_DIR
-ENV MERCURY_CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=$MERCURY_DIR -DCMAKE_C_COMPILER=mpicc -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DNA_USE_OFI=ON -DNA_USE_SM=OFF -DNA_OFI_TESTING_PROTOCOL=tcp "
-RUN git checkout v2.2.0 \
-    mkdir -p build 
-WORKDIR ${MERCURY_SRC_DIR}/build
-RUN cmake $MERCURY_CMAKE_FLAGS ../ && \
+    make check && \
+    cd $MERCURY_SRC_DIR && \
+    git checkout v2.2.0 && \
+    mkdir -p build && \
+    cd ${MERCURY_SRC_DIR}/build && \
+    cmake $MERCURY_CMAKE_FLAGS ../ && \
     make -j 16 && make install && \
-    ctest
-
-# Set the environment variables
-ENV LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH"
-ENV PATH="$MERCURY_DIR/include:$MERCURY_DIR/lib:$PATH"
-RUN echo 'export LD_LIBRARY_PATH=$MERCURY_DIR/lib:$LD_LIBRARY_PATH' >> $WORK_SPACE/pdc_env.sh \
-    echo 'export PATH=$MERCURY_DIR/include:$MERCURY_DIR/lib:$PATH' >> $WORK_SPACE/pdc_env.sh
-
+    ctest && \
+    apt-get autoremove -y && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* && \
+    rm -rf /tmp/* && \
+    rm -rf /var/tmp/* 
 
 ENV PDC_CMAKE_FLAGS="-DBUILD_MPI_TESTING=ON -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DCMAKE_INSTALL_PREFIX=$PDC_DIR -DPDC_ENABLE_MPI=ON -DMERCURY_DIR=$MERCURY_DIR -DCMAKE_C_COMPILER=mpicc -DMPI_RUN_CMD=mpiexec "
 

.github/workflows/clang-format-check.yml

@@ -9,7 +9,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: Run clang-format style check for C programs.
-      uses: DoozyX/clang-format-lint-action@v0.11
+      uses: DoozyX/clang-format-lint-action@v0.18.1
       with:
         source: '.'
         extensions: 'c,h,cpp,hpp'

.gitignore

@@ -19,3 +19,9 @@ build
 
 # ignore docs build dir
 docs/build
+
+# ignore venv
+**/*venv
+
+# ignore .zed
+.zed

.gitlab-ci.yml

@@ -23,7 +23,7 @@ perlmutter-no-cache-build:
         SUPERCOMPUTER: "perlmutter"
         MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install"
     script:
-        - module load libfabric/1.15.2.0
+        - module load libfabric
         - module list
         - mkdir -p ${PDC_BUILD_PATH}/perlmutter/no-cache
         - cd ${PDC_BUILD_PATH}/perlmutter/no-cache
@@ -51,7 +51,7 @@ perlmutter-cache-build:
         SUPERCOMPUTER: "perlmutter"
         MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install"
     script:
-        - module load libfabric/1.15.2.0
+        - module load libfabric
         - module list
         - mkdir -p ${PDC_BUILD_PATH}/perlmutter/cache
         - cd ${PDC_BUILD_PATH}/perlmutter/cache
@@ -374,7 +374,7 @@ perlmutter-metrics-build:
         SUPERCOMPUTER: "perlmutter"
         MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install"
     script:
-        - module load libfabric/1.15.2.0
+        - module load libfabric
         - module list
         - mkdir -p ${PDC_BUILD_PATH}/perlmutter/metrics
         - cd ${PDC_BUILD_PATH}/perlmutter/metrics

CMakeLists.txt

@@ -55,7 +55,7 @@ endif()
 # Version information
 #------------------------------------------------------------------------------
 set(PDC_VERSION_MAJOR "0")
-set(PDC_VERSION_MINOR "1")
+set(PDC_VERSION_MINOR "6")
 set(PDC_VERSION_PATCH "0")
 set(PDC_PACKAGE "pdc")
 set(PDC_PACKAGE_NAME "PDC")
@@ -198,6 +198,9 @@ if(SUPPRESS_IGNORABLE_WARNINGS)
     set(SUPPRESSED_LIST "-Wno-deprecated-non-prototype" ${SUPPRESSED_LIST})
   else()
     set(SUPPRESSED_LIST "-Wno-maybe-uninitialized" ${SUPPRESSED_LIST})
+    set(SUPPRESSED_LIST "-Wno-alloc-size-larger-than" ${SUPPRESSED_LIST})
+    # set(SUPPRESSED_LIST "-Wno-use-after-free" ${SUPPRESSED_LIST})
+    set(SUPPRESSED_LIST "-Wno-stringop-truncation" ${SUPPRESSED_LIST})
   endif()
   set(SUPPRESSED_LIST "-Wno-sign-compare" ${SUPPRESSED_LIST})
   set(SUPPRESSED_LIST "-Wno-format" ${SUPPRESSED_LIST})
@@ -207,6 +210,8 @@ if(SUPPRESS_IGNORABLE_WARNINGS)
   set(SUPPRESSED_LIST "-Wno-unused-function" ${SUPPRESSED_LIST})
   set(SUPPRESSED_LIST "-Wno-unused-result" ${SUPPRESSED_LIST})
   set(SUPPRESSED_LIST "-Wno-unused-but-set-variable" ${SUPPRESSED_LIST})
+  set(SUPPRESSED_LIST "-Wno-inline" ${SUPPRESSED_LIST})
+  set(SUPPRESSED_LIST "-Wno-array-bounds" ${SUPPRESSED_LIST})
 
   if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang")
     add_compile_options(${SUPPRESSED_LIST})
@@ -383,10 +388,10 @@ endif()
 option(PDC_SERVER_CACHE "Enable Server Caching." ON)
 if(PDC_SERVER_CACHE)
   set(PDC_SERVER_CACHE 1)
-  set(PDC_SERVER_CACHE_MAX_GB "3" CACHE STRING "Max GB for server cache")
-  set(PDC_SERVER_CACHE_FLUSH_TIME "30" CACHE STRING "Flush time for server cache")
+  set(PDC_SERVER_CACHE_MAX_GB "32" CACHE STRING "Max GB for server cache")
+  set(PDC_SERVER_IDLE_CACHE_FLUSH_TIME "3" CACHE STRING "Idle time to initiate flush from server cache")
 
-  add_compile_definitions(PDC_SERVER_CACHE_MAX_GB=${PDC_SERVER_CACHE_MAX_GB} PDC_SERVER_CACHE_FLUSH_TIME=${PDC_SERVER_CACHE_FLUSH_TIME})
+  add_compile_definitions(PDC_SERVER_CACHE_MAX_GB=${PDC_SERVER_CACHE_MAX_GB} PDC_SERVER_IDLE_CACHE_FLUSH_TIME=${PDC_SERVER_IDLE_CACHE_FLUSH_TIME})
 endif()
 
 
@@ -482,6 +487,14 @@ if(PDC_ENABLE_SQLITE3)
     set(ENABLE_SQLITE3 1)
 endif()
 
+#-----------------------------------------------------------------------------
+# ZFP option
+#-----------------------------------------------------------------------------
+option(PDC_ENABLE_ZFP "Enable ZFP." OFF)
+if(PDC_ENABLE_ZFP)
+    set(ENABLE_ZFP 1)
+endif()
+
 # Check availability of symbols
 #-----------------------------------------------------------------------------
 check_symbol_exists(malloc_usable_size "malloc.h" HAVE_MALLOC_USABLE_SIZE)
@@ -663,4 +676,4 @@ add_custom_target(format
   | xargs -0 clang-format -i -style=file && echo "... done"
   COMMENT "clang-format all source codes"
   VERBATIM 
-)
+)

benchmark/idioms/CMakeLists.txt

@@ -0,0 +1,68 @@
+cmake_minimum_required (VERSION 2.8.12)
+
+# Setup cmake policies.
+foreach(p
+    CMP0012
+    CMP0013
+    CMP0014
+    CMP0022 # CMake 2.8.12
+    CMP0025 # CMake 3.0
+    CMP0053 # CMake 3.1
+    CMP0054 # CMake 3.1
+    CMP0074 # CMake 3.12
+    CMP0075 # CMake 3.12
+    CMP0083 # CMake 3.14
+    CMP0093 # CMake 3.15
+  )
+  if(POLICY ${p})
+    cmake_policy(SET ${p} NEW)
+  endif()
+endforeach()
+
+project(METADATA_JSON_LOADER C)
+
+set(JMD_LDR_EXT_INCLUDE_DIRS "")
+set(JMD_LDR_EXT_LIBRARIES "")
+
+set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "Choose the type of build." FORCE)
+# set(JMD_DEBUG ON CACHE BOOL "Enable debug mode.")
+option(JMD_DEBUG_MODE "Enable debug mode." ON)
+if(JMD_DEBUG_MODE)
+  add_definitions(-DJMD_DEBUG=1)
+endif()
+
+option(USE_SYSTEM_MPI "Use system-installed MPI." ON)
+if(USE_SYSTEM_MPI)
+  find_package(MPI)
+  if(MPI_FOUND)
+    add_definitions(-DJMD_LDR_ENABLE_MPI=1)
+    SET(CMAKE_C_COMPILER ${MPI_C_COMPILER})
+    SET(CMAKE_CXX_COMPILER ${MPI_CXX_COMPILER})
+    set(JMD_LDR_EXT_INCLUDE_DIRS ${MPI_C_INCLUDE_PATH}
+      ${JMD_LDR_EXT_INCLUDE_DIRS}
+      )
+    set(JMD_LDR_EXT_LIBRARIES ${MPI_C_LIBRARIES} ${JMD_LDR_EXT_LIBRARIES})
+  endif()
+endif()
+
+find_package(PDC REQUIRED)
+if(PDC_FOUND)
+    #message(STATUS "PDC include directory: ${PDC_INCLUDE_DIR}")
+  set(JMD_LDR_EXT_INCLUDE_DIRS ${PDC_INCLUDE_DIR}
+    ${JMD_LDR_EXT_INCLUDE_DIRS}
+    )
+  set(JMD_LDR_EXT_LIBRARIES pdc ${JMD_LDR_EXT_LIBRARIES})
+endif()
+
+
+add_library(cjson cjson/cJSON.c)
+
+add_executable(metadata_json_loader 
+            metadata_json_loader.c 
+            metadata_json_processor.h 
+            metadata_json_printer.c 
+            metadata_json_printer.h 
+            metadata_json_importer.c 
+            metadata_json_importer.h)
+target_link_libraries(metadata_json_loader ${PDC_EXT_LIB_DEPENDENCIES} pdc cjson ${JMD_LDR_EXT_LIBRARIES})
+target_include_directories(metadata_json_loader PUBLIC ${PDC_EXT_INCLUDE_DEPENDENCIES} ${JMD_LDR_EXT_INCLUDE_DIRS})