Merge pull request #2305 from DARMA-tasking/2291-make-lb-data-example…

…s-always-up-to-date #2291: Make LB data examples always up to date
DARMA-tasking · Jul 16, 2024 · f5476ba · f5476ba
2 parents 4d44a33 + f52d668
commit f5476ba
Show file tree

Hide file tree

Showing 36 changed files with 425 additions and 321 deletions.
diff --git a/.gitignore b/.gitignore
@@ -21,6 +21,7 @@ compile_commands.json
 *.lo
 *.o
 *.obj
+*.pyc
 
 # Precompiled Headers
 *.gch

diff --git a/ci/azure/azure-clang-10-ubuntu-mpich.yml b/ci/azure/azure-clang-10-ubuntu-mpich.yml
@@ -57,7 +57,7 @@ variables:
   VT_DEBUG_VERBOSE: 0
   VT_KOKKOS_ENABLED: 0
   CMAKE_CXX_STANDARD: 17
-  TEST_LB_SCHEMA: 0
+  VT_CI_TEST_LB_SCHEMA: 0
   CACHE: "$(Agent.TempDirectory)/cache/"
   cache_name: ubuntu-clang-10-cache
   volume_name: ubuntu-cpp

diff --git a/ci/azure/azure-clang-11-ubuntu-mpich.yml b/ci/azure/azure-clang-11-ubuntu-mpich.yml
@@ -57,7 +57,7 @@ variables:
   VT_DEBUG_VERBOSE: 0
   VT_KOKKOS_ENABLED: 0
   CMAKE_CXX_STANDARD: 17
-  TEST_LB_SCHEMA: 0
+  VT_CI_TEST_LB_SCHEMA: 0
   CACHE: "$(Agent.TempDirectory)/cache/"
   cache_name: ubuntu-clang-11-cache
   volume_name: ubuntu-cpp

diff --git a/ci/azure/azure-clang-12-ubuntu-mpich.yml b/ci/azure/azure-clang-12-ubuntu-mpich.yml
@@ -57,7 +57,7 @@ variables:
   VT_DEBUG_VERBOSE: 0
   VT_KOKKOS_ENABLED: 0
   CMAKE_CXX_STANDARD: 17
-  TEST_LB_SCHEMA: 0
+  VT_CI_TEST_LB_SCHEMA: 0
   CACHE: "$(Agent.TempDirectory)/cache/"
   cache_name: ubuntu-clang-12-cache
   volume_name: ubuntu-cpp

diff --git a/ci/azure/azure-clang-13-ubuntu-mpich.yml b/ci/azure/azure-clang-13-ubuntu-mpich.yml
@@ -57,7 +57,7 @@ variables:
   VT_DEBUG_VERBOSE: 0
   VT_KOKKOS_ENABLED: 0
   CMAKE_CXX_STANDARD: 17
-  TEST_LB_SCHEMA: 0
+  VT_CI_TEST_LB_SCHEMA: 0
   CACHE: "$(Agent.TempDirectory)/cache/"
   cache_name: ubuntu-clang-13-cache
   volume_name: ubuntu-cpp

diff --git a/ci/azure/azure-clang-14-ubuntu-mpich.yml b/ci/azure/azure-clang-14-ubuntu-mpich.yml
@@ -57,7 +57,7 @@ variables:
   VT_DEBUG_VERBOSE: 1
   VT_KOKKOS_ENABLED: 0
   CMAKE_CXX_STANDARD: 17
-  TEST_LB_SCHEMA: 0
+  VT_CI_TEST_LB_SCHEMA: 0
   CACHE: "$(Agent.TempDirectory)/cache/"
   cache_name: ubuntu-clang-14-cache
   volume_name: ubuntu-cpp

diff --git a/ci/azure/azure-clang-9-ubuntu-mpich.yml b/ci/azure/azure-clang-9-ubuntu-mpich.yml
@@ -57,7 +57,7 @@ variables:
   VT_DEBUG_VERBOSE: 0
   VT_KOKKOS_ENABLED: 0
   CMAKE_CXX_STANDARD: 17
-  TEST_LB_SCHEMA: 0
+  VT_CI_TEST_LB_SCHEMA: 0
   CACHE: "$(Agent.TempDirectory)/cache/"
   cache_name: ubuntu-clang-9-cache
   volume_name: ubuntu-cpp

diff --git a/ci/azure/azure-clang-alpine-mpich.yml b/ci/azure/azure-clang-alpine-mpich.yml
@@ -57,7 +57,7 @@ variables:
   VT_DEBUG_VERBOSE: 0
   VT_KOKKOS_ENABLED: 0
   CMAKE_CXX_STANDARD: 17
-  TEST_LB_SCHEMA: 0
+  VT_CI_TEST_LB_SCHEMA: 0
   CACHE: "$(Agent.TempDirectory)/cache/"
   cache_name: alpine-clang-13-cache
   volume_name: alpine-cpp

diff --git a/ci/azure/azure-gcc-10-ubuntu-openmpi.yml b/ci/azure/azure-gcc-10-ubuntu-openmpi.yml
@@ -57,7 +57,7 @@ variables:
   VT_DEBUG_VERBOSE: 0
   VT_KOKKOS_ENABLED: 0
   CMAKE_CXX_STANDARD: 17
-  TEST_LB_SCHEMA: 0
+  VT_CI_TEST_LB_SCHEMA: 0
   CACHE: "$(Agent.TempDirectory)/cache/"
   cache_name: ubuntu-gcc-10-cache
   volume_name: ubuntu-cpp

diff --git a/ci/azure/azure-gcc-11-ubuntu-mpich.yml b/ci/azure/azure-gcc-11-ubuntu-mpich.yml
@@ -57,7 +57,7 @@ variables:
   VT_DEBUG_VERBOSE: 0
   VT_KOKKOS_ENABLED: 0
   CMAKE_CXX_STANDARD: 17
-  TEST_LB_SCHEMA: 0
+  VT_CI_TEST_LB_SCHEMA: 0
   CACHE: "$(Agent.TempDirectory)/cache/"
   cache_name: ubuntu-gcc-11-cache
   volume_name: ubuntu-cpp

diff --git a/ci/azure/azure-gcc-12-ubuntu-mpich.yml b/ci/azure/azure-gcc-12-ubuntu-mpich.yml
@@ -57,7 +57,7 @@ variables:
   VT_DEBUG_VERBOSE: 1
   VT_KOKKOS_ENABLED: 1
   CMAKE_CXX_STANDARD: 17
-  TEST_LB_SCHEMA: 0
+  VT_CI_TEST_LB_SCHEMA: 0
   CACHE: "$(Agent.TempDirectory)/cache/"
   cache_name: ubuntu-gcc-12-cache
   volume_name: ubuntu-cpp

diff --git a/ci/azure/azure-gcc-8-ubuntu-mpich.yml b/ci/azure/azure-gcc-8-ubuntu-mpich.yml
@@ -57,7 +57,7 @@ variables:
   VT_DEBUG_VERBOSE: 0
   VT_KOKKOS_ENABLED: 0
   CMAKE_CXX_STANDARD: 17
-  TEST_LB_SCHEMA: 0
+  VT_CI_TEST_LB_SCHEMA: 0
   CACHE: "$(Agent.TempDirectory)/cache/"
   cache_name: ubuntu-gcc-8-cache
   volume_name: ubuntu-cpp

diff --git a/ci/azure/azure-gcc-9-ubuntu-mpich.yml b/ci/azure/azure-gcc-9-ubuntu-mpich.yml
@@ -57,7 +57,7 @@ variables:
   VT_DEBUG_VERBOSE: 0
   VT_KOKKOS_ENABLED: 0
   CMAKE_CXX_STANDARD: 17
-  TEST_LB_SCHEMA: 1
+  VT_CI_TEST_LB_SCHEMA: 1
   CACHE: "$(Agent.TempDirectory)/cache/"
   cache_name: ubuntu-gcc-9-cache
   volume_name: ubuntu-cpp

diff --git a/ci/azure/azure-intel-oneapi-icpc-ubuntu-mpich.yml b/ci/azure/azure-intel-oneapi-icpc-ubuntu-mpich.yml
@@ -57,7 +57,7 @@ variables:
   VT_DEBUG_VERBOSE: 0
   VT_KOKKOS_ENABLED: 0
   CMAKE_CXX_STANDARD: 17
-  TEST_LB_SCHEMA: 0
+  VT_CI_TEST_LB_SCHEMA: 0
   CACHE: "$(Agent.TempDirectory)/cache/"
   cache_name: ubuntu-intel-oneapi-icpc-cache
   volume_name: ubuntu-cpp

diff --git a/ci/azure/azure-intel-oneapi-icpx-ubuntu-mpich.yml b/ci/azure/azure-intel-oneapi-icpx-ubuntu-mpich.yml
@@ -57,7 +57,7 @@ variables:
   VT_DEBUG_VERBOSE: 1
   VT_KOKKOS_ENABLED: 0
   CMAKE_CXX_STANDARD: 17
-  TEST_LB_SCHEMA: 0
+  VT_CI_TEST_LB_SCHEMA: 0
   CACHE: "$(Agent.TempDirectory)/cache/"
   cache_name: ubuntu-intel-oneapi-icpx-cache
   volume_name: ubuntu-cpp

diff --git a/ci/azure/azure-nvidia-11-2-ubuntu-mpich.yml b/ci/azure/azure-nvidia-11-2-ubuntu-mpich.yml
@@ -57,7 +57,7 @@ variables:
   VT_DEBUG_VERBOSE: 0
   VT_KOKKOS_ENABLED: 0
   CMAKE_CXX_STANDARD: 17
-  TEST_LB_SCHEMA: 0
+  VT_CI_TEST_LB_SCHEMA: 0
   CACHE: "$(Agent.TempDirectory)/cache/"
   cache_name: ubuntu-nvidia-11.2-cache
   volume_name: ubuntu-cpp

diff --git a/ci/azure/azure-nvidia-12-ubuntu-mpich.yml b/ci/azure/azure-nvidia-12-ubuntu-mpich.yml
@@ -57,7 +57,7 @@ variables:
   VT_DEBUG_VERBOSE: 1
   VT_KOKKOS_ENABLED: 0
   CMAKE_CXX_STANDARD: 17
-  TEST_LB_SCHEMA: 0
+  VT_CI_TEST_LB_SCHEMA: 0
   CACHE: "$(Agent.TempDirectory)/cache/"
   cache_name: ubuntu-nvidia-12-cache
   volume_name: ubuntu-cpp

diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh
@@ -147,6 +147,7 @@ cmake -G "${CMAKE_GENERATOR:-Ninja}" \
       -DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH:-}" \
       -DCMAKE_INSTALL_PREFIX="$VT_BUILD/install" \
       -Dvt_ci_build="${VT_CI_BUILD:-0}" \
+      -Dvt_ci_generate_lb_files="${VT_CI_TEST_LB_SCHEMA:-0}" \
       -Dvt_debug_verbose="${VT_DEBUG_VERBOSE:-0}" \
       -Dvt_tests_num_nodes="${VT_TESTS_NUM_NODES:-}" \
       -Dvt_external_fmt="${VT_EXTERNAL_FMT:-0}" \

diff --git a/ci/docker/ubuntu-gnu-cpp.dockerfile b/ci/docker/ubuntu-gnu-cpp.dockerfile
@@ -87,7 +87,8 @@ RUN apt-get update -y -q && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-RUN pip3 install schema
+RUN pip3 install --upgrade pip \
+    && pip3 install schema deepdiff
 
 FROM base as build
 COPY . /vt

diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh
@@ -23,7 +23,7 @@ then
     popd
 fi
 
-if test "${TEST_LB_SCHEMA:-0}" -eq 1
+if test "${VT_CI_TEST_LB_SCHEMA:-0}" -eq 1
 then
     echo "Validating schema of json files..."
     "${VT}/scripts/check_lb_data_files.sh" "${VT_BUILD}" "${VT}"

diff --git a/cmake/configure_options.cmake b/cmake/configure_options.cmake
@@ -150,6 +150,7 @@ define_option(vt_werror_enabled "-Werror" "Build VT with -Werror enabled" OFF em
 define_option(vt_build_tests "tests" "Build VT tests" ON empty_feature)
 define_option(vt_build_tools "tools" "Build VT tools" ON empty_feature)
 define_option(vt_build_examples "examples" "Build VT examples" ON empty_feature)
+define_option(vt_ci_generate_lb_files "generate lb files" "Enable generation of LBDatafiles when runnning examples" OFF empty_feature)
 
 option(vt_external_fmt "Build VT with external fmt" OFF)
 if(${vt_external_fmt})

diff --git a/cmake/load_doxygen.cmake b/cmake/load_doxygen.cmake
@@ -19,6 +19,7 @@ if (${vt_doxygen_enabled})
     set(DOXYGEN_CHECKPOINT_SRC_DIR "${CMAKE_CURRENT_SOURCE_DIR}/lib/checkpoint/src")
     set(DOXYGEN_DOCS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/docs/")
     set(DOXYGEN_EXAMPLE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/examples/")
+    set(DOXYGEN_SCRIPTS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/scripts/")
     set(DOXYGEN_TUTORIAL_DIR "${CMAKE_CURRENT_SOURCE_DIR}/tutorial/")
     set(DOXYGEN_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/docs/")
     set(DOXYGEN_MAIN_PAGE "${CMAKE_CURRENT_SOURCE_DIR}/src/vt.md")

diff --git a/cmake/test_vt.cmake b/cmake/test_vt.cmake
@@ -17,16 +17,20 @@ function(run_executable_with_mpi)
     WRAPPER_EXECUTABLE
   )
   set(
-    multiValueArg
+    multiValueArgs
     TARGET_ARGS
     WRAPPER_ARGS
   )
-  set(allKeywords ${noValOption} ${singleValArg} ${multiValueArg})
 
   cmake_parse_arguments(
     ARG "${noValOption}" "${singleValArg}" "${multiValueArgs}" ${ARGN}
   )
 
+  # Stop the configurtion if there are any unparsed arguments
+  if (ARG_UNPARSED_ARGUMENTS)
+    message(FATAL_ERROR "found unparsed arguments: ${ARG_UNPARSED_ARGUMENTS}")
+  endif()
+
   if (NOT DEFINED ARG_EXECUTE_WITH_WRAPPER)
     set(ARG_WRAPPER_EXECUTABLE "")
     set(ARG_WRAPPER_ARGS "")
@@ -62,12 +66,17 @@ function(build_mpi_proc_test_list)
 
   set(noValOption)
   set(singleValArg MAX_PROC VARIABLE_OUT )
-  set(multiValueArg)
-  set(allKeywords ${noValOption} ${singleValArg} ${multiValueArg})
+  set(multiValueArgs)
+  set(allKeywords ${noValOption} ${singleValArg} ${multiValueArgs})
   cmake_parse_arguments(
     ARG "${noValOption}" "${singleValArg}" "${multiValueArgs}" ${ARGN}
   )
 
+  # Stop the configurtion if there are any unparsed arguments
+  if (ARG_UNPARSED_ARGUMENTS)
+    message(FATAL_ERROR "found unparsed arguments: ${ARG_UNPARSED_ARGUMENTS}")
+  endif()
+
   if (NOT DEFINED ARG_MAX_PROC)
     # Default to 8 processors
     set(ARG_MAX_PROC "8")
@@ -108,6 +117,16 @@ macro(add_test_for_example_vt test_target test_exec test_list)
         list(APPEND EXEC_ARGS "--vt_trace")
     endif()
 
+    # Append parameters required for the examples to output LBDatafiles.
+    if (vt_ci_generate_lb_files)
+      list(APPEND EXEC_ARGS
+        "--vt_lb_interval=1"
+        "--vt_lb_data"
+        "--vt_lb_data_compress=false"
+        "--vt_lb_data_file=${test_name}_${PROC}_LBDatafile.%p.json"
+        "--vt_lb_data_dir=.")
+    endif()
+
     run_executable_with_mpi(
       TARGET_EXECUTABLE            ${test_name}
       TARGET_ARGS                  ${EXEC_ARGS}

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -115,7 +115,7 @@ x-vtopts: &vtopts
   BUILD_SHARED_LIBS: ${VT_BUILD_SHARED_LIBS:-0}
   VT_INCLUSION_TYPE: ${VT_INCLUSION:-TPL}
   CODECOV_TOKEN: ${CODECOV_TOKEN:-}
-  TEST_LB_SCHEMA: ${TEST_LB_SCHEMA:-0}
+  VT_CI_TEST_LB_SCHEMA: ${VT_CI_TEST_LB_SCHEMA:-0}
   CMAKE_CXX_STANDARD: ${CMAKE_CXX_STANDARD:-17}
 
 services:

diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in
@@ -936,6 +936,7 @@ EXCLUDE_SYMBOLS        =
 
 EXAMPLE_PATH           = "@DOXYGEN_EXAMPLE_DIR@" \
                          "@DOXYGEN_TUTORIAL_DIR@" \
+                         "@DOXYGEN_SCRIPTS_DIR@" \
                          "@DOXYGEN_CHECKPOINT_EXAMPLE_DIR@"
 
 # If the value of the EXAMPLE_PATH tag contains directories, you can use the
-Original file line number
+Diff line change
@@ Expand Up / @@ -21,6 +21,7 @@ compile_commands.json @@
     *.lo
     *.o
     *.obj
+    *.pyc
     # Precompiled Headers
     *.gch
@@ Expand Down @@