#2174: integrate vt tv api into lb data export

CMakeLists.txt

@@ -60,11 +60,6 @@ set(PROJECT_EXAMPLE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/examples)
 # Import the linking macros for VT-related targets
 include(cmake/link_vt.cmake)
 
-# Load packages that are required for core VT build
-include(cmake/load_packages.cmake)
-
-include(cmake/check_compiler.cmake)
-
 option(vt_gold_linker_enabled "Build VT using the `gold' linker" ON)
 
 option(vt_unity_build_enabled "Build VT with Unity/Jumbo mode enabled" OFF)
@@ -93,6 +88,11 @@ option(vt_werror_enabled "Build VT with -Werror enabled" OFF)
 include(cmake/nvcc_no_deprecated_gpu_targets.cmake)
 include(cmake/load_bundled_libraries.cmake)
 
+# Load packages that are required for core VT build
+# We need to do this after bundled libraries so we avoid library duplication
+include(cmake/load_packages.cmake)
+include(cmake/check_compiler.cmake)
+
 option(vt_trace_only "Build VT with trace-only mode enabled" OFF)
 if (vt_trace_only)
   message(STATUS "Building additional target for VT in trace-only mode")

ci/build_cpp.sh

@@ -73,6 +73,19 @@ else
     fi
 fi
 
+if test "${VT_TV_ENABLED}" -eq 1
+then
+    if test -d "${source_dir}/lib/vt-tv"
+    then
+        { echo "vt-tv already in lib... not downloading"; } 2>/dev/null
+    else
+        cd "${source_dir}/lib"
+        vt_tv_rev="master"
+        git clone -b "${vt_tv_rev}" --depth 1 https://github.com/DARMA-tasking/vt-tv.git
+        cd -
+    fi
+fi
+
 if test "${VT_ZOLTAN_ENABLED:-0}" -eq 1
 then
     export Zoltan_DIR=${ZOLTAN_DIR:-""}

ci/deps/vtk.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+set -exo pipefail
+
+if test $# -lt 2
+then
+    echo "usage: ./$0 <vtk-version> <installation-dir>"
+    exit 1
+fi
+
+vtk_version="$1"
+vtk_tar_name="v${vtk_version}.tar.gz"
+vtk_name="VTK-${vtk_version}"
+install_dir="${2-}"
+build_dir=/vtk-build/
+
+wget "https://github.com/Kitware/VTK/archive/refs/tags/$vtk_tar_name"
+tar xzf ${vtk_tar_name}
+rm ${vtk_tar_name}
+cd ${vtk_name}
+
+mkdir -p ${build_dir}
+mkdir -p ${install_dir}
+
+cd ${build_dir}
+cmake \
+  -DCMAKE_INSTALL_PREFIX:FILEPATH=${install_dir} \
+  ../${vtk_name}
+cmake --build ${build_dir} --target install

ci/docker/ubuntu-gnu-cpp.dockerfile

@@ -32,6 +32,11 @@ RUN apt-get update -y -q && \
     wget \
     zlib1g \
     zlib1g-dev \
+    libncurses5-dev \
+    m4 \
+    libgl1-mesa-dev \
+    libglu1-mesa-dev \
+    mesa-common-dev \
     brotli \
     python3 \
     python3-brotli \
@@ -64,6 +69,11 @@ RUN ./cmake.sh 3.23.4 ${arch}
 ENV PATH=/cmake/bin/:$PATH
 ENV LESSCHARSET=utf-8
 
+COPY ./ci/deps/vtk.sh vtk.sh
+RUN if [ "$VT_TV_ENABLED" -eq 1]; then \
+      ./vtk.sh 9.2.2 /vtk-install; \
+    fi
+
 COPY ./ci/deps/mpich.sh mpich.sh
 RUN if [ "$ubuntu" = "18.04" ]; then \
       ./mpich.sh 3.3.2 -j4; else \

cmake/configure_options.cmake

@@ -98,6 +98,14 @@ else()
     set(vt_feature_cmake_mimalloc "0")
 endif()
 
+if (TV_LIBRARY)
+  message(STATUS "vt-tv found and enabled")
+  set(vt_feature_cmake_tv "1")
+else()
+  message(STATUS "vt-tv not found")
+  set(vt_feature_cmake_tv "0")
+endif()
+
 option(vt_mpi_guards "Build VT with poison MPI calls: code invoked from VT callbacks cannot invoke MPI functions" ON)
 
 if ((vt_mpi_guards OR vt_trace_only) AND PERL_FOUND)

cmake/link_vt.cmake

@@ -33,6 +33,7 @@ function(link_target_with_vt)
     LINK_FORT
     LINK_JSON
     LINK_BROTLI
+    LINK_VT_TV
   )
   set(
     multiValueArg
@@ -70,6 +71,14 @@ function(link_target_with_vt)
     endif()
   endif()
 
+  if (NOT DEFINED ARG_LINK_VT_TV AND ${ARG_DEFAULT_LINK_SET} OR ARG_LINK_VT_TV)
+    if (TV_LIBRARY)
+      target_link_libraries(
+        ${ARG_TARGET} PUBLIC ${ARG_BUILD_TYPE} ${TV_LIBRARY}
+      )
+    endif()
+  endif()
+
   if (NOT DEFINED ARG_LINK_ZOLTAN AND ${ARG_DEFAULT_LINK_SET} OR ARG_LINK_ZOLTAN)
     if (vt_zoltan_enabled)
       if (${ARG_DEBUG_LINK})

cmake/load_local_packages.cmake

@@ -15,3 +15,9 @@ else()
 endif()
 
 set(CHECKPOINT_LIBRARY vt::lib::checkpoint)
+
+if (EXISTS "${PROJECT_LIB_DIR}/vt-tv")
+  set(vt_tv_python_bindings_enabled OFF)
+  add_subdirectory(${PROJECT_LIB_DIR}/vt-tv)
+  set(TV_LIBRARY vt::lib::vt-tv)
+endif()

cmake_config.h.in

@@ -65,6 +65,7 @@
 #define vt_feature_cmake_production_build    @vt_feature_cmake_production_build@
 #define vt_feature_cmake_debug_verbose       @vt_feature_cmake_debug_verbose@
 #define vt_feature_cmake_rdma_tests          @vt_feature_cmake_rdma_tests@
+#define vt_feature_cmake_tv                  @vt_feature_cmake_tv@
 
 
 #define vt_detected_max_num_nodes @cmake_detected_max_num_nodes@

docker-compose.yml

@@ -167,6 +167,7 @@ x-vtopts: &vtopts
   CODECOV_TOKEN: ${CODECOV_TOKEN:-}
   TEST_LB_SCHEMA: ${TEST_LB_SCHEMA:-0}
   CMAKE_CXX_STANDARD: ${CMAKE_CXX_STANDARD:-17}
+  VT_TV_ENABLED: ${VT_TV:-0}
 
 services:
   ##############################################################################

src/vt/configs/arguments/app_config.h

@@ -228,6 +228,9 @@ struct AppConfig {
 
   bool vt_debug_print_flush = false;
 
+  bool vt_tv = false;
+  std::string vt_tv_config_file = "";
+
   bool vt_user_1 = false;
   bool vt_user_2 = false;
   bool vt_user_3 = false;
@@ -389,6 +392,9 @@ struct AppConfig {
 
       | vt_debug_print_flush
 
+      | vt_tv
+      | vt_tv_config_file
+
       | vt_user_1
       | vt_user_2
       | vt_user_3

src/vt/configs/arguments/args.cc

@@ -671,6 +671,20 @@ void addRuntimeArgs(CLI::App& app, AppConfig& appConfig) {
   a3->group(configRuntime);
 }
 
+void addTVArgs(CLI::App& app, AppConfig& appConfig) {
+  auto tv_enabled = "Enable vt-tv visualization/mesh streaming";
+  auto tv_file = "File name for YAML vt-tv configuraton file";
+
+  auto a1 = app.add_flag("--vt_tv", appConfig.vt_tv, tv_enabled);
+  auto a2 = app.add_option(
+    "--vt_tv_config_file", appConfig.vt_tv_config_file, tv_file
+  );
+
+  auto configTV = "vt-tv Configuration";
+  a1->group(configTV);
+  a2->group(configTV);
+}
+
 void addThreadingArgs(CLI::App& app, AppConfig& appConfig) {
 #if (vt_feature_fcontext != 0)
   auto ult_disable = "Disable running handlers in user-level threads";
@@ -768,6 +782,7 @@ std::tuple<int, std::string> ArgConfig::parseToConfig(
   addSchedulerArgs(app, appConfig);
   addConfigFileArgs(app, appConfig);
   addRuntimeArgs(app, appConfig);
+  addTVArgs(app, appConfig);
   addThreadingArgs(app, appConfig);
 
   std::tuple<int, std::string> result = parseArguments(app, argc, argv, appConfig);

src/vt/configs/features/features_defines.h

@@ -72,6 +72,7 @@
 #define vt_feature_libfort             0 || vt_feature_cmake_libfort
 #define vt_feature_production_build    0 || vt_feature_cmake_production_build
 #define vt_feature_debug_verbose       0 || vt_feature_cmake_debug_verbose
+#define vt_feature_tv                  0 || vt_feature_cmake_tv
 
 #define vt_check_enabled(test_option) (vt_feature_ ## test_option != 0)
 

src/vt/vrt/collection/balance/lb_data_holder.cc

@@ -44,6 +44,10 @@
 #include "vt/vrt/collection/balance/lb_data_holder.h"
 #include "vt/context/context.h"
 
+#if vt_check_enabled(tv)
+#  include <vt-tv/api/info.h>
+#endif
+
 #include <nlohmann/json.hpp>
 
 namespace vt { namespace vrt { namespace collection { namespace balance {
@@ -221,6 +225,100 @@ std::unique_ptr<nlohmann::json> LBDataHolder::toJson(PhaseType phase) const {
   return std::make_unique<json>(std::move(j));
 }
 
+#if vt_check_enabled(tv)
+std::unique_ptr<vt::tv::PhaseWork> LBDataHolder::toTV(PhaseType phase) const {
+  using vt::tv::PhaseWork;
+  using vt::tv::ObjectWork;
+  using vt::tv::ObjectCommunicator;
+
+  std::unordered_map<ElementIDType, ObjectWork> objects;
+
+  if (node_data_.find(phase) != node_data_.end()) {
+    for (auto&& elm : node_data_.at(phase)) {
+      ElementIDStruct id = elm.first;
+      TimeType whole_phase_load = elm.second.whole_phase_load;
+      auto const& subphase_loads = elm.second.subphase_loads;
+
+      typename DataMapType::mapped_type user_defined;
+      if (
+        user_defined_lb_info_.find(phase) != user_defined_lb_info_.end() and
+        user_defined_lb_info_.at(phase).find(id) !=
+        user_defined_lb_info_.at(phase).end()
+      ) {
+        user_defined = user_defined_lb_info_.at(phase).at(id);
+      }
+      std::unordered_map<SubphaseType, double> subphase_map;
+      for (std::size_t i = 0; i < subphase_loads.size(); i++) {
+        subphase_map[i] = subphase_loads[i];
+      }
+      objects.try_emplace(
+        id.id,
+        id.id, whole_phase_load, std::move(subphase_map), std::move(user_defined)
+      );
+    }
+  }
+
+  if (node_comm_.find(phase) != node_comm_.end()) {
+    for (auto&& elm : node_comm_.at(phase)) {
+      auto const& key = elm.first;
+      auto const& volume = elm.second;
+      auto const& bytes = volume.bytes;
+      switch(key.cat_) {
+      case elm::CommCategory::SendRecv: {
+        auto from_id = key.fromObj();
+        auto to_id = key.toObj();
+
+        if (objects.find(from_id.id) != objects.end()) {
+          objects.at(from_id.id).addSentCommunications(to_id.id, bytes);
+        } else if (objects.find(to_id.id) != objects.end()) {
+          objects.at(to_id.id).addReceivedCommunications(from_id.id, bytes);
+        }
+        break;
+      }
+      default:
+        // skip all other communications for now
+        break;
+      }
+    }
+  }
+
+  return std::make_unique<PhaseWork>(phase, objects);;
+}
+
+std::unordered_map<ElementIDType, tv::ObjectInfo> LBDataHolder::getObjInfo(
+  PhaseType phase
+) const {
+  std::unordered_map<ElementIDType, tv::ObjectInfo> map;
+  if (node_data_.find(phase) != node_data_.end()) {
+    for (auto&& elm : node_data_.at(phase)) {
+      ElementIDStruct id = elm.first;
+
+      bool is_collection = false;
+      bool is_objgroup = false;
+
+      std::vector<uint64_t> idx;
+      if (node_idx_.find(id) != node_idx_.end()) {
+        is_collection = true;
+        idx = std::get<1>(node_idx_.find(id)->second);
+      }
+
+      if (node_objgroup_.find(id) != node_objgroup_.end()) {
+        is_objgroup = true;
+      }
+
+      tv::ObjectInfo oi{
+        id.id, id.getHomeNode(), id.isMigratable(), std::move(idx)
+      };
+      oi.setIsCollection(is_collection);
+      oi.setIsObjGroup(is_objgroup);
+      map[id.id] = std::move(oi);
+    }
+  }
+  return map;
+}
+
+#endif
+
 LBDataHolder::LBDataHolder(nlohmann::json const& j)
   : count_(0)
 {

src/vt/vrt/collection/balance/lb_data_holder.h

@@ -48,6 +48,10 @@
 #include "vt/vrt/collection/balance/lb_common.h"
 #include "vt/elm/elm_comm.h"
 
+#if vt_check_enabled(tv)
+#  include <vt-tv/api/info.h>
+#endif
+
 #include <unordered_map>
 #include <memory>
 #include <variant>
@@ -95,6 +99,26 @@ struct LBDataHolder {
    */
   std::unique_ptr<nlohmann::json> toJson(PhaseType phase) const;
 
+#if vt_check_enabled(tv)
+  /**
+   * \brief Generate vt-tv data structure for visualization
+   *
+   * \param[in] phase the phase to generate
+   *
+   * \return a \c vt::tv::PhaseWork data structure
+   */
+  std::unique_ptr<tv::PhaseWork> toTV(PhaseType phase) const;
+
+  /**
+   * \brief Get all object info mapped here for a given phase
+   *
+   * \return map with object info
+   */
+  std::unordered_map<ElementIDType, tv::ObjectInfo> getObjInfo(
+    PhaseType phase
+  ) const;
+#endif
+
   /**
    * \brief Output a LB phase's metdadata to JSON
    *