nv-morpheus · Jan 24, 2023 · Dec 2, 2022 · Dec 2, 2022 · Dec 2, 2022 · Dec 2, 2022
@@ -70,7 +70,7 @@ body:
     id: env-printout
     attributes:
       label: Full env printout
-      description: Please run and paste the output of the `./scripts/print_env.sh` script here, to gather any other relevant environment details
+      description: Please run and paste the output of the `./external/utilities/scripts/print_env.sh` script here, to gather any other relevant environment details
       render: shell
 
   - type: textarea

@@ -2,3 +2,7 @@
 	path = external/morpheus-visualizations
 	url = https://github.com/nv-morpheus/morpheus-visualizations.git
 	branch = branch-22.11
+[submodule "morpheus_utils"]
+	path = external/utilities
+	url = ../utilities.git
+	branch = branch-23.01 
@@ -25,6 +25,7 @@ option(MORPHEUS_BUILD_BENCHMARKS "Whether or not to build benchmarks" OFF)
 option(MORPHEUS_BUILD_EXAMPLES "Whether or not to build examples" OFF)
 option(MORPHEUS_BUILD_TESTS "Whether or not to build tests" OFF)
 option(MORPHEUS_BUILD_PYTHON_STUBS "Whether or not to generated .pyi stub files for C++ Python modules. Disable to avoid requiring loading the NVIDIA GPU Driver during build" ON)
+option(MORPHEUS_ENABLE_DEBUG_INFO "Enable printing debug information" OFF)
 option(MORPHEUS_USE_CCACHE "Enable caching compilation results with ccache" OFF)
 option(MORPHEUS_USE_CLANG_TIDY "Enable running clang-tidy as part of the build process" OFF)
 option(MORPHEUS_USE_CONDA "Enables finding dependencies via conda instead of vcpkg.
@@ -33,6 +34,7 @@ option(MORPHEUS_USE_IWYU "Enable running include-what-you-use as part of the bui
 option(MORPHEUS_BUILD_DOCS "Enable building of API documentation" OFF)
 set(MORPHEUS_PY_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/wheel" CACHE STRING "Location to install the python directory")
 
+set(MORPHEUS_RAPIDS_CMAKE_VERSION "22.08" CACHE STRING "Sets default versions for RAPIDS CMake.")
 set(MORPHEUS_CACHE_DIR "${CMAKE_SOURCE_DIR}/.cache" CACHE PATH "Directory to contain all CPM and CCache data")
 mark_as_advanced(MORPHEUS_CACHE_DIR)
 
@@ -46,37 +48,65 @@ set(CMAKE_INSTALL_RPATH "$ORIGIN")
 
 enable_testing()
 
-# CMake path
-list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
-list(APPEND CMAKE_PREFIX_PATH "${CMAKE_SOURCE_DIR}/cmake")
+if (MORPHEUS_USE_IWYU AND MORPHEUS_USE_CCACHE)
+  message(FATAL_ERROR "MORPHEUS_USE_IWYU and MORPHEUS_USE_CCACHE cannot be set simultaneously")
+endif()
 
-# Add the RAPIDS cmake helper scripts
-include(import-rapids-cmake)
+# MRC CMake path and module extensions
+set(MORPHEUS_CMAKE_MODULE_PATH_EXTENSIONS
+    "${CMAKE_CURRENT_SOURCE_DIR}/cmake"
+    "${CMAKE_CURRENT_SOURCE_DIR}/external/utilities/cmake"
+    "${CMAKE_CURRENT_SOURCE_DIR}/external/utilities/cmake/morpheus_utils/package_search"
+    )
+
+set(MORPHEUS_CMAKE_PREFIX_PATH_EXTENSIONS
+    "${CMAKE_CURRENT_SOURCE_DIR}/cmake"
+    )
+
+# Prepend path and prefix updates so they take priority in this scope.
+list(PREPEND CMAKE_MODULE_PATH "${MORPHEUS_CMAKE_MODULE_PATH_EXTENSIONS}")
+list(PREPEND CMAKE_PREFIX_PATH "${MORPHEUS_CMAKE_PREFIX_PATH_EXTENSIONS}")
+
+# Load morpheus utils and update CMake paths
+set(MORPHEUS_UTILS_RAPIDS_CMAKE_VERSION ${MORPHEUS_RAPIDS_CMAKE_VERSION})
+include(morpheus_utils/load)
+
+# Configure project package manager
+morpheus_utils_initialize_package_manager(
+    MORPHEUS_USE_CONDA
+    BUILD_SHARED_LIBS
+)
 
 # Configure CUDA architecture
-include(configure_cuda_architecture)
+# NOTE: This MUST occur before any 'project' calls because of rapids_cmake requirements.
+if (DEFINED MORPHEUS_CUDA_ARCHITECTURES)
+  set(CMAKE_CUDA_ARCHITECTURES "${MORPHEUS_CUDA_ARCHITECTURES}")
+endif()
+
+morpheus_utils_initialize_cuda_arch(morpheus)
 
 # Set a default build type if none was specified
 rapids_cmake_build_type(Release)
 
-# Toolchain selection (Conda/VCPKG/custom) and configuration
-include(cmake/configure_toolchain.cmake)
-
 # Project definition
-project(morpheus LANGUAGES C CXX CUDA)
+project(morpheus
+    VERSION 23.01.00
+    LANGUAGES C CXX CUDA)
+
+rapids_cmake_write_version_file(${CMAKE_BINARY_DIR}/autogenerated/include/mrc/version.hpp)
 
 # Ccache configuration
-include(setup_cache)
+include(environment/init_ccache)
 
 # Configure all dependencies
-include(cmake/dependencies.cmake)
+include(dependencies)
 
 ####################################
 # - Post dependencies setup --------
-include(cmake/setup_compiler.cmake)
+morpheus_utils_compiler_set_defaults(MORPHEUS_USE_CLANG_TIDY)
 
 # Setup IWYU if enabled
-include(cmake/setup_iwyu.cmake)
+include(environment/init_iwyu)
 
 # To make it easier for CI to find output files, set the default executable suffix to .x if not set
 if("${CMAKE_EXECUTABLE_SUFFIX}" STREQUAL "")
@@ -94,8 +124,26 @@ add_custom_target(${PROJECT_NAME}_style_checks
 )
 
 # Include the main morpheus code
+morpheus_utils_create_python_package(morpheus
+    PROJECT_DIRECTORY "${CMAKE_SOURCE_DIR}"
+    SOURCE_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/morpheus"
+)
+
 add_subdirectory(morpheus)
 
+# Complete the python package
+set(extra_args "")
+
+if(MORPHEUS_PYTHON_INPLACE_BUILD)
+  list(APPEND extra_args "IS_INPLACE")
+endif()
+
+if(MORPHEUS_PYTHON_PERFORM_INSTALL)
+  list(APPEND extra_args "INSTALL_WHEEL")
+endif()
+
+morpheus_utils_build_python_package(morpheus ${extra_args})
+
 if(MORPHEUS_BUILD_EXAMPLES)
   add_subdirectory(examples)
 endif()
@@ -104,64 +152,27 @@ if(MORPHEUS_BUILD_DOCS)
   add_subdirectory(docs)
 endif()
 
-# Get all of the python files to copy to the build directory
-file(GLOB_RECURSE MORPHEUS_PYTHON_FILES
-  RELATIVE ${PROJECT_SOURCE_DIR}
-  LIST_DIRECTORIES FALSE
-  CONFIGURE_DEPENDS
-  "morpheus/*.py"
-  "morpheus/py.typed"
-)
-
-set(MORPHEUS_ROOT_PYTHON_FILES
-  "pyproject.toml"
-  "setup.cfg"
-  "setup.py"
-  "versioneer.py"
-  "MANIFEST.in"
-)
-
-# Add the root python files to the list
-list(PREPEND MORPHEUS_PYTHON_FILES ${MORPHEUS_ROOT_PYTHON_FILES})
-
-set(OUTPUT_PYTHON_FILES "")
-
-# For each file, copy if its different
-foreach(py_file ${MORPHEUS_PYTHON_FILES})
-
-  add_custom_command(
-    OUTPUT ${PROJECT_BINARY_DIR}/${py_file}
-    COMMAND ${CMAKE_COMMAND} -E copy_if_different ${PROJECT_SOURCE_DIR}/${py_file} ${PROJECT_BINARY_DIR}/${py_file}
-    DEPENDS ${PROJECT_SOURCE_DIR}/${py_file}
-    COMMENT "Copying ${py_file} to ${PROJECT_BINARY_DIR}/${py_file}"
+if(MORPHEUS_ENABLE_DEBUG_INFO)
+  morpheus_utils_print_all_targets()
+
+  morpheus_utils_print_target_properties(
+      TARGETS
+        cuda_utils
+        morpheus
+        stages
+        messages
+        common
+        file_types
+      WRITE_TO_FILE
   )
 
-  list(APPEND OUTPUT_PYTHON_FILES ${PROJECT_BINARY_DIR}/${py_file})
-
-endforeach()
-
-add_custom_target(copy_python_source ALL
-  DEPENDS ${OUTPUT_PYTHON_FILES}
-)
-
-# Manually install the below files. install(DIRECTORY) doesn't work well and
-# makes it impossible to get these files and MORPHEUS_PYTHON_FILES in one command.
-install(
-  FILES ${MORPHEUS_ROOT_PYTHON_FILES}
-  DESTINATION ${MORPHEUS_PY_INSTALL_DIR}
-  COMPONENT Wheel
-)
-
-# Use the install(DIRECTORY) to keep the files tructure
-install(
-  DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/morpheus
-  DESTINATION ${MORPHEUS_PY_INSTALL_DIR}
-  COMPONENT Wheel
-  FILES_MATCHING
-    PATTERN "*.py"
-    PATTERN "py.typed"
-    PATTERN "data/*"
-)
+  morpheus_utils_print_global_properties(
+      WRITE_TO_FILE
+  )
+endif()
 
+# Cleanup the environment after we exit this scope
+list(REMOVE_ITEM CMAKE_PREFIX_PATH "${MORPHEUS_CMAKE_PREFIX_PATH_EXTENSIONS}")
+list(REMOVE_ITEM CMAKE_MODULE_PATH "${MORPHEUS_CMAKE_MODULE_PATH_EXTENSIONS}")
 
 list(POP_BACK CMAKE_MESSAGE_CONTEXT)
@@ -14,10 +14,10 @@
 # limitations under the License.
 
 c_compiler_version:
-  - 9.4
+  - 11.2
 
 cxx_compiler_version:
-  - 9.4
+  - 11.2
 
 cuda_compiler:
   - nvcc
@@ -34,3 +34,4 @@ boost:
 
 rapids_version:
   - 22.08
+  - 22.10
@@ -40,7 +40,7 @@ CMAKE_ARGS="-DMORPHEUS_USE_CONDA=ON ${CMAKE_ARGS}"
 CMAKE_ARGS="-DMORPHEUS_USE_CCACHE=ON ${CMAKE_ARGS}"
 CMAKE_ARGS="-DMORPHEUS_BUILD_PYTHON_STUBS=${MORPHEUS_BUILD_PYTHON_STUBS=-"ON"} ${CMAKE_ARGS}"
 CMAKE_ARGS="-DMORPHEUS_PYTHON_INPLACE_BUILD=ON ${CMAKE_ARGS}"
-CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES=-"ALL"} ${CMAKE_ARGS}"
+CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES=-"all"} ${CMAKE_ARGS}"
 CMAKE_ARGS="-DPython_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}"
 
 if [[ "${USE_SCCACHE}" == "1" ]]; then

@@ -26,6 +26,14 @@ function get_version() {
    echo "$(git describe --tags | grep -o -E '^([^-]*?)')"
 }
 
+# Color variables
+export b="\033[0;36m"
+export g="\033[0;32m"
+export r="\033[0;31m"
+export e="\033[0;90m"
+export y="\033[0;33m"
+export x="\033[0m"
+
 # Change this to switch between build/mambabuild/debug
 export CONDA_COMMAND=${CONDA_COMMAND:-"mambabuild"}
 

@@ -28,6 +28,7 @@ cmake --version
 ninja --version
 
 rapids-logger "Configuring cmake for Morpheus"
+git submodule update --init --recursive
 cmake -B build -G Ninja ${CMAKE_BUILD_ALL_FEATURES} \
     -DCCACHE_PROGRAM_PATH=$(which sccache) \
     -DCMAKE_BUILD_RPATH_USE_ORIGIN=ON .

@@ -28,6 +28,8 @@ python ${MORPHEUS_ROOT}/ci/scripts/copyright.py --verify-apache-v2 --git-diff-co
 rapids-logger "Runing Python style checks"
 ${MORPHEUS_ROOT}/ci/scripts/python_checks.sh
 
+git submodule update --init --recursive
+
 rapids-logger "Configuring cmake for Morpheus"
 cmake -B build -G Ninja ${CMAKE_BUILD_ALL_FEATURES} -DCCACHE_PROGRAM_PATH=$(which sccache) .
 

@@ -60,7 +60,7 @@ export SCCACHE_REGION="${AWS_DEFAULT_REGION}"
 export SCCACHE_IDLE_TIMEOUT=32768
 #export SCCACHE_LOG=debug
 
-export CMAKE_BUILD_ALL_FEATURES="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON -DMORPHEUS_BUILD_BENCHMARKS=ON -DMORPHEUS_BUILD_EXAMPLES=ON -DMORPHEUS_BUILD_TESTS=ON -DMORPHEUS_USE_CONDA=ON -DMORPHEUS_PYTHON_INPLACE_BUILD=OFF -DMORPHEUS_BUILD_PYTHON_STUBS=ON -DMORPHEUS_USE_CCACHE=ON"
+export CMAKE_BUILD_ALL_FEATURES="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON -DMORPHEUS_CUDA_ARCHITECTURES=60;70;75;80 -DMORPHEUS_BUILD_BENCHMARKS=ON -DMORPHEUS_BUILD_EXAMPLES=ON -DMORPHEUS_BUILD_TESTS=ON -DMORPHEUS_USE_CONDA=ON -DMORPHEUS_PYTHON_INPLACE_BUILD=OFF -DMORPHEUS_BUILD_PYTHON_STUBS=ON -DMORPHEUS_USE_CCACHE=ON"
 
 export FETCH_STATUS=0
 

@@ -35,6 +35,8 @@ ${MORPHEUS_ROOT}/scripts/fetch_data.py fetch docs examples
 rapids-logger "Installing Documentation dependencies"
 mamba env update -f ${MORPHEUS_ROOT}/docs/conda_docs.yml
 
+git submodule update --init --recursive
+
 rapids-logger "Configuring for docs"
 cmake -B build -G Ninja ${CMAKE_BUILD_ALL_FEATURES} -DMORPHEUS_BUILD_DOCS=ON .