Refactor digital_fingerprinting stages and add morpheus-split conda recipe (core, dfp, llm)

.github/workflows/ci_pipe.yml

@@ -21,16 +21,12 @@ on:
       run_check:
         required: true
         type: boolean
-      conda_core_run_build:
-        description: 'Runs the stage to build the morpheus-core conda package'
-        required: true
-        type: boolean
       conda_upload_label:
         description: 'The label to use when uploading the morpheus conda packages. Leave empty to disable uploading'
         required: true
         type: string
       conda_run_build:
-        description: 'Runs the conda-build stage to build the conda package with all morpheus components'
+        description: 'Runs the conda-build stage to build the morpheus conda packages'
         required: true
         type: boolean
       container:
@@ -204,9 +200,9 @@ jobs:
         shell: bash
         run: ./morpheus/ci/scripts/github/docs.sh
 
-  package-core:
-    name: Package Core
-    if: ${{ inputs.conda_core_run_build }}
+  package:
+    name: Conda Package
+    if: ${{ inputs.conda_run_build }}
     needs: [documentation, test]
     runs-on: linux-amd64-cpu16
     timeout-minutes: 60
@@ -240,38 +236,4 @@ jobs:
           CONDA_TOKEN: "${{ secrets.CONDA_TOKEN }}"
           SCRIPT_ARGS: "${{ inputs.conda_upload_label != '' && 'upload' || '' }}"
           CONDA_PKG_LABEL: "${{ inputs.conda_upload_label }}"
-        run: ./morpheus/ci/scripts/github/conda_core.sh $SCRIPT_ARGS
-
-  package:
-    name: Package All
-    if: ${{ inputs.conda_run_build }}
-    needs: [check, documentation, test]
-    runs-on: linux-amd64-cpu16
-    timeout-minutes: 60
-    container:
-      credentials:
-        username: '$oauthtoken'
-        password: ${{ secrets.NGC_API_KEY }}
-      image: ${{ inputs.container }}
-    strategy:
-      fail-fast: true
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          lfs: false
-          path: 'morpheus'
-          fetch-depth: 0
-          submodules: 'recursive'
-
-      - name: Get AWS credentials using OIDC
-        uses: aws-actions/configure-aws-credentials@v1-node16
-        with:
-          role-to-assume: ${{ vars.AWS_ROLE_ARN }}
-          aws-region: ${{ vars.AWS_REGION }}
-          role-duration-seconds: 43200 # 12h
-
-      - name: conda
-        shell: bash
-        run: ./morpheus/ci/scripts/github/conda.sh
+        run: ./morpheus/ci/scripts/github/conda_libs.sh $SCRIPT_ARGS

.github/workflows/pr.yaml

@@ -68,7 +68,6 @@ jobs:
       is_main_branch: ${{ github.ref_name == 'main' }}
       is_dev_branch: ${{ startsWith(github.ref_name, 'branch-') }}
       has_conda_build_label: ${{ steps.get-pr-info.outcome == 'success' && contains(fromJSON(steps.get-pr-info.outputs.pr-info).labels.*.name, 'conda-build') || false }}
-      has_conda_core_build_label: ${{ steps.get-pr-info.outcome == 'success' && contains(fromJSON(steps.get-pr-info.outputs.pr-info).labels.*.name, 'conda-core-build') || false }}
       has_skip_ci_label: ${{ steps.get-pr-info.outcome == 'success' && contains(fromJSON(steps.get-pr-info.outputs.pr-info).labels.*.name, 'skip-ci') || false }}
       pr_info: ${{ steps.get-pr-info.outcome == 'success' && steps.get-pr-info.outputs.pr-info || '' }}
 
@@ -90,14 +89,11 @@ jobs:
     with:
       # Run checks for any PR branch
       run_check: ${{ fromJSON(needs.prepare.outputs.is_pr) }}
-      # Build morpheus-core conda package. This is done for main/dev branches and
-      # for PRs with the conda-core-build label
-      conda_core_run_build: ${{ !fromJSON(needs.prepare.outputs.is_pr) || fromJSON(needs.prepare.outputs.has_conda_core_build_label) }}
-      # Upload morpheus-core conda package only for non PR branches. Use 'main' for main branch and 'dev' for all other branches
-      conda_upload_label: ${{ !fromJSON(needs.prepare.outputs.is_pr) && (fromJSON(needs.prepare.outputs.is_main_branch) && 'main' || 'dev') || '' }}
-      # Run morpheus conda package, with all components. This is done for main/dev
-      # branches and for PRs with the conda-build label.
+      # Build conda packages for all the morpheus libraries - core, dfp, llm. This is
+      # done for main/dev branches and for PRs with the conda-build label
       conda_run_build: ${{ !fromJSON(needs.prepare.outputs.is_pr) || fromJSON(needs.prepare.outputs.has_conda_build_label) }}
+      # Upload morpheus conda packages only for non PR branches. Use 'main' for main branch and 'dev' for all other branches
+      conda_upload_label: ${{ !fromJSON(needs.prepare.outputs.is_pr) && (fromJSON(needs.prepare.outputs.is_main_branch) && 'main' || 'dev') || '' }}
       container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-240614
       test_container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-240614
     secrets:

CMakeLists.txt

@@ -22,6 +22,8 @@ option(BUILD_SHARED_LIBS "Default value for whether or not to build shared or st
 option(MORPHEUS_BUILD_BENCHMARKS "Whether or not to build benchmarks" OFF)
 option(MORPHEUS_BUILD_DOCS "Enable building of API documentation" OFF)
 option(MORPHEUS_BUILD_EXAMPLES "Whether or not to build examples" OFF)
+option(MORPHEUS_BUILD_MORPHEUS_CORE "Whether or not to build morpheus_core" ON)
+option(MORPHEUS_BUILD_MORPHEUS_DFP "Whether or not to build morpheus_dfp" ON)
 option(MORPHEUS_BUILD_MORPHEUS_LLM "Whether or not to build morpheus_llm" ON)
 option(MORPHEUS_BUILD_TESTS "Whether or not to build tests" OFF)
 option(MORPHEUS_ENABLE_DEBUG_INFO "Enable printing debug information" OFF)

ci/conda/recipes/morpheus-core/morpheus_core_build.sh → ci/conda/recipes/morpheus-libs/cmake_common.sh

@@ -1,3 +1,4 @@
+
 # SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -16,26 +17,10 @@
 # It is assumed that this script is executed from the root of the repo directory by conda-build
 # (https://conda-forge.org/docs/maintainer/knowledge_base.html#using-cmake)
 
-# Need to ensure this value is set before checking it in the if block
-MORPHEUS_SUPPORT_DOCA=-OFF
-MORPHEUS_BUILD_MORPHEUS_LLM=-OFF
-
 # This will store all of the cmake args. Make sure to prepend args to allow
 # incoming values to overwrite them
-CMAKE_ARGS=${CMAKE_ARGS:-""}
-
-export CCACHE_BASEDIR=$(realpath ${SRC_DIR}/..)
-export USE_SCCACHE=${USE_SCCACHE:-""}
-
-# Check for some mrc environment variables. Append to front of args to allow users to overwrite them
-if [[ -n "${MORPHEUS_CACHE_DIR}" ]]; then
-   # Set the cache variable, then set the Staging prefix to allow for host searching
-   CMAKE_ARGS="-DMORPHEUS_CACHE_DIR=${MORPHEUS_CACHE_DIR} ${CMAKE_ARGS}"
-
-   # Double check that the cache dir has been created
-   mkdir -p ${MORPHEUS_CACHE_DIR}
-fi
 
+# CMake flags common across all libraries
 CMAKE_ARGS="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON ${CMAKE_ARGS}"
 CMAKE_ARGS="-DCMAKE_INSTALL_PREFIX=$PREFIX ${CMAKE_ARGS}"
 CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib ${CMAKE_ARGS}"
@@ -51,38 +36,12 @@ CMAKE_ARGS="-DPython_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}"
 CMAKE_ARGS="-DPYTHON_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}" # for pybind11
 CMAKE_ARGS="--log-level=VERBOSE ${CMAKE_ARGS}"
 
-if [[ "${USE_SCCACHE}" == "1" ]]; then
-   CMAKE_ARGS="-DCCACHE_PROGRAM_PATH=$(which sccache) ${CMAKE_ARGS}"
-fi
-
-echo "CC          : ${CC}"
-echo "CXX         : ${CXX}"
-echo "CUDAHOSTCXX : ${CUDAHOSTCXX}"
-echo "CUDA        : ${CUDA}"
-echo "CMAKE_ARGS  : ${CMAKE_ARGS}"
-
-echo "========Begin Env========"
-env
-echo "========End Env========"
-
-BUILD_DIR="build-conda"
+# Append to front of args to allow users to overwrite them
+if [[ -n "${MORPHEUS_CACHE_DIR}" ]]; then
+   # Set the cache variable, then set the Staging prefix to allow for host searching
+   CMAKE_ARGS="-DMORPHEUS_CACHE_DIR=${MORPHEUS_CACHE_DIR} ${CMAKE_ARGS}"
 
-# Check if the build directory already exists. And if so, delete the
-# CMakeCache.txt and CMakeFiles to ensure a clean configuration
-if [[ -d "./${BUILD_DIR}" ]]; then
-   echo "Deleting old CMake files at ./${BUILD_DIR}"
-   rm -rf "./${BUILD_DIR}/CMakeCache.txt"
-   rm -rf "./${BUILD_DIR}/CMakeFiles"
+   # Double check that the cache dir has been created
+   mkdir -p ${MORPHEUS_CACHE_DIR}
 fi
 
-# Run configure
-cmake -B ${BUILD_DIR} \
-   ${CMAKE_ARGS} \
-   --log-level=verbose \
-   .
-
-# Build the components
-cmake --build ${BUILD_DIR} -j${PARALLEL_LEVEL:-$(nproc)} --target install
-
-# Install just the mprpheus core python wheel components
-${PYTHON} -m pip install -vv ${BUILD_DIR}/python/morpheus/dist/*.whl

ci/conda/recipes/morpheus-core/meta.yaml → ci/conda/recipes/morpheus-libs/meta.yaml

@@ -13,19 +13,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# This recipe splits into packages - morpheus-core, morpheus-dfp and morpheus-llm
 {% set version = environ.get('GIT_VERSION', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %}
 {% set minor_version =  version.split('.')[0] + '.' + version.split('.')[1] %}
 
 package:
-  name: morpheus-core
+  name: morpheus-split
   version: {{ version }}
 
 source:
   git_url: ../../../..
 
 outputs:
-
+  ############################### morpheus-core #############################
   - name: morpheus-core
+    type: conda_v2
     build:
       number: {{ GIT_DESCRIBE_NUMBER }}
       string: cuda_{{ cuda_compiler_version }}_py{{ python }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
@@ -35,8 +37,6 @@ outputs:
         - CMAKE_CUDA_ARCHITECTURES
         - MORPHEUS_CACHE_DIR
         - MORPHEUS_PYTHON_BUILD_STUBS
-        - MORPHEUS_SUPPORT_DOCA
-        - MORPHEUS_BUILD_MORPHEUS_LLM
         - PARALLEL_LEVEL
       run_exports:
         - {{ pin_subpackage("morpheus-core", max_pin="x.x") }}
@@ -47,14 +47,12 @@ outputs:
         - {{ compiler("c") }}
         - {{ compiler("cuda") }}
         - {{ compiler("cxx") }}
-        - automake =1.16.5 # Needed for DOCA build
         - ccache
         - cmake =3.27
         - cuda-cudart-dev {{ cuda_compiler_version }}.* # Needed by CMake to compile a test application
         - cuda-version {{ cuda_compiler_version }}.*
-        - libtool # Needed for DOCA build
         - ninja =1.11
-        - pkg-config =0.29 # for mrc cmake
+        - pkg-config =0.29
         - sysroot_linux-64 =2.17
       host:
         # CUDA dependencies
@@ -130,11 +128,82 @@ outputs:
       imports:
         - morpheus
       commands:
-          - echo # make sure test requirements get installed
+          - echo # pytest will be added post re-factoring
+
+  ############################### morpheus-dfp #############################
+  - name: morpheus-dfp
+    type: conda_v2
+    build:
+      number: {{ GIT_DESCRIBE_NUMBER }}
+      string: cuda_{{ cuda_compiler_version }}_py{{ python }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
+      script_env:
+        - CMAKE_CUDA_ARCHITECTURES
+        - MORPHEUS_CACHE_DIR
+        - MORPHEUS_PYTHON_BUILD_STUBS
+        - PARALLEL_LEVEL
+      run_exports:
+        - {{ pin_subpackage("morpheus-dfp", max_pin="x.x") }}
+    script: morpheus_dfp_build.sh
+
+    requirements:
+      build:
+        - ccache
+        - cmake =3.27
+      host:
+        - {{ pin_subpackage('morpheus-core', exact=True) }}
+        - pip
+        - python {{ python }}
+        - scikit-build 0.17.6
+        - versioneer-518
+      run:
+        - {{ pin_subpackage('morpheus-core', exact=True) }}
+
+    #test: Tests will be added post test refactoring
+
+  ############################### morpheus-llm #############################
+  - name: morpheus-llm
+    type: conda_v2
+    build:
+      number: {{ GIT_DESCRIBE_NUMBER }}
+      string: cuda_{{ cuda_compiler_version }}_py{{ python }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
+      script_env:
+        - CMAKE_CUDA_ARCHITECTURES
+        - MORPHEUS_CACHE_DIR
+        - MORPHEUS_PYTHON_BUILD_STUBS
+        - PARALLEL_LEVEL
+      run_exports:
+        - {{ pin_subpackage("morpheus-llm", max_pin="x.x") }}
+    script: morpheus_llm_build.sh
+
+    requirements:
+      build:
+        - {{ compiler("c") }}
+        - {{ compiler("cxx") }}
+        - ccache
+        - cmake =3.27
+        - ninja =1.11
+        - pkg-config =0.29
+      host:
+        # morpheus-core has to be at the top. changing that order will result in different
+        # package versions getting installed creating unexpected version conflicts.
+        - {{ pin_subpackage('morpheus-core', exact=True) }}
+        - cython 3.0.*
+        - glog 0.6.*
+        - pip
+        - pybind11-stubgen 0.10.5
+        - python {{ python }}
+        - rapidjson 1.1.0
+        - scikit-build 0.17.6
+        - versioneer-518
+        - zlib 1.2.13 # required to build triton client
+      run:
+        - {{ pin_subpackage('morpheus-core', exact=True) }}
 
+    #test: Tests will be added post test refactoring
+
 about:
   home: https://github.com/nv-morpheus/Morpheus
   license: Apache-2.0
   license_family: Apache
   license_file: LICENSE
-  summary: Morpheus Cybersecurity Core Library
+  summary: Morpheus Cybersecurity Library

ci/conda/recipes/morpheus-libs/morpheus_core_build.sh

@@ -0,0 +1,77 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# It is assumed that this script is executed from the root of the repo directory by conda-build
+# (https://conda-forge.org/docs/maintainer/knowledge_base.html#using-cmake)
+
+# This will store all of the cmake args. Make sure to prepend args to allow
+# incoming values to overwrite them
+
+source $RECIPE_DIR/cmake_common.sh
+
+CMAKE_ARGS=${CMAKE_ARGS:-""}
+
+export CCACHE_BASEDIR=$(realpath ${SRC_DIR}/..)
+export USE_SCCACHE=${USE_SCCACHE:-""}
+
+if [[ -n "${MORPHEUS_CACHE_DIR}" ]]; then
+   # Set the cache variable, then set the Staging prefix to allow for host searching
+   CMAKE_ARGS="-DMORPHEUS_CACHE_DIR=${MORPHEUS_CACHE_DIR} ${CMAKE_ARGS}"
+
+   # Double check that the cache dir has been created
+   mkdir -p ${MORPHEUS_CACHE_DIR}
+fi
+
+# Enable core. Core is enabled by default and this is to just highlight that it is on
+CMAKE_ARGS="-DMORPHEUS_BUILD_MORPHEUS_CORE=ON ${CMAKE_ARGS}"
+
+# Disable dfp, llm and doca
+CMAKE_ARGS="-DMORPHEUS_SUPPORT_DOCA=OFF ${CMAKE_ARGS}"
+CMAKE_ARGS="-DMORPHEUS_BUILD_MORPHEUS_DFP=OFF ${CMAKE_ARGS}"
+CMAKE_ARGS="-DMORPHEUS_BUILD_MORPHEUS_LLM=OFF ${CMAKE_ARGS}"
+
+if [[ "${USE_SCCACHE}" == "1" ]]; then
+   CMAKE_ARGS="-DCCACHE_PROGRAM_PATH=$(which sccache) ${CMAKE_ARGS}"
+fi
+
+echo "CC          : ${CC}"
+echo "CXX         : ${CXX}"
+echo "CUDAHOSTCXX : ${CUDAHOSTCXX}"
+echo "CUDA        : ${CUDA}"
+echo "CMAKE_ARGS  : ${CMAKE_ARGS}"
+
+echo "========Begin Env========"
+env
+echo "========End Env========"
+
+BUILD_DIR="build-conda-core"
+
+# remove the old build directory
+if [[ -d "./${BUILD_DIR}" ]]; then
+   echo "Deleting old build dir at ./${BUILD_DIR}"
+   rm -rf "./${BUILD_DIR}/"
+fi
+
+# Run configure
+cmake -B ${BUILD_DIR} \
+   ${CMAKE_ARGS} \
+   --log-level=verbose \
+   .
+
+# Build the components
+cmake --build ${BUILD_DIR} -j${PARALLEL_LEVEL:-$(nproc)} --target install
+
+# Install just the morpheus core python wheel components
+${PYTHON} -m pip install -vv ${BUILD_DIR}/python/morpheus/dist/*.whl