Add runtime environment output to fix building the release container

conda/environments/all_cuda-121_arch-x86_64.yaml

@@ -85,6 +85,7 @@ dependencies:
 - pytest-cov
 - pytest=7.4.4
 - python-confluent-kafka>=1.9.2,<1.10.0a0
+- python-docx==1.1.0
 - python-graphviz
 - python=3.10
 - pytorch-cuda
@@ -113,6 +114,7 @@ dependencies:
 - pip:
   - --find-links https://data.dgl.ai/wheels-test/repo.html
   - --find-links https://data.dgl.ai/wheels/cu121/repo.html
+  - PyMuPDF==1.23.21
   - databricks-connect
   - dgl
   - dglgo

conda/environments/dev_cuda-121_arch-x86_64.yaml

@@ -66,6 +66,7 @@ dependencies:
 - pytest-cov
 - pytest=7.4.4
 - python-confluent-kafka>=1.9.2,<1.10.0a0
+- python-docx==1.1.0
 - python-graphviz
 - python=3.10
 - pytorch-cuda
@@ -88,6 +89,7 @@ dependencies:
 - yapf=0.40.1
 - zlib=1.2.13
 - pip:
+  - PyMuPDF==1.23.21
   - databricks-connect
   - milvus==2.3.5
   - pymilvus==2.3.6

conda/environments/examples_cuda-121_arch-x86_64.yaml

@@ -41,7 +41,6 @@ dependencies:
 - pip
 - pypdf=3.17.4
 - python-confluent-kafka>=1.9.2,<1.10.0a0
-- python-docx==1.1.0
 - python-graphviz
 - python=3.10
 - pytorch-cuda
@@ -59,12 +58,12 @@ dependencies:
 - pip:
   - --find-links https://data.dgl.ai/wheels-test/repo.html
   - --find-links https://data.dgl.ai/wheels/cu121/repo.html
+  - PyMuPDF==1.23.21
   - databricks-connect
   - dgl
   - dglgo
   - google-search-results==2.4
   - milvus==2.3.5
   - nemollm
   - pymilvus==2.3.6
-  - PyMuPDF==1.23.21
 name: examples_cuda-121_arch-x86_64

conda/environments/runtime_cuda-121_arch-x86_64.yaml

@@ -0,0 +1,41 @@
+# This file is generated by `rapids-dependency-file-generator`.
+# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
+channels:
+- conda-forge
+- huggingface
+- rapidsai
+- rapidsai-nightly
+- nvidia
+- nvidia/label/dev
+- pytorch
+- defaults
+dependencies:
+- appdirs
+- click >=8
+- datacompy=0.10
+- dill=0.3.7
+- elasticsearch==8.9.0
+- feedparser=6.0.10
+- grpcio=1.59
+- mlflow=2.9.2
+- networkx=2.8.8
+- numpydoc=1.5
+- nvtabular=23.08.00
+- pip
+- python-confluent-kafka>=1.9.2,<1.10.0a0
+- python-graphviz
+- python=3.10
+- pytorch-cuda
+- pytorch=*=*cuda*
+- requests-cache=1.1
+- scikit-learn=1.3.2
+- sqlalchemy
+- tqdm=4
+- typing_utils=0.1
+- watchdog=3.0
+- websockets
+- pip:
+  - databricks-connect
+  - milvus==2.3.5
+  - pymilvus==2.3.6
+name: runtime_cuda-121_arch-x86_64

dependencies.yaml

@@ -124,7 +124,7 @@ files:
       - example-llm-vdb-upload
 
   runtime:
-    output: none
+    output: conda
     matrix:
       cuda: ["12.1"]
       arch: [x86_64]

docker/Dockerfile

@@ -39,9 +39,9 @@
 
 # Args used in FROM commands must come first
 ARG FROM_IMAGE="nvidia/cuda"
-ARG CUDA_MAJOR_VER=11
-ARG CUDA_MINOR_VER=8
-ARG CUDA_REV_VER=0
+ARG CUDA_MAJOR_VER=12
+ARG CUDA_MINOR_VER=1
+ARG CUDA_REV_VER=1
 ARG LINUX_DISTRO=ubuntu
 ARG LINUX_VER=22.04
 
@@ -176,13 +176,13 @@ ARG CUDA_MINOR_VER
 ARG CONDA_CHANNEL_ALIAS
 
 # Copy the development dependencies file
-COPY ${MORPHEUS_ROOT_HOST}/docker/conda/environments/cuda${CUDA_MAJOR_VER}.${CUDA_MINOR_VER}_dev.yml ./docker/conda/environments/
+COPY ${MORPHEUS_ROOT_HOST}/conda/environments/dev_cuda-${CUDA_MAJOR_VER}${CUDA_MINOR_VER}_arch-x86_64.yaml ./conda/environments/dev.yaml
 
 # Update the morpheus environment
 RUN --mount=type=cache,id=conda_pkgs,target=/opt/conda/pkgs,sharing=locked \
     # Temp add channel_alias to get around conda 404 errors
     conda config --env --set channel_alias ${CONDA_CHANNEL_ALIAS} &&\
-    /opt/conda/bin/mamba env update -n morpheus --file docker/conda/environments/cuda${CUDA_MAJOR_VER}.${CUDA_MINOR_VER}_dev.yml &&\
+    /opt/conda/bin/conda env update --solver=libmamba -n morpheus --file conda/environments/dev.yaml &&\
     # Remove channel_alias to use the normal channel in the container
     conda config --env --remove-key channel_alias
 
@@ -232,7 +232,7 @@ ARG MORPHEUS_ROOT_HOST
 ARG CUDA_MAJOR_VER
 ARG CUDA_MINOR_VER
 
-COPY ${MORPHEUS_ROOT_HOST}/docker/conda/environments/cuda${CUDA_MAJOR_VER}.${CUDA_MINOR_VER}_runtime.yml ./docker/conda/environments/
+COPY ${MORPHEUS_ROOT_HOST}/conda/environments/runtime_cuda-${CUDA_MAJOR_VER}${CUDA_MINOR_VER}_arch-x86_64.yaml ./conda/environments/runtime.yaml
 
 # Mount Morpheus conda package build in `conda_bld_morpheus`
 RUN --mount=type=bind,from=conda_bld_morpheus,source=/opt/conda/conda-bld,target=/opt/conda/conda-bld \
@@ -244,13 +244,17 @@ RUN --mount=type=bind,from=conda_bld_morpheus,source=/opt/conda/conda-bld,target
     # Install morpheus
     CONDA_ALWAYS_YES=true /opt/conda/bin/mamba install -n morpheus \
         -c local \
+        -c conda-forge \
+        -c huggingface \
         -c rapidsai \
-        -c nvidia/label/cuda-12.1.0 \
-        -c nvidia/label/dev \
+        -c rapidsai-nightly \
         -c nvidia \
+        -c nvidia/label/dev \
         -c pytorch \
-        -c conda-forge morpheus &&\
-    /opt/conda/bin/mamba env update -n morpheus --file docker/conda/environments/cuda${CUDA_MAJOR_VER}.${CUDA_MINOR_VER}_runtime.yml && \
+        -c defaults \
+        morpheus &&\
+
+    /opt/conda/bin/conda env update --solver=libmamba -n morpheus --file conda/environments/runtime.yaml && \
     conda clean -afy
 
 # Only copy specific files/folders over that are necessary for runtime
@@ -259,7 +263,8 @@ COPY "${MORPHEUS_ROOT_HOST}/docs" "./docs"
 COPY "${MORPHEUS_ROOT_HOST}/examples" "./examples"
 COPY "${MORPHEUS_ROOT_HOST}/models" "./models"
 COPY "${MORPHEUS_ROOT_HOST}/scripts" "./scripts"
-COPY ["${MORPHEUS_ROOT_HOST}/*.md", "${MORPHEUS_ROOT_HOST}/LICENSE", "./"]
+COPY "${MORPHEUS_ROOT_HOST}/*.md" "./"
+COPY "${MORPHEUS_ROOT_HOST}/LICENSE" "./"
 
 # Use morpheus by default
 CMD [ "morpheus" ]

docker/build_container.sh

@@ -28,18 +28,16 @@ DOCKER_TARGET=${DOCKER_TARGET:-"runtime"}
 DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""}
 
 # Build args
-CUDA_MAJOR_VER=${CUDA_MAJOR_VER:-11}
-CUDA_MINOR_VER=${CUDA_MINOR_VER:-8}
-CUDA_REV_VER=${CUDA_REV_VER:-0}
+CUDA_MAJOR_VER=${CUDA_MAJOR_VER:-12}
+CUDA_MINOR_VER=${CUDA_MINOR_VER:-1}
+CUDA_REV_VER=${CUDA_REV_VER:-1}
 DOCA_ARTIFACTS_HOST=${DOCA_ARTIFACTS_HOST:-""}
 DOCA_REPO_HOST=${DOCA_REPO_HOST:-""}
 FROM_IMAGE=${FROM_IMAGE:-"nvidia/cuda"}
 LINUX_DISTRO=${LINUX_DISTRO:-ubuntu}
 LINUX_VER=${LINUX_VER:-22.04}
 MORPHEUS_SUPPORT_DOCA=${MORPHEUS_SUPPORT_DOCA:-"OFF"}
 PYTHON_VER=${PYTHON_VER:-3.10}
-RAPIDS_VER=${RAPIDS_VER:-24.02}
-TENSORRT_VERSION=${TENSORRT_VERSION:-8.2.1.3}
 
 # Determine the relative path from $PWD to $MORPHEUS_ROOT
 MORPHEUS_ROOT_HOST=${MORPHEUS_ROOT_HOST:-"$(realpath --relative-to=${PWD} ${MORPHEUS_ROOT})"}
@@ -58,8 +56,6 @@ DOCKER_ARGS="${DOCKER_ARGS} --build-arg LINUX_VER=${LINUX_VER}"
 DOCKER_ARGS="${DOCKER_ARGS} --build-arg MORPHEUS_ROOT_HOST=${MORPHEUS_ROOT_HOST}"
 DOCKER_ARGS="${DOCKER_ARGS} --build-arg MORPHEUS_SUPPORT_DOCA=${MORPHEUS_SUPPORT_DOCA}"
 DOCKER_ARGS="${DOCKER_ARGS} --build-arg PYTHON_VER=${PYTHON_VER}"
-DOCKER_ARGS="${DOCKER_ARGS} --build-arg RAPIDS_VER=${RAPIDS_VER}"
-DOCKER_ARGS="${DOCKER_ARGS} --build-arg TENSORRT_VERSION=${TENSORRT_VERSION}"
 DOCKER_ARGS="${DOCKER_ARGS} --network=host"
 
 # Last add any extra args (duplicates override earlier ones)
@@ -80,11 +76,9 @@ echo "   LINUX_VER            : ${LINUX_VER}"
 echo "   MORPHEUS_ROOT_HOST   : ${MORPHEUS_ROOT_HOST}"
 echo "   MORPHEUS_SUPPORT_DOCA: ${MORPHEUS_SUPPORT_DOCA}"
 echo "   PYTHON_VER           : ${PYTHON_VER}"
-echo "   RAPIDS_VER           : ${RAPIDS_VER}"
-echo "   TENSORRT_VERSION     : ${TENSORRT_VERSION}"
 
 echo ""
-echo "   COMMAND: docker build ${DOCKER_ARGS} -f docker/Dockerfile ."
+echo "   COMMAND: docker build ${DOCKER_ARGS} -f ${SCRIPT_DIR}/Dockerfile ."
 echo "   Note: add '--progress plain' to DOCKER_EXTRA_ARGS to show all container build output"
 
 docker build ${DOCKER_ARGS} -f ${SCRIPT_DIR}/Dockerfile .

docs/README.md

@@ -22,11 +22,7 @@ Additional packages required for building the documentation are defined in `./co
 ## Install Additional Dependencies
 From the root of the Morpheus repo:
 ```bash
-export CUDA_VER=11.8
-mamba install -n base -c conda-forge conda-merge
-conda run -n base --live-stream conda-merge docker/conda/environments/cuda${CUDA_VER}_dev.yml \
-  docs/conda_docs.yml > .tmp/merged.yml \
-  && mamba env update -n ${CONDA_DEFAULT_ENV} --file .tmp/merged.yml
+conda env update --solver=libmamba -n morpheus --file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune
 ```
 
 ## Build Morpheus and Documentation

docs/source/developer_guide/contributing.md

@@ -205,7 +205,7 @@ git submodule update --init --recursive
 
 1. Create the Morpheus Conda environment
    ```bash
-   mamba env create -f ./docker/conda/environments/cuda${CUDA_VER}_dev.yml
+   conda env create --solver=libmamba -n morpheus --file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml
    conda activate morpheus
    ```
 

morpheus/llm/services/nemo_llm_service.py

@@ -25,10 +25,8 @@
 IMPORT_EXCEPTION = None
 IMPORT_ERROR_MESSAGE = (
     "NemoLLM not found. Install it and other additional dependencies by running the following command:\n"
-    "`mamba install -n base -c conda-forge conda-merge`\n"
-    "`conda run -n base --live-stream conda-merge docker/conda/environments/cuda${CUDA_VER}_dev.yml "
-    "  docker/conda/environments/cuda${CUDA_VER}_examples.yml"
-    "  > .tmp/merged.yml && mamba env update -n morpheus --file .tmp/merged.yml`")
+    "`conda env update --solver=libmamba -n morpheus "
+    "--file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`")
 
 try:
     import nemollm

morpheus/llm/services/openai_chat_service.py

@@ -23,13 +23,10 @@
 logger = logging.getLogger(__name__)
 
 IMPORT_EXCEPTION = None
-IMPORT_ERROR_MESSAGE = (
-    "OpenAIChatService & OpenAIChatClient require the openai package to be installed. "
-    "Install it by running the following command:\n"
-    "`mamba install -n base -c conda-forge conda-merge`\n"
-    "`conda run -n base --live-stream conda-merge docker/conda/environments/cuda${CUDA_VER}_dev.yml "
-    "  docker/conda/environments/cuda${CUDA_VER}_examples.yml"
-    "  > .tmp/merged.yml && mamba env update -n morpheus --file .tmp/merged.yml`")
+IMPORT_ERROR_MESSAGE = ("OpenAIChatService & OpenAIChatClient require the openai package to be installed. "
+                        "Install it by running the following command:\n"
+                        "`conda env update --solver=libmamba -n morpheus "
+                        "--file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`")
 
 try:
     import openai

morpheus/stages/input/arxiv_source.py

@@ -36,10 +36,8 @@
 
 IMPORT_ERROR_MESSAGE = (
     "ArxivSource requires additional dependencies to be installed. Install them by running the following command: "
-    "`mamba install -n base -c conda-forge conda-merge`\n"
-    "`conda run -n base --live-stream conda-merge docker/conda/environments/cuda${CUDA_VER}_dev.yml "
-    "  docker/conda/environments/cuda${CUDA_VER}_examples.yml"
-    "  > .tmp/merged.yml && mamba env update -n morpheus --file .tmp/merged.yml`")
+    "`conda env update --solver=libmamba -n morpheus"
+    "--file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`")
 
 
 @register_stage("from-arxiv")
@@ -48,10 +46,8 @@ class ArxivSource(PreallocatorMixin, SingleOutputSource):
     Source stage that downloads PDFs from arxiv and converts them to dataframes.
 
     This stage requires several additional dependencies to be installed. Install them by running the following command:
-    mamba install -n base -c conda-forge conda-merge
-    conda run -n base --live-stream conda-merge docker/conda/environments/cuda${CUDA_VER}_dev.yml
-    docker/conda/environments/cuda${CUDA_VER}_examples.yml
-    > .tmp/merged.yml && mamba env update -n morpheus --file .tmp/merged.yml
+    `conda env update --solver=libmamba -n morpheus "
+    "--file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`
 
     Parameters
     ----------

tests/conftest.py

@@ -1032,10 +1032,8 @@ def nemollm_fixture(fail_missing: bool):
     Fixture to ensure nemollm is installed
     """
     skip_reason = ("Tests for the NeMoLLMService require the nemollm package to be installed, to install this run:\n"
-                   "`mamba install -n base -c conda-forge conda-merge`\n"
-                   "`conda run -n base --live-stream conda-merge docker/conda/environments/cuda${CUDA_VER}_dev.yml "
-                   "  docker/conda/environments/cuda${CUDA_VER}_examples.yml"
-                   "  > .tmp/merged.yml && mamba env update -n morpheus --file .tmp/merged.yml`")
+                   "`conda env update --solver=libmamba -n morpheus "
+                   "--file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`")
     yield import_or_skip("nemollm", reason=skip_reason, fail_missing=fail_missing)
 
 
@@ -1045,10 +1043,8 @@ def openai_fixture(fail_missing: bool):
     Fixture to ensure openai is installed
     """
     skip_reason = ("Tests for the OpenAIChatService require the openai package to be installed, to install this run:\n"
-                   "`mamba install -n base -c conda-forge conda-merge`\n"
-                   "`conda run -n base --live-stream conda-merge docker/conda/environments/cuda${CUDA_VER}_dev.yml "
-                   "  docker/conda/environments/cuda${CUDA_VER}_examples.yml"
-                   "  > .tmp/merged.yml && mamba env update -n morpheus --file .tmp/merged.yml`")
+                   "`conda env update --solver=libmamba -n morpheus "
+                   "--file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`")
     yield import_or_skip("openai", reason=skip_reason, fail_missing=fail_missing)
 
 

tests/examples/llm/common/conftest.py

@@ -59,9 +59,6 @@ def nemollm_fixture(fail_missing: bool):
     """
 
     skip_reason = ("Tests for the WebScraperStage require the langchain package to be installed, to install this run:\n"
-                   "`mamba install -n base -c conda-forge conda-merge`\n"
-                   "`conda run -n base --live-stream conda-merge docker/conda/environments/cuda${CUDA_VER}_dev.yml "
-                   "  docker/conda/environments/cuda${CUDA_VER}_examples.yml"
-                   "  > .tmp/merged.yml && mamba env update -n morpheus --file .tmp/merged.yml`")
-
+                   "`conda env update --solver=libmamba -n morpheus "
+                   "--file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`")
     yield import_or_skip("langchain", reason=skip_reason, fail_missing=fail_missing)

tests/stages/arxiv/conftest.py

@@ -22,10 +22,8 @@
 
 SKIP_REASON = ("Tests for the arxiv_source require a number of packages not installed in the Morpheus development "
                "environment. To install these run:\n"
-               "`mamba install -n base -c conda-forge conda-merge`\n"
-               "`conda run -n base --live-stream conda-merge docker/conda/environments/cuda${CUDA_VER}_dev.yml "
-               "  docker/conda/environments/cuda${CUDA_VER}_examples.yml"
-               "  > .tmp/merged.yml && mamba env update -n morpheus --file .tmp/merged.yml`")
+               "`conda env update --solver=libmamba -n morpheus "
+               "--file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`")
 
 
 @pytest.fixture(name="arxiv", autouse=True, scope='session')