Add runtime environment output to fix building the release container (#…

…1496) Fixes the release container build. ## By Submitting this PR I confirm: - I am familiar with the [Contributing Guidelines](https://github.com/nv-morpheus/Morpheus/blob/main/docs/source/developer_guide/contributing.md). - When the PR is ready for review, new or existing tests cover these changes. - When the PR is ready for review, the documentation is up to date with these changes. Authors: - Christopher Harris (https://github.com/cwharris) Approvers: - Michael Demoret (https://github.com/mdemoret-nv) URL: #1496
nv-morpheus · Feb 7, 2024 · 6388da1 · 6388da1
1 parent 44825d9
commit 6388da1
Show file tree

Hide file tree

Showing 15 changed files with 87 additions and 66 deletions.
diff --git a/conda/environments/all_cuda-121_arch-x86_64.yaml b/conda/environments/all_cuda-121_arch-x86_64.yaml
@@ -85,6 +85,7 @@ dependencies:
 - pytest-cov
 - pytest=7.4.4
 - python-confluent-kafka>=1.9.2,<1.10.0a0
+- python-docx==1.1.0
 - python-graphviz
 - python=3.10
 - pytorch-cuda
@@ -113,6 +114,7 @@ dependencies:
 - pip:
  - --find-links https://data.dgl.ai/wheels-test/repo.html
  - --find-links https://data.dgl.ai/wheels/cu121/repo.html
+ - PyMuPDF==1.23.21
  - databricks-connect
  - dgl
  - dglgo

diff --git a/conda/environments/dev_cuda-121_arch-x86_64.yaml b/conda/environments/dev_cuda-121_arch-x86_64.yaml
@@ -66,6 +66,7 @@ dependencies:
 - pytest-cov
 - pytest=7.4.4
 - python-confluent-kafka>=1.9.2,<1.10.0a0
+- python-docx==1.1.0
 - python-graphviz
 - python=3.10
 - pytorch-cuda
@@ -88,6 +89,7 @@ dependencies:
 - yapf=0.40.1
 - zlib=1.2.13
 - pip:
+ - PyMuPDF==1.23.21
  - databricks-connect
  - milvus==2.3.5
  - pymilvus==2.3.6

diff --git a/conda/environments/examples_cuda-121_arch-x86_64.yaml b/conda/environments/examples_cuda-121_arch-x86_64.yaml
@@ -41,7 +41,6 @@ dependencies:
 - pip
 - pypdf=3.17.4
 - python-confluent-kafka>=1.9.2,<1.10.0a0
-- python-docx==1.1.0
 - python-graphviz
 - python=3.10
 - pytorch-cuda
@@ -59,12 +58,12 @@ dependencies:
 - pip:
  - --find-links https://data.dgl.ai/wheels-test/repo.html
  - --find-links https://data.dgl.ai/wheels/cu121/repo.html
+ - PyMuPDF==1.23.21
  - databricks-connect
  - dgl
  - dglgo
  - google-search-results==2.4
  - milvus==2.3.5
  - nemollm
  - pymilvus==2.3.6
- - PyMuPDF==1.23.21
 name: examples_cuda-121_arch-x86_64
diff --git a/conda/environments/runtime_cuda-121_arch-x86_64.yaml b/conda/environments/runtime_cuda-121_arch-x86_64.yaml
@@ -0,0 +1,41 @@
+# This file is generated by `rapids-dependency-file-generator`.
+# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
+channels:
+- conda-forge
+- huggingface
+- rapidsai
+- rapidsai-nightly
+- nvidia
+- nvidia/label/dev
+- pytorch
+- defaults
+dependencies:
+- appdirs
+- click >=8
+- datacompy=0.10
+- dill=0.3.7
+- elasticsearch==8.9.0
+- feedparser=6.0.10
+- grpcio=1.59
+- mlflow=2.9.2
+- networkx=2.8.8
+- numpydoc=1.5
+- nvtabular=23.08.00
+- pip
+- python-confluent-kafka>=1.9.2,<1.10.0a0
+- python-graphviz
+- python=3.10
+- pytorch-cuda
+- pytorch=*=*cuda*
+- requests-cache=1.1
+- scikit-learn=1.3.2
+- sqlalchemy
+- tqdm=4
+- typing_utils=0.1
+- watchdog=3.0
+- websockets
+- pip:
+ - databricks-connect
+ - milvus==2.3.5
+ - pymilvus==2.3.6
+name: runtime_cuda-121_arch-x86_64
diff --git a/dependencies.yaml b/dependencies.yaml
@@ -124,7 +124,7 @@ files:
  - example-llm-vdb-upload
 
  runtime:
- output: none
+ output: conda
  matrix:
  cuda: ["12.1"]
  arch: [x86_64]

diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -39,9 +39,9 @@
 
 # Args used in FROM commands must come first
 ARG FROM_IMAGE="nvidia/cuda"
-ARG CUDA_MAJOR_VER=11
-ARG CUDA_MINOR_VER=8
-ARG CUDA_REV_VER=0
+ARG CUDA_MAJOR_VER=12
+ARG CUDA_MINOR_VER=1
+ARG CUDA_REV_VER=1
 ARG LINUX_DISTRO=ubuntu
 ARG LINUX_VER=22.04
 
@@ -176,13 +176,13 @@ ARG CUDA_MINOR_VER
 ARG CONDA_CHANNEL_ALIAS
 
 # Copy the development dependencies file
-COPY ${MORPHEUS_ROOT_HOST}/docker/conda/environments/cuda${CUDA_MAJOR_VER}.${CUDA_MINOR_VER}_dev.yml ./docker/conda/environments/
+COPY ${MORPHEUS_ROOT_HOST}/conda/environments/dev_cuda-${CUDA_MAJOR_VER}${CUDA_MINOR_VER}_arch-x86_64.yaml ./conda/environments/dev.yaml
 
 # Update the morpheus environment
 RUN --mount=type=cache,id=conda_pkgs,target=/opt/conda/pkgs,sharing=locked \
  # Temp add channel_alias to get around conda 404 errors
  conda config --env --set channel_alias ${CONDA_CHANNEL_ALIAS} &&\
- /opt/conda/bin/mamba env update -n morpheus --file docker/conda/environments/cuda${CUDA_MAJOR_VER}.${CUDA_MINOR_VER}_dev.yml &&\
+ /opt/conda/bin/conda env update --solver=libmamba -n morpheus --file conda/environments/dev.yaml &&\
  # Remove channel_alias to use the normal channel in the container
  conda config --env --remove-key channel_alias
 
@@ -232,7 +232,7 @@ ARG MORPHEUS_ROOT_HOST
 ARG CUDA_MAJOR_VER
 ARG CUDA_MINOR_VER
 
-COPY ${MORPHEUS_ROOT_HOST}/docker/conda/environments/cuda${CUDA_MAJOR_VER}.${CUDA_MINOR_VER}_runtime.yml ./docker/conda/environments/
+COPY ${MORPHEUS_ROOT_HOST}/conda/environments/runtime_cuda-${CUDA_MAJOR_VER}${CUDA_MINOR_VER}_arch-x86_64.yaml ./conda/environments/runtime.yaml
 
 # Mount Morpheus conda package build in `conda_bld_morpheus`
 RUN --mount=type=bind,from=conda_bld_morpheus,source=/opt/conda/conda-bld,target=/opt/conda/conda-bld \
@@ -244,13 +244,17 @@ RUN --mount=type=bind,from=conda_bld_morpheus,source=/opt/conda/conda-bld,target
  # Install morpheus
  CONDA_ALWAYS_YES=true /opt/conda/bin/mamba install -n morpheus \
  -c local \
+ -c conda-forge \
+ -c huggingface \
  -c rapidsai \
- -c nvidia/label/cuda-12.1.0 \
- -c nvidia/label/dev \
+ -c rapidsai-nightly \
  -c nvidia \
+ -c nvidia/label/dev \
  -c pytorch \
- -c conda-forge morpheus &&\
- /opt/conda/bin/mamba env update -n morpheus --file docker/conda/environments/cuda${CUDA_MAJOR_VER}.${CUDA_MINOR_VER}_runtime.yml && \
+ -c defaults \
+ morpheus &&\
+
+ /opt/conda/bin/conda env update --solver=libmamba -n morpheus --file conda/environments/runtime.yaml && \
  conda clean -afy
 
 # Only copy specific files/folders over that are necessary for runtime
@@ -259,7 +263,8 @@ COPY "${MORPHEUS_ROOT_HOST}/docs" "./docs"
 COPY "${MORPHEUS_ROOT_HOST}/examples" "./examples"
 COPY "${MORPHEUS_ROOT_HOST}/models" "./models"
 COPY "${MORPHEUS_ROOT_HOST}/scripts" "./scripts"
-COPY ["${MORPHEUS_ROOT_HOST}/*.md", "${MORPHEUS_ROOT_HOST}/LICENSE", "./"]
+COPY "${MORPHEUS_ROOT_HOST}/*.md" "./"
+COPY "${MORPHEUS_ROOT_HOST}/LICENSE" "./"
 
 # Use morpheus by default
 CMD [ "morpheus" ]

diff --git a/docker/build_container.sh b/docker/build_container.sh
@@ -28,18 +28,16 @@ DOCKER_TARGET=${DOCKER_TARGET:-"runtime"}
 DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""}
 
 # Build args
-CUDA_MAJOR_VER=${CUDA_MAJOR_VER:-11}
-CUDA_MINOR_VER=${CUDA_MINOR_VER:-8}
-CUDA_REV_VER=${CUDA_REV_VER:-0}
+CUDA_MAJOR_VER=${CUDA_MAJOR_VER:-12}
+CUDA_MINOR_VER=${CUDA_MINOR_VER:-1}
+CUDA_REV_VER=${CUDA_REV_VER:-1}
 DOCA_ARTIFACTS_HOST=${DOCA_ARTIFACTS_HOST:-""}
 DOCA_REPO_HOST=${DOCA_REPO_HOST:-""}
 FROM_IMAGE=${FROM_IMAGE:-"nvidia/cuda"}
 LINUX_DISTRO=${LINUX_DISTRO:-ubuntu}
 LINUX_VER=${LINUX_VER:-22.04}
 MORPHEUS_SUPPORT_DOCA=${MORPHEUS_SUPPORT_DOCA:-"OFF"}
 PYTHON_VER=${PYTHON_VER:-3.10}
-RAPIDS_VER=${RAPIDS_VER:-24.02}
-TENSORRT_VERSION=${TENSORRT_VERSION:-8.2.1.3}
 
 # Determine the relative path from $PWD to $MORPHEUS_ROOT
 MORPHEUS_ROOT_HOST=${MORPHEUS_ROOT_HOST:-"$(realpath --relative-to=${PWD} ${MORPHEUS_ROOT})"}
@@ -58,8 +56,6 @@ DOCKER_ARGS="${DOCKER_ARGS} --build-arg LINUX_VER=${LINUX_VER}"
 DOCKER_ARGS="${DOCKER_ARGS} --build-arg MORPHEUS_ROOT_HOST=${MORPHEUS_ROOT_HOST}"
 DOCKER_ARGS="${DOCKER_ARGS} --build-arg MORPHEUS_SUPPORT_DOCA=${MORPHEUS_SUPPORT_DOCA}"
 DOCKER_ARGS="${DOCKER_ARGS} --build-arg PYTHON_VER=${PYTHON_VER}"
-DOCKER_ARGS="${DOCKER_ARGS} --build-arg RAPIDS_VER=${RAPIDS_VER}"
-DOCKER_ARGS="${DOCKER_ARGS} --build-arg TENSORRT_VERSION=${TENSORRT_VERSION}"
 DOCKER_ARGS="${DOCKER_ARGS} --network=host"
 
 # Last add any extra args (duplicates override earlier ones)
@@ -80,11 +76,9 @@ echo " LINUX_VER : ${LINUX_VER}"
 echo " MORPHEUS_ROOT_HOST : ${MORPHEUS_ROOT_HOST}"
 echo " MORPHEUS_SUPPORT_DOCA: ${MORPHEUS_SUPPORT_DOCA}"
 echo " PYTHON_VER : ${PYTHON_VER}"
-echo " RAPIDS_VER : ${RAPIDS_VER}"
-echo " TENSORRT_VERSION : ${TENSORRT_VERSION}"
 
 echo ""
-echo " COMMAND: docker build ${DOCKER_ARGS} -f docker/Dockerfile ."
+echo " COMMAND: docker build ${DOCKER_ARGS} -f ${SCRIPT_DIR}/Dockerfile ."
 echo " Note: add '--progress plain' to DOCKER_EXTRA_ARGS to show all container build output"
 
 docker build ${DOCKER_ARGS} -f ${SCRIPT_DIR}/Dockerfile .
diff --git a/docs/README.md b/docs/README.md
@@ -22,11 +22,7 @@ Additional packages required for building the documentation are defined in `./co
 ## Install Additional Dependencies
 From the root of the Morpheus repo:
 ```bash
-export CUDA_VER=11.8
-mamba install -n base -c conda-forge conda-merge
-conda run -n base --live-stream conda-merge docker/conda/environments/cuda${CUDA_VER}_dev.yml \
- docs/conda_docs.yml > .tmp/merged.yml \
- && mamba env update -n ${CONDA_DEFAULT_ENV} --file .tmp/merged.yml
+conda env update --solver=libmamba -n morpheus --file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune
 ```
 
 ## Build Morpheus and Documentation

diff --git a/docs/source/developer_guide/contributing.md b/docs/source/developer_guide/contributing.md
@@ -205,7 +205,7 @@ git submodule update --init --recursive
 
 1. Create the Morpheus Conda environment
  ```bash
- mamba env create -f ./docker/conda/environments/cuda${CUDA_VER}_dev.yml
+ conda env create --solver=libmamba -n morpheus --file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml
  conda activate morpheus
  ```
 

diff --git a/morpheus/llm/services/nemo_llm_service.py b/morpheus/llm/services/nemo_llm_service.py
@@ -25,10 +25,8 @@
 IMPORT_EXCEPTION = None
 IMPORT_ERROR_MESSAGE = (
  "NemoLLM not found. Install it and other additional dependencies by running the following command:\n"
- "`mamba install -n base -c conda-forge conda-merge`\n"
- "`conda run -n base --live-stream conda-merge docker/conda/environments/cuda${CUDA_VER}_dev.yml "
- " docker/conda/environments/cuda${CUDA_VER}_examples.yml"
- " > .tmp/merged.yml && mamba env update -n morpheus --file .tmp/merged.yml`")
+ "`conda env update --solver=libmamba -n morpheus "
+ "--file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`")
 
 try:
  import nemollm

diff --git a/morpheus/llm/services/openai_chat_service.py b/morpheus/llm/services/openai_chat_service.py
@@ -23,13 +23,10 @@
 logger = logging.getLogger(__name__)
 
 IMPORT_EXCEPTION = None
-IMPORT_ERROR_MESSAGE = (
- "OpenAIChatService & OpenAIChatClient require the openai package to be installed. "
- "Install it by running the following command:\n"
- "`mamba install -n base -c conda-forge conda-merge`\n"
- "`conda run -n base --live-stream conda-merge docker/conda/environments/cuda${CUDA_VER}_dev.yml "
- " docker/conda/environments/cuda${CUDA_VER}_examples.yml"
- " > .tmp/merged.yml && mamba env update -n morpheus --file .tmp/merged.yml`")
+IMPORT_ERROR_MESSAGE = ("OpenAIChatService & OpenAIChatClient require the openai package to be installed. "
+ "Install it by running the following command:\n"
+ "`conda env update --solver=libmamba -n morpheus "
+ "--file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`")
 
 try:
  import openai

diff --git a/morpheus/stages/input/arxiv_source.py b/morpheus/stages/input/arxiv_source.py
@@ -36,10 +36,8 @@
 
 IMPORT_ERROR_MESSAGE = (
  "ArxivSource requires additional dependencies to be installed. Install them by running the following command: "
- "`mamba install -n base -c conda-forge conda-merge`\n"
- "`conda run -n base --live-stream conda-merge docker/conda/environments/cuda${CUDA_VER}_dev.yml "
- " docker/conda/environments/cuda${CUDA_VER}_examples.yml"
- " > .tmp/merged.yml && mamba env update -n morpheus --file .tmp/merged.yml`")
+ "`conda env update --solver=libmamba -n morpheus"
+ "--file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`")
 
 
 @register_stage("from-arxiv")
@@ -48,10 +46,8 @@ class ArxivSource(PreallocatorMixin, SingleOutputSource):
  Source stage that downloads PDFs from arxiv and converts them to dataframes.
 
  This stage requires several additional dependencies to be installed. Install them by running the following command:
- mamba install -n base -c conda-forge conda-merge
- conda run -n base --live-stream conda-merge docker/conda/environments/cuda${CUDA_VER}_dev.yml
- docker/conda/environments/cuda${CUDA_VER}_examples.yml
- > .tmp/merged.yml && mamba env update -n morpheus --file .tmp/merged.yml
+ `conda env update --solver=libmamba -n morpheus "
+ "--file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`
 
  Parameters
  ----------

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -1032,10 +1032,8 @@ def nemollm_fixture(fail_missing: bool):
  Fixture to ensure nemollm is installed
  """
  skip_reason = ("Tests for the NeMoLLMService require the nemollm package to be installed, to install this run:\n"
- "`mamba install -n base -c conda-forge conda-merge`\n"
- "`conda run -n base --live-stream conda-merge docker/conda/environments/cuda${CUDA_VER}_dev.yml "
- " docker/conda/environments/cuda${CUDA_VER}_examples.yml"
- " > .tmp/merged.yml && mamba env update -n morpheus --file .tmp/merged.yml`")
+ "`conda env update --solver=libmamba -n morpheus "
+ "--file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`")
  yield import_or_skip("nemollm", reason=skip_reason, fail_missing=fail_missing)
 
 
@@ -1045,10 +1043,8 @@ def openai_fixture(fail_missing: bool):
  Fixture to ensure openai is installed
  """
  skip_reason = ("Tests for the OpenAIChatService require the openai package to be installed, to install this run:\n"
- "`mamba install -n base -c conda-forge conda-merge`\n"
- "`conda run -n base --live-stream conda-merge docker/conda/environments/cuda${CUDA_VER}_dev.yml "
- " docker/conda/environments/cuda${CUDA_VER}_examples.yml"
- " > .tmp/merged.yml && mamba env update -n morpheus --file .tmp/merged.yml`")
+ "`conda env update --solver=libmamba -n morpheus "
+ "--file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`")
  yield import_or_skip("openai", reason=skip_reason, fail_missing=fail_missing)
 
 

diff --git a/tests/examples/llm/common/conftest.py b/tests/examples/llm/common/conftest.py
@@ -59,9 +59,6 @@ def nemollm_fixture(fail_missing: bool):
  """
 
  skip_reason = ("Tests for the WebScraperStage require the langchain package to be installed, to install this run:\n"
- "`mamba install -n base -c conda-forge conda-merge`\n"
- "`conda run -n base --live-stream conda-merge docker/conda/environments/cuda${CUDA_VER}_dev.yml "
- " docker/conda/environments/cuda${CUDA_VER}_examples.yml"
- " > .tmp/merged.yml && mamba env update -n morpheus --file .tmp/merged.yml`")
-
+ "`conda env update --solver=libmamba -n morpheus "
+ "--file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`")
  yield import_or_skip("langchain", reason=skip_reason, fail_missing=fail_missing)
diff --git a/tests/stages/arxiv/conftest.py b/tests/stages/arxiv/conftest.py
@@ -22,10 +22,8 @@
 
 SKIP_REASON = ("Tests for the arxiv_source require a number of packages not installed in the Morpheus development "
  "environment. To install these run:\n"
- "`mamba install -n base -c conda-forge conda-merge`\n"
- "`conda run -n base --live-stream conda-merge docker/conda/environments/cuda${CUDA_VER}_dev.yml "
- " docker/conda/environments/cuda${CUDA_VER}_examples.yml"
- " > .tmp/merged.yml && mamba env update -n morpheus --file .tmp/merged.yml`")
+ "`conda env update --solver=libmamba -n morpheus "
+ "--file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`")
 
 
 @pytest.fixture(name="arxiv", autouse=True, scope='session')