From 6388da183d9e1c61c230a1123b6f33deddd0e23f Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Wed, 7 Feb 2024 13:32:42 -0600 Subject: [PATCH] Add runtime environment output to fix building the release container (#1496) Fixes the release container build. ## By Submitting this PR I confirm: - I am familiar with the [Contributing Guidelines](https://github.com/nv-morpheus/Morpheus/blob/main/docs/source/developer_guide/contributing.md). - When the PR is ready for review, new or existing tests cover these changes. - When the PR is ready for review, the documentation is up to date with these changes. Authors: - Christopher Harris (https://github.com/cwharris) Approvers: - Michael Demoret (https://github.com/mdemoret-nv) URL: https://github.com/nv-morpheus/Morpheus/pull/1496 --- .../all_cuda-121_arch-x86_64.yaml | 2 + .../dev_cuda-121_arch-x86_64.yaml | 2 + .../examples_cuda-121_arch-x86_64.yaml | 3 +- .../runtime_cuda-121_arch-x86_64.yaml | 41 +++++++++++++++++++ dependencies.yaml | 2 +- docker/Dockerfile | 27 +++++++----- docker/build_container.sh | 14 ++----- docs/README.md | 6 +-- docs/source/developer_guide/contributing.md | 2 +- morpheus/llm/services/nemo_llm_service.py | 6 +-- morpheus/llm/services/openai_chat_service.py | 11 ++--- morpheus/stages/input/arxiv_source.py | 12 ++---- tests/conftest.py | 12 ++---- tests/examples/llm/common/conftest.py | 7 +--- tests/stages/arxiv/conftest.py | 6 +-- 15 files changed, 87 insertions(+), 66 deletions(-) create mode 100644 conda/environments/runtime_cuda-121_arch-x86_64.yaml diff --git a/conda/environments/all_cuda-121_arch-x86_64.yaml b/conda/environments/all_cuda-121_arch-x86_64.yaml index 21d67c4764..5c3329da82 100644 --- a/conda/environments/all_cuda-121_arch-x86_64.yaml +++ b/conda/environments/all_cuda-121_arch-x86_64.yaml @@ -85,6 +85,7 @@ dependencies: - pytest-cov - pytest=7.4.4 - python-confluent-kafka>=1.9.2,<1.10.0a0 +- python-docx==1.1.0 - python-graphviz - python=3.10 - pytorch-cuda @@ -113,6 +114,7 @@ dependencies: - pip: - --find-links https://data.dgl.ai/wheels-test/repo.html - --find-links https://data.dgl.ai/wheels/cu121/repo.html + - PyMuPDF==1.23.21 - databricks-connect - dgl - dglgo diff --git a/conda/environments/dev_cuda-121_arch-x86_64.yaml b/conda/environments/dev_cuda-121_arch-x86_64.yaml index 8b17122044..f88cc79f93 100644 --- a/conda/environments/dev_cuda-121_arch-x86_64.yaml +++ b/conda/environments/dev_cuda-121_arch-x86_64.yaml @@ -66,6 +66,7 @@ dependencies: - pytest-cov - pytest=7.4.4 - python-confluent-kafka>=1.9.2,<1.10.0a0 +- python-docx==1.1.0 - python-graphviz - python=3.10 - pytorch-cuda @@ -88,6 +89,7 @@ dependencies: - yapf=0.40.1 - zlib=1.2.13 - pip: + - PyMuPDF==1.23.21 - databricks-connect - milvus==2.3.5 - pymilvus==2.3.6 diff --git a/conda/environments/examples_cuda-121_arch-x86_64.yaml b/conda/environments/examples_cuda-121_arch-x86_64.yaml index f291f4280e..04cef6f3ac 100644 --- a/conda/environments/examples_cuda-121_arch-x86_64.yaml +++ b/conda/environments/examples_cuda-121_arch-x86_64.yaml @@ -41,7 +41,6 @@ dependencies: - pip - pypdf=3.17.4 - python-confluent-kafka>=1.9.2,<1.10.0a0 -- python-docx==1.1.0 - python-graphviz - python=3.10 - pytorch-cuda @@ -59,6 +58,7 @@ dependencies: - pip: - --find-links https://data.dgl.ai/wheels-test/repo.html - --find-links https://data.dgl.ai/wheels/cu121/repo.html + - PyMuPDF==1.23.21 - databricks-connect - dgl - dglgo @@ -66,5 +66,4 @@ dependencies: - milvus==2.3.5 - nemollm - pymilvus==2.3.6 - - PyMuPDF==1.23.21 name: examples_cuda-121_arch-x86_64 diff --git a/conda/environments/runtime_cuda-121_arch-x86_64.yaml b/conda/environments/runtime_cuda-121_arch-x86_64.yaml new file mode 100644 index 0000000000..fb388052a6 --- /dev/null +++ b/conda/environments/runtime_cuda-121_arch-x86_64.yaml @@ -0,0 +1,41 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- conda-forge +- huggingface +- rapidsai +- rapidsai-nightly +- nvidia +- nvidia/label/dev +- pytorch +- defaults +dependencies: +- appdirs +- click >=8 +- datacompy=0.10 +- dill=0.3.7 +- elasticsearch==8.9.0 +- feedparser=6.0.10 +- grpcio=1.59 +- mlflow=2.9.2 +- networkx=2.8.8 +- numpydoc=1.5 +- nvtabular=23.08.00 +- pip +- python-confluent-kafka>=1.9.2,<1.10.0a0 +- python-graphviz +- python=3.10 +- pytorch-cuda +- pytorch=*=*cuda* +- requests-cache=1.1 +- scikit-learn=1.3.2 +- sqlalchemy +- tqdm=4 +- typing_utils=0.1 +- watchdog=3.0 +- websockets +- pip: + - databricks-connect + - milvus==2.3.5 + - pymilvus==2.3.6 +name: runtime_cuda-121_arch-x86_64 diff --git a/dependencies.yaml b/dependencies.yaml index 75b6cfdf1c..bff273259b 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -124,7 +124,7 @@ files: - example-llm-vdb-upload runtime: - output: none + output: conda matrix: cuda: ["12.1"] arch: [x86_64] diff --git a/docker/Dockerfile b/docker/Dockerfile index 00187ee1e6..db93d08486 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -39,9 +39,9 @@ # Args used in FROM commands must come first ARG FROM_IMAGE="nvidia/cuda" -ARG CUDA_MAJOR_VER=11 -ARG CUDA_MINOR_VER=8 -ARG CUDA_REV_VER=0 +ARG CUDA_MAJOR_VER=12 +ARG CUDA_MINOR_VER=1 +ARG CUDA_REV_VER=1 ARG LINUX_DISTRO=ubuntu ARG LINUX_VER=22.04 @@ -176,13 +176,13 @@ ARG CUDA_MINOR_VER ARG CONDA_CHANNEL_ALIAS # Copy the development dependencies file -COPY ${MORPHEUS_ROOT_HOST}/docker/conda/environments/cuda${CUDA_MAJOR_VER}.${CUDA_MINOR_VER}_dev.yml ./docker/conda/environments/ +COPY ${MORPHEUS_ROOT_HOST}/conda/environments/dev_cuda-${CUDA_MAJOR_VER}${CUDA_MINOR_VER}_arch-x86_64.yaml ./conda/environments/dev.yaml # Update the morpheus environment RUN --mount=type=cache,id=conda_pkgs,target=/opt/conda/pkgs,sharing=locked \ # Temp add channel_alias to get around conda 404 errors conda config --env --set channel_alias ${CONDA_CHANNEL_ALIAS} &&\ - /opt/conda/bin/mamba env update -n morpheus --file docker/conda/environments/cuda${CUDA_MAJOR_VER}.${CUDA_MINOR_VER}_dev.yml &&\ + /opt/conda/bin/conda env update --solver=libmamba -n morpheus --file conda/environments/dev.yaml &&\ # Remove channel_alias to use the normal channel in the container conda config --env --remove-key channel_alias @@ -232,7 +232,7 @@ ARG MORPHEUS_ROOT_HOST ARG CUDA_MAJOR_VER ARG CUDA_MINOR_VER -COPY ${MORPHEUS_ROOT_HOST}/docker/conda/environments/cuda${CUDA_MAJOR_VER}.${CUDA_MINOR_VER}_runtime.yml ./docker/conda/environments/ +COPY ${MORPHEUS_ROOT_HOST}/conda/environments/runtime_cuda-${CUDA_MAJOR_VER}${CUDA_MINOR_VER}_arch-x86_64.yaml ./conda/environments/runtime.yaml # Mount Morpheus conda package build in `conda_bld_morpheus` RUN --mount=type=bind,from=conda_bld_morpheus,source=/opt/conda/conda-bld,target=/opt/conda/conda-bld \ @@ -244,13 +244,17 @@ RUN --mount=type=bind,from=conda_bld_morpheus,source=/opt/conda/conda-bld,target # Install morpheus CONDA_ALWAYS_YES=true /opt/conda/bin/mamba install -n morpheus \ -c local \ + -c conda-forge \ + -c huggingface \ -c rapidsai \ - -c nvidia/label/cuda-12.1.0 \ - -c nvidia/label/dev \ + -c rapidsai-nightly \ -c nvidia \ + -c nvidia/label/dev \ -c pytorch \ - -c conda-forge morpheus &&\ - /opt/conda/bin/mamba env update -n morpheus --file docker/conda/environments/cuda${CUDA_MAJOR_VER}.${CUDA_MINOR_VER}_runtime.yml && \ + -c defaults \ + morpheus &&\ + + /opt/conda/bin/conda env update --solver=libmamba -n morpheus --file conda/environments/runtime.yaml && \ conda clean -afy # Only copy specific files/folders over that are necessary for runtime @@ -259,7 +263,8 @@ COPY "${MORPHEUS_ROOT_HOST}/docs" "./docs" COPY "${MORPHEUS_ROOT_HOST}/examples" "./examples" COPY "${MORPHEUS_ROOT_HOST}/models" "./models" COPY "${MORPHEUS_ROOT_HOST}/scripts" "./scripts" -COPY ["${MORPHEUS_ROOT_HOST}/*.md", "${MORPHEUS_ROOT_HOST}/LICENSE", "./"] +COPY "${MORPHEUS_ROOT_HOST}/*.md" "./" +COPY "${MORPHEUS_ROOT_HOST}/LICENSE" "./" # Use morpheus by default CMD [ "morpheus" ] diff --git a/docker/build_container.sh b/docker/build_container.sh index 3874804aa1..b3cdaf32b4 100755 --- a/docker/build_container.sh +++ b/docker/build_container.sh @@ -28,9 +28,9 @@ DOCKER_TARGET=${DOCKER_TARGET:-"runtime"} DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""} # Build args -CUDA_MAJOR_VER=${CUDA_MAJOR_VER:-11} -CUDA_MINOR_VER=${CUDA_MINOR_VER:-8} -CUDA_REV_VER=${CUDA_REV_VER:-0} +CUDA_MAJOR_VER=${CUDA_MAJOR_VER:-12} +CUDA_MINOR_VER=${CUDA_MINOR_VER:-1} +CUDA_REV_VER=${CUDA_REV_VER:-1} DOCA_ARTIFACTS_HOST=${DOCA_ARTIFACTS_HOST:-""} DOCA_REPO_HOST=${DOCA_REPO_HOST:-""} FROM_IMAGE=${FROM_IMAGE:-"nvidia/cuda"} @@ -38,8 +38,6 @@ LINUX_DISTRO=${LINUX_DISTRO:-ubuntu} LINUX_VER=${LINUX_VER:-22.04} MORPHEUS_SUPPORT_DOCA=${MORPHEUS_SUPPORT_DOCA:-"OFF"} PYTHON_VER=${PYTHON_VER:-3.10} -RAPIDS_VER=${RAPIDS_VER:-24.02} -TENSORRT_VERSION=${TENSORRT_VERSION:-8.2.1.3} # Determine the relative path from $PWD to $MORPHEUS_ROOT MORPHEUS_ROOT_HOST=${MORPHEUS_ROOT_HOST:-"$(realpath --relative-to=${PWD} ${MORPHEUS_ROOT})"} @@ -58,8 +56,6 @@ DOCKER_ARGS="${DOCKER_ARGS} --build-arg LINUX_VER=${LINUX_VER}" DOCKER_ARGS="${DOCKER_ARGS} --build-arg MORPHEUS_ROOT_HOST=${MORPHEUS_ROOT_HOST}" DOCKER_ARGS="${DOCKER_ARGS} --build-arg MORPHEUS_SUPPORT_DOCA=${MORPHEUS_SUPPORT_DOCA}" DOCKER_ARGS="${DOCKER_ARGS} --build-arg PYTHON_VER=${PYTHON_VER}" -DOCKER_ARGS="${DOCKER_ARGS} --build-arg RAPIDS_VER=${RAPIDS_VER}" -DOCKER_ARGS="${DOCKER_ARGS} --build-arg TENSORRT_VERSION=${TENSORRT_VERSION}" DOCKER_ARGS="${DOCKER_ARGS} --network=host" # Last add any extra args (duplicates override earlier ones) @@ -80,11 +76,9 @@ echo " LINUX_VER : ${LINUX_VER}" echo " MORPHEUS_ROOT_HOST : ${MORPHEUS_ROOT_HOST}" echo " MORPHEUS_SUPPORT_DOCA: ${MORPHEUS_SUPPORT_DOCA}" echo " PYTHON_VER : ${PYTHON_VER}" -echo " RAPIDS_VER : ${RAPIDS_VER}" -echo " TENSORRT_VERSION : ${TENSORRT_VERSION}" echo "" -echo " COMMAND: docker build ${DOCKER_ARGS} -f docker/Dockerfile ." +echo " COMMAND: docker build ${DOCKER_ARGS} -f ${SCRIPT_DIR}/Dockerfile ." echo " Note: add '--progress plain' to DOCKER_EXTRA_ARGS to show all container build output" docker build ${DOCKER_ARGS} -f ${SCRIPT_DIR}/Dockerfile . diff --git a/docs/README.md b/docs/README.md index bed7acf1be..c6ad45283b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -22,11 +22,7 @@ Additional packages required for building the documentation are defined in `./co ## Install Additional Dependencies From the root of the Morpheus repo: ```bash -export CUDA_VER=11.8 -mamba install -n base -c conda-forge conda-merge -conda run -n base --live-stream conda-merge docker/conda/environments/cuda${CUDA_VER}_dev.yml \ - docs/conda_docs.yml > .tmp/merged.yml \ - && mamba env update -n ${CONDA_DEFAULT_ENV} --file .tmp/merged.yml +conda env update --solver=libmamba -n morpheus --file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune ``` ## Build Morpheus and Documentation diff --git a/docs/source/developer_guide/contributing.md b/docs/source/developer_guide/contributing.md index 271d8fe46d..7b38f322f6 100644 --- a/docs/source/developer_guide/contributing.md +++ b/docs/source/developer_guide/contributing.md @@ -205,7 +205,7 @@ git submodule update --init --recursive 1. Create the Morpheus Conda environment ```bash - mamba env create -f ./docker/conda/environments/cuda${CUDA_VER}_dev.yml + conda env create --solver=libmamba -n morpheus --file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml conda activate morpheus ``` diff --git a/morpheus/llm/services/nemo_llm_service.py b/morpheus/llm/services/nemo_llm_service.py index bfa464f622..5dede8a240 100644 --- a/morpheus/llm/services/nemo_llm_service.py +++ b/morpheus/llm/services/nemo_llm_service.py @@ -25,10 +25,8 @@ IMPORT_EXCEPTION = None IMPORT_ERROR_MESSAGE = ( "NemoLLM not found. Install it and other additional dependencies by running the following command:\n" - "`mamba install -n base -c conda-forge conda-merge`\n" - "`conda run -n base --live-stream conda-merge docker/conda/environments/cuda${CUDA_VER}_dev.yml " - " docker/conda/environments/cuda${CUDA_VER}_examples.yml" - " > .tmp/merged.yml && mamba env update -n morpheus --file .tmp/merged.yml`") + "`conda env update --solver=libmamba -n morpheus " + "--file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`") try: import nemollm diff --git a/morpheus/llm/services/openai_chat_service.py b/morpheus/llm/services/openai_chat_service.py index 27f3316abb..098625c538 100644 --- a/morpheus/llm/services/openai_chat_service.py +++ b/morpheus/llm/services/openai_chat_service.py @@ -23,13 +23,10 @@ logger = logging.getLogger(__name__) IMPORT_EXCEPTION = None -IMPORT_ERROR_MESSAGE = ( - "OpenAIChatService & OpenAIChatClient require the openai package to be installed. " - "Install it by running the following command:\n" - "`mamba install -n base -c conda-forge conda-merge`\n" - "`conda run -n base --live-stream conda-merge docker/conda/environments/cuda${CUDA_VER}_dev.yml " - " docker/conda/environments/cuda${CUDA_VER}_examples.yml" - " > .tmp/merged.yml && mamba env update -n morpheus --file .tmp/merged.yml`") +IMPORT_ERROR_MESSAGE = ("OpenAIChatService & OpenAIChatClient require the openai package to be installed. " + "Install it by running the following command:\n" + "`conda env update --solver=libmamba -n morpheus " + "--file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`") try: import openai diff --git a/morpheus/stages/input/arxiv_source.py b/morpheus/stages/input/arxiv_source.py index 71299804c9..1b03b299fa 100644 --- a/morpheus/stages/input/arxiv_source.py +++ b/morpheus/stages/input/arxiv_source.py @@ -36,10 +36,8 @@ IMPORT_ERROR_MESSAGE = ( "ArxivSource requires additional dependencies to be installed. Install them by running the following command: " - "`mamba install -n base -c conda-forge conda-merge`\n" - "`conda run -n base --live-stream conda-merge docker/conda/environments/cuda${CUDA_VER}_dev.yml " - " docker/conda/environments/cuda${CUDA_VER}_examples.yml" - " > .tmp/merged.yml && mamba env update -n morpheus --file .tmp/merged.yml`") + "`conda env update --solver=libmamba -n morpheus" + "--file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`") @register_stage("from-arxiv") @@ -48,10 +46,8 @@ class ArxivSource(PreallocatorMixin, SingleOutputSource): Source stage that downloads PDFs from arxiv and converts them to dataframes. This stage requires several additional dependencies to be installed. Install them by running the following command: - mamba install -n base -c conda-forge conda-merge - conda run -n base --live-stream conda-merge docker/conda/environments/cuda${CUDA_VER}_dev.yml - docker/conda/environments/cuda${CUDA_VER}_examples.yml - > .tmp/merged.yml && mamba env update -n morpheus --file .tmp/merged.yml + `conda env update --solver=libmamba -n morpheus " + "--file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune` Parameters ---------- diff --git a/tests/conftest.py b/tests/conftest.py index 76752f2feb..6599f53ce3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1032,10 +1032,8 @@ def nemollm_fixture(fail_missing: bool): Fixture to ensure nemollm is installed """ skip_reason = ("Tests for the NeMoLLMService require the nemollm package to be installed, to install this run:\n" - "`mamba install -n base -c conda-forge conda-merge`\n" - "`conda run -n base --live-stream conda-merge docker/conda/environments/cuda${CUDA_VER}_dev.yml " - " docker/conda/environments/cuda${CUDA_VER}_examples.yml" - " > .tmp/merged.yml && mamba env update -n morpheus --file .tmp/merged.yml`") + "`conda env update --solver=libmamba -n morpheus " + "--file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`") yield import_or_skip("nemollm", reason=skip_reason, fail_missing=fail_missing) @@ -1045,10 +1043,8 @@ def openai_fixture(fail_missing: bool): Fixture to ensure openai is installed """ skip_reason = ("Tests for the OpenAIChatService require the openai package to be installed, to install this run:\n" - "`mamba install -n base -c conda-forge conda-merge`\n" - "`conda run -n base --live-stream conda-merge docker/conda/environments/cuda${CUDA_VER}_dev.yml " - " docker/conda/environments/cuda${CUDA_VER}_examples.yml" - " > .tmp/merged.yml && mamba env update -n morpheus --file .tmp/merged.yml`") + "`conda env update --solver=libmamba -n morpheus " + "--file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`") yield import_or_skip("openai", reason=skip_reason, fail_missing=fail_missing) diff --git a/tests/examples/llm/common/conftest.py b/tests/examples/llm/common/conftest.py index b6f5f7a3d8..2a1e58ba22 100644 --- a/tests/examples/llm/common/conftest.py +++ b/tests/examples/llm/common/conftest.py @@ -59,9 +59,6 @@ def nemollm_fixture(fail_missing: bool): """ skip_reason = ("Tests for the WebScraperStage require the langchain package to be installed, to install this run:\n" - "`mamba install -n base -c conda-forge conda-merge`\n" - "`conda run -n base --live-stream conda-merge docker/conda/environments/cuda${CUDA_VER}_dev.yml " - " docker/conda/environments/cuda${CUDA_VER}_examples.yml" - " > .tmp/merged.yml && mamba env update -n morpheus --file .tmp/merged.yml`") - + "`conda env update --solver=libmamba -n morpheus " + "--file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`") yield import_or_skip("langchain", reason=skip_reason, fail_missing=fail_missing) diff --git a/tests/stages/arxiv/conftest.py b/tests/stages/arxiv/conftest.py index cbcbce01b1..fb1fdad7be 100644 --- a/tests/stages/arxiv/conftest.py +++ b/tests/stages/arxiv/conftest.py @@ -22,10 +22,8 @@ SKIP_REASON = ("Tests for the arxiv_source require a number of packages not installed in the Morpheus development " "environment. To install these run:\n" - "`mamba install -n base -c conda-forge conda-merge`\n" - "`conda run -n base --live-stream conda-merge docker/conda/environments/cuda${CUDA_VER}_dev.yml " - " docker/conda/environments/cuda${CUDA_VER}_examples.yml" - " > .tmp/merged.yml && mamba env update -n morpheus --file .tmp/merged.yml`") + "`conda env update --solver=libmamba -n morpheus " + "--file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`") @pytest.fixture(name="arxiv", autouse=True, scope='session')