Update Dockerfile to match current Morpheus Dockerfile

anomalous-auth-detection/requirements.txt

@@ -1,7 +1,4 @@
+--find-links https://download.pytorch.org/whl/cu118
+
 dgl==0.9.1
-matplotlib==3.6.2
-numpy==1.19.2
-pandas==1.2.1
-scikit_learn==1.1.3
-torch==1.7.1
-xgboost==1.7.1
+torch==2.0.1

appshield-dga-detection/inference/requirements.txt

@@ -1,7 +1,5 @@
 keras==2.10.0
-numpy==1.23.1
 onnx==1.12.0
 onnxruntime==1.12.1
-pandas==1.2.4
 tensorflow==2.10.0
 tldextract==3.3.1

appshield-dga-detection/training-tuning/requirements.txt

@@ -1,8 +1,6 @@
-matplotlib==3.4.2
-numpy==1.23.1
-pandas==1.2.4
-scikit_learn==1.1.2
+--find-links https://download.pytorch.org/whl/cu118
+
 tensorflow==2.10.0
 tldextract==3.3.1
-torch==1.13.1
+torch==2.0.1
 swifter==1.3.4

dga-detection/requirements.txt

@@ -1 +1,3 @@
-torch==1.13.1
+--find-links https://download.pytorch.org/whl/cu118
+
+torch==2.0.1

docker/Dockerfile

@@ -17,50 +17,73 @@
 
 
 # Args used in FROM commands must come first
-ARG FROM_IMAGE="gpuci/miniforge-cuda"
-ARG CUDA_VER=11.8
+ARG FROM_IMAGE="nvidia/cuda"
+ARG CUDA_MAJOR_VER=11
+ARG CUDA_MINOR_VER=8
+ARG CUDA_REV_VER=0
 ARG LINUX_DISTRO=ubuntu
-ARG LINUX_VER=20.04
+ARG LINUX_VER=22.04
+
+# =====Global Options=====
+ARG PYTHON_VER=3.10
+# Allows running the docker build from a different directory than MORPHEUS_ROOT. Needed for using Morpheus in a
+# submodule
+
+# Supply a channel alias to use for conda. This is needed if the conda channels go down
+ARG CONDA_CHANNEL_ALIAS="https://conda.anaconda.org"
 
 # ============ Stage: base ============
 # Configure the base conda environment
-FROM ${FROM_IMAGE}:${CUDA_VER}-devel-${LINUX_DISTRO}${LINUX_VER} AS base
+FROM ${FROM_IMAGE}:${CUDA_MAJOR_VER}.${CUDA_MINOR_VER}.${CUDA_REV_VER}-base-${LINUX_DISTRO}${LINUX_VER} AS base
 
-# Required arguments
-ARG IMAGE_TYPE=base
-ARG RAPIDS_CHANNEL=rapidsai-nightly
-ARG RAPIDS_VER=23.02
-ARG PYTHON_VER=3.10
-ARG CONDA_CHANNEL=rapidsai
+ARG CUDA_MAJOR_VER
+ARG CUDA_MINOR_VER
+ARG PYTHON_VER
 
-# Temp option to set conda channel_alias property during build
-ARG CONDA_CHANNEL_ALIAS
+ENV DEBIAN_FRONTEND=noninteractive
 
-# Capture argument used for FROM
-ARG CUDA_VER
+# Remove the file which breaks apt caching since we will handle it ourselves. See
+# https://vsupalov.com/buildkit-cache-mount-dockerfile/
+RUN rm -f /etc/apt/apt.conf.d/docker-clean
 
 # Install dependencies to build
-RUN apt-get update &&\
+RUN --mount=type=cache,id=apt,target=/var/cache/apt \
+ apt-get update &&\
  apt-get upgrade -y &&\
- curl -sL https://deb.nodesource.com/setup_16.x | bash - &&\
  apt-get install --no-install-recommends -y \
- build-essential pkg-config curl unzip tar zip openssh-client bc jq nodejs git-lfs \
+ bc \
+ build-essential \
+ curl \
+ git-lfs \
+ jq \
+ nodejs \
+ openssh-client \
+ pkg-config \
+ tar \
+ unzip \
+ zip \
  && rm -rf /var/lib/apt/lists/*
 
 # Enables "source activate conda"
 SHELL ["/bin/bash", "-c"]
 
+ENV CUDA_PATH=/usr/local/cuda
+
 # All code will be under /workspace
-WORKDIR /workspace
+ENV MORPHEUS_ROOT=/workspace
+WORKDIR ${MORPHEUS_ROOT}
+
+COPY --from=condaforge/mambaforge:latest /opt/conda /opt/conda
+
+ENV PATH="${PATH}:/opt/conda/bin"
 
 # Install mamba to speed the solve up
-RUN conda config --set ssl_verify false &&\
+RUN --mount=type=cache,id=conda_pkgs,target=/opt/conda/pkgs,sharing=locked \
+ conda config --set ssl_verify false &&\
  conda config --add pkgs_dirs /opt/conda/pkgs &&\
- conda config --env --add channels conda-forge &&\
  # Install mamba, boa and git here. Conda build breaks with other git installs
- /opt/conda/bin/conda install -y -n base -c conda-forge "mamba >=0.22" "boa >=0.10" "git >=2.35.3" "git-lfs" "python=${PYTHON_VER}" &&\
- source activate base &&\
- git lfs install
+ /opt/conda/bin/mamba install -y -n base -c conda-forge "boa" "git >=2.35.3" "python=${PYTHON_VER}" "tini=0.19" &&\
+ source activate base
  # conda clean -afy
 
 # ============ Stage: conda_env ============
@@ -70,13 +93,14 @@ FROM base as runtime
 # Create a base environment
 RUN --mount=type=cache,id=conda_pkgs,target=/opt/conda/pkgs,sharing=locked \
  # Create the environment and install as little dependencies as possible
- CONDA_ALWAYS_YES=true /opt/conda/bin/mamba create -n mor_exp -c rapidsai -c conda-forge -c nvidia \
- rapids=${RAPIDS_VER} python=${PYTHON_VER} cudatoolkit=${CUDA_VER} &&\
+ CONDA_ALWAYS_YES=true /opt/conda/bin/mamba create -n mor_exp -c conda-forge \
+  python=${PYTHON_VER} && \
  # Clean and activate
  # conda clean -afy && \
- sed -i 's/conda activate base/conda activate mor_exp/g' ~/.bashrc
+ conda init bash && \
+ echo "conda activate mor_exp" >> ~/.bashrc
 
-# Set the permenant conda channels to use for morpheus experimental
+# Set the permenant conda channels to use for morpheus-experimental
 RUN source activate mor_exp &&\
  conda config --env --add channels conda-forge &&\
  conda config --env --add channels nvidia &&\
@@ -90,11 +114,11 @@ ENTRYPOINT [ "/workspace/docker/entrypoint.sh" ]
 SHELL ["/bin/bash", "-c"]
 
 COPY docker/conda/environments/requirements.txt ./docker/conda/environments/
-COPY docker/conda/environments/cuda${CUDA_VER}_runtime.yml ./docker/conda/environments/
+COPY docker/conda/environments/cuda${CUDA_MAJOR_VER}.${CUDA_MINOR_VER}_runtime.yml ./docker/conda/environments/
 
 RUN --mount=type=cache,id=conda_pkgs,target=/opt/conda/pkgs,sharing=locked
 RUN source activate mor_exp
-RUN CONDA_ALWAYS_YES=true /opt/conda/bin/mamba env update -n mor_exp --file ./docker/conda/environments/cuda${CUDA_VER}_runtime.yml
+RUN CONDA_ALWAYS_YES=true /opt/conda/bin/mamba env update -n mor_exp --file ./docker/conda/environments/cuda${CUDA_MAJOR_VER}.${CUDA_MINOR_VER}_runtime.yml
 RUN conda clean -afy
 
 # Only copy specific files/folders over that are necessary for runtime
@@ -103,6 +127,7 @@ COPY "./anomalous-auth-detection" "./anomalous-auth-detection"
 COPY "./appshield-dga-detection" "./appshield-dga-detection"
 COPY "./asset-clustering" "./asset-clustering"
 COPY "./dga-detection" "./dga-detection"
+COPY "./gnn-asset-clustering" "./gnn-asset-clustering"
 COPY "./ids-detection" "./ids-detection"
 COPY "./log-sequence-ad" "./log-sequence-ad"
 COPY "./operational-technology" "./operational-technology"

docker/build_container.sh

@@ -20,22 +20,29 @@ DOCKER_IMAGE_NAME=${DOCKER_IMAGE_NAME:-"nvcr.io/nvidia/morpheus/mor_exp"}
 DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG:-"$(git describe --tags --abbrev=0)-runtime"}
 DOCKER_TARGET=${DOCKER_TARGET:-"runtime"}
 
+DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""}
+
 # Build args
-FROM_IMAGE=${FROM_IMAGE:-"gpuci/miniforge-cuda"}
-CUDA_VER=${CUDA_VER:-11.8}
+CUDA_MAJOR_VER=${CUDA_MAJOR_VER:-11}
+CUDA_MINOR_VER=${CUDA_MINOR_VER:-8}
+CUDA_REV_VER=${CUDA_REV_VER:-0}
+FROM_IMAGE=${FROM_IMAGE:-"nvidia/cuda"}
 LINUX_DISTRO=${LINUX_DISTRO:-ubuntu}
-LINUX_VER=${LINUX_VER:-20.04}
-RAPIDS_VER=${RAPIDS_VER:-23.02}
+LINUX_VER=${LINUX_VER:-22.04}
 PYTHON_VER=${PYTHON_VER:-3.10}
+RAPIDS_VER=${RAPIDS_VER:-23.06}
 
+# Build the docker arguments
 DOCKER_ARGS="-t ${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}"
 DOCKER_ARGS="${DOCKER_ARGS} --target ${DOCKER_TARGET}"
+DOCKER_ARGS="${DOCKER_ARGS} --build-arg CUDA_MAJOR_VER=${CUDA_MAJOR_VER}"
+DOCKER_ARGS="${DOCKER_ARGS} --build-arg CUDA_MINOR_VER=${CUDA_MINOR_VER}"
+DOCKER_ARGS="${DOCKER_ARGS} --build-arg CUDA_REV_VER=${CUDA_REV_VER}"
 DOCKER_ARGS="${DOCKER_ARGS} --build-arg FROM_IMAGE=${FROM_IMAGE}"
-DOCKER_ARGS="${DOCKER_ARGS} --build-arg CUDA_VER=${CUDA_VER}"
 DOCKER_ARGS="${DOCKER_ARGS} --build-arg LINUX_DISTRO=${LINUX_DISTRO}"
 DOCKER_ARGS="${DOCKER_ARGS} --build-arg LINUX_VER=${LINUX_VER}"
-DOCKER_ARGS="${DOCKER_ARGS} --build-arg RAPIDS_VER=${RAPIDS_VER}"
 DOCKER_ARGS="${DOCKER_ARGS} --build-arg PYTHON_VER=${PYTHON_VER}"
+DOCKER_ARGS="${DOCKER_ARGS} --build-arg RAPIDS_VER=${RAPIDS_VER}"
 DOCKER_ARGS="${DOCKER_ARGS} --network=host"
 
 # Last add any extra args (duplicates override earlier ones)
@@ -44,15 +51,18 @@ DOCKER_ARGS="${DOCKER_ARGS} ${DOCKER_EXTRA_ARGS}"
 # Export buildkit variable
 export DOCKER_BUILDKIT=1
 
-echo "Building morpheus experimental:${DOCKER_TAG}..."
-echo " FROM_IMAGE : ${FROM_IMAGE}"
-echo " CUDA_VER : ${CUDA_VER}"
-echo " LINUX_DISTRO : ${LINUX_DISTRO}"
-echo " LINUX_VER : ${LINUX_VER}"
-echo " RAPIDS_VER : ${RAPIDS_VER}"
-echo " PYTHON_VER : ${PYTHON_VER}"
+echo "Building morpheus:${DOCKER_TAG} with args..."
+echo " CUDA_MAJOR_VER : ${CUDA_MAJOR_VER}"
+echo " CUDA_MINOR_VER : ${CUDA_MINOR_VER}"
+echo " CUDA_REV_VER : ${CUDA_REV_VER}"
+echo " FROM_IMAGE : ${FROM_IMAGE}"
+echo " LINUX_DISTRO : ${LINUX_DISTRO}"
+echo " LINUX_VER : ${LINUX_VER}"
+echo " PYTHON_VER : ${PYTHON_VER}"
+echo " RAPIDS_VER : ${RAPIDS_VER}"
+
 echo ""
 echo " COMMAND: docker build ${DOCKER_ARGS} -f docker/Dockerfile ."
-echo " Note: add '--progress plain' to DOCKER_ARGS to show all container build output"
+echo " Note: add '--progress plain' to DOCKER_EXTRA_ARGS to show all container build output"
 
 docker build ${DOCKER_ARGS} -f docker/Dockerfile .

docker/conda/environments/cuda11.8_runtime.yml

@@ -20,11 +20,13 @@ channels:
  - rapidsai-nightly
  - conda-forge
 dependencies:
+ - cudatoolkit=11.8
  - dask
  - dill
  - matplotlib
  - nb_conda_kernels
  - pip
+ - rapids=23.06
  ####### Morpheus Pip Dependencies (keep sorted!) #######
  - pip:
  # Ensure all runtime requirements are installed using the requirements file

gnn-asset-clustering/requirements.txt

@@ -1,12 +1,8 @@
+--find-links https://download.pytorch.org/whl/cu118
+
 catboost==1.2
 data_science_utils==1.7.1
-matplotlib==3.7.1
-munkres==1.1.4
-networkx==3.1
-numpy==1.23.5
 pacmap==0.7.0
 pandas==1.3.5
 pyvis==0.3.2
-scikit_learn==1.2.2
-scipy==1.10.1
-torch==2.0.1+cu118
+torch==2.0.1

log-sequence-ad/requirements.txt

@@ -1,5 +1,5 @@
+--find-links https://download.pytorch.org/whl/cu118
+
 gensim==3.8.0
 nltk==3.8
-numpy==1.23.5
-pandas==1.5.2
-torch==1.12
+torch==2.0.1