From cb2341a750f187e31952c5d0cedda15029f585ae Mon Sep 17 00:00:00 2001 From: Julio Perez Date: Mon, 2 May 2022 16:24:09 -0400 Subject: [PATCH 1/3] add entrypoint to all containers --- docker/inference/dockerfile.ctr | 1 + docker/inference/dockerfile.tf | 1 + docker/inference/dockerfile.torch | 1 + docker/training/dockerfile.ctr | 1 + docker/training/dockerfile.tf | 1 + docker/training/dockerfile.torch | 1 + 6 files changed, 6 insertions(+) diff --git a/docker/inference/dockerfile.ctr b/docker/inference/dockerfile.ctr index 17a109d0e..b049c7bf1 100644 --- a/docker/inference/dockerfile.ctr +++ b/docker/inference/dockerfile.ctr @@ -381,3 +381,4 @@ RUN rm -rf /repos HEALTHCHECK NONE CMD ["/bin/bash"] +ENTRYPOINT ["/bin/bash", "-c", "/opt/nvidia/nvidia_entrypoint.sh"] diff --git a/docker/inference/dockerfile.tf b/docker/inference/dockerfile.tf index 1e55df0d1..d4c96b7c9 100644 --- a/docker/inference/dockerfile.tf +++ b/docker/inference/dockerfile.tf @@ -205,3 +205,4 @@ RUN rm -rf /repos HEALTHCHECK NONE CMD ["/bin/bash"] +ENTRYPOINT ["/bin/bash", "-c", "/opt/nvidia/nvidia_entrypoint.sh"] diff --git a/docker/inference/dockerfile.torch b/docker/inference/dockerfile.torch index ca0bbb54b..544e39a5f 100644 --- a/docker/inference/dockerfile.torch +++ b/docker/inference/dockerfile.torch @@ -199,3 +199,4 @@ RUN rm -rf /repos HEALTHCHECK NONE CMD ["/bin/bash"] +ENTRYPOINT ["/bin/bash", "-c", "/opt/nvidia/nvidia_entrypoint.sh"] diff --git a/docker/training/dockerfile.ctr b/docker/training/dockerfile.ctr index 9f9ec804b..e9234f918 100644 --- a/docker/training/dockerfile.ctr +++ b/docker/training/dockerfile.ctr @@ -273,3 +273,4 @@ RUN rm -rf /usr/local/share/jupyter/lab/staging/node_modules/node-fetch HEALTHCHECK NONE CMD ["/bin/bash"] +ENTRYPOINT ["/bin/bash", "-c", "/opt/nvidia/nvidia_entrypoint.sh"] diff --git a/docker/training/dockerfile.tf b/docker/training/dockerfile.tf index 724920708..cc8dc1580 100644 --- a/docker/training/dockerfile.tf +++ b/docker/training/dockerfile.tf @@ -123,3 +123,4 @@ RUN rm -rf /usr/local/share/jupyter/lab/staging/node_modules/node-fetch HEALTHCHECK NONE CMD ["/bin/bash"] +ENTRYPOINT ["/bin/bash", "-c", "/opt/nvidia/nvidia_entrypoint.sh"] diff --git a/docker/training/dockerfile.torch b/docker/training/dockerfile.torch index c7c80c7b8..e8f2ab2db 100644 --- a/docker/training/dockerfile.torch +++ b/docker/training/dockerfile.torch @@ -86,3 +86,4 @@ RUN rm -rf /opt/conda/share/jupyter/lab/staging/node_modules/node-fetch HEALTHCHECK NONE CMD ["/bin/bash"] +ENTRYPOINT ["/bin/bash", "-c", "/opt/nvidia/nvidia_entrypoint.sh"] From fa45e642ef7d9ce1fc47cb5e2cee67df8d6c85f1 Mon Sep 17 00:00:00 2001 From: Julio Perez Date: Wed, 4 May 2022 20:22:40 -0400 Subject: [PATCH 2/3] remove -e for pip installs --- docker/inference/dockerfile.ctr | 8 ++++---- docker/inference/dockerfile.tf | 10 +++++----- docker/inference/dockerfile.torch | 8 ++++---- docker/training/dockerfile.ctr | 6 +++--- docker/training/dockerfile.tf | 2 +- docker/training/dockerfile.torch | 8 ++++---- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/docker/inference/dockerfile.ctr b/docker/inference/dockerfile.ctr index 19bc00a87..59a731d30 100644 --- a/docker/inference/dockerfile.ctr +++ b/docker/inference/dockerfile.ctr @@ -192,7 +192,7 @@ RUN git clone https://github.com/rapidsai/cudf.git build-env && cd build-env/ && # Install Merlin Core RUN git clone https://github.com/NVIDIA-Merlin/core.git /core/ && \ - cd /core/ && git checkout ${CORE_VER} && pip install -e . --no-deps + cd /core/ && git checkout ${CORE_VER} && pip install . --no-deps ENV PYTHONPATH=$PYTHONPATH:/core # Install Merlin Systems @@ -203,17 +203,17 @@ ENV PYTHONPATH=$PYTHONPATH:/systems # Install NVTabular ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION='python' RUN git clone https://github.com/NVIDIA-Merlin/NVTabular.git /nvtabular/ && \ - cd /nvtabular/ && git checkout ${NVTAB_VER} && pip install -e . --no-deps + cd /nvtabular/ && git checkout ${NVTAB_VER} && pip install . --no-deps ENV PYTHONPATH=$PYTHONPATH:/nvtabular # Install Transformers4Rec RUN git clone https://github.com/NVIDIA-Merlin/Transformers4Rec.git /transformers4rec && \ - cd /transformers4rec/ && git checkout ${TF4REC_VER} && pip install -e . --no-deps + cd /transformers4rec/ && git checkout ${TF4REC_VER} && pip install . --no-deps ENV PYTHONPATH=$PYTHONPATH:/transformers4rec # Install Models RUN git clone https://github.com/NVIDIA-Merlin/Models.git /models/ && \ - cd /models/ && git checkout ${MODELS_VER} && pip install -e . --no-deps; + cd /models/ && git checkout ${MODELS_VER} && pip install . --no-deps; ENV PYTHONPATH=$PYTHONPATH:/models # Add Merlin Repo diff --git a/docker/inference/dockerfile.tf b/docker/inference/dockerfile.tf index d4c96b7c9..e642ada57 100644 --- a/docker/inference/dockerfile.tf +++ b/docker/inference/dockerfile.tf @@ -85,7 +85,7 @@ RUN git clone --branch v1.9.2 https://github.com/gabime/spdlog.git build-env && # Install arrow ENV ARROW_HOME=/usr/local -RUN git clone --branch apache-arrow-5.0.0 --recurse-submodules https://github.com/apache/arrow.git build-env && \ +RUN git clone --branch apache-arrow-6.0.1 --recurse-submodules https://github.com/apache/arrow.git build-env && \ pushd build-env && \ export PARQUET_TEST_DATA="${PWD}/cpp/submodules/parquet-testing/data" && \ export ARROW_TEST_DATA="${PWD}/testing/data" && \ @@ -156,7 +156,7 @@ RUN git clone https://github.com/rapidsai/cudf.git build-env && cd build-env/ && # Install Merlin Core RUN git clone https://github.com/NVIDIA-Merlin/core.git /core/ && \ - cd /core/ && git checkout ${CORE_VER} && pip install -e . --no-deps + cd /core/ && git checkout ${CORE_VER} && pip install . --no-deps ENV PYTHONPATH=$PYTHONPATH:/core # Install Merlin Systems @@ -167,17 +167,17 @@ ENV PYTHONPATH=$PYTHONPATH:/systems # Install NVTabular ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION='python' RUN git clone https://github.com/NVIDIA-Merlin/NVTabular.git /nvtabular/ && \ - cd /nvtabular/ && git checkout ${NVTAB_VER} && pip install -e . --no-deps + cd /nvtabular/ && git checkout ${NVTAB_VER} && pip install . --no-deps ENV PYTHONPATH=$PYTHONPATH:/nvtabular # Install Transformers4Rec RUN git clone https://github.com/NVIDIA-Merlin/Transformers4Rec.git /transformers4rec && \ - cd /transformers4rec/ && git checkout ${TF4REC_VER} && pip install -e . --no-deps + cd /transformers4rec/ && git checkout ${TF4REC_VER} && pip install . --no-deps ENV PYTHONPATH=$PYTHONPATH:/transformers4rec # Install Models RUN git clone https://github.com/NVIDIA-Merlin/Models.git /models/ && \ - cd /models/ && git checkout ${MODELS_VER} && pip install -e . --no-deps; + cd /models/ && git checkout ${MODELS_VER} && pip install . --no-deps; ENV PYTHONPATH=$PYTHONPATH:/models # Add Merlin Repo diff --git a/docker/inference/dockerfile.torch b/docker/inference/dockerfile.torch index 544e39a5f..241b60008 100644 --- a/docker/inference/dockerfile.torch +++ b/docker/inference/dockerfile.torch @@ -150,7 +150,7 @@ RUN git clone https://github.com/rapidsai/cudf.git build-env && cd build-env/ && # Install Merlin Core RUN git clone https://github.com/NVIDIA-Merlin/core.git /core/ && \ - cd /core/ && git checkout ${CORE_VER} && pip install -e . --no-deps + cd /core/ && git checkout ${CORE_VER} && pip install . --no-deps ENV PYTHONPATH=$PYTHONPATH:/core # Install Merlin Systems @@ -161,17 +161,17 @@ ENV PYTHONPATH=$PYTHONPATH:/systems # Install NVTabular ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION='python' RUN git clone https://github.com/NVIDIA-Merlin/NVTabular.git /nvtabular/ && \ - cd /nvtabular/ && git checkout ${NVTAB_VER} && pip install -e . --no-deps + cd /nvtabular/ && git checkout ${NVTAB_VER} && pip install . --no-deps ENV PYTHONPATH=$PYTHONPATH:/nvtabular # Install Transformers4Rec RUN git clone https://github.com/NVIDIA-Merlin/Transformers4Rec.git /transformers4rec && \ - cd /transformers4rec/ && git checkout ${TF4REC_VER} && pip install -e . --no-deps + cd /transformers4rec/ && git checkout ${TF4REC_VER} && pip install . --no-deps ENV PYTHONPATH=$PYTHONPATH:/transformers4rec # Install Models RUN git clone https://github.com/NVIDIA-Merlin/Models.git /models/ && \ - cd /models/ && git checkout ${MODELS_VER} && pip install -e . --no-deps; + cd /models/ && git checkout ${MODELS_VER} && pip install . --no-deps; ENV PYTHONPATH=$PYTHONPATH:/models # Add Merlin Repo diff --git a/docker/training/dockerfile.ctr b/docker/training/dockerfile.ctr index 4bda7a84e..d3372b53e 100644 --- a/docker/training/dockerfile.ctr +++ b/docker/training/dockerfile.ctr @@ -95,17 +95,17 @@ ENV PYTHONPATH=$PYTHONPATH:/systems # Install NVTabular ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION='python' RUN git clone https://github.com/NVIDIA-Merlin/NVTabular.git /nvtabular/ && \ - cd /nvtabular/ && git checkout ${NVTAB_VER} && pip install -e . --no-deps + cd /nvtabular/ && git checkout ${NVTAB_VER} && pip install . --no-deps ENV PYTHONPATH=$PYTHONPATH:/nvtabular # Install Transformers4Rec RUN git clone https://github.com/NVIDIA-Merlin/Transformers4Rec.git /transformers4rec && \ - cd /transformers4rec/ && git checkout ${TF4REC_VER} && pip install -e . --no-deps + cd /transformers4rec/ && git checkout ${TF4REC_VER} && pip install . --no-deps ENV PYTHONPATH=$PYTHONPATH:/transformers4rec # Install Models RUN git clone https://github.com/NVIDIA-Merlin/Models.git /models/ && \ - cd /models/ && git checkout ${MODELS_VER} && pip install -e . --no-deps + cd /models/ && git checkout ${MODELS_VER} && pip install . --no-deps ENV PYTHONPATH=$PYTHONPATH:/models # Add Merlin Repo diff --git a/docker/training/dockerfile.tf b/docker/training/dockerfile.tf index cc8dc1580..61cfe4ee2 100644 --- a/docker/training/dockerfile.tf +++ b/docker/training/dockerfile.tf @@ -80,7 +80,7 @@ ENV PYTHONPATH=$PYTHONPATH:/transformers4rec # Install Models RUN git clone https://github.com/NVIDIA-Merlin/Models.git /models/ && \ - cd /models/ && git checkout ${MODELS_VER} && pip install -e . --no-deps + cd /models/ && git checkout ${MODELS_VER} && pip install . --no-deps ENV PYTHONPATH=$PYTHONPATH:/models # Add Merlin Repo diff --git a/docker/training/dockerfile.torch b/docker/training/dockerfile.torch index e8f2ab2db..ecfa1b913 100644 --- a/docker/training/dockerfile.torch +++ b/docker/training/dockerfile.torch @@ -52,7 +52,7 @@ RUN pip install git+https://github.com/rapidsai/asvdb.git@main # Install Merlin Core RUN git clone https://github.com/NVIDIA-Merlin/core.git /core/ && \ - cd /core/ && git checkout ${CORE_VER} && pip install -e . --no-deps + cd /core/ && git checkout ${CORE_VER} && pip install . --no-deps ENV PYTHONPATH=$PYTHONPATH:/core # Install Merlin Systems @@ -63,17 +63,17 @@ ENV PYTHONPATH=$PYTHONPATH:/systems # Install NVTabular ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION='python' RUN git clone https://github.com/NVIDIA-Merlin/NVTabular.git /nvtabular/ && \ - cd /nvtabular/ && git checkout ${NVTAB_VER} && pip install -e . --no-deps + cd /nvtabular/ && git checkout ${NVTAB_VER} && pip install . --no-deps ENV PYTHONPATH=$PYTHONPATH:/nvtabular # Install Transformers4Rec RUN git clone https://github.com/NVIDIA-Merlin/Transformers4Rec.git /transformers4rec && \ - cd /transformers4rec/ && git checkout ${TF4REC_VER} && pip install -e . --no-deps + cd /transformers4rec/ && git checkout ${TF4REC_VER} && pip install . --no-deps ENV PYTHONPATH=$PYTHONPATH:/transformers4rec # Install Models RUN git clone https://github.com/NVIDIA-Merlin/Models.git /models/ && \ - cd /models/ && git checkout ${MODELS_VER} && pip install -e . --no-deps + cd /models/ && git checkout ${MODELS_VER} && pip install . --no-deps ENV PYTHONPATH=$PYTHONPATH:/models # Add Merlin Repo From a2237e607c24d2027d7be5eefc1204bc6dc0fac5 Mon Sep 17 00:00:00 2001 From: Julio Perez Date: Tue, 10 May 2022 14:32:36 -0400 Subject: [PATCH 3/3] remove all -e pip installs --- docker/inference/dockerfile.ctr | 2 +- docker/inference/dockerfile.tf | 2 +- docker/inference/dockerfile.torch | 2 +- docker/training/dockerfile.ctr | 4 ++-- docker/training/dockerfile.tf | 6 +++--- docker/training/dockerfile.torch | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docker/inference/dockerfile.ctr b/docker/inference/dockerfile.ctr index 5a4a0fda1..548b09bcd 100644 --- a/docker/inference/dockerfile.ctr +++ b/docker/inference/dockerfile.ctr @@ -197,7 +197,7 @@ ENV PYTHONPATH=$PYTHONPATH:/core # Install Merlin Systems RUN git clone https://github.com/NVIDIA-Merlin/systems.git /systems/ && \ - cd /systems/ && git checkout ${SYSTEMS_VER} && pip install --no-deps -e . + cd /systems/ && git checkout ${SYSTEMS_VER} && pip install --no-deps . ENV PYTHONPATH=$PYTHONPATH:/systems # Install NVTabular diff --git a/docker/inference/dockerfile.tf b/docker/inference/dockerfile.tf index e642ada57..9fa70519b 100644 --- a/docker/inference/dockerfile.tf +++ b/docker/inference/dockerfile.tf @@ -161,7 +161,7 @@ ENV PYTHONPATH=$PYTHONPATH:/core # Install Merlin Systems RUN git clone https://github.com/NVIDIA-Merlin/systems.git /systems/ && \ - cd /systems/ && git checkout ${SYSTEMS_VER} && pip install --no-deps -e . + cd /systems/ && git checkout ${SYSTEMS_VER} && pip install --no-deps . ENV PYTHONPATH=$PYTHONPATH:/systems # Install NVTabular diff --git a/docker/inference/dockerfile.torch b/docker/inference/dockerfile.torch index 44288f8fd..5ef0ad25c 100644 --- a/docker/inference/dockerfile.torch +++ b/docker/inference/dockerfile.torch @@ -155,7 +155,7 @@ ENV PYTHONPATH=$PYTHONPATH:/core # Install Merlin Systems RUN git clone https://github.com/NVIDIA-Merlin/systems.git /systems/ && \ - cd /systems/ && git checkout ${SYSTEMS_VER} && pip install --no-deps -e . + cd /systems/ && git checkout ${SYSTEMS_VER} && pip install --no-deps . ENV PYTHONPATH=$PYTHONPATH:/systems # Install NVTabular diff --git a/docker/training/dockerfile.ctr b/docker/training/dockerfile.ctr index f19269a09..85d7f685f 100644 --- a/docker/training/dockerfile.ctr +++ b/docker/training/dockerfile.ctr @@ -84,12 +84,12 @@ RUN pip install git+https://github.com/rapidsai/asvdb.git@main # Install Merlin Core RUN git clone https://github.com/NVIDIA-Merlin/core.git /core/ && \ - cd /core/ && git checkout ${CORE_VER} && pip install --no-deps -e . + cd /core/ && git checkout ${CORE_VER} && pip install --no-deps . ENV PYTHONPATH=$PYTHONPATH:/core # Install Merlin Systems RUN git clone https://github.com/NVIDIA-Merlin/systems.git /systems/ && \ - cd /systems/ && git checkout ${SYSTEMS_VER} && pip install --no-deps -e . + cd /systems/ && git checkout ${SYSTEMS_VER} && pip install --no-deps . ENV PYTHONPATH=$PYTHONPATH:/systems # Install NVTabular diff --git a/docker/training/dockerfile.tf b/docker/training/dockerfile.tf index f7f04b309..00ed3b3c6 100644 --- a/docker/training/dockerfile.tf +++ b/docker/training/dockerfile.tf @@ -54,12 +54,12 @@ RUN pip install git+https://github.com/rapidsai/asvdb.git@main # Install Merlin Core RUN git clone https://github.com/NVIDIA-Merlin/core.git /core/ && \ - cd /core/ && git checkout ${CORE_VER} && pip install --no-deps -e . + cd /core/ && git checkout ${CORE_VER} && pip install --no-deps . ENV PYTHONPATH=$PYTHONPATH:/core # Install Merlin Systems RUN git clone https://github.com/NVIDIA-Merlin/systems.git /systems/ && \ - cd /systems/ && git checkout ${SYSTEMS_VER} && pip install --no-deps -e . + cd /systems/ && git checkout ${SYSTEMS_VER} && pip install --no-deps . ENV PYTHONPATH=$PYTHONPATH:/systems ARG INSTALL_NVT=true @@ -67,7 +67,7 @@ ARG INSTALL_NVT=true ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION='python' RUN if [ "$INSTALL_NVT" == "true" ]; then \ git clone https://github.com/NVIDIA-Merlin/NVTabular.git /nvtabular/ && \ - cd /nvtabular/ && git checkout ${NVTAB_VER} && pip install --no-deps -e .; \ + cd /nvtabular/ && git checkout ${NVTAB_VER} && pip install --no-deps .; \ fi ENV PYTHONPATH=$PYTHONPATH:/nvtabular diff --git a/docker/training/dockerfile.torch b/docker/training/dockerfile.torch index a5d490745..f47ed44ae 100644 --- a/docker/training/dockerfile.torch +++ b/docker/training/dockerfile.torch @@ -57,7 +57,7 @@ ENV PYTHONPATH=$PYTHONPATH:/core # Install Merlin Systems RUN git clone https://github.com/NVIDIA-Merlin/systems.git /systems/ && \ - cd /systems/ && git checkout ${SYSTEMS_VER} && pip install --no-deps -e . + cd /systems/ && git checkout ${SYSTEMS_VER} && pip install --no-deps . ENV PYTHONPATH=$PYTHONPATH:/systems # Install NVTabular