Skip to content

Commit

Permalink
Update dockerfiles and make compatible with Renovate (kubeflow/kubefl…
Browse files Browse the repository at this point in the history
…ow#5968)

* Update dockerfiles and make compatible with Renovate

* Set memory for jupyter pytorch to the same as jupyter tensorflow

* Update protobuf

* Remove conda version and use substring expansion

* Update SQLAlchemy

* Update dill
  • Loading branch information
davidspek authored and Adembc committed Jun 23, 2024
1 parent f68d648 commit 7431c75
Show file tree
Hide file tree
Showing 19 changed files with 66 additions and 65 deletions.
21 changes: 11 additions & 10 deletions notebooks/example-notebook-servers/codeserver-python/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,15 +1,14 @@
FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/codeserver:master-ebc0c4f0
FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/codeserver:master-434b10ab

USER root

# args - software versions
ARG CODESERVER_PYTHON_VERSION="2021.3.658691958"
ARG CONDA_VERSION="4.9.2"
ARG CODESERVER_PYTHON_VERSION=2021.5.842923320
ARG MINIFORGE_ARCH="x86_64"
ARG MINIFORGE_SHA256="91d5aa5f732b5e02002a371196a2607f839bab166970ea06e6ecc602cb446848"
ARG MINIFORGE_VERSION="${CONDA_VERSION}-7"
ARG PIP_VERSION="21.0.1"
ARG PYTHON_VERSION="3.8.8"
# renovate: datasource=github-tags depName=conda-forge/miniforge versioning=loose
ARG MINIFORGE_VERSION=4.10.1-4
ARG PIP_VERSION=21.1.2
ARG PYTHON_VERSION=3.8.10

# setup environment for conda
ENV CONDA_DIR /opt/conda
Expand All @@ -21,16 +20,18 @@ USER $NB_UID

# install - conda, pip, python
RUN curl -sL "https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-${MINIFORGE_ARCH}.sh" -o /tmp/Miniforge3.sh \
&& echo "${MINIFORGE_SHA256} /tmp/Miniforge3.sh" | sha256sum --check \
&& curl -sL "https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-${MINIFORGE_ARCH}.sh.sha256" -o /tmp/Miniforge3.sh.sha256 \
&& echo "$(cat /tmp/Miniforge3.sh.sha256 | awk '{ print $1; }') /tmp/Miniforge3.sh" | sha256sum --check \
&& rm /tmp/Miniforge3.sh.sha256 \
&& /bin/bash /tmp/Miniforge3.sh -b -f -p ${CONDA_DIR} \
&& rm /tmp/Miniforge3.sh \
&& conda config --system --set auto_update_conda false \
&& conda config --system --set show_channel_urls true \
&& echo "conda ${CONDA_VERSION}" >> ${CONDA_DIR}/conda-meta/pinned \
&& echo "conda ${MINIFORGE_VERSION:0:-2}" >> ${CONDA_DIR}/conda-meta/pinned \
&& echo "python ${PYTHON_VERSION}" >> ${CONDA_DIR}/conda-meta/pinned \
&& conda install -y -q \
python=${PYTHON_VERSION} \
conda=${CONDA_VERSION} \
conda=${MINIFORGE_VERSION:0:-2} \
pip=${PIP_VERSION} \
&& conda update -y -q --all \
&& conda clean -a -f -y \
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
kfp==1.0.4
kfp-server-api==1.0.4
kfserving==0.4.1
kfp==1.6.3
kfp-server-api==1.6.0
kfserving==0.5.1
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
--find-links https://download.pytorch.org/whl/torch_stable.html
torch==1.8.1+cpu
torchvision==0.9.1+cpu
torchaudio==0.8.1
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter:master-ebc0c4f0
FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter:master-434b10ab

# install - requirements.txt
COPY --chown=jovyan:users requirements-cpu.txt /tmp/requirements.txt
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir -f https://download.pytorch.org/whl/torch_stable.html \
COPY --chown=jovyan:users cpu-requirements.txt /tmp/requirements.txt
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
--find-links https://download.pytorch.org/whl/torch_stable.html
torch==1.8.1+cu111
torchvision==0.9.1+cu111
torchaudio==0.8.1
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter:master-ebc0c4f0
FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter:master-434b10ab

# nvidia configs
ENV NVIDIA_VISIBLE_DEVICES all
ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64

# install - requirements.txt
COPY --chown=jovyan:users requirements-cuda.txt /tmp/requirements.txt
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir -f https://download.pytorch.org/whl/torch_stable.html \
COPY --chown=jovyan:users cuda-requirements.txt /tmp/requirements.txt
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt

This file was deleted.

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter:master-ebc0c4f0
FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter:master-434b10ab

# install - requirements.txt
COPY --chown=jovyan:users requirements.txt /tmp/requirements.txt
Expand Down
28 changes: 14 additions & 14 deletions notebooks/example-notebook-servers/jupyter-scipy/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,33 +1,33 @@
# kubeflow packages
kfp==1.0.4
kfp-server-api==1.0.4
kfserving==0.4.1
kfp==1.6.3
kfp-server-api==1.6.0
kfserving==0.5.1

# scipy packages
# https://github.com/jupyter/docker-stacks/blob/master/scipy-notebook/Dockerfile
beautifulsoup4==4.9.3
bokeh==2.3.1
bokeh==2.3.2
#Bottleneck==1.3.2 Could not build wheels for Bottleneck which use PEP 517 and cannot be installed directly
cloudpickle==1.6.0
cython==0.29.22
dask==2021.4.0
dill==0.3.3
cython==0.29.23
dask==2021.6.0
dill==0.3.4
h5py==3.2.1
ipympl==0.7.0
ipywidgets==7.6.3
matplotlib==3.4.1
matplotlib==3.4.2
numba==0.53.1
numexpr==2.7.3
pandas==1.2.3
pandas==1.2.4
patsy==0.5.1
protobuf==3.15.7
protobuf==3.17.3
scikit-image==0.18.1
scikit-learn==0.24.1
scipy==1.6.2
scikit-learn==0.24.2
scipy==1.6.3
seaborn==0.11.1
SQLAlchemy==1.4.5
SQLAlchemy==1.4.18
statsmodels==0.12.2
sympy==1.7.1
sympy==1.8
tables==3.6.1
vincent==0.4.4
xlrd==2.0.1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
tensorflow==2.5.0
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter:master-ebc0c4f0
FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter:master-434b10ab

# install - requirements.txt
COPY --chown=jovyan:users requirements-cpu.txt /tmp/requirements.txt
COPY --chown=jovyan:users cpu-requirements.txt /tmp/requirements.txt
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
tensorflow-gpu==2.5.0
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter:master-ebc0c4f0
FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter:master-434b10ab

USER root

# needed for LIBNVINFER
ARG OLD_CUDA_VERSION=11.1
# args - software versions
ARG CUDA_VERSION=11.1
ARG CUDA_COMPART_VERSION=455.45.01-1
ARG CUDA_CUDART_VERSION=11.1.74-1
ARG CUDNN_VERSION=8.0.5.39-1
ARG LIBNVINFER_VERSION=7.2.2-1
ARG CUDA_VERSION=11.2
ARG CUDA_COMPAT_VERSION=460.73.01-1
ARG CUDA_CUDART_VERSION=11.2.152-1
ARG CUDNN_VERSION=8.1.0.77-1
ARG LIBNVINFER_VERSION=7.2.3-1

# we need bash's env var character substitution
SHELL ["/bin/bash", "-c"]
Expand All @@ -18,7 +20,7 @@ RUN curl -sL "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu200
&& echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /" > /etc/apt/sources.list.d/cuda.list \
&& apt-get -yq update \
&& apt-get -yq install --no-install-recommends \
cuda-compat-${CUDA_VERSION/./-}=${CUDA_COMPART_VERSION} \
cuda-compat-${CUDA_VERSION/./-}=${CUDA_COMPAT_VERSION} \
cuda-cudart-${CUDA_VERSION/./-}=${CUDA_CUDART_VERSION} \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* \
Expand Down Expand Up @@ -48,12 +50,12 @@ RUN curl -sL "https://developer.download.nvidia.com/compute/machine-learning/rep
libcusparse-${CUDA_VERSION/./-} \
libfreetype6-dev \
libhdf5-serial-dev \
libnvinfer7=${LIBNVINFER_VERSION}+cuda${CUDA_VERSION} \
libnvinfer-plugin7=${LIBNVINFER_VERSION}+cuda${CUDA_VERSION} \
libnvinfer7=${LIBNVINFER_VERSION}+cuda${OLD_CUDA_VERSION} \
libnvinfer-plugin7=${LIBNVINFER_VERSION}+cuda${OLD_CUDA_VERSION} \
libzmq3-dev \
pkg-config \
# can't be used until NVIDIA updates (requires python < 3.7)
# pthon3-libnvinfer=${LIBNVINFER_VERSION}+cuda${CUDA_VERSION} \
# python3-libnvinfer=${LIBNVINFER_VERSION}+cuda${CUDA_VERSION} \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*

Expand All @@ -75,6 +77,6 @@ RUN ln -s $(which python3) /usr/local/bin/python
USER $NB_UID

# install - requirements.txt
COPY --chown=jovyan:users requirements-cuda.txt /tmp/requirements.txt
COPY --chown=jovyan:users cuda-requirements.txt /tmp/requirements.txt
RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
&& rm -f /tmp/requirements.txt

This file was deleted.

This file was deleted.

6 changes: 2 additions & 4 deletions notebooks/example-notebook-servers/jupyter/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@ FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/base:master-6c41e72b
USER root

# args - software versions
# renovate: datasource=github-tags depName=conda/conda versioning=semver
ARG CONDA_VERSION=4.10.1
ARG MINIFORGE_ARCH="x86_64"
# renovate: datasource=github-tags depName=conda-forge/miniforge versioning=loose
ARG MINIFORGE_VERSION=4.10.1-4
Expand Down Expand Up @@ -44,11 +42,11 @@ RUN curl -sL "https://github.com/conda-forge/miniforge/releases/download/${MINIF
&& rm /tmp/Miniforge3.sh \
&& conda config --system --set auto_update_conda false \
&& conda config --system --set show_channel_urls true \
&& echo "conda ${CONDA_VERSION}" >> ${CONDA_DIR}/conda-meta/pinned \
&& echo "conda ${MINIFORGE_VERSION:0:-2}" >> ${CONDA_DIR}/conda-meta/pinned \
&& echo "python ${PYTHON_VERSION}" >> ${CONDA_DIR}/conda-meta/pinned \
&& conda install -y -q \
python=${PYTHON_VERSION} \
conda=${CONDA_VERSION} \
conda=${MINIFORGE_VERSION:0:-2} \
pip=${PIP_VERSION} \
&& conda update -y -q --all \
&& conda clean -a -f -y \
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/rstudio:master-ebc0c4f0
FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/rstudio:master-434b10ab

# args - software versions
ARG R_TIDYVERSE_VERSION="1.3.0"
ARG R_TIDYVERSE_VERSION="1.3.1"

# switch to root user for conda installation
# (only necessary when installing r-tidyverse with Kaniko)
Expand Down
6 changes: 2 additions & 4 deletions notebooks/example-notebook-servers/rstudio/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@ FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/base:master-6c41e72b
USER root

# args - software versions
# renovate: datasource=github-tags depName=conda/conda versioning=semver
ARG CONDA_VERSION=4.10.1
ARG MINIFORGE_ARCH="x86_64"
# renovate: datasource=github-tags depName=conda-forge/miniforge versioning=loose
ARG MINIFORGE_VERSION=4.10.1-4
Expand Down Expand Up @@ -57,12 +55,12 @@ RUN curl -sL "https://github.com/conda-forge/miniforge/releases/download/${MINIF
&& rm /tmp/Miniforge3.sh \
&& conda config --system --set auto_update_conda false \
&& conda config --system --set show_channel_urls true \
&& echo "conda ${CONDA_VERSION}" >> ${CONDA_DIR}/conda-meta/pinned \
&& echo "conda ${MINIFORGE_VERSION:0:-2}" >> ${CONDA_DIR}/conda-meta/pinned \
&& echo "python ${PYTHON_VERSION}" >> ${CONDA_DIR}/conda-meta/pinned \
&& echo "r-base ${R_BASE_VERSION}" >> ${CONDA_DIR}/conda-meta/pinned \
&& conda install -y -q \
python=${PYTHON_VERSION} \
conda=${CONDA_VERSION} \
conda=${MINIFORGE_VERSION:0:-2} \
pip=${PIP_VERSION} \
r-base=${R_BASE_VERSION} \
r-reticulate=${R_RETICULATE_VERSION} \
Expand Down

0 comments on commit 7431c75

Please sign in to comment.