Update dockerfiles and make compatible with Renovate (kubeflow#5968)

* Update dockerfiles and make compatible with Renovate

* Set memory for jupyter pytorch to the same as jupyter tensorflow

* Update protobuf

* Remove conda version and use substring expansion

* Update SQLAlchemy

* Update dill

components/example-notebook-servers/codeserver-python/Dockerfile

@@ -1,15 +1,14 @@
-FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/codeserver:master-ebc0c4f0
+FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/codeserver:master-434b10ab
 
 USER root
 
 # args - software versions
-ARG CODESERVER_PYTHON_VERSION="2021.3.658691958"
-ARG CONDA_VERSION="4.9.2"
+ARG CODESERVER_PYTHON_VERSION=2021.5.842923320
 ARG MINIFORGE_ARCH="x86_64"
-ARG MINIFORGE_SHA256="91d5aa5f732b5e02002a371196a2607f839bab166970ea06e6ecc602cb446848"
-ARG MINIFORGE_VERSION="${CONDA_VERSION}-7"
-ARG PIP_VERSION="21.0.1"
-ARG PYTHON_VERSION="3.8.8"
+ # renovate: datasource=github-tags depName=conda-forge/miniforge versioning=loose
+ARG MINIFORGE_VERSION=4.10.1-4
+ARG PIP_VERSION=21.1.2
+ARG PYTHON_VERSION=3.8.10
 
 # setup environment for conda
 ENV CONDA_DIR /opt/conda
@@ -21,16 +20,18 @@ USER $NB_UID
 
 # install - conda, pip, python
 RUN curl -sL "https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-${MINIFORGE_ARCH}.sh" -o /tmp/Miniforge3.sh \
- && echo "${MINIFORGE_SHA256} /tmp/Miniforge3.sh" | sha256sum --check \
+ && curl -sL "https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-${MINIFORGE_ARCH}.sh.sha256" -o /tmp/Miniforge3.sh.sha256 \
+ && echo "$(cat /tmp/Miniforge3.sh.sha256 | awk '{ print $1; }') /tmp/Miniforge3.sh" | sha256sum --check \
+ && rm /tmp/Miniforge3.sh.sha256 \
  && /bin/bash /tmp/Miniforge3.sh -b -f -p ${CONDA_DIR} \
  && rm /tmp/Miniforge3.sh \
  && conda config --system --set auto_update_conda false \
  && conda config --system --set show_channel_urls true \
- && echo "conda ${CONDA_VERSION}" >> ${CONDA_DIR}/conda-meta/pinned \
+ && echo "conda ${MINIFORGE_VERSION:0:-2}" >> ${CONDA_DIR}/conda-meta/pinned \
  && echo "python ${PYTHON_VERSION}" >> ${CONDA_DIR}/conda-meta/pinned \
  && conda install -y -q \
     python=${PYTHON_VERSION} \
-    conda=${CONDA_VERSION} \
+    conda=${MINIFORGE_VERSION:0:-2} \
     pip=${PIP_VERSION} \
  && conda update -y -q --all \
  && conda clean -a -f -y \

components/example-notebook-servers/codeserver-python/requirements.txt

@@ -1,3 +1,3 @@
-kfp==1.0.4
-kfp-server-api==1.0.4
-kfserving==0.4.1
+kfp==1.6.3
+kfp-server-api==1.6.0
+kfserving==0.5.1

components/example-notebook-servers/jupyter-pytorch/cpu-requirements.txt

@@ -0,0 +1,4 @@
+--find-links https://download.pytorch.org/whl/torch_stable.html
+torch==1.8.1+cpu
+torchvision==0.9.1+cpu
+torchaudio==0.8.1

components/example-notebook-servers/jupyter-pytorch/cpu.Dockerfile

@@ -1,6 +1,6 @@
-FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter:master-ebc0c4f0
+FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter:master-434b10ab
 
 # install - requirements.txt
-COPY --chown=jovyan:users requirements-cpu.txt /tmp/requirements.txt
-RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir -f https://download.pytorch.org/whl/torch_stable.html \
+COPY --chown=jovyan:users cpu-requirements.txt /tmp/requirements.txt
+RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
  && rm -f /tmp/requirements.txt

components/example-notebook-servers/jupyter-pytorch/cuda-requirements.txt

@@ -0,0 +1,4 @@
+--find-links https://download.pytorch.org/whl/torch_stable.html
+torch==1.8.1+cu111
+torchvision==0.9.1+cu111
+torchaudio==0.8.1

components/example-notebook-servers/jupyter-pytorch/cuda.Dockerfile

@@ -1,11 +1,11 @@
-FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter:master-ebc0c4f0
+FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter:master-434b10ab
 
 # nvidia configs
 ENV NVIDIA_VISIBLE_DEVICES all
 ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
 ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64
 
 # install - requirements.txt
-COPY --chown=jovyan:users requirements-cuda.txt /tmp/requirements.txt
-RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir -f https://download.pytorch.org/whl/torch_stable.html \
+COPY --chown=jovyan:users cuda-requirements.txt /tmp/requirements.txt
+RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
  && rm -f /tmp/requirements.txt

components/example-notebook-servers/jupyter-scipy/Dockerfile

@@ -1,4 +1,4 @@
-FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter:master-ebc0c4f0
+FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter:master-434b10ab
 
 # install - requirements.txt
 COPY --chown=jovyan:users requirements.txt /tmp/requirements.txt

components/example-notebook-servers/jupyter-scipy/requirements.txt

@@ -1,33 +1,33 @@
 # kubeflow packages
-kfp==1.0.4
-kfp-server-api==1.0.4
-kfserving==0.4.1
+kfp==1.6.3
+kfp-server-api==1.6.0
+kfserving==0.5.1
 
 # scipy packages
 # https://github.com/jupyter/docker-stacks/blob/master/scipy-notebook/Dockerfile
 beautifulsoup4==4.9.3
-bokeh==2.3.1
+bokeh==2.3.2
 #Bottleneck==1.3.2 Could not build wheels for Bottleneck which use PEP 517 and cannot be installed directly
 cloudpickle==1.6.0
-cython==0.29.22
-dask==2021.4.0
-dill==0.3.3
+cython==0.29.23
+dask==2021.6.0
+dill==0.3.4
 h5py==3.2.1
 ipympl==0.7.0
 ipywidgets==7.6.3
-matplotlib==3.4.1
+matplotlib==3.4.2
 numba==0.53.1
 numexpr==2.7.3
-pandas==1.2.3
+pandas==1.2.4
 patsy==0.5.1
-protobuf==3.15.7
+protobuf==3.17.3
 scikit-image==0.18.1
-scikit-learn==0.24.1
-scipy==1.6.2
+scikit-learn==0.24.2
+scipy==1.6.3
 seaborn==0.11.1
-SQLAlchemy==1.4.5
+SQLAlchemy==1.4.18
 statsmodels==0.12.2
-sympy==1.7.1
+sympy==1.8
 tables==3.6.1
 vincent==0.4.4
 xlrd==2.0.1

components/example-notebook-servers/jupyter-tensorflow/cpu-requirements.txt

@@ -0,0 +1 @@
+tensorflow==2.5.0

components/example-notebook-servers/jupyter-tensorflow/cpu.Dockerfile

@@ -1,6 +1,6 @@
-FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter:master-ebc0c4f0
+FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter:master-434b10ab
 
 # install - requirements.txt
-COPY --chown=jovyan:users requirements-cpu.txt /tmp/requirements.txt
+COPY --chown=jovyan:users cpu-requirements.txt /tmp/requirements.txt
 RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
  && rm -f /tmp/requirements.txt

components/example-notebook-servers/jupyter-tensorflow/cuda-requirements.txt

@@ -0,0 +1 @@
+tensorflow-gpu==2.5.0

components/example-notebook-servers/jupyter-tensorflow/cuda.Dockerfile

@@ -1,13 +1,15 @@
-FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter:master-ebc0c4f0
+FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter:master-434b10ab
 
 USER root
 
+# needed for LIBNVINFER
+ARG OLD_CUDA_VERSION=11.1
 # args - software versions
-ARG CUDA_VERSION=11.1
-ARG CUDA_COMPART_VERSION=455.45.01-1
-ARG CUDA_CUDART_VERSION=11.1.74-1
-ARG CUDNN_VERSION=8.0.5.39-1
-ARG LIBNVINFER_VERSION=7.2.2-1
+ARG CUDA_VERSION=11.2
+ARG CUDA_COMPAT_VERSION=460.73.01-1
+ARG CUDA_CUDART_VERSION=11.2.152-1
+ARG CUDNN_VERSION=8.1.0.77-1
+ARG LIBNVINFER_VERSION=7.2.3-1
 
 # we need bash's env var character substitution
 SHELL ["/bin/bash", "-c"]
@@ -18,7 +20,7 @@ RUN curl -sL "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu200
  && echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /" > /etc/apt/sources.list.d/cuda.list \
  && apt-get -yq update \
  && apt-get -yq install --no-install-recommends \
-    cuda-compat-${CUDA_VERSION/./-}=${CUDA_COMPART_VERSION} \
+    cuda-compat-${CUDA_VERSION/./-}=${CUDA_COMPAT_VERSION} \
     cuda-cudart-${CUDA_VERSION/./-}=${CUDA_CUDART_VERSION} \
  && apt-get clean \
  && rm -rf /var/lib/apt/lists/* \
@@ -48,12 +50,12 @@ RUN curl -sL "https://developer.download.nvidia.com/compute/machine-learning/rep
     libcusparse-${CUDA_VERSION/./-} \
     libfreetype6-dev \
     libhdf5-serial-dev \
-    libnvinfer7=${LIBNVINFER_VERSION}+cuda${CUDA_VERSION} \
-    libnvinfer-plugin7=${LIBNVINFER_VERSION}+cuda${CUDA_VERSION} \
+    libnvinfer7=${LIBNVINFER_VERSION}+cuda${OLD_CUDA_VERSION} \
+    libnvinfer-plugin7=${LIBNVINFER_VERSION}+cuda${OLD_CUDA_VERSION} \
     libzmq3-dev \
     pkg-config \
     # can't be used until NVIDIA updates (requires python < 3.7)
-    # pthon3-libnvinfer=${LIBNVINFER_VERSION}+cuda${CUDA_VERSION} \
+    # python3-libnvinfer=${LIBNVINFER_VERSION}+cuda${CUDA_VERSION} \
  && apt-get clean \
  && rm -rf /var/lib/apt/lists/*
 
@@ -75,6 +77,6 @@ RUN ln -s $(which python3) /usr/local/bin/python
 USER $NB_UID
 
 # install - requirements.txt
-COPY --chown=jovyan:users requirements-cuda.txt /tmp/requirements.txt
+COPY --chown=jovyan:users cuda-requirements.txt /tmp/requirements.txt
 RUN python3 -m pip install -r /tmp/requirements.txt --quiet --no-cache-dir \
  && rm -f /tmp/requirements.txt

components/example-notebook-servers/jupyter/Dockerfile

@@ -3,8 +3,6 @@ FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/base:master-6c41e72b
 USER root
 
 # args - software versions
- # renovate: datasource=github-tags depName=conda/conda versioning=semver
-ARG CONDA_VERSION=4.10.1
 ARG MINIFORGE_ARCH="x86_64"
  # renovate: datasource=github-tags depName=conda-forge/miniforge versioning=loose
 ARG MINIFORGE_VERSION=4.10.1-4
@@ -44,11 +42,11 @@ RUN curl -sL "https://github.com/conda-forge/miniforge/releases/download/${MINIF
  && rm /tmp/Miniforge3.sh \
  && conda config --system --set auto_update_conda false \
  && conda config --system --set show_channel_urls true \
- && echo "conda ${CONDA_VERSION}" >> ${CONDA_DIR}/conda-meta/pinned \
+ && echo "conda ${MINIFORGE_VERSION:0:-2}" >> ${CONDA_DIR}/conda-meta/pinned \
  && echo "python ${PYTHON_VERSION}" >> ${CONDA_DIR}/conda-meta/pinned \
  && conda install -y -q \
     python=${PYTHON_VERSION} \
-    conda=${CONDA_VERSION} \
+    conda=${MINIFORGE_VERSION:0:-2} \
     pip=${PIP_VERSION} \
  && conda update -y -q --all \
  && conda clean -a -f -y \

components/example-notebook-servers/rstudio-tidyverse/Dockerfile

@@ -1,7 +1,7 @@
-FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/rstudio:master-ebc0c4f0
+FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/rstudio:master-434b10ab
 
 # args - software versions
-ARG R_TIDYVERSE_VERSION="1.3.0"
+ARG R_TIDYVERSE_VERSION="1.3.1"
 
 # switch to root user for conda installation
 # (only necessary when installing r-tidyverse with Kaniko)

components/example-notebook-servers/rstudio/Dockerfile

@@ -3,8 +3,6 @@ FROM public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/base:master-6c41e72b
 USER root
 
 # args - software versions
- # renovate: datasource=github-tags depName=conda/conda versioning=semver
-ARG CONDA_VERSION=4.10.1
 ARG MINIFORGE_ARCH="x86_64"
  # renovate: datasource=github-tags depName=conda-forge/miniforge versioning=loose
 ARG MINIFORGE_VERSION=4.10.1-4
@@ -57,12 +55,12 @@ RUN curl -sL "https://github.com/conda-forge/miniforge/releases/download/${MINIF
  && rm /tmp/Miniforge3.sh \
  && conda config --system --set auto_update_conda false \
  && conda config --system --set show_channel_urls true \
- && echo "conda ${CONDA_VERSION}" >> ${CONDA_DIR}/conda-meta/pinned \
+ && echo "conda ${MINIFORGE_VERSION:0:-2}" >> ${CONDA_DIR}/conda-meta/pinned \
  && echo "python ${PYTHON_VERSION}" >> ${CONDA_DIR}/conda-meta/pinned \
  && echo "r-base ${R_BASE_VERSION}" >> ${CONDA_DIR}/conda-meta/pinned \
  && conda install -y -q \
     python=${PYTHON_VERSION} \
-    conda=${CONDA_VERSION} \
+    conda=${MINIFORGE_VERSION:0:-2} \
     pip=${PIP_VERSION} \
     r-base=${R_BASE_VERSION} \
     r-reticulate=${R_RETICULATE_VERSION} \

py/kubeflow/kubeflow/cd/notebook_servers/notebook_server_jupyter_pytorch.py

@@ -14,4 +14,5 @@ def create_workflow(name=None, namespace=None, bucket=None, **kwargs):
                          context="components/example-notebook-servers/jupyter-pytorch/",
                          destination=config.NOTEBOOK_SERVER_JUPYTER_PYTORCH,
                          second_dockerfile="components/example-notebook-servers/jupyter-pytorch/cuda.Dockerfile",
-                         second_destination=config.NOTEBOOK_SERVER_JUPYTER_PYTORCH_CUDA)
+                         second_destination=config.NOTEBOOK_SERVER_JUPYTER_PYTORCH_CUDA,
+                         mem_override="8Gi")

py/kubeflow/kubeflow/cd/notebook_servers/notebook_server_jupyter_pytorch_full.py

@@ -14,4 +14,5 @@ def create_workflow(name=None, namespace=None, bucket=None, **kwargs):
                          context="components/example-notebook-servers/jupyter-pytorch-full/",
                          destination=config.NOTEBOOK_SERVER_JUPYTER_PYTORCH_FULL,
                          second_dockerfile="components/example-notebook-servers/jupyter-pytorch-full/cuda.Dockerfile",
-                         second_destination=config.NOTEBOOK_SERVER_JUPYTER_PYTORCH_CUDA_FULL)
+                         second_destination=config.NOTEBOOK_SERVER_JUPYTER_PYTORCH_CUDA_FULL,
+                         mem_override="8Gi")

py/kubeflow/kubeflow/ci/notebook_servers/notebook_server_jupyter_pytorch_full_tests.py

@@ -12,7 +12,7 @@ def __init__(self, name=None, namespace=None, bucket=None,
     def build(self):
         """Build the Argo workflow graph"""
         workflow = self.build_init_workflow(exit_dag=False)
-        task_template = self.build_task_template()
+        task_template = self.build_task_template(mem_override="8Gi")
 
         # Test building notebook-server-jupyter-pytorch-full images using Kaniko
         dockerfile = ("%s/components/example-notebook-servers"

py/kubeflow/kubeflow/ci/notebook_servers/notebook_server_jupyter_pytorch_tests.py

@@ -12,7 +12,7 @@ def __init__(self, name=None, namespace=None, bucket=None,
     def build(self):
         """Build the Argo workflow graph"""
         workflow = self.build_init_workflow(exit_dag=False)
-        task_template = self.build_task_template()
+        task_template = self.build_task_template(mem_override="8Gi")
 
         # Test building notebook-server-jupyter-pytorch images using Kaniko
         dockerfile = ("%s/components/example-notebook-servers"