From 80fc33bca7ee110de604a6fca7f8c74432879cf2 Mon Sep 17 00:00:00 2001 From: Meghna Baijal Date: Wed, 3 Aug 2022 00:02:51 -0700 Subject: [PATCH 1/6] Update TF and other versions --- .../pytorch/requirements.txt | 18 +++++++++--------- .../tensorflow/cpu.Dockerfile | 2 +- .../tensorflow/cuda.Dockerfile | 2 +- .../tensorflow/requirements.txt | 18 +++++++++--------- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/components/notebook-dockerfiles/pytorch/requirements.txt b/components/notebook-dockerfiles/pytorch/requirements.txt index 0540cca051..ed380e3154 100644 --- a/components/notebook-dockerfiles/pytorch/requirements.txt +++ b/components/notebook-dockerfiles/pytorch/requirements.txt @@ -2,31 +2,31 @@ # Jupyter related packages jupyter-console==6.4.4 -jupyterlab==3.4.3 -jupyterlab-git==0.37.1 +jupyterlab==3.4.4 +jupyterlab-git==0.38.0 notebook==6.4.12 -ipykernel==6.15.0 +ipykernel==6.15.1 # Extensions for jupyter notebook widgetsnbextension==3.6.1 ipywidgets==7.7.1 # Kubeflow Related Packages -kfp==1.8.12 -kfp-server-api==1.8.2 +kfp==1.8.13 +kfp-server-api==1.8.3 kfserving==0.6.1 kubeflow-training==1.4.0 # Common data science packages h5py==3.7.0 -pandas==1.4.2 +pandas==1.4.3 matplotlib==3.5.2 xgboost==1.6.1 ipympl==0.9.1 # AWS related packages -awscli==1.25.15 -boto3==1.24.15 +awscli==1.25.44 +boto3==1.24.44 -# pytorch packages +# pytorch packages, latest version reinstalls Pytorch. fastai==2.7.3 \ No newline at end of file diff --git a/components/notebook-dockerfiles/tensorflow/cpu.Dockerfile b/components/notebook-dockerfiles/tensorflow/cpu.Dockerfile index f8b99b660b..fa56e752e7 100644 --- a/components/notebook-dockerfiles/tensorflow/cpu.Dockerfile +++ b/components/notebook-dockerfiles/tensorflow/cpu.Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=763104351884.dkr.ecr.us-west-2.amazonaws.com/tensorflow-training:2.6.3-cpu-py38-ubuntu20.04-v1.8-2022-06-23-16-58-08 +ARG BASE_IMAGE=763104351884.dkr.ecr.us-west-2.amazonaws.com/tensorflow-training:2.9.1-cpu-py39-ubuntu20.04-e3-v1.2 FROM $BASE_IMAGE diff --git a/components/notebook-dockerfiles/tensorflow/cuda.Dockerfile b/components/notebook-dockerfiles/tensorflow/cuda.Dockerfile index 25441cfa2c..f0d858cd4e 100644 --- a/components/notebook-dockerfiles/tensorflow/cuda.Dockerfile +++ b/components/notebook-dockerfiles/tensorflow/cuda.Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=763104351884.dkr.ecr.us-west-2.amazonaws.com/tensorflow-training:2.6.3-gpu-py38-cu112-ubuntu20.04-v1.8-2022-06-23-16-57-41 +ARG BASE_IMAGE=763104351884.dkr.ecr.us-west-2.amazonaws.com/tensorflow-training:2.9.1-gpu-py39-cu112-ubuntu20.04-e3-v1.2 FROM $BASE_IMAGE diff --git a/components/notebook-dockerfiles/tensorflow/requirements.txt b/components/notebook-dockerfiles/tensorflow/requirements.txt index 1efbe35f0b..576c87da93 100644 --- a/components/notebook-dockerfiles/tensorflow/requirements.txt +++ b/components/notebook-dockerfiles/tensorflow/requirements.txt @@ -2,31 +2,31 @@ # Jupyter related packages jupyter-console==6.4.4 -jupyterlab==3.4.3 -jupyterlab-git==0.37.1 +jupyterlab==3.4.4 +jupyterlab-git==0.38.0 notebook==6.4.12 -ipykernel==6.15.0 +ipykernel==6.15.1 # Extensions for jupyter notebook widgetsnbextension==3.6.1 ipywidgets==7.7.1 # Kubeflow Related Packages -kfp==1.8.12 -kfp-server-api==1.8.2 +kfp==1.8.13 +kfp-server-api==1.8.3 kfserving==0.6.1 kubeflow-training==1.4.0 # Common data science packages h5py==3.7.0 -pandas==1.4.2 +pandas==1.4.3 matplotlib==3.5.2 xgboost==1.6.1 ipympl==0.9.1 # AWS related packages -awscli==1.25.15 -boto3==1.24.15 +awscli==1.25.44 +boto3==1.24.44 # TF Packages -keras==2.6.0 \ No newline at end of file +keras==2.9.0 \ No newline at end of file From f8f8006de9a4f06006db5c83603d11d9402a140f Mon Sep 17 00:00:00 2001 From: Meghna Baijal Date: Wed, 14 Sep 2022 13:12:23 -0700 Subject: [PATCH 2/6] Update Notebook Images for 1.6 --- .../apps/jupyter-web-app/configs/spawner_ui_config.yaml | 6 +++--- components/notebook-dockerfiles/pytorch/requirements.txt | 1 + .../notebook-dockerfiles/tensorflow/requirements.txt | 1 + tests/e2e/tests/test_notebook_images.py | 8 ++++---- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/awsconfigs/apps/jupyter-web-app/configs/spawner_ui_config.yaml b/awsconfigs/apps/jupyter-web-app/configs/spawner_ui_config.yaml index 76dde7c196..e87a71a315 100644 --- a/awsconfigs/apps/jupyter-web-app/configs/spawner_ui_config.yaml +++ b/awsconfigs/apps/jupyter-web-app/configs/spawner_ui_config.yaml @@ -20,9 +20,9 @@ spawnerFormDefaults: value: public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-tensorflow:2.6.3-cpu-py38-ubuntu20.04-v1.8 # The list of available standard container Images options: - - kubeflownotebookswg/jupyter-scipy:v1.6.0 - - public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-tensorflow:2.6.3-gpu-py38-cu112-ubuntu20.04-v1.8 - - public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-tensorflow:2.6.3-cpu-py38-ubuntu20.04-v1.8 + - public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter-scipy:v1.5.0 + - public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-tensorflow:2.9.1-gpu-py39-cu112-ubuntu20.04-e3-v1.2 + - public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-tensorflow:2.9.1-cpu-py39-ubuntu20.04-e3-v1.2 - public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-pytorch:1.11.0-gpu-py38-cu115-ubuntu20.04-e3-v1.1 - public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-pytorch:1.11.0-cpu-py38-ubuntu20.04-e3-v1.1 imageGroupOne: diff --git a/components/notebook-dockerfiles/pytorch/requirements.txt b/components/notebook-dockerfiles/pytorch/requirements.txt index ed380e3154..3cd376aef2 100644 --- a/components/notebook-dockerfiles/pytorch/requirements.txt +++ b/components/notebook-dockerfiles/pytorch/requirements.txt @@ -16,6 +16,7 @@ kfp==1.8.13 kfp-server-api==1.8.3 kfserving==0.6.1 kubeflow-training==1.4.0 +kubeflow-katib==0.14.0 # Common data science packages h5py==3.7.0 diff --git a/components/notebook-dockerfiles/tensorflow/requirements.txt b/components/notebook-dockerfiles/tensorflow/requirements.txt index 576c87da93..4c2cc6bab4 100644 --- a/components/notebook-dockerfiles/tensorflow/requirements.txt +++ b/components/notebook-dockerfiles/tensorflow/requirements.txt @@ -16,6 +16,7 @@ kfp==1.8.13 kfp-server-api==1.8.3 kfserving==0.6.1 kubeflow-training==1.4.0 +kubeflow-katib==0.14.0 # Common data science packages h5py==3.7.0 diff --git a/tests/e2e/tests/test_notebook_images.py b/tests/e2e/tests/test_notebook_images.py index 350076c210..d125048855 100644 --- a/tests/e2e/tests/test_notebook_images.py +++ b/tests/e2e/tests/test_notebook_images.py @@ -24,16 +24,16 @@ NOTEBOOK_IMAGES = [ "public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter-scipy:v1.5.0", - "public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-tensorflow:2.6.3-gpu-py38-cu112-ubuntu20.04-v1.8", - "public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-tensorflow:2.6.3-cpu-py38-ubuntu20.04-v1.8", + "public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-tensorflow:2.9.1-gpu-py39-cu112-ubuntu20.04-e3-v1.2", + "public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-tensorflow:2.9.1-cpu-py39-ubuntu20.04-e3-v1.2", "public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-pytorch:1.11.0-gpu-py38-cu115-ubuntu20.04-e3-v1.1", "public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-pytorch:1.11.0-cpu-py38-ubuntu20.04-e3-v1.1", ] testdata = [ ("scipy", NOTEBOOK_IMAGES[0], "sanity_check.ipynb", "Hello World!"), - ("tf-gpu", NOTEBOOK_IMAGES[1], "verify_tensorflow_installation.ipynb", "2.6.3"), - ("tf-cpu", NOTEBOOK_IMAGES[2], "verify_tensorflow_installation.ipynb", "2.6.3"), + ("tf-gpu", NOTEBOOK_IMAGES[1], "verify_tensorflow_installation.ipynb", "2.9.1"), + ("tf-cpu", NOTEBOOK_IMAGES[2], "verify_tensorflow_installation.ipynb", "2.9.1"), ( "pytorch-gpu", NOTEBOOK_IMAGES[3], From a8c65e4ddf389f733d23e644765206f6c5191719 Mon Sep 17 00:00:00 2001 From: Meghna Baijal Date: Wed, 14 Sep 2022 13:32:29 -0700 Subject: [PATCH 3/6] Fix protobuf dependency conflict --- components/notebook-dockerfiles/tensorflow/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/notebook-dockerfiles/tensorflow/requirements.txt b/components/notebook-dockerfiles/tensorflow/requirements.txt index 4c2cc6bab4..9a65de3c74 100644 --- a/components/notebook-dockerfiles/tensorflow/requirements.txt +++ b/components/notebook-dockerfiles/tensorflow/requirements.txt @@ -30,4 +30,5 @@ awscli==1.25.44 boto3==1.24.44 # TF Packages -keras==2.9.0 \ No newline at end of file +keras==2.9.0 +protobuf==3.19.5 \ No newline at end of file From a41c7e95066fff59af6bac6797fe8f1ffc78317a Mon Sep 17 00:00:00 2001 From: Meghna Baijal Date: Wed, 14 Sep 2022 14:48:20 -0700 Subject: [PATCH 4/6] Update the scipy image and default TF version --- .../apps/jupyter-web-app/configs/spawner_ui_config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/awsconfigs/apps/jupyter-web-app/configs/spawner_ui_config.yaml b/awsconfigs/apps/jupyter-web-app/configs/spawner_ui_config.yaml index e87a71a315..18d29199cc 100644 --- a/awsconfigs/apps/jupyter-web-app/configs/spawner_ui_config.yaml +++ b/awsconfigs/apps/jupyter-web-app/configs/spawner_ui_config.yaml @@ -17,10 +17,10 @@ spawnerFormDefaults: image: # The container Image for the user's Jupyter Notebook - value: public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-tensorflow:2.6.3-cpu-py38-ubuntu20.04-v1.8 + value: public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-tensorflow:2.9.1-cpu-py39-ubuntu20.04-e3-v1.2 # The list of available standard container Images options: - - public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter-scipy:v1.5.0 + - public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter-scipy:v1.6.0 - public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-tensorflow:2.9.1-gpu-py39-cu112-ubuntu20.04-e3-v1.2 - public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-tensorflow:2.9.1-cpu-py39-ubuntu20.04-e3-v1.2 - public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-pytorch:1.11.0-gpu-py38-cu115-ubuntu20.04-e3-v1.1 From cd4cc155a6287e155db75f9505c582a9d5d6b717 Mon Sep 17 00:00:00 2001 From: Meghna Baijal Date: Wed, 14 Sep 2022 23:12:11 -0700 Subject: [PATCH 5/6] Update Pytorch to 1.12 --- .../apps/jupyter-web-app/configs/spawner_ui_config.yaml | 4 ++-- components/notebook-dockerfiles/pytorch/cpu.Dockerfile | 2 +- components/notebook-dockerfiles/pytorch/cuda.Dockerfile | 2 +- tests/e2e/tests/test_notebook_images.py | 8 ++++---- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/awsconfigs/apps/jupyter-web-app/configs/spawner_ui_config.yaml b/awsconfigs/apps/jupyter-web-app/configs/spawner_ui_config.yaml index 18d29199cc..0e852f355d 100644 --- a/awsconfigs/apps/jupyter-web-app/configs/spawner_ui_config.yaml +++ b/awsconfigs/apps/jupyter-web-app/configs/spawner_ui_config.yaml @@ -23,8 +23,8 @@ spawnerFormDefaults: - public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter-scipy:v1.6.0 - public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-tensorflow:2.9.1-gpu-py39-cu112-ubuntu20.04-e3-v1.2 - public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-tensorflow:2.9.1-cpu-py39-ubuntu20.04-e3-v1.2 - - public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-pytorch:1.11.0-gpu-py38-cu115-ubuntu20.04-e3-v1.1 - - public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-pytorch:1.11.0-cpu-py38-ubuntu20.04-e3-v1.1 + - public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-pytorch:1.12.0-gpu-py38-cu116-ubuntu20.04-ec2 + - public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-pytorch:1.12.0-cpu-py38-ubuntu20.04-ec2 imageGroupOne: # The container Image for the user's Group One Server # The annotation `notebooks.kubeflow.org/http-rewrite-uri: /` diff --git a/components/notebook-dockerfiles/pytorch/cpu.Dockerfile b/components/notebook-dockerfiles/pytorch/cpu.Dockerfile index 6f9287a845..4e8dd4a2f7 100644 --- a/components/notebook-dockerfiles/pytorch/cpu.Dockerfile +++ b/components/notebook-dockerfiles/pytorch/cpu.Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training:1.11.0-cpu-py38-ubuntu20.04-e3-v1.1 +ARG BASE_IMAGE=763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training:1.12.0-cpu-py38-ubuntu20.04-ec2 FROM $BASE_IMAGE ARG NB_USER=jovyan diff --git a/components/notebook-dockerfiles/pytorch/cuda.Dockerfile b/components/notebook-dockerfiles/pytorch/cuda.Dockerfile index ed4b095943..9187c1781b 100644 --- a/components/notebook-dockerfiles/pytorch/cuda.Dockerfile +++ b/components/notebook-dockerfiles/pytorch/cuda.Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training:1.11.0-gpu-py38-cu115-ubuntu20.04-e3-v1.1 +ARG BASE_IMAGE=763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training:1.12.0-gpu-py38-cu116-ubuntu20.04-ec2 FROM $BASE_IMAGE diff --git a/tests/e2e/tests/test_notebook_images.py b/tests/e2e/tests/test_notebook_images.py index d125048855..d18a6d96ac 100644 --- a/tests/e2e/tests/test_notebook_images.py +++ b/tests/e2e/tests/test_notebook_images.py @@ -26,8 +26,8 @@ "public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter-scipy:v1.5.0", "public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-tensorflow:2.9.1-gpu-py39-cu112-ubuntu20.04-e3-v1.2", "public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-tensorflow:2.9.1-cpu-py39-ubuntu20.04-e3-v1.2", - "public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-pytorch:1.11.0-gpu-py38-cu115-ubuntu20.04-e3-v1.1", - "public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-pytorch:1.11.0-cpu-py38-ubuntu20.04-e3-v1.1", + "public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-pytorch:1.12.0-gpu-py38-cu116-ubuntu20.04-ec2", + "public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-pytorch:1.12.0-cpu-py38-ubuntu20.04-ec2", ] testdata = [ @@ -38,13 +38,13 @@ "pytorch-gpu", NOTEBOOK_IMAGES[3], "verify_pytorch_installation.ipynb", - "1.11.0+cu115", + "1.12.0+cu116", ), ( "pytorch-cpu", NOTEBOOK_IMAGES[4], "verify_pytorch_installation.ipynb", - "1.11.0+cpu", + "1.12.0+cpu", ), ] From e52a889b8c599bfe651f8f905be0115872c7149c Mon Sep 17 00:00:00 2001 From: Meghna Baijal Date: Thu, 15 Sep 2022 17:59:03 -0700 Subject: [PATCH 6/6] fix for pytorch 1.12 and scipy --- .../apps/jupyter-web-app/configs/spawner_ui_config.yaml | 2 +- components/notebook-dockerfiles/pytorch/requirements.txt | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/awsconfigs/apps/jupyter-web-app/configs/spawner_ui_config.yaml b/awsconfigs/apps/jupyter-web-app/configs/spawner_ui_config.yaml index 0e852f355d..f7cd025f4d 100644 --- a/awsconfigs/apps/jupyter-web-app/configs/spawner_ui_config.yaml +++ b/awsconfigs/apps/jupyter-web-app/configs/spawner_ui_config.yaml @@ -20,7 +20,7 @@ spawnerFormDefaults: value: public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-tensorflow:2.9.1-cpu-py39-ubuntu20.04-e3-v1.2 # The list of available standard container Images options: - - public.ecr.aws/j1r0q0g6/notebooks/notebook-servers/jupyter-scipy:v1.6.0 + - kubeflownotebookswg/jupyter-scipy:v1.6.0 - public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-tensorflow:2.9.1-gpu-py39-cu112-ubuntu20.04-e3-v1.2 - public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-tensorflow:2.9.1-cpu-py39-ubuntu20.04-e3-v1.2 - public.ecr.aws/c9e4w0g3/notebook-servers/jupyter-pytorch:1.12.0-gpu-py38-cu116-ubuntu20.04-ec2 diff --git a/components/notebook-dockerfiles/pytorch/requirements.txt b/components/notebook-dockerfiles/pytorch/requirements.txt index 3cd376aef2..8f1c38698c 100644 --- a/components/notebook-dockerfiles/pytorch/requirements.txt +++ b/components/notebook-dockerfiles/pytorch/requirements.txt @@ -29,5 +29,6 @@ ipympl==0.9.1 awscli==1.25.44 boto3==1.24.44 -# pytorch packages, latest version reinstalls Pytorch. -fastai==2.7.3 \ No newline at end of file +# Pytorch packages +# a version mismatch for fastai can cause a different version of torch to get installed, be careful. +fastai==2.7.9 \ No newline at end of file