Bump Python version to 3.9 (#1731)

* bump Python to 3.9

* modify script to build container image

* fix example for enas

* update scripts to modify image name in ci

* review: change docker build command

* review: use new tf-mnist-with-example in Ci for tfjob

* review: refactor tf-mnist-with-summaries

* review: remove Dockerfile.ppc64le for new-ui

* review: update docs related tf-mnist-with-summaries

* TFEventMetricsCollector supports TF>=2.0 and stop supporting TF <=1.x

* review: add help command to scripts/v1beta1/build.sh

Co-authored-by: Andrey Velichkevich <andrey.velichkevich@gmail.com>

* fix unit test for tfevent-metricscollector

* review: generate tf event files on CI

* add test command to Makefile

* update publish-trial-images

* update update-images.sh

* reduce batch size

Co-authored-by: Andrey Velichkevich <andrey.velichkevich@gmail.com>

.github/workflows/publish-trial-images.yaml

@@ -32,6 +32,8 @@ jobs:
             dockerfile: examples/v1beta1/trial-images/mxnet-mnist/Dockerfile
           - trial-name: pytorch-mnist
             dockerfile: examples/v1beta1/trial-images/pytorch-mnist/Dockerfile
+          - trial-name: tf-mnist-with-summaries
+            dockerfile: examples/v1beta1/trial-images/tf-mnist-with-summaries/Dockerfile
           - trial-name: enas-cnn-cifar10-gpu
             dockerfile: examples/v1beta1/trial-images/enas-cnn-cifar10/Dockerfile.gpu
           - trial-name: enas-cnn-cifar10-cpu

.github/workflows/test-python.yaml

@@ -16,24 +16,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.7
-
-      - name: Install Packages
-        run: |
-          pip install -r test/unit/v1beta1/requirements.txt
-
-          pip install -r cmd/suggestion/chocolate/v1beta1/requirements.txt
-          pip install -r cmd/suggestion/hyperopt/v1beta1/requirements.txt
-          pip install -r cmd/suggestion/skopt/v1beta1/requirements.txt
-          pip install -r cmd/suggestion/optuna/v1beta1/requirements.txt
-          pip install -r cmd/suggestion/nas/enas/v1beta1/requirements.txt
-          pip install -r cmd/suggestion/hyperband/v1beta1/requirements.txt
-          pip install -r cmd/suggestion/nas/darts/v1beta1/requirements.txt
-
-          pip install -r cmd/earlystopping/medianstop/v1beta1/requirements.txt
+          python-version: 3.9
 
       - name: Run Python test
-        run: |
-          export PYTHONPATH=$(pwd):$(pwd)/pkg/apis/manager/v1beta1/python:$(pwd)/pkg/apis/manager/health/python
-          pytest ./test/unit/v1beta1/suggestion
-          pytest ./test/unit/v1beta1/earlystopping
+        run: make pytest

.gitignore

@@ -6,6 +6,7 @@ __pycache__/
 *.egg-info
 build/
 *.charm
+test/unit/v1beta1/metricscollector/testdata
 
 # SDK generator JAR file
 hack/gen-python-sdk/openapi-generator-cli.jar

Makefile

@@ -1,6 +1,12 @@
 HAS_LINT := $(shell command -v golangci-lint;)
 COMMIT := v1beta1-$(shell git rev-parse --short=7 HEAD)
 KATIB_REGISTRY := docker.io/kubeflowkatib
+CPU_ARCH ?= amd64
+
+# for pytest
+PYTHONPATH := $(PYTHONPATH):$(CURDIR)/pkg/apis/manager/v1beta1/python:$(CURDIR)/pkg/apis/manager/health/python
+PYTHONPATH := $(PYTHONPATH):$(CURDIR)/pkg/metricscollector/v1beta1/common:$(CURDIR)/pkg/metricscollector/v1beta1/tfevent-metricscollector
+TEST_TENSORFLOW_EVENT_FILE_PATH ?= $(CURDIR)/test/unit/v1beta1/metricscollector/testdata/tfevent-metricscollector/logs
 
 # Run tests
 .PHONY: test
@@ -49,10 +55,10 @@ endif
 
 # Build images for the Katib v1beta1 components.
 build: generate
-ifeq ($(and $(REGISTRY),$(TAG)),)
-	$(error REGISTRY and TAG must be set. Usage: make build REGISTRY=<registry> TAG=<tag>)
+ifeq ($(and $(REGISTRY),$(TAG),$(CPU_ARCH)),)
+	$(error REGISTRY and TAG must be set. Usage: make build REGISTRY=<registry> TAG=<tag> CPU_ARCH=<cpu-architecture>)
 endif
-	bash scripts/v1beta1/build.sh $(REGISTRY) $(TAG)
+	bash scripts/v1beta1/build.sh $(REGISTRY) $(TAG) $(CPU_ARCH)
 
 # Build and push Katib images from the latest master commit.
 push-latest: generate
@@ -94,3 +100,25 @@ prettier-check:
 # Update boilerplate for the source code.
 update-boilerplate:
 	./hack/boilerplate/update-boilerplate.sh
+
+prepare-pytest:
+	pip install -r test/unit/v1beta1/requirements.txt
+	pip install -r cmd/suggestion/chocolate/v1beta1/requirements.txt
+	pip install -r cmd/suggestion/hyperopt/v1beta1/requirements.txt
+	pip install -r cmd/suggestion/skopt/v1beta1/requirements.txt
+	pip install -r cmd/suggestion/optuna/v1beta1/requirements.txt
+	pip install -r cmd/suggestion/hyperband/v1beta1/requirements.txt
+	pip install -r cmd/suggestion/nas/enas/v1beta1/requirements.txt
+	pip install -r cmd/suggestion/nas/darts/v1beta1/requirements.txt
+	pip install -r cmd/earlystopping/medianstop/v1beta1/requirements.txt
+	pip install -r cmd/metricscollector/v1beta1/tfevent-metricscollector/requirements.txt
+
+prepare-pytest-testdata:
+ifeq ("$(wildcard $(TEST_TENSORFLOW_EVENT_FILE_PATH))", "")
+	python examples/v1beta1/trial-images/tf-mnist-with-summaries/mnist.py --epochs 5 --batch-size 200 --log-path $(TEST_TENSORFLOW_EVENT_FILE_PATH)
+endif
+
+pytest: prepare-pytest prepare-pytest-testdata
+	PYTHONPATH=$(PYTHONPATH) pytest ./test/unit/v1beta1/suggestion
+	PYTHONPATH=$(PYTHONPATH) pytest ./test/unit/v1beta1/earlystopping
+	PYTHONPATH=$(PYTHONPATH) pytest ./test/unit/v1beta1/metricscollector

cmd/earlystopping/medianstop/v1beta1/Dockerfile

@@ -1,12 +1,13 @@
-FROM python:3.6
+FROM python:3.9
 
 ENV TARGET_DIR /opt/katib
 ENV EARLY_STOPPING_DIR cmd/earlystopping/medianstop/v1beta1
 
 RUN if [ "$(uname -m)" = "ppc64le" ] || [ "$(uname -m)" = "aarch64" ]; then \
-  apt-get -y update && \
-  apt-get -y install gfortran libopenblas-dev liblapack-dev && \
-  pip install cython; \
+    apt-get -y update && \
+    apt-get -y install gfortran libopenblas-dev liblapack-dev && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*; \
   fi
 
 ADD ./pkg/ ${TARGET_DIR}/pkg/

cmd/earlystopping/medianstop/v1beta1/requirements.txt

@@ -1,4 +1,5 @@
-grpcio==1.23.0
-protobuf==3.9.1
+grpcio==1.41.1
+protobuf==3.19.1
 googleapis-common-protos==1.6.0
 kubernetes==11.0.0
+cython>=0.29.24

cmd/metricscollector/v1beta1/tfevent-metricscollector/Dockerfile

@@ -1,7 +1,25 @@
-FROM tensorflow/tensorflow:1.11.0
-RUN pip install rfc3339 grpcio googleapis-common-protos
-ADD . /usr/src/app/github.com/kubeflow/katib
-WORKDIR /usr/src/app/github.com/kubeflow/katib/cmd/metricscollector/v1beta1/tfevent-metricscollector/
+FROM python:3.9
+
+ENV TARGET_DIR /opt/katib
+ENV METRICS_COLLECTOR_DIR cmd/metricscollector/v1beta1/tfevent-metricscollector
+# tensorflow community build for aarch64
+# https://github.com/tensorflow/build#tensorflow-builds
+ENV PIP_EXTRA_INDEX_URL https://snapshots.linaro.org/ldcg/python-cache/
+
+ADD ./pkg/ ${TARGET_DIR}/pkg/
+ADD ./${METRICS_COLLECTOR_DIR}/ ${TARGET_DIR}/${METRICS_COLLECTOR_DIR}/
+WORKDIR  ${TARGET_DIR}/${METRICS_COLLECTOR_DIR}
+
+RUN if [ "$(uname -m)" = "aarch64" ]; then \
+    pip install tensorflow-aarch64==2.7.0; \
+    else \
+    pip install tensorflow==2.7.0; \
+    fi;
 RUN pip install --no-cache-dir -r requirements.txt
-ENV PYTHONPATH /usr/src/app/github.com/kubeflow/katib:/usr/src/app/github.com/kubeflow/katib/pkg/apis/manager/v1beta1/python:/usr/src/app/github.com/kubeflow/katib/pkg/metricscollector/v1beta1/tfevent-metricscollector/:/usr/src/app/github.com/kubeflow/katib/pkg/metricscollector/v1beta1/common/
+
+RUN chgrp -R 0 ${TARGET_DIR} \
+  && chmod -R g+rwX ${TARGET_DIR}
+
+ENV PYTHONPATH ${TARGET_DIR}:${TARGET_DIR}/pkg/apis/manager/v1beta1/python:${TARGET_DIR}/pkg/metricscollector/v1beta1/tfevent-metricscollector/::${TARGET_DIR}/pkg/metricscollector/v1beta1/common/
+
 ENTRYPOINT ["python", "main.py"]

cmd/metricscollector/v1beta1/tfevent-metricscollector/Dockerfile.ppc64le

@@ -1,5 +1,4 @@
-FROM ibmcom/tensorflow-ppc64le:1.14.0-py3
-RUN pip install rfc3339 grpcio googleapis-common-protos
+FROM ibmcom/tensorflow-ppc64le:2.2.0-py3
 ADD . /usr/src/app/github.com/kubeflow/katib
 WORKDIR /usr/src/app/github.com/kubeflow/katib/cmd/metricscollector/v1beta1/tfevent-metricscollector/
 RUN pip install --no-cache-dir -r requirements.txt

cmd/metricscollector/v1beta1/tfevent-metricscollector/requirements.txt

@@ -1 +1,4 @@
-psutil==5.6.6
+psutil==5.8.0
+rfc3339>=6.2
+grpcio==1.41.1
+googleapis-common-protos==1.6.0

cmd/suggestion/chocolate/v1beta1/Dockerfile

@@ -1,15 +1,16 @@
-FROM python:3.6
+FROM python:3.9
 
 ENV TARGET_DIR /opt/katib
 ENV SUGGESTION_DIR cmd/suggestion/chocolate/v1beta1
+ENV GRPC_HEALTH_PROBE_VERSION v0.3.1
 
 RUN if [ "$(uname -m)" = "ppc64le" ] || [ "$(uname -m)" = "aarch64" ]; then \
     apt-get -y update && \
     apt-get -y install gfortran libopenblas-dev liblapack-dev && \
-    pip install cython 'numpy>=1.13.3'; \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*; \
     fi
-RUN GRPC_HEALTH_PROBE_VERSION=v0.3.1 && \
-    if [ "$(uname -m)" = "ppc64le" ]; then \
+RUN if [ "$(uname -m)" = "ppc64le" ]; then \
     wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-ppc64le; \
     elif [ "$(uname -m)" = "aarch64" ]; then \
     wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-arm64; \
@@ -21,6 +22,9 @@ RUN GRPC_HEALTH_PROBE_VERSION=v0.3.1 && \
 ADD ./pkg/ ${TARGET_DIR}/pkg/
 ADD ./${SUGGESTION_DIR}/ ${TARGET_DIR}/${SUGGESTION_DIR}/
 WORKDIR  ${TARGET_DIR}/${SUGGESTION_DIR}
+RUN if [ "$(uname -m)" = "aarch64" ]; then \
+    sed -i -e '$a git+https://github.com/fmder/ghalton@master' -e '/^ghalton/d' requirements.txt; \
+    fi;
 RUN pip install --no-cache-dir -r requirements.txt
 
 RUN chgrp -R 0 ${TARGET_DIR} \

cmd/suggestion/chocolate/v1beta1/requirements.txt

@@ -1,11 +1,12 @@
-grpcio==1.23.0
+grpcio==1.41.1
 cloudpickle==0.5.6
-numpy>=1.13.3
-scikit-learn>=0.19.0
-scipy>=0.19.1
+numpy>=1.20.0
+scikit-learn>=0.24.0
+scipy>=1.5.4
 forestci==0.3
-protobuf==3.9.1
+protobuf==3.19.1
 googleapis-common-protos==1.6.0
-SQLAlchemy==1.3.8
+SQLAlchemy==1.4.26
 git+https://github.com/AIworx-Labs/chocolate@master
-ghalton>=0.6
+ghalton>=0.6.2
+cython>=0.29.24

cmd/suggestion/goptuna/v1beta1/Dockerfile

@@ -1,6 +1,8 @@
 # Build the Goptuna Suggestion.
 FROM golang:alpine AS build-env
 
+ENV GRPC_HEALTH_PROBE_VERSION v0.3.1
+
 WORKDIR /go/src/github.com/kubeflow/katib
 
 # Download packages.
@@ -22,8 +24,7 @@ RUN if [ "$(uname -m)" = "ppc64le" ]; then \
   fi
 
 # Add GRPC health probe.
-RUN GRPC_HEALTH_PROBE_VERSION=v0.3.1 && \
-  if [ "$(uname -m)" = "ppc64le" ]; then \
+RUN if [ "$(uname -m)" = "ppc64le" ]; then \
   wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-ppc64le; \
   elif [ "$(uname -m)" = "aarch64" ]; then \
   wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-arm64; \

cmd/suggestion/hyperband/v1beta1/Dockerfile

@@ -1,16 +1,17 @@
-FROM python:3.6
+FROM python:3.9
 
 ENV TARGET_DIR /opt/katib
 ENV SUGGESTION_DIR cmd/suggestion/hyperband/v1beta1
+ENV GRPC_HEALTH_PROBE_VERSION v0.3.1
 
 RUN if [ "$(uname -m)" = "ppc64le" ] || [ "$(uname -m)" = "aarch64" ]; then \
     apt-get -y update && \
     apt-get -y install gfortran libopenblas-dev liblapack-dev && \
-    pip install cython; \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*; \
     fi
 
-RUN GRPC_HEALTH_PROBE_VERSION=v0.3.1 && \
-    if [ "$(uname -m)" = "ppc64le" ]; then \
+RUN if [ "$(uname -m)" = "ppc64le" ]; then \
     wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-ppc64le; \
     elif [ "$(uname -m)" = "aarch64" ]; then \
     wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-arm64; \

cmd/suggestion/hyperband/v1beta1/requirements.txt

@@ -1,8 +1,9 @@
-grpcio==1.23.0
+grpcio==1.41.1
 cloudpickle==0.5.6
-numpy>=1.13.3
-scikit-learn>=0.19.0
-scipy>=0.19.1
+numpy>=1.20.0
+scikit-learn>=0.24.0
+scipy>=1.5.4
 forestci==0.3
-protobuf==3.9.1
+protobuf==3.19.1
 googleapis-common-protos==1.6.0
+cython>=0.29.24

cmd/suggestion/hyperopt/v1beta1/Dockerfile

@@ -1,16 +1,17 @@
-FROM python:3.6
+FROM python:3.9
 
 ENV TARGET_DIR /opt/katib
 ENV SUGGESTION_DIR cmd/suggestion/hyperopt/v1beta1
+ENV GRPC_HEALTH_PROBE_VERSION v0.3.1
 
 RUN if [ "$(uname -m)" = "ppc64le" ] || [ "$(uname -m)" = "aarch64" ]; then \
     apt-get -y update && \
     apt-get -y install gfortran libopenblas-dev liblapack-dev && \
-    pip install cython; \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*; \
     fi
 
-RUN GRPC_HEALTH_PROBE_VERSION=v0.3.1 && \
-    if [ "$(uname -m)" = "ppc64le" ]; then \
+RUN if [ "$(uname -m)" = "ppc64le" ]; then \
     wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-ppc64le; \
     elif [ "$(uname -m)" = "aarch64" ]; then \
     wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-arm64; \

cmd/suggestion/hyperopt/v1beta1/requirements.txt

@@ -1,9 +1,10 @@
-grpcio==1.23.0
+grpcio==1.41.1
 cloudpickle==0.5.6
-numpy>=1.13.3
-scikit-learn>=0.19.0
-scipy>=0.19.1
+numpy>=1.20.0
+scikit-learn>=0.24.0
+scipy>=1.5.4
 forestci==0.3
-protobuf==3.9.1
+protobuf==3.19.1
 googleapis-common-protos==1.6.0
-hyperopt==0.2.3
+hyperopt==0.2.5
+cython>=0.29.24

cmd/suggestion/nas/darts/v1beta1/Dockerfile

@@ -1,16 +1,17 @@
-FROM python:3.6
+FROM python:3.9
 
 ENV TARGET_DIR /opt/katib
 ENV SUGGESTION_DIR cmd/suggestion/nas/darts/v1beta1
+ENV GRPC_HEALTH_PROBE_VERSION v0.3.1
 
 RUN if [ "$(uname -m)" = "ppc64le" ] || [ "$(uname -m)" = "aarch64" ]; then \
     apt-get -y update && \
     apt-get -y install gfortran libopenblas-dev liblapack-dev && \
-    pip install cython; \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*; \
     fi
 
-RUN GRPC_HEALTH_PROBE_VERSION=v0.3.1 && \
-    if [ "$(uname -m)" = "ppc64le" ]; then \
+RUN if [ "$(uname -m)" = "ppc64le" ]; then \
     wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-ppc64le; \
     elif [ "$(uname -m)" = "aarch64" ]; then \
     wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-arm64; \
@@ -30,4 +31,3 @@ RUN chgrp -R 0 ${TARGET_DIR} \
 ENV PYTHONPATH ${TARGET_DIR}:${TARGET_DIR}/pkg/apis/manager/v1beta1/python:${TARGET_DIR}/pkg/apis/manager/health/python
 
 ENTRYPOINT ["python", "main.py"]
-

cmd/suggestion/nas/darts/v1beta1/requirements.txt

@@ -1,3 +1,4 @@
-grpcio==1.23.0
-protobuf==3.9.1
+grpcio==1.41.1
+protobuf==3.19.1
 googleapis-common-protos==1.6.0
+cython>=0.29.24