bump Python to 3.9

.github/workflows/test-python.yaml

@@ -16,7 +16,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.7
+          python-version: 3.9
 
       - name: Install Packages
         run: |

cmd/earlystopping/medianstop/v1beta1/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.6
+FROM python:3.9
 
 ENV TARGET_DIR /opt/katib
 ENV EARLY_STOPPING_DIR cmd/earlystopping/medianstop/v1beta1

cmd/earlystopping/medianstop/v1beta1/requirements.txt

@@ -1,4 +1,4 @@
-grpcio==1.23.0
-protobuf==3.9.1
+grpcio==1.41.1
+protobuf==3.19.1
 googleapis-common-protos==1.6.0
 kubernetes==11.0.0

cmd/metricscollector/v1beta1/tfevent-metricscollector/Dockerfile

@@ -1,7 +1,25 @@
-FROM tensorflow/tensorflow:1.11.0
-RUN pip install rfc3339 grpcio googleapis-common-protos
-ADD . /usr/src/app/github.com/kubeflow/katib
-WORKDIR /usr/src/app/github.com/kubeflow/katib/cmd/metricscollector/v1beta1/tfevent-metricscollector/
+FROM python:3.9
+
+ENV TARGET_DIR /opt/katib
+ENV METRICS_COLLECTOR_DIR cmd/metricscollector/v1beta1/tfevent-metricscollector
+# tensorflow community build for aarch64
+# https://github.com/tensorflow/build#tensorflow-builds
+ENV PIP_EXTRA_INDEX_URL https://snapshots.linaro.org/ldcg/python-cache/
+
+ADD ./pkg/ ${TARGET_DIR}/pkg/
+ADD ./${METRICS_COLLECTOR_DIR}/ ${TARGET_DIR}/${METRICS_COLLECTOR_DIR}/
+WORKDIR  ${TARGET_DIR}/${METRICS_COLLECTOR_DIR}
+
+RUN if [ "$(uname -m)" = "aarch64" ]; then \
+    pip install tensorflow-aarch64==2.7.0; \
+    else \
+    pip install tensorflow==2.7.0; \
+    fi;
 RUN pip install --no-cache-dir -r requirements.txt
-ENV PYTHONPATH /usr/src/app/github.com/kubeflow/katib:/usr/src/app/github.com/kubeflow/katib/pkg/apis/manager/v1beta1/python:/usr/src/app/github.com/kubeflow/katib/pkg/metricscollector/v1beta1/tfevent-metricscollector/:/usr/src/app/github.com/kubeflow/katib/pkg/metricscollector/v1beta1/common/
+
+RUN chgrp -R 0 ${TARGET_DIR} \
+  && chmod -R g+rwX ${TARGET_DIR}
+
+ENV PYTHONPATH ${TARGET_DIR}:${TARGET_DIR}/pkg/apis/manager/v1beta1/python:${TARGET_DIR}/pkg/metricscollector/v1beta1/tfevent-metricscollector/::${TARGET_DIR}/pkg/metricscollector/v1beta1/common/
+
 ENTRYPOINT ["python", "main.py"]

cmd/metricscollector/v1beta1/tfevent-metricscollector/Dockerfile.ppc64le

@@ -1,4 +1,4 @@
-FROM ibmcom/tensorflow-ppc64le:1.14.0-py3
+FROM ibmcom/tensorflow-ppc64le:2.2.0-py3
 RUN pip install rfc3339 grpcio googleapis-common-protos
 ADD . /usr/src/app/github.com/kubeflow/katib
 WORKDIR /usr/src/app/github.com/kubeflow/katib/cmd/metricscollector/v1beta1/tfevent-metricscollector/

cmd/metricscollector/v1beta1/tfevent-metricscollector/requirements.txt

@@ -1 +1,4 @@
 psutil==5.6.6
+rfc3339
+grpcio
+googleapis-common-protos

cmd/suggestion/chocolate/v1beta1/Dockerfile

@@ -1,15 +1,17 @@
-FROM python:3.6
+FROM python:3.9
 
 ENV TARGET_DIR /opt/katib
 ENV SUGGESTION_DIR cmd/suggestion/chocolate/v1beta1
+ENV GRPC_HEALTH_PROBE_VERSION v0.3.1
 
 RUN if [ "$(uname -m)" = "ppc64le" ] || [ "$(uname -m)" = "aarch64" ]; then \
     apt-get -y update && \
     apt-get -y install gfortran libopenblas-dev liblapack-dev && \
-    pip install cython 'numpy>=1.13.3'; \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* && \
+    pip install cython; \
     fi
-RUN GRPC_HEALTH_PROBE_VERSION=v0.3.1 && \
-    if [ "$(uname -m)" = "ppc64le" ]; then \
+RUN if [ "$(uname -m)" = "ppc64le" ]; then \
     wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-ppc64le; \
     elif [ "$(uname -m)" = "aarch64" ]; then \
     wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-arm64; \

cmd/suggestion/chocolate/v1beta1/requirements.txt

@@ -1,11 +1,11 @@
-grpcio==1.23.0
+grpcio==1.41.1
 cloudpickle==0.5.6
-numpy>=1.13.3
-scikit-learn>=0.19.0
-scipy>=0.19.1
+numpy>=1.20.0
+scikit-learn>=0.24.0
+scipy>=1.5.4
 forestci==0.3
-protobuf==3.9.1
+protobuf==3.19.1
 googleapis-common-protos==1.6.0
-SQLAlchemy==1.3.8
+SQLAlchemy==1.4.26
 git+https://github.com/AIworx-Labs/chocolate@master
-ghalton>=0.6
+ghalton>=0.6.2

cmd/suggestion/goptuna/v1beta1/Dockerfile

@@ -1,6 +1,8 @@
 # Build the Goptuna Suggestion.
 FROM golang:alpine AS build-env
 
+ENV GRPC_HEALTH_PROBE_VERSION v0.3.1
+
 WORKDIR /go/src/github.com/kubeflow/katib
 
 # Download packages.
@@ -22,8 +24,7 @@ RUN if [ "$(uname -m)" = "ppc64le" ]; then \
   fi
 
 # Add GRPC health probe.
-RUN GRPC_HEALTH_PROBE_VERSION=v0.3.1 && \
-  if [ "$(uname -m)" = "ppc64le" ]; then \
+RUN if [ "$(uname -m)" = "ppc64le" ]; then \
   wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-ppc64le; \
   elif [ "$(uname -m)" = "aarch64" ]; then \
   wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-arm64; \

cmd/suggestion/hyperband/v1beta1/Dockerfile

@@ -1,16 +1,18 @@
-FROM python:3.6
+FROM python:3.9
 
 ENV TARGET_DIR /opt/katib
 ENV SUGGESTION_DIR cmd/suggestion/hyperband/v1beta1
+ENV GRPC_HEALTH_PROBE_VERSION v0.3.1
 
 RUN if [ "$(uname -m)" = "ppc64le" ] || [ "$(uname -m)" = "aarch64" ]; then \
     apt-get -y update && \
     apt-get -y install gfortran libopenblas-dev liblapack-dev && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* && \
     pip install cython; \
     fi
 
-RUN GRPC_HEALTH_PROBE_VERSION=v0.3.1 && \
-    if [ "$(uname -m)" = "ppc64le" ]; then \
+RUN if [ "$(uname -m)" = "ppc64le" ]; then \
     wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-ppc64le; \
     elif [ "$(uname -m)" = "aarch64" ]; then \
     wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-arm64; \

cmd/suggestion/hyperband/v1beta1/requirements.txt

@@ -1,8 +1,8 @@
-grpcio==1.23.0
+grpcio==1.41.1
 cloudpickle==0.5.6
-numpy>=1.13.3
-scikit-learn>=0.19.0
-scipy>=0.19.1
+numpy>=1.20.0
+scikit-learn>=0.24.0
+scipy>=1.5.4
 forestci==0.3
-protobuf==3.9.1
+protobuf==3.19.1
 googleapis-common-protos==1.6.0

cmd/suggestion/hyperopt/v1beta1/Dockerfile

@@ -1,16 +1,18 @@
-FROM python:3.6
+FROM python:3.9
 
 ENV TARGET_DIR /opt/katib
 ENV SUGGESTION_DIR cmd/suggestion/hyperopt/v1beta1
+ENV GRPC_HEALTH_PROBE_VERSION v0.3.1
 
 RUN if [ "$(uname -m)" = "ppc64le" ] || [ "$(uname -m)" = "aarch64" ]; then \
     apt-get -y update && \
     apt-get -y install gfortran libopenblas-dev liblapack-dev && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* && \
     pip install cython; \
     fi
 
-RUN GRPC_HEALTH_PROBE_VERSION=v0.3.1 && \
-    if [ "$(uname -m)" = "ppc64le" ]; then \
+RUN if [ "$(uname -m)" = "ppc64le" ]; then \
     wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-ppc64le; \
     elif [ "$(uname -m)" = "aarch64" ]; then \
     wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-arm64; \

cmd/suggestion/hyperopt/v1beta1/requirements.txt

@@ -1,9 +1,9 @@
-grpcio==1.23.0
+grpcio==1.41.1
 cloudpickle==0.5.6
-numpy>=1.13.3
-scikit-learn>=0.19.0
-scipy>=0.19.1
+numpy>=1.20.0
+scikit-learn>=0.24.0
+scipy>=1.5.4
 forestci==0.3
-protobuf==3.9.1
+protobuf==3.19.1
 googleapis-common-protos==1.6.0
-hyperopt==0.2.3
+hyperopt==0.2.5

cmd/suggestion/nas/darts/v1beta1/Dockerfile

@@ -1,16 +1,18 @@
-FROM python:3.6
+FROM python:3.9
 
 ENV TARGET_DIR /opt/katib
 ENV SUGGESTION_DIR cmd/suggestion/nas/darts/v1beta1
+ENV GRPC_HEALTH_PROBE_VERSION v0.3.1
 
 RUN if [ "$(uname -m)" = "ppc64le" ] || [ "$(uname -m)" = "aarch64" ]; then \
     apt-get -y update && \
     apt-get -y install gfortran libopenblas-dev liblapack-dev && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* && \
     pip install cython; \
     fi
 
-RUN GRPC_HEALTH_PROBE_VERSION=v0.3.1 && \
-    if [ "$(uname -m)" = "ppc64le" ]; then \
+RUN if [ "$(uname -m)" = "ppc64le" ]; then \
     wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-ppc64le; \
     elif [ "$(uname -m)" = "aarch64" ]; then \
     wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-arm64; \
@@ -30,4 +32,3 @@ RUN chgrp -R 0 ${TARGET_DIR} \
 ENV PYTHONPATH ${TARGET_DIR}:${TARGET_DIR}/pkg/apis/manager/v1beta1/python:${TARGET_DIR}/pkg/apis/manager/health/python
 
 ENTRYPOINT ["python", "main.py"]
-

cmd/suggestion/nas/darts/v1beta1/requirements.txt

@@ -1,3 +1,3 @@
-grpcio==1.23.0
-protobuf==3.9.1
+grpcio==1.41.1
+protobuf==3.19.1
 googleapis-common-protos==1.6.0

cmd/suggestion/nas/enas/v1beta1/Dockerfile

@@ -1,16 +1,24 @@
-FROM python:3.6
+FROM python:3.9
 
 ENV TARGET_DIR /opt/katib
 ENV SUGGESTION_DIR cmd/suggestion/nas/enas/v1beta1
+ENV GRPC_HEALTH_PROBE_VERSION v0.3.1
+# tensorflow community build for aarch64
+# https://github.com/tensorflow/build#tensorflow-builds
+ENV PIP_EXTRA_INDEX_URL https://snapshots.linaro.org/ldcg/python-cache/
 
-RUN if [ "$(uname -m)" = "ppc64le" ]; then \
+RUN if [ "$(uname -m)" = "ppc64le" ] || [ "$(uname -m)" = "aarch64" ]; then \
     apt-get -y update && \
     apt-get -y install gfortran libopenblas-dev liblapack-dev && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* && \
     pip install cython; \
     fi
-RUN GRPC_HEALTH_PROBE_VERSION=v0.3.1 && \
-    if [ "$(uname -m)" = "ppc64le" ]; then \
+
+RUN if [ "$(uname -m)" = "ppc64le" ]; then \
     wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-ppc64le; \
+    elif [ "$(uname -m)" = "aarch64" ]; then \
+    wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-arm64; \
     else \
     wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64; \
     fi && \
@@ -19,6 +27,10 @@ RUN GRPC_HEALTH_PROBE_VERSION=v0.3.1 && \
 ADD ./pkg/ ${TARGET_DIR}/pkg/
 ADD ./${SUGGESTION_DIR}/ ${TARGET_DIR}/${SUGGESTION_DIR}/
 WORKDIR  ${TARGET_DIR}/${SUGGESTION_DIR}
+
+RUN if [ "$(uname -m)" = "aarch64" ]; then \
+    sed -i 's/tensorflow==/tensorflow-aarch64==/' requirements.txt; \
+    fi;
 RUN pip install --no-cache-dir -r requirements.txt
 
 RUN chgrp -R 0 ${TARGET_DIR} \

cmd/suggestion/nas/enas/v1beta1/requirements.txt

@@ -1,4 +1,4 @@
-grpcio==1.23.0
-protobuf==3.9.1
+grpcio==1.41.1
+protobuf==3.19.1
 googleapis-common-protos==1.6.0
-tensorflow==1.15.4
+tensorflow==2.7.0

cmd/suggestion/optuna/v1beta1/Dockerfile

@@ -2,14 +2,15 @@ FROM python:3.9
 
 ENV TARGET_DIR /opt/katib
 ENV SUGGESTION_DIR cmd/suggestion/optuna/v1beta1
+ENV GRPC_HEALTH_PROBE_VERSION v0.3.1
 
 RUN if [ "$(uname -m)" = "ppc64le" ] || [ "$(uname -m)" = "aarch64" ]; then \
     apt-get -y update && \
-    apt-get -y install gfortran libopenblas-dev liblapack-dev; \
+    apt-get -y install gfortran libopenblas-dev liblapack-dev && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*; \
     fi
-
-RUN GRPC_HEALTH_PROBE_VERSION=v0.3.1 && \
-    if [ "$(uname -m)" = "ppc64le" ]; then \
+RUN if [ "$(uname -m)" = "ppc64le" ]; then \
     wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-ppc64le; \
     elif [ "$(uname -m)" = "aarch64" ]; then \
     wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-arm64; \

cmd/suggestion/optuna/v1beta1/requirements.txt

@@ -1,4 +1,4 @@
-grpcio==1.39.0
-protobuf==3.17.3
+grpcio==1.41.1
+protobuf==3.19.1
 googleapis-common-protos==1.53.0
-optuna>=2.8.0
+optuna>=2.8.0