[CI] Upgrade CI to Python 3.9

docker/Dockerfile.ci_arm

@@ -53,10 +53,10 @@ ENV PATH /opt/sccache:$PATH
 COPY install/ubuntu2204_install_llvm.sh /install/ubuntu2204_install_llvm.sh
 RUN bash /install/ubuntu2204_install_llvm.sh
 
-ENV TVM_VENV /venv/apache-tvm-py3.8
+ENV TVM_VENV /venv/apache-tvm-py3.9
 COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
 COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
-RUN bash /install/ubuntu_install_python.sh 3.8
+RUN bash /install/ubuntu_install_python.sh 3.9
 ENV PATH ${TVM_VENV}/bin:$PATH
 ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 
@@ -71,14 +71,6 @@ RUN bash /install/ubuntu_install_tensorflow_aarch64.sh
 COPY install/ubuntu_install_tflite.sh /install/ubuntu_install_tflite.sh
 RUN bash /install/ubuntu_install_tflite.sh
 
-# Caffe deps
-COPY install/ubuntu_install_boost.sh /install/ubuntu_install_boost.sh
-RUN bash /install/ubuntu_install_boost.sh
-
-# Caffe
-COPY install/ubuntu_install_caffe.sh /install/ubuntu_install_caffe.sh
-RUN bash /install/ubuntu_install_caffe.sh
-
 # ONNX
 COPY install/ubuntu_install_onnx.sh /install/ubuntu_install_onnx.sh
 RUN bash /install/ubuntu_install_onnx.sh

docker/Dockerfile.ci_cortexm

@@ -30,15 +30,15 @@ COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
 RUN bash /install/ubuntu_install_core.sh
 
 COPY install/ubuntu_install_cmake_source.sh /install/ubuntu_install_cmake_source.sh
-RUN bash /install/ubuntu_install_cmake_source.sh 3.20.0
+RUN bash /install/ubuntu_install_cmake_source.sh 3.20.0 9c06b2ddf7c337e31d8201f6ebcd3bba86a9a033976a9aee207fe0c6971f4755
 
 COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh
 RUN bash /install/ubuntu_install_googletest.sh
 
-ENV TVM_VENV /venv/apache-tvm-py3.8
+ENV TVM_VENV /venv/apache-tvm-py3.9
 COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
 COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
-RUN bash /install/ubuntu_install_python.sh 3.8
+RUN bash /install/ubuntu_install_python.sh 3.9
 ENV PATH ${TVM_VENV}/bin:$PATH
 ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 

docker/Dockerfile.ci_cpu

@@ -34,10 +34,10 @@ RUN bash /install/ubuntu_install_cmake_source.sh
 COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh
 RUN bash /install/ubuntu_install_googletest.sh
 
-ENV TVM_VENV /venv/apache-tvm-py3.8
+ENV TVM_VENV /venv/apache-tvm-py3.9
 COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
 COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
-RUN bash /install/ubuntu_install_python.sh 3.8
+RUN bash /install/ubuntu_install_python.sh 3.9
 ENV PATH ${TVM_VENV}/bin:$PATH
 ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 
@@ -109,14 +109,6 @@ RUN bash /install/ubuntu_install_jax.sh "cpu"
 COPY install/ubuntu_download_arm_compute_lib_binaries.sh /install/ubuntu_download_arm_compute_lib_binaries.sh
 RUN bash /install/ubuntu_download_arm_compute_lib_binaries.sh
 
-# Caffe deps
-COPY install/ubuntu_install_boost.sh /install/ubuntu_install_boost.sh
-RUN bash /install/ubuntu_install_boost.sh
-
-# Caffe
-COPY install/ubuntu_install_caffe.sh /install/ubuntu_install_caffe.sh
-RUN bash /install/ubuntu_install_caffe.sh
-
 # Github Arm(R) Ethos(TM)-N NPU driver
 COPY install/ubuntu_install_ethosn_driver_stack.sh /install/ubuntu_install_ethosn_driver_stack.sh
 RUN bash /install/ubuntu_install_ethosn_driver_stack.sh

docker/Dockerfile.ci_gpu

@@ -41,10 +41,10 @@ RUN bash /install/ubuntu_install_cmake_source.sh
 COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh
 RUN bash /install/ubuntu_install_googletest.sh /googletest
 
-ENV TVM_VENV /venv/apache-tvm-py3.8
+ENV TVM_VENV /venv/apache-tvm-py3.9
 COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
 COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
-RUN bash /install/ubuntu_install_python.sh 3.8
+RUN bash /install/ubuntu_install_python.sh 3.9
 ENV PATH ${TVM_VENV}/bin:$PATH
 ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 

docker/Dockerfile.ci_hexagon

@@ -37,10 +37,10 @@ RUN bash /install/ubuntu_install_cmake_source.sh
 COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh
 RUN bash /install/ubuntu_install_googletest.sh
 
-ENV TVM_VENV /venv/apache-tvm-py3.8
+ENV TVM_VENV /venv/apache-tvm-py3.9
 COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
 COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
-RUN bash /install/ubuntu_install_python.sh 3.8
+RUN bash /install/ubuntu_install_python.sh 3.9
 ENV PATH ${TVM_VENV}/bin:$PATH
 ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 

docker/Dockerfile.ci_i386

@@ -49,7 +49,7 @@ ENV CARGO_HOME /opt/rust
 ENV PATH $PATH:$CARGO_HOME/bin
 ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 
-ENV TVM_VENV /venv/apache-tvm-py3.8
+ENV TVM_VENV /venv/apache-tvm-py3.9
 COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
 COPY install/ubuntu2004_install_python.sh /install/ubuntu2004_install_python.sh
 RUN bash /install/ubuntu2004_install_python.sh

docker/Dockerfile.ci_lint

@@ -29,10 +29,10 @@ RUN bash /install/ubuntu_setup_tz.sh
 
 RUN apt-install-and-clear -y wget git sudo make parallel
 
-ENV TVM_VENV /venv/apache-tvm-py3.8
+ENV TVM_VENV /venv/apache-tvm-py3.9
 COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
 COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
-RUN bash /install/ubuntu_install_python.sh 3.8
+RUN bash /install/ubuntu_install_python.sh 3.9
 ENV PATH ${TVM_VENV}/bin:$PATH
 ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 

docker/Dockerfile.ci_minimal

@@ -38,10 +38,10 @@ RUN bash /install/ubuntu_install_cmake_source.sh
 COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh
 RUN bash /install/ubuntu_install_googletest.sh
 
-ENV TVM_VENV /venv/apache-tvm-py3.8
+ENV TVM_VENV /venv/apache-tvm-py3.9
 COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
 COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
-RUN bash /install/ubuntu_install_python.sh 3.8
+RUN bash /install/ubuntu_install_python.sh 3.9
 ENV PATH ${TVM_VENV}/bin:$PATH
 ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 

docker/Dockerfile.ci_riscv

@@ -35,10 +35,10 @@ RUN bash /install/ubuntu_install_cmake_source.sh
 COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh
 RUN bash /install/ubuntu_install_googletest.sh
 
-ENV TVM_VENV /venv/apache-tvm-py3.8
+ENV TVM_VENV /venv/apache-tvm-py3.9
 COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
 COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
-RUN bash /install/ubuntu_install_python.sh 3.8
+RUN bash /install/ubuntu_install_python.sh 3.9
 ENV PATH ${TVM_VENV}/bin:$PATH
 ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 

docker/Dockerfile.ci_wasm

@@ -32,10 +32,10 @@ RUN bash /install/ubuntu_install_cmake_source.sh
 COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh
 RUN bash /install/ubuntu_install_googletest.sh
 
-ENV TVM_VENV /venv/apache-tvm-py3.8
+ENV TVM_VENV /venv/apache-tvm-py3.9
 COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
 COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
-RUN bash /install/ubuntu_install_python.sh 3.8
+RUN bash /install/ubuntu_install_python.sh 3.9
 ENV PATH ${TVM_VENV}/bin:$PATH
 ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 

docker/Dockerfile.demo_android

@@ -28,10 +28,10 @@ RUN bash /install/ubuntu_setup_tz.sh
 COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
 RUN bash /install/ubuntu_install_core.sh
 
-ENV TVM_VENV /venv/apache-tvm-py3.8
+ENV TVM_VENV /venv/apache-tvm-py3.9
 COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
 COPY install/ubuntu_install_python.sh /install/ubuntu1804_install_python.sh
-RUN bash /install/ubuntu1804_install_python.sh 3.8
+RUN bash /install/ubuntu1804_install_python.sh 3.9
 ENV PATH ${TVM_VENV}/bin:$PATH
 ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 

docker/Dockerfile.demo_rocm

@@ -26,10 +26,10 @@ RUN bash /install/ubuntu_setup_tz.sh
 COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
 RUN bash /install/ubuntu_install_core.sh
 
-ENV TVM_VENV /venv/apache-tvm-py3.8
+ENV TVM_VENV /venv/apache-tvm-py3.9
 COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
 COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
-RUN bash /install/ubuntu_install_python.sh 3.8
+RUN bash /install/ubuntu_install_python.sh 3.9
 ENV PATH ${TVM_VENV}/bin:$PATH
 ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 

docker/Dockerfile.demo_vitis_ai

@@ -32,10 +32,10 @@ RUN bash /install/ubuntu_install_core.sh
 COPY install/ubuntu_install_vitis_ai_core.sh /install/ubuntu_install_vitis_ai_core.sh
 RUN bash /install/ubuntu_install_vitis_ai_core.sh
 
-ENV TVM_VENV /venv/apache-tvm-py3.8
+ENV TVM_VENV /venv/apache-tvm-py3.9
 COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
 COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
-RUN bash /install/ubuntu_install_python.sh 3.8
+RUN bash /install/ubuntu_install_python.sh 3.9
 ENV PATH ${TVM_VENV}/bin:$PATH
 ENV PYTHONNOUSERSITE 1  # Disable .local directory from affecting CI.
 

docker/install/ubuntu2004_install_python.sh

@@ -30,15 +30,15 @@ trap cleanup 0
 # Install python and pip. Don't modify this to add Python package dependencies,
 # instead modify install_python_package.sh
 apt-get update
-apt-install-and-clear -y python3.8 python3.8-dev python3-pip
-update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1
+apt-install-and-clear -y python3.9 python3.9-dev python3-pip
+update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1
 
 # Pin pip and setuptools versions
 # Hashes generated via:
 #   $ pip download <package>==<version>
 #   $ pip hash --algorithm sha256 <package>.whl
 cat <<EOF > base-requirements.txt
-pip==23.3.2 --hash=sha256:5052d7889c1f9d05224cd41741acb7c5d6fa735ab34e339624a614eaaa7e7d76
-setuptools==58.4.0 --hash=sha256:e8b1d3127a0441fb99a130bcc3c2bf256c2d3ead3aba8fd400e5cbbaf788e036
+pip==24.2 --hash=sha256:2cd581cf58ab7fcfca4ce8efa6dcacd0de5bf8d0a3eb9ec927e07405f4d9e2a2
+setuptools==75.1.0 --hash=sha256:35ab7fd3bcd95e6b7fd704e4a1539513edad446c097797f2985e0e4b960772f2
 EOF
 pip3 install -r base-requirements.txt

docker/install/ubuntu_install_cmake_source.sh

@@ -20,19 +20,21 @@ set -e
 set -u
 set -o pipefail
 
-if [ -z ${1+x} ]; then
-    version=3.24.0
-else
-    version=$1
-fi
+CMAKE_VERSION="3.30.4"
+CMAKE_SHA256="c759c97274f1e7aaaafcb1f0d261f9de9bf3a5d6ecb7e2df616324a46fe704b2"
 
-v=$(echo $version | sed 's/\(.*\)\..*/\1/g')
-echo "Installing cmake $version ($v)"
-wget https://cmake.org/files/v${v}/cmake-${version}.tar.gz
-tar xvf cmake-${version}.tar.gz
-cd cmake-${version}
-./bootstrap
-make -j$(nproc)
-make install
-cd ..
-rm -rf cmake-${version} cmake-${version}.tar.gz
+# parse argument
+CMAKE_VERSION=${1:-$CMAKE_VERSION}
+CMAKE_SHA256=${2:-$CMAKE_SHA256}
+
+v=$(echo $CMAKE_VERSION | sed 's/\(.*\)\..*/\1/g')
+echo "Installing cmake $CMAKE_VERSION ($v)"
+wget https://cmake.org/files/v${v}/cmake-${CMAKE_VERSION}.tar.gz
+echo "$CMAKE_SHA256" cmake-${CMAKE_VERSION}.tar.gz | sha256sum -c
+tar xvf cmake-${CMAKE_VERSION}.tar.gz
+pushd cmake-${CMAKE_VERSION}
+  ./bootstrap
+  make -j$(nproc)
+  make install
+popd
+rm -rf cmake-${CMAKE_VERSION} cmake-${CMAKE_VERSION}.tar.gz

docker/install/ubuntu_install_jax.sh

@@ -20,16 +20,18 @@ set -e
 set -u
 set -o pipefail
 
-# Install jax and jaxlib
+JAX_VERSION=0.4.30
+
+# Install jaxlib
 if [ "$1" == "cuda" ]; then
-    pip3 install --upgrade \
-        jaxlib~=0.4.9 \
-        "jax[cuda11_pip]~=0.4.9" -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html
+    pip install -U \
+        "jax[cuda12]~=${JAX_VERSION}" \
+        jaxlib~=${JAX_VERSION}
 else
-    pip3 install --upgrade \
-        jaxlib~=0.4.9 \
-        "jax[cpu]~=0.4.9"
+    pip3 install -U \
+        jax~=${JAX_VERSION} \
+        jaxlib~=${JAX_VERSION}
 fi
 
 # Install flax
-pip3 install flax~=0.6.9
+pip3 install flax~=0.8.5

docker/install/ubuntu_install_llvm_from_source.sh

@@ -63,7 +63,7 @@ cmake \
     -DLLVM_ENABLE_PROJECTS=mlir \
     -DLLVM_USE_INTEL_JITEVENTS=ON \
     -DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN=ON \
-    -DPYTHON_EXECUTABLE="$(which python3.8)" \
+    -DPYTHON_EXECUTABLE="$(which python3.9)" \
     -GNinja \
     ..
 ninja install

docker/install/ubuntu_install_python.sh

@@ -33,10 +33,13 @@ if [ "$#" -lt 1 ]; then
 fi
 PYTHON_VERSION=$1
 
-if [ "${PYTHON_VERSION}" != "3.7" ] && [ "${PYTHON_VERSION}" != "3.8" ]; then
-    echo "Only 3.7 and 3.8 versions are supported in this script."
-    exit -1
-fi
+case "$PYTHON_VERSION" in
+    3.7|3.8|3.9) ;;
+    *)
+        echo "Only 3.7, 3.8, and 3.9 versions are supported in this script."
+        exit -1
+        ;;
+esac
 
 apt-get update
 
@@ -47,22 +50,23 @@ apt-install-and-clear -y \
 apt-install-and-clear -y software-properties-common
 
 release=$(lsb_release -sc)
-if [ "${release}" == "bionic" ]; then
-    if [ "${PYTHON_VERSION}" == "3.8" ]; then
-        add-apt-repository -y ppa:deadsnakes/ppa
-    fi
-elif [ "${release}" == "focal" ]; then
-    if [ "${PYTHON_VERSION}" == "3.7" ]; then
-        add-apt-repository -y ppa:deadsnakes/ppa
-    fi
-elif [ "${release}" == "jammy" ]; then
-    if [ "${PYTHON_VERSION}" == "3.8" ]; then
-        add-apt-repository -y ppa:deadsnakes/ppa
-    fi
-else
-    echo "Don't know which version of python to install for lsb-release ${release}"
-    exit 2
-fi
+case "${release}" in
+    bionic)
+        [ "${PYTHON_VERSION}" == "3.8" ] && add-apt-repository -y ppa:deadsnakes/ppa
+        ;;
+    focal)
+        [ "${PYTHON_VERSION}" == "3.7" ] && add-apt-repository -y ppa:deadsnakes/ppa
+        ;;
+    jammy)
+        if [ "${PYTHON_VERSION}" == "3.8" ] || [ "${PYTHON_VERSION}" == "3.9" ]; then
+            add-apt-repository -y ppa:deadsnakes/ppa
+        fi
+        ;;
+    *)
+        echo "Don't know which version of python to install for lsb-release ${release}"
+        exit 2
+        ;;
+esac
 
 # Install python and pip. Don't modify this to add Python package dependencies,
 # instead modify install_python_package.sh
@@ -84,7 +88,6 @@ export PYTHONNOUSERSITE=1
 venv_dir="$(python3 -c "import os.path;print(os.path.dirname(\"${TVM_VENV}\"))")"
 mkdir -p "${venv_dir}"
 python3 -mvenv "${TVM_VENV}"
-. "${TVM_VENV}/bin/activate"
 
 # NOTE: Only in python3.9 does venv guarantee it creates the python3.X binary.
 # This is needed so that cmake's find_package(PythonInterp) works inside the venv.
@@ -95,15 +98,15 @@ fi
 
 # Update pip to match version used to produce requirements-hashed.txt. This step
 # is necessary so that pip's dependency solver is recent.
-pip_spec=$(cat /install/python/bootstrap/lockfiles/constraints-${PYTHON_VERSION}.txt | grep 'pip==')
-pip3 install -U --require-hashes -r <(echo "${pip_spec}") \
+pip_spec=$(tac /install/python/bootstrap/lockfiles/constraints-${PYTHON_VERSION}.txt | grep -m 1 'pip==')
+${TVM_VENV}/bin/pip install -U --require-hashes -r <(echo "${pip_spec}") \
      -c /install/python/bootstrap/lockfiles/constraints-${PYTHON_VERSION}.txt
 
 # Python configuration
-pip3 config set global.no-cache-dir true  # Never cache packages
+${TVM_VENV}/bin/pip config set global.no-cache-dir true  # Never cache packages
 
 # Now install the remaining base packages.
-pip3 install \
+${TVM_VENV}/bin/pip install \
      --require-hashes \
      -r /install/python/bootstrap/lockfiles/constraints-${PYTHON_VERSION}.txt
 
@@ -114,7 +117,6 @@ setfacl -R -m group:tvm-venv:rwx "${TVM_VENV}"
 
 # Prevent further use of pip3 via the system.
 # There may be multiple (i.e. from python3-pip apt package and pip3 install -U).
-deactivate
 while [ "$(which pip3)" != "" ]; do
     rm "$(which pip3)"
 done