Skip to content

Commit

Permalink
upgrade ci
Browse files Browse the repository at this point in the history
  • Loading branch information
mshr-h committed Sep 30, 2024
1 parent d9ee637 commit af6630c
Show file tree
Hide file tree
Showing 29 changed files with 762 additions and 164 deletions.
4 changes: 2 additions & 2 deletions docker/Dockerfile.ci_arm
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,10 @@ ENV PATH /opt/sccache:$PATH
COPY install/ubuntu2204_install_llvm.sh /install/ubuntu2204_install_llvm.sh
RUN bash /install/ubuntu2204_install_llvm.sh

ENV TVM_VENV /venv/apache-tvm-py3.8
ENV TVM_VENV /venv/apache-tvm-py3.9
COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
RUN bash /install/ubuntu_install_python.sh 3.8
RUN bash /install/ubuntu_install_python.sh 3.9
ENV PATH ${TVM_VENV}/bin:$PATH
ENV PYTHONNOUSERSITE 1 # Disable .local directory from affecting CI.

Expand Down
6 changes: 3 additions & 3 deletions docker/Dockerfile.ci_cortexm
Original file line number Diff line number Diff line change
Expand Up @@ -30,15 +30,15 @@ COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
RUN bash /install/ubuntu_install_core.sh

COPY install/ubuntu_install_cmake_source.sh /install/ubuntu_install_cmake_source.sh
RUN bash /install/ubuntu_install_cmake_source.sh 3.20.0
RUN bash /install/ubuntu_install_cmake_source.sh 3.20.0 9c06b2ddf7c337e31d8201f6ebcd3bba86a9a033976a9aee207fe0c6971f4755

COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh
RUN bash /install/ubuntu_install_googletest.sh

ENV TVM_VENV /venv/apache-tvm-py3.8
ENV TVM_VENV /venv/apache-tvm-py3.9
COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
RUN bash /install/ubuntu_install_python.sh 3.8
RUN bash /install/ubuntu_install_python.sh 3.9
ENV PATH ${TVM_VENV}/bin:$PATH
ENV PYTHONNOUSERSITE 1 # Disable .local directory from affecting CI.

Expand Down
12 changes: 2 additions & 10 deletions docker/Dockerfile.ci_cpu
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,10 @@ RUN bash /install/ubuntu_install_cmake_source.sh
COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh
RUN bash /install/ubuntu_install_googletest.sh

ENV TVM_VENV /venv/apache-tvm-py3.8
ENV TVM_VENV /venv/apache-tvm-py3.9
COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
RUN bash /install/ubuntu_install_python.sh 3.8
RUN bash /install/ubuntu_install_python.sh 3.9
ENV PATH ${TVM_VENV}/bin:$PATH
ENV PYTHONNOUSERSITE 1 # Disable .local directory from affecting CI.

Expand Down Expand Up @@ -109,14 +109,6 @@ RUN bash /install/ubuntu_install_jax.sh "cpu"
COPY install/ubuntu_download_arm_compute_lib_binaries.sh /install/ubuntu_download_arm_compute_lib_binaries.sh
RUN bash /install/ubuntu_download_arm_compute_lib_binaries.sh

# Caffe deps
COPY install/ubuntu_install_boost.sh /install/ubuntu_install_boost.sh
RUN bash /install/ubuntu_install_boost.sh

# Caffe
COPY install/ubuntu_install_caffe.sh /install/ubuntu_install_caffe.sh
RUN bash /install/ubuntu_install_caffe.sh

# Github Arm(R) Ethos(TM)-N NPU driver
COPY install/ubuntu_install_ethosn_driver_stack.sh /install/ubuntu_install_ethosn_driver_stack.sh
RUN bash /install/ubuntu_install_ethosn_driver_stack.sh
Expand Down
4 changes: 2 additions & 2 deletions docker/Dockerfile.ci_gpu
Original file line number Diff line number Diff line change
Expand Up @@ -41,10 +41,10 @@ RUN bash /install/ubuntu_install_cmake_source.sh
COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh
RUN bash /install/ubuntu_install_googletest.sh /googletest

ENV TVM_VENV /venv/apache-tvm-py3.8
ENV TVM_VENV /venv/apache-tvm-py3.9
COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
RUN bash /install/ubuntu_install_python.sh 3.8
RUN bash /install/ubuntu_install_python.sh 3.9
ENV PATH ${TVM_VENV}/bin:$PATH
ENV PYTHONNOUSERSITE 1 # Disable .local directory from affecting CI.

Expand Down
4 changes: 2 additions & 2 deletions docker/Dockerfile.ci_hexagon
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,10 @@ RUN bash /install/ubuntu_install_cmake_source.sh
COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh
RUN bash /install/ubuntu_install_googletest.sh

ENV TVM_VENV /venv/apache-tvm-py3.8
ENV TVM_VENV /venv/apache-tvm-py3.9
COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
RUN bash /install/ubuntu_install_python.sh 3.8
RUN bash /install/ubuntu_install_python.sh 3.9
ENV PATH ${TVM_VENV}/bin:$PATH
ENV PYTHONNOUSERSITE 1 # Disable .local directory from affecting CI.

Expand Down
2 changes: 1 addition & 1 deletion docker/Dockerfile.ci_i386
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ ENV CARGO_HOME /opt/rust
ENV PATH $PATH:$CARGO_HOME/bin
ENV PYTHONNOUSERSITE 1 # Disable .local directory from affecting CI.

ENV TVM_VENV /venv/apache-tvm-py3.8
ENV TVM_VENV /venv/apache-tvm-py3.9
COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
COPY install/ubuntu2004_install_python.sh /install/ubuntu2004_install_python.sh
RUN bash /install/ubuntu2004_install_python.sh
Expand Down
4 changes: 2 additions & 2 deletions docker/Dockerfile.ci_lint
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,10 @@ RUN bash /install/ubuntu_setup_tz.sh

RUN apt-install-and-clear -y wget git sudo make parallel

ENV TVM_VENV /venv/apache-tvm-py3.8
ENV TVM_VENV /venv/apache-tvm-py3.9
COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
RUN bash /install/ubuntu_install_python.sh 3.8
RUN bash /install/ubuntu_install_python.sh 3.9
ENV PATH ${TVM_VENV}/bin:$PATH
ENV PYTHONNOUSERSITE 1 # Disable .local directory from affecting CI.

Expand Down
4 changes: 2 additions & 2 deletions docker/Dockerfile.ci_minimal
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,10 @@ RUN bash /install/ubuntu_install_cmake_source.sh
COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh
RUN bash /install/ubuntu_install_googletest.sh

ENV TVM_VENV /venv/apache-tvm-py3.8
ENV TVM_VENV /venv/apache-tvm-py3.9
COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
RUN bash /install/ubuntu_install_python.sh 3.8
RUN bash /install/ubuntu_install_python.sh 3.9
ENV PATH ${TVM_VENV}/bin:$PATH
ENV PYTHONNOUSERSITE 1 # Disable .local directory from affecting CI.

Expand Down
4 changes: 2 additions & 2 deletions docker/Dockerfile.ci_riscv
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,10 @@ RUN bash /install/ubuntu_install_cmake_source.sh
COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh
RUN bash /install/ubuntu_install_googletest.sh

ENV TVM_VENV /venv/apache-tvm-py3.8
ENV TVM_VENV /venv/apache-tvm-py3.9
COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
RUN bash /install/ubuntu_install_python.sh 3.8
RUN bash /install/ubuntu_install_python.sh 3.9
ENV PATH ${TVM_VENV}/bin:$PATH
ENV PYTHONNOUSERSITE 1 # Disable .local directory from affecting CI.

Expand Down
4 changes: 2 additions & 2 deletions docker/Dockerfile.ci_wasm
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,10 @@ RUN bash /install/ubuntu_install_cmake_source.sh
COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh
RUN bash /install/ubuntu_install_googletest.sh

ENV TVM_VENV /venv/apache-tvm-py3.8
ENV TVM_VENV /venv/apache-tvm-py3.9
COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
RUN bash /install/ubuntu_install_python.sh 3.8
RUN bash /install/ubuntu_install_python.sh 3.9
ENV PATH ${TVM_VENV}/bin:$PATH
ENV PYTHONNOUSERSITE 1 # Disable .local directory from affecting CI.

Expand Down
4 changes: 2 additions & 2 deletions docker/Dockerfile.demo_android
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,10 @@ RUN bash /install/ubuntu_setup_tz.sh
COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
RUN bash /install/ubuntu_install_core.sh

ENV TVM_VENV /venv/apache-tvm-py3.8
ENV TVM_VENV /venv/apache-tvm-py3.9
COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
COPY install/ubuntu_install_python.sh /install/ubuntu1804_install_python.sh
RUN bash /install/ubuntu1804_install_python.sh 3.8
RUN bash /install/ubuntu1804_install_python.sh 3.9
ENV PATH ${TVM_VENV}/bin:$PATH
ENV PYTHONNOUSERSITE 1 # Disable .local directory from affecting CI.

Expand Down
4 changes: 2 additions & 2 deletions docker/Dockerfile.demo_rocm
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,10 @@ RUN bash /install/ubuntu_setup_tz.sh
COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
RUN bash /install/ubuntu_install_core.sh

ENV TVM_VENV /venv/apache-tvm-py3.8
ENV TVM_VENV /venv/apache-tvm-py3.9
COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
RUN bash /install/ubuntu_install_python.sh 3.8
RUN bash /install/ubuntu_install_python.sh 3.9
ENV PATH ${TVM_VENV}/bin:$PATH
ENV PYTHONNOUSERSITE 1 # Disable .local directory from affecting CI.

Expand Down
4 changes: 2 additions & 2 deletions docker/Dockerfile.demo_vitis_ai
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,10 @@ RUN bash /install/ubuntu_install_core.sh
COPY install/ubuntu_install_vitis_ai_core.sh /install/ubuntu_install_vitis_ai_core.sh
RUN bash /install/ubuntu_install_vitis_ai_core.sh

ENV TVM_VENV /venv/apache-tvm-py3.8
ENV TVM_VENV /venv/apache-tvm-py3.9
COPY python/bootstrap/lockfiles /install/python/bootstrap/lockfiles
COPY install/ubuntu_install_python.sh /install/ubuntu_install_python.sh
RUN bash /install/ubuntu_install_python.sh 3.8
RUN bash /install/ubuntu_install_python.sh 3.9
ENV PATH ${TVM_VENV}/bin:$PATH
ENV PYTHONNOUSERSITE 1 # Disable .local directory from affecting CI.

Expand Down
8 changes: 4 additions & 4 deletions docker/install/ubuntu2004_install_python.sh
Original file line number Diff line number Diff line change
Expand Up @@ -30,15 +30,15 @@ trap cleanup 0
# Install python and pip. Don't modify this to add Python package dependencies,
# instead modify install_python_package.sh
apt-get update
apt-install-and-clear -y python3.8 python3.8-dev python3-pip
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1
apt-install-and-clear -y python3.9 python3.9-dev python3-pip
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1

# Pin pip and setuptools versions
# Hashes generated via:
# $ pip download <package>==<version>
# $ pip hash --algorithm sha256 <package>.whl
cat <<EOF > base-requirements.txt
pip==23.3.2 --hash=sha256:5052d7889c1f9d05224cd41741acb7c5d6fa735ab34e339624a614eaaa7e7d76
setuptools==58.4.0 --hash=sha256:e8b1d3127a0441fb99a130bcc3c2bf256c2d3ead3aba8fd400e5cbbaf788e036
pip==24.2 --hash=sha256:2cd581cf58ab7fcfca4ce8efa6dcacd0de5bf8d0a3eb9ec927e07405f4d9e2a2
setuptools==75.1.0 --hash=sha256:35ab7fd3bcd95e6b7fd704e4a1539513edad446c097797f2985e0e4b960772f2
EOF
pip3 install -r base-requirements.txt
18 changes: 9 additions & 9 deletions docker/install/ubuntu_install_boost.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,23 +19,23 @@
set -euxo pipefail

cleanup() {
rm -rf /boost_1_67_0 /boost_1_67_0.tar.gz
rm -rf /boost_1_86_0 /boost_1_86_0.tar.gz
}

trap cleanup 0

# NOTE: by default, tvm-venv python is used. Install boost on the system.
PATH=${PATH/${TVM_VENV}\/bin:/}

curl -LO https://archives.boost.io/release/1.67.0/source/boost_1_67_0.tar.gz
BOOST_HASH=8c247e040303a97895cee9c9407ef205e2c3ab09f0b8320997835ad6221dff23a87231629498ccfd0acca473f74e9ec27b8bd774707b062228df1e5f72d44c92
echo "$BOOST_HASH" boost_1_67_0.tar.gz | sha512sum -c
tar -xf boost_1_67_0.tar.gz
curl -LO https://archives.boost.io/release/1.86.0/source/boost_1_86_0.tar.gz
BOOST_HASH=2575e74ffc3ef1cd0babac2c1ee8bdb5782a0ee672b1912da40e5b4b591ca01f
echo "$BOOST_HASH" boost_1_86_0.tar.gz | sha256sum -c
tar -xf boost_1_86_0.tar.gz

pushd boost_1_67_0
./bootstrap.sh --with-python="$(which python3.8)"
./b2 install --with-python --with-system --with-filesystem --with-thread --with-regex
pushd boost_1_86_0
./bootstrap.sh --with-python="$(which python3.9)"
./b2 install --with-python --with-system --with-filesystem --with-thread --with-regex
popd

ln -s /usr/local/lib/libboost_python38.so.1.67.0 /usr/local/lib/libboost_python.so
ln -s /usr/local/lib/libboost_python38.so.1.86.0 /usr/local/lib/libboost_python.so
ln -s /usr/local/lib/libboost_python38.a /usr/local/lib/libboost_python.a
32 changes: 17 additions & 15 deletions docker/install/ubuntu_install_cmake_source.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,19 +20,21 @@ set -e
set -u
set -o pipefail

if [ -z ${1+x} ]; then
version=3.24.0
else
version=$1
fi
CMAKE_VERSION="3.30.4"
CMAKE_SHA256="c759c97274f1e7aaaafcb1f0d261f9de9bf3a5d6ecb7e2df616324a46fe704b2"

v=$(echo $version | sed 's/\(.*\)\..*/\1/g')
echo "Installing cmake $version ($v)"
wget https://cmake.org/files/v${v}/cmake-${version}.tar.gz
tar xvf cmake-${version}.tar.gz
cd cmake-${version}
./bootstrap
make -j$(nproc)
make install
cd ..
rm -rf cmake-${version} cmake-${version}.tar.gz
# parse argument
CMAKE_VERSION=${1:-$CMAKE_VERSION}
CMAKE_SHA256=${2:-$CMAKE_SHA256}

v=$(echo $CMAKE_VERSION | sed 's/\(.*\)\..*/\1/g')
echo "Installing cmake $CMAKE_VERSION ($v)"
wget https://cmake.org/files/v${v}/cmake-${CMAKE_VERSION}.tar.gz
echo "$CMAKE_SHA256" cmake-${CMAKE_VERSION}.tar.gz | sha256sum -c
tar xvf cmake-${CMAKE_VERSION}.tar.gz
pushd cmake-${CMAKE_VERSION}
./bootstrap
make -j$(nproc)
make install
popd
rm -rf cmake-${CMAKE_VERSION} cmake-${CMAKE_VERSION}.tar.gz
2 changes: 1 addition & 1 deletion docker/install/ubuntu_install_llvm_from_source.sh
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ cmake \
-DLLVM_ENABLE_PROJECTS=mlir \
-DLLVM_USE_INTEL_JITEVENTS=ON \
-DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN=ON \
-DPYTHON_EXECUTABLE="$(which python3.8)" \
-DPYTHON_EXECUTABLE="$(which python3.9)" \
-GNinja \
..
ninja install
Expand Down
54 changes: 28 additions & 26 deletions docker/install/ubuntu_install_python.sh
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,13 @@ if [ "$#" -lt 1 ]; then
fi
PYTHON_VERSION=$1

if [ "${PYTHON_VERSION}" != "3.7" ] && [ "${PYTHON_VERSION}" != "3.8" ]; then
echo "Only 3.7 and 3.8 versions are supported in this script."
exit -1
fi
case "$PYTHON_VERSION" in
3.7|3.8|3.9) ;;
*)
echo "Only 3.7, 3.8, and 3.9 versions are supported in this script."
exit -1
;;
esac

apt-get update

Expand All @@ -47,22 +50,23 @@ apt-install-and-clear -y \
apt-install-and-clear -y software-properties-common

release=$(lsb_release -sc)
if [ "${release}" == "bionic" ]; then
if [ "${PYTHON_VERSION}" == "3.8" ]; then
add-apt-repository -y ppa:deadsnakes/ppa
fi
elif [ "${release}" == "focal" ]; then
if [ "${PYTHON_VERSION}" == "3.7" ]; then
add-apt-repository -y ppa:deadsnakes/ppa
fi
elif [ "${release}" == "jammy" ]; then
if [ "${PYTHON_VERSION}" == "3.8" ]; then
add-apt-repository -y ppa:deadsnakes/ppa
fi
else
echo "Don't know which version of python to install for lsb-release ${release}"
exit 2
fi
case "${release}" in
bionic)
[ "${PYTHON_VERSION}" == "3.8" ] && add-apt-repository -y ppa:deadsnakes/ppa
;;
focal)
[ "${PYTHON_VERSION}" == "3.7" ] && add-apt-repository -y ppa:deadsnakes/ppa
;;
jammy)
if [ "${PYTHON_VERSION}" == "3.8" ] || [ "${PYTHON_VERSION}" == "3.9" ]; then
add-apt-repository -y ppa:deadsnakes/ppa
fi
;;
*)
echo "Don't know which version of python to install for lsb-release ${release}"
exit 2
;;
esac

# Install python and pip. Don't modify this to add Python package dependencies,
# instead modify install_python_package.sh
Expand All @@ -84,7 +88,6 @@ export PYTHONNOUSERSITE=1
venv_dir="$(python3 -c "import os.path;print(os.path.dirname(\"${TVM_VENV}\"))")"
mkdir -p "${venv_dir}"
python3 -mvenv "${TVM_VENV}"
. "${TVM_VENV}/bin/activate"

# NOTE: Only in python3.9 does venv guarantee it creates the python3.X binary.
# This is needed so that cmake's find_package(PythonInterp) works inside the venv.
Expand All @@ -95,15 +98,15 @@ fi

# Update pip to match version used to produce requirements-hashed.txt. This step
# is necessary so that pip's dependency solver is recent.
pip_spec=$(cat /install/python/bootstrap/lockfiles/constraints-${PYTHON_VERSION}.txt | grep 'pip==')
pip3 install -U --require-hashes -r <(echo "${pip_spec}") \
pip_spec=$(tac /install/python/bootstrap/lockfiles/constraints-${PYTHON_VERSION}.txt | grep -m 1 'pip==')
${TVM_VENV}/bin/pip install -U --require-hashes -r <(echo "${pip_spec}") \
-c /install/python/bootstrap/lockfiles/constraints-${PYTHON_VERSION}.txt

# Python configuration
pip3 config set global.no-cache-dir true # Never cache packages
${TVM_VENV}/bin/pip config set global.no-cache-dir true # Never cache packages

# Now install the remaining base packages.
pip3 install \
${TVM_VENV}/bin/pip install \
--require-hashes \
-r /install/python/bootstrap/lockfiles/constraints-${PYTHON_VERSION}.txt

Expand All @@ -114,7 +117,6 @@ setfacl -R -m group:tvm-venv:rwx "${TVM_VENV}"

# Prevent further use of pip3 via the system.
# There may be multiple (i.e. from python3-pip apt package and pip3 install -U).
deactivate
while [ "$(which pip3)" != "" ]; do
rm "$(which pip3)"
done
Expand Down
Loading

0 comments on commit af6630c

Please sign in to comment.