From a7ab1afe3a6be8080a7116d3bd1f9f603c3f14cf Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Mon, 10 Jun 2024 22:57:24 -0400 Subject: [PATCH] ci(build): use uv for cibuildwheel (#3695) Should save 1 minute per build on average. ## Summary by CodeRabbit - **New Features** - Introduced platform-specific constraints for `h5py` when using TensorFlow, ensuring better compatibility and performance. - **Chores** - Updated build configuration to include the `uv` tool for building wheels. - Modified `.gitignore` to exclude the `.uv/` directory. --------- Signed-off-by: Jinzhe Zeng --- .github/workflows/build_wheel.yml | 23 +++++++---------------- .gitignore | 1 + backend/find_tensorflow.py | 7 +++++++ pyproject.toml | 7 +++++++ 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index 934d37ff7d..7fada27493 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -74,23 +74,19 @@ jobs: with: # https://github.com/pypa/setuptools_scm/issues/480 fetch-depth: 0 + - name: Install uv + run: curl -LsSf https://astral.sh/uv/install.sh | sh + if: runner.os != 'Linux' - uses: docker/setup-qemu-action@v3 name: Setup QEMU if: matrix.platform_id == 'manylinux_aarch64' && matrix.os == 'ubuntu-latest' # detect version in advance. See #3168 - - uses: actions/setup-python@v5 - name: Install Python - with: - python-version: '3.11' - cache: 'pip' - if: matrix.dp_pkg_name == 'deepmd-kit-cu11' - run: | - python -m pip install setuptools_scm - python -c "from setuptools_scm import get_version;print('SETUPTOOLS_SCM_PRETEND_VERSION='+get_version())" >> $GITHUB_ENV + echo "SETUPTOOLS_SCM_PRETEND_VERSION=$(pipx run uv tool run --from setuptools_scm python -m setuptools_scm)" >> $GITHUB_ENV rm -rf .git if: matrix.dp_pkg_name == 'deepmd-kit-cu11' - name: Build wheels - uses: pypa/cibuildwheel@v2.18 + uses: pypa/cibuildwheel@v2.19 env: CIBW_BUILD_VERBOSITY: 1 CIBW_ARCHS: all @@ -98,6 +94,7 @@ jobs: DP_VARIANT: ${{ matrix.dp_variant }} CUDA_VERSION: ${{ matrix.cuda_version }} DP_PKG_NAME: ${{ matrix.dp_pkg_name }} + CIBW_BUILD_FRONTEND: 'build[uv]' - uses: actions/upload-artifact@v4 with: name: cibw-cp${{ matrix.python }}-${{ matrix.platform_id }}-cu${{ matrix.cuda_version }}-${{ strategy.job-index }} @@ -109,14 +106,8 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: actions/setup-python@v5 - name: Install Python - with: - python-version: '3.11' - cache: 'pip' - - run: python -m pip install build - name: Build sdist - run: python -m build --sdist + run: pipx run uv tool run --with build[uv] --from build python -m build --installer uv --sdist - uses: actions/upload-artifact@v4 with: diff --git a/.gitignore b/.gitignore index 5e30cf3167..c531a76177 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,4 @@ build_cc_tests build_c_tests build_c/ libdeepmd_c/ +.uv/ diff --git a/backend/find_tensorflow.py b/backend/find_tensorflow.py index 6d13f7fee5..8ba62c9814 100644 --- a/backend/find_tensorflow.py +++ b/backend/find_tensorflow.py @@ -156,11 +156,18 @@ def get_tf_requirement(tf_version: str = "") -> dict: "tensorflow; platform_machine=='aarch64' or (platform_machine=='arm64' and platform_system == 'Darwin')", # https://github.com/tensorflow/tensorflow/issues/61830 "tensorflow-cpu!=2.15.*; platform_system=='Windows'", + # TODO: build(wheel): unpin h5py on aarch64 + # Revert after https://github.com/h5py/h5py/issues/2408 is fixed; + # or set UV_PREFER_BINARY when https://github.com/astral-sh/uv/issues/1794 is resolved. + # 3.6.0 is the first version to have aarch64 wheels. + "h5py>=3.6.0,<3.11.0; platform_system=='Linux' and platform_machine=='aarch64'", *extra_requires, ], "gpu": [ "tensorflow", "tensorflow-metal; platform_machine=='arm64' and platform_system == 'Darwin'", + # See above. + "h5py>=3.6.0,<3.11.0; platform_system=='Linux' and platform_machine=='aarch64'", *extra_requires, ], **extra_select, diff --git a/pyproject.toml b/pyproject.toml index f22ff40fa2..ebb60b1712 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ dependencies = [ 'typing_extensions; python_version < "3.8"', 'importlib_metadata>=1.4; python_version < "3.8"', 'h5py', + "h5py>=3.6.0,<3.11.0; platform_system=='Linux' and platform_machine=='aarch64'", 'wcmatch', 'packaging', 'ml_dtypes', @@ -232,6 +233,12 @@ before-all = [ """rpm --import https://repo.almalinux.org/almalinux/RPM-GPG-KEY-AlmaLinux""", """{ if [ "$(uname -m)" = "x86_64" ] ; then yum config-manager --add-repo http://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo && yum install -y cuda-nvcc-${CUDA_VERSION/./-} cuda-cudart-devel-${CUDA_VERSION/./-}; fi }""", "yum install -y mpich-devel", + # uv is not available in the old manylinux image + """{ if [ "$(uname -m)" = "x86_64" ] ; then pipx install uv; fi }""", +] +before-build = [ + # old build doesn't support uv + """{ if [ "$(uname -m)" = "x86_64" ] ; then uv pip install --system -U build; fi }""", ] [tool.cibuildwheel.windows]