From c7d0aa805e7a3864e70a45608434b30b915d5891 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Jul 2024 08:54:02 -0700 Subject: [PATCH 1/6] Bump pypa/cibuildwheel from 2.19.1 to 2.19.2 (#59208) Bumps [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel) from 2.19.1 to 2.19.2. - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.19.1...v2.19.2) --- updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> (cherry picked from commit ad09dc6108896e175979c247cff2878d259acf3d) --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 4bd9068e91b67..4cb1260f9911e 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -140,7 +140,7 @@ jobs: run: echo "sdist_name=$(cd ./dist && ls -d */)" >> "$GITHUB_ENV" - name: Build wheels - uses: pypa/cibuildwheel@v2.17.0 + uses: pypa/cibuildwheel@v2.19.2 with: package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }} env: From e1dfdfd08e8496857d942493930b805279f9fa58 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Mon, 22 Jul 2024 22:26:45 +0200 Subject: [PATCH 2/6] Upload 3.13 & free-threaded nightly wheels (#59136) * Upload free-threaded nightly wheels on Linux and macOS * Consolidate jobs into one * Install build dependencies in before-build and pass --no-build-isolation * Fix {project} placeholder in cibuildwheel config * Correctly quote echo CIBW_BUILD_FRONTEND command * Run echo -e * Add {package} to before-build * Include cibw script in sdist & add matrix value for build frontend * Change manifest and gitattributes * Change gitattributes * Install verioneer in before-build * Add cibw_before_test to install nightly NumPy * Expand before-test to musl * Better comments plus always run before-build/before-test on 3.13 * Add --no-build-isolation in 3.13 as well * Install nightly numpy before windows tests * Address feedback; add todo for NumPy nightly and move default outside matrix * Set build_frontend to 'build' in pyodide build --------- Co-authored-by: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> (cherry picked from commit 7c0ee27e6c00e9645154583917de0f385190d8d8) --- .gitattributes | 6 +++++- .github/workflows/wheels.yml | 21 ++++++++++++++++++--- MANIFEST.in | 4 ++++ pyproject.toml | 12 +++++------- scripts/cibw_before_build.sh | 9 +++++++++ scripts/cibw_before_test.sh | 8 ++++++++ 6 files changed, 49 insertions(+), 11 deletions(-) create mode 100644 scripts/cibw_before_build.sh create mode 100644 scripts/cibw_before_test.sh diff --git a/.gitattributes b/.gitattributes index 19c6fd2fd1d47..b3d70ca8b24fb 100644 --- a/.gitattributes +++ b/.gitattributes @@ -68,7 +68,7 @@ ci export-ignore doc export-ignore gitpod export-ignore MANIFEST.in export-ignore -scripts export-ignore +scripts/** export-ignore typings export-ignore web export-ignore CITATION.cff export-ignore @@ -82,3 +82,7 @@ setup.py export-ignore # csv_dir_path fixture checks the existence of the directory # exclude the whole directory to avoid running related tests in sdist pandas/tests/io/parser/data export-ignore + +# Include cibw script in sdist since it's needed for building wheels +scripts/cibw_before_build.sh -export-ignore +scripts/cibw_before_test.sh -export-ignore diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 4cb1260f9911e..546fdfb260951 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -99,7 +99,19 @@ jobs: - [macos-14, macosx_arm64] - [windows-2022, win_amd64] # TODO: support PyPy? - python: [["cp39", "3.9"], ["cp310", "3.10"], ["cp311", "3.11"], ["cp312", "3.12"]] + python: [["cp39", "3.9"], ["cp310", "3.10"], ["cp311", "3.11"], ["cp312", "3.12"], ["cp313", "3.13"], ["cp313t", "3.13"]] + include: + # TODO: Remove this plus installing build deps in cibw_before_build.sh + # and test deps in cibw_before_test.sh after pandas can be built with a released NumPy/Cython + - python: ["cp313", "3.13"] + cibw_build_frontend: 'pip; args: --no-build-isolation' + - python: ["cp313t", "3.13"] + cibw_build_frontend: 'pip; args: --no-build-isolation' + # TODO: Build free-threaded wheels for Windows + exclude: + - buildplat: [windows-2022, win_amd64] + python: ["cp313t", "3.13"] + env: IS_PUSH: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }} IS_SCHEDULE_DISPATCH: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} @@ -146,6 +158,7 @@ jobs: env: CIBW_PRERELEASE_PYTHONS: True CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }} + CIBW_BUILD_FRONTEND: ${{ matrix.cibw_build_frontend || 'pip' }} - name: Set up Python uses: mamba-org/setup-micromamba@v1 @@ -168,15 +181,17 @@ jobs: - name: Test Windows Wheels if: ${{ matrix.buildplat[1] == 'win_amd64' }} shell: pwsh + # TODO: Remove NumPy nightly install when there's a 3.13 wheel on PyPI run: | $TST_CMD = @" python -m pip install hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0; + ${{ matrix.python[1] == '3.13' && 'python -m pip install -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy;' }} python -m pip install `$(Get-Item pandas\wheelhouse\*.whl); python -c `'import pandas as pd; pd.test(extra_args=[`\"--no-strict-data-files`\", `\"-m not clipboard and not single_cpu and not slow and not network and not db`\"])`'; "@ # add rc to the end of the image name if the Python version is unreleased - docker pull python:${{ matrix.python[1] == '3.12' && '3.12-rc' || format('{0}-windowsservercore', matrix.python[1]) }} - docker run --env PANDAS_CI='1' -v ${PWD}:C:\pandas python:${{ matrix.python[1] == '3.12' && '3.12-rc' || format('{0}-windowsservercore', matrix.python[1]) }} powershell -Command $TST_CMD + docker pull python:${{ matrix.python[1] == '3.13' && '3.13-rc' || format('{0}-windowsservercore', matrix.python[1]) }} + docker run --env PANDAS_CI='1' -v ${PWD}:C:\pandas python:${{ matrix.python[1] == '3.13' && '3.13-rc' || format('{0}-windowsservercore', matrix.python[1]) }} powershell -Command $TST_CMD - uses: actions/upload-artifact@v4 with: diff --git a/MANIFEST.in b/MANIFEST.in index 9894381ed6252..f586d457eaaf8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -62,3 +62,7 @@ prune pandas/tests/io/parser/data # Selectively re-add *.cxx files that were excluded above graft pandas/_libs/src graft pandas/_libs/include + +# Include cibw script in sdist since it's needed for building wheels +include scripts/cibw_before_build.sh +include scripts/cibw_before_test.sh diff --git a/pyproject.toml b/pyproject.toml index db9f055799ab0..22c4688436e0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -153,25 +153,23 @@ setup = ['--vsenv'] # For Windows skip = "cp36-* cp37-* cp38-* pp* *_i686 *_ppc64le *_s390x" build-verbosity = "3" environment = {LDFLAGS="-Wl,--strip-all"} -# TODO: remove this once numpy 2.0 proper releases -# and specify numpy 2.0 as a dependency in [build-system] requires in pyproject.toml -before-build = "pip install numpy==2.0.0rc1" test-requires = "hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0" test-command = """ PANDAS_CI='1' python -c 'import pandas as pd; \ pd.test(extra_args=["-m not clipboard and not single_cpu and not slow and not network and not db", "-n 2", "--no-strict-data-files"]); \ pd.test(extra_args=["-m not clipboard and single_cpu and not slow and not network and not db", "--no-strict-data-files"]);' \ """ +free-threaded-support = true +before-build = "bash {package}/scripts/cibw_before_build.sh" +before-test = "bash {package}/scripts/cibw_before_test.sh" [tool.cibuildwheel.windows] -# TODO: remove this once numpy 2.0 proper releases -# and specify numpy 2.0 as a dependency in [build-system] requires in pyproject.toml -before-build = "pip install delvewheel numpy==2.0.0rc1" +before-build = "pip install delvewheel && bash {package}/scripts/cibw_before_build.sh" repair-wheel-command = "delvewheel repair -w {dest_dir} {wheel}" [[tool.cibuildwheel.overrides]] select = "*-musllinux*" -before-test = "apk update && apk add musl-locales" +before-test = "apk update && apk add musl-locales && bash {package}/scripts/cibw_before_test.sh" [[tool.cibuildwheel.overrides]] select = "*-win*" diff --git a/scripts/cibw_before_build.sh b/scripts/cibw_before_build.sh new file mode 100644 index 0000000000000..f3049b27ed5d1 --- /dev/null +++ b/scripts/cibw_before_build.sh @@ -0,0 +1,9 @@ +# TODO: Delete when there's PyPI NumPy/Cython releases the support Python 3.13. +# If free-threading support is not included in those releases, this script will have +# to whether this runs for a free-threaded build instead. +PYTHON_VERSION="$(python -c "import sys; print(f'{sys.version_info.major}{sys.version_info.minor}')")" +if [[ $PYTHON_VERSION == "313" ]]; then + python -m pip install -U pip + python -m pip install -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy cython + python -m pip install ninja meson-python versioneer[toml] +fi diff --git a/scripts/cibw_before_test.sh b/scripts/cibw_before_test.sh new file mode 100644 index 0000000000000..7d1b143881ced --- /dev/null +++ b/scripts/cibw_before_test.sh @@ -0,0 +1,8 @@ +# TODO: Delete when there's PyPI NumPy/Cython releases the support Python 3.13. +# If free-threading support is not included in those releases, this script will have +# to whether this runs for a free-threaded build instead. +PYTHON_VERSION="$(python -c "import sys; print(f'{sys.version_info.major}{sys.version_info.minor}')")" +if [[ $PYTHON_VERSION == "313" ]]; then + python -m pip install -U pip + python -m pip install -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy +fi From 112a95b1ee88911a0b17ac4b45ad4d9fc74e4d6b Mon Sep 17 00:00:00 2001 From: Ewout ter Hoeven Date: Tue, 6 Aug 2024 03:08:35 +0200 Subject: [PATCH 3/6] CI: Update to cibuildwheel 2.20.0 (#59401) cibuildwheel 2.20.0 uses the ABI stable Python 3.13.0rc1 and build Python 3.13 wheels by default, which allows removing the `CIBW_PRERELEASE_PYTHONS` flag. Co-authored-by: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> (cherry picked from commit 70bb855cbbc75b52adcb127c84e0a35d2cd796a9) --- .circleci/config.yml | 2 +- .github/workflows/wheels.yml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0748d6550fe2d..e66352c519383 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -72,7 +72,7 @@ jobs: name: Build aarch64 wheels no_output_timeout: 30m # Sometimes the tests won't generate any output, make sure the job doesn't get killed by that command: | - pip3 install cibuildwheel==2.15.0 + pip3 install cibuildwheel==2.20.0 cibuildwheel --prerelease-pythons --output-dir wheelhouse environment: diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 546fdfb260951..3dff33e881d56 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -156,7 +156,6 @@ jobs: with: package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }} env: - CIBW_PRERELEASE_PYTHONS: True CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }} CIBW_BUILD_FRONTEND: ${{ matrix.cibw_build_frontend || 'pip' }} From 30f073e393fe5dd2cddc40d7ac6f16d0812fe09e Mon Sep 17 00:00:00 2001 From: Thomas Li <47963215+lithomas1@users.noreply.github.com> Date: Wed, 18 Sep 2024 13:51:48 -0400 Subject: [PATCH 4/6] Update wheels.yml --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index fdaa9c96081af..26620b72cb713 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -152,7 +152,7 @@ jobs: run: echo "sdist_name=$(cd ./dist && ls -d */)" >> "$GITHUB_ENV" - name: Build wheels - uses: pypa/cibuildwheel@v2.19.2 + uses: pypa/cibuildwheel@v2.20.0 with: package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }} env: From 584b7e812a9c69a439686ffed994ef42964d5007 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 19 Sep 2024 00:46:07 +0200 Subject: [PATCH 5/6] BLD/RLS: build wheels with released numpy/cython for Python 3.13 (#59819) (cherry picked from commit 22372175e04f05f73521cab1b26f0818d6766717) --- .github/workflows/wheels.yml | 6 +----- MANIFEST.in | 1 - pyproject.toml | 3 +-- scripts/cibw_before_build.sh | 8 +++----- scripts/cibw_before_test.sh | 8 -------- 5 files changed, 5 insertions(+), 21 deletions(-) delete mode 100644 scripts/cibw_before_test.sh diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 26620b72cb713..284559f42610a 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -102,9 +102,7 @@ jobs: python: [["cp39", "3.9"], ["cp310", "3.10"], ["cp311", "3.11"], ["cp312", "3.12"], ["cp313", "3.13"], ["cp313t", "3.13"]] include: # TODO: Remove this plus installing build deps in cibw_before_build.sh - # and test deps in cibw_before_test.sh after pandas can be built with a released NumPy/Cython - - python: ["cp313", "3.13"] - cibw_build_frontend: 'pip; args: --no-build-isolation' + # after pandas can be built with a released NumPy/Cython - python: ["cp313t", "3.13"] cibw_build_frontend: 'pip; args: --no-build-isolation' # TODO: Build free-threaded wheels for Windows @@ -180,11 +178,9 @@ jobs: - name: Test Windows Wheels if: ${{ matrix.buildplat[1] == 'win_amd64' }} shell: pwsh - # TODO: Remove NumPy nightly install when there's a 3.13 wheel on PyPI run: | $TST_CMD = @" python -m pip install hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytz<2024.2; - ${{ matrix.python[1] == '3.13' && 'python -m pip install -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy;' }} python -m pip install `$(Get-Item pandas\wheelhouse\*.whl); python -c `'import pandas as pd; pd.test(extra_args=[`\"--no-strict-data-files`\", `\"-m not clipboard and not single_cpu and not slow and not network and not db`\"])`'; "@ diff --git a/MANIFEST.in b/MANIFEST.in index f586d457eaaf8..a7d7d7eb4e062 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -65,4 +65,3 @@ graft pandas/_libs/include # Include cibw script in sdist since it's needed for building wheels include scripts/cibw_before_build.sh -include scripts/cibw_before_test.sh diff --git a/pyproject.toml b/pyproject.toml index e7b4a8e61c7f3..2a8e63caaf37a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -162,7 +162,6 @@ test-command = """ """ free-threaded-support = true before-build = "bash {package}/scripts/cibw_before_build.sh" -before-test = "bash {package}/scripts/cibw_before_test.sh" [tool.cibuildwheel.windows] before-build = "pip install delvewheel && bash {package}/scripts/cibw_before_build.sh" @@ -178,7 +177,7 @@ test-command = """ [[tool.cibuildwheel.overrides]] select = "*-musllinux*" -before-test = "apk update && apk add musl-locales && bash {package}/scripts/cibw_before_test.sh" +before-test = "apk update && apk add musl-locales" [[tool.cibuildwheel.overrides]] select = "*-win*" diff --git a/scripts/cibw_before_build.sh b/scripts/cibw_before_build.sh index f3049b27ed5d1..6186340807f8f 100644 --- a/scripts/cibw_before_build.sh +++ b/scripts/cibw_before_build.sh @@ -1,8 +1,6 @@ -# TODO: Delete when there's PyPI NumPy/Cython releases the support Python 3.13. -# If free-threading support is not included in those releases, this script will have -# to whether this runs for a free-threaded build instead. -PYTHON_VERSION="$(python -c "import sys; print(f'{sys.version_info.major}{sys.version_info.minor}')")" -if [[ $PYTHON_VERSION == "313" ]]; then +# TODO: Delete when there's a PyPI Cython release that supports free-threaded Python 3.13. +FREE_THREADED_BUILD="$(python -c"import sysconfig; print(bool(sysconfig.get_config_var('Py_GIL_DISABLED')))")" +if [[ $FREE_THREADED_BUILD == "True" ]]; then python -m pip install -U pip python -m pip install -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy cython python -m pip install ninja meson-python versioneer[toml] diff --git a/scripts/cibw_before_test.sh b/scripts/cibw_before_test.sh deleted file mode 100644 index 7d1b143881ced..0000000000000 --- a/scripts/cibw_before_test.sh +++ /dev/null @@ -1,8 +0,0 @@ -# TODO: Delete when there's PyPI NumPy/Cython releases the support Python 3.13. -# If free-threading support is not included in those releases, this script will have -# to whether this runs for a free-threaded build instead. -PYTHON_VERSION="$(python -c "import sys; print(f'{sys.version_info.major}{sys.version_info.minor}')")" -if [[ $PYTHON_VERSION == "313" ]]; then - python -m pip install -U pip - python -m pip install -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy -fi From 93fbcd8c275cd8692389988f52a2da1eaa87cf3f Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 19 Sep 2024 00:58:11 +0200 Subject: [PATCH 6/6] enable prerelease again --- .gitattributes | 1 - .github/workflows/wheels.yml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index b3d70ca8b24fb..2655d0d018d4f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -85,4 +85,3 @@ pandas/tests/io/parser/data export-ignore # Include cibw script in sdist since it's needed for building wheels scripts/cibw_before_build.sh -export-ignore -scripts/cibw_before_test.sh -export-ignore diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 284559f42610a..41417622c3ef2 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -154,6 +154,7 @@ jobs: with: package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }} env: + CIBW_PRERELEASE_PYTHONS: True CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }} CIBW_BUILD_FRONTEND: ${{ matrix.cibw_build_frontend || 'pip' }}