GH-47438: [Python][Packaging] Set up wheel building for Python 3.14 (#47616)

AlenkaF · web-flow · commit f268c43ccd86 · 2025-10-14T06:17:08.000+02:00
### Rationale for this change Python 3.14 is currently in a prerelease status and is expected to have a final release in October this year (https://peps.python.org/pep-0745/). We should ensure we are fully ready to support Python 3.14 for the PyArrow 22 release. ### What changes are included in this PR? This PR updates wheels for Python 3.14. ### Are these changes tested? Tested in the CI and with extended builds. ### Are there any user-facing changes? No, but users will be able to use PyArrow with Python 3.14. * GitHub Issue: #47438 --- Todo: - Update the image revision name in `.env` - Add 3.14 conda build ([arrow/dev/tasks/tasks.yml](https://github.com/apache/arrow/blob/d803afcc43f5d132506318fd9e162d33b2c3d4cd/dev/tasks/tasks.yml#L809)) when conda-forge/pyarrow-feedstock#156 is merged Follow-ups: - #47437 Authored-by: AlenkaF <frim.alenka@gmail.com> Signed-off-by: AlenkaF <frim.alenka@gmail.com>
diff --git a/.env b/.env
@@ -102,8 +102,8 @@ VCPKG="4334d8b4c8916018600212ab4dd4bbdc343065d1"    # 2025.09.17 Release
 # ci/docker/python-*-windows-*.dockerfile or the vcpkg config.
 # This is a workaround for our CI problem that "archery docker build" doesn't
 # use pulled built images in dev/tasks/python-wheels/github.windows.yml.
-PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2025-09-04
-PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION=2025-09-04
+PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2025-10-13
+PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION=2025-10-13
 
 # Use conanio/${CONAN_BASE}:{CONAN_VERSION} for "docker compose run --rm conan".
 # See https://github.com/conan-io/conan-docker-tools#readme and
diff --git a/ci/docker/python-free-threaded-wheel-manylinux-test-imports.dockerfile b/ci/docker/python-free-threaded-wheel-manylinux-test-imports.dockerfile
@@ -18,18 +18,20 @@
 ARG base
 FROM ${base}
 
+ARG python_version=3.13
+
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update -y -q && \
     apt install -y -q --no-install-recommends software-properties-common gpg-agent && \
     add-apt-repository -y ppa:deadsnakes/ppa && \
     apt-get update -y -q && \
-    apt install -y -q --no-install-recommends python3.13-dev python3.13-nogil python3.13-venv && \
+    apt install -y -q --no-install-recommends python${python_version}-dev python${python_version}-nogil python${python_version}-venv && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists*
 
 ENV ARROW_PYTHON_VENV /arrow-dev
-RUN python3.13t -m venv ${ARROW_PYTHON_VENV}
+RUN python${python_version}t -m venv ${ARROW_PYTHON_VENV}
 
 ENV PYTHON_GIL 0
 ENV PATH "${ARROW_PYTHON_VENV}/bin:${PATH}"
diff --git a/ci/docker/python-free-threaded-wheel-manylinux-test-unittests.dockerfile b/ci/docker/python-free-threaded-wheel-manylinux-test-unittests.dockerfile
@@ -18,6 +18,8 @@
 ARG base
 FROM ${base}
 
+ARG python_version=3.13
+
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update -y -q && \
@@ -27,14 +29,14 @@ RUN apt-get update -y -q && \
     apt install -y -q --no-install-recommends \
         build-essential \
         libffi-dev \
-        python3.13-dev \
-        python3.13-nogil \
-        python3.13-venv && \
+        python${python_version}-dev \
+        python${python_version}-nogil \
+        python${python_version}-venv && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists*
 
 ENV ARROW_PYTHON_VENV /arrow-dev
-RUN python3.13t -m venv ${ARROW_PYTHON_VENV}
+RUN python${python_version}t -m venv ${ARROW_PYTHON_VENV}
 
 ENV PYTHON_GIL 0
 ENV PATH "${ARROW_PYTHON_VENV}/bin:${PATH}"
diff --git a/ci/docker/python-free-threaded-wheel-musllinux-test-imports.dockerfile b/ci/docker/python-free-threaded-wheel-musllinux-test-imports.dockerfile
@@ -18,6 +18,9 @@
 ARG base
 FROM ${base}
 
+ARG python_version=3.13
+ARG python_patch_version=3.13.7
+
 RUN apk add --no-cache \
     bash \
     build-base \
@@ -34,19 +37,19 @@ RUN apk add --no-cache \
     tzdata \
     zlib-dev
 
-# Install Python3.13.2 without GIL
-RUN wget https://github.com/python/cpython/archive/refs/tags/v3.13.2.tar.gz && \
-    tar -xzf v3.13.2.tar.gz && \
-    rm v3.13.2.tar.gz && \
-    cd cpython-3.13.2/ && \
+# Install Python without GIL
+RUN wget https://github.com/python/cpython/archive/refs/tags/v${python_patch_version}.tar.gz && \
+    tar -xzf v${python_patch_version}.tar.gz && \
+    rm v${python_patch_version}.tar.gz && \
+    cd cpython-${python_patch_version}/ && \
     ./configure --disable-gil --with-ensurepip && \
     make -j && \
     make install && \
     cd ../ && \
-    rm -rf cpython-3.13.2/
+    rm -rf cpython-${python_patch_version}/
 
 ENV ARROW_PYTHON_VENV /arrow-dev
-RUN python3.13t -m venv ${ARROW_PYTHON_VENV}
+RUN python${python_version}t -m venv ${ARROW_PYTHON_VENV}
 
 ENV PYTHON_GIL 0
 ENV PATH "${ARROW_PYTHON_VENV}/bin:${PATH}"
diff --git a/ci/docker/python-free-threaded-wheel-musllinux-test-unittests.dockerfile b/ci/docker/python-free-threaded-wheel-musllinux-test-unittests.dockerfile
@@ -18,6 +18,9 @@
 ARG base
 FROM ${base}
 
+ARG python_version=3.13
+ARG python_patch_version=3.13.7
+
 RUN apk add --no-cache \
     bash \
     build-base \
@@ -34,19 +37,19 @@ RUN apk add --no-cache \
     tzdata \
     zlib-dev
 
-# Install Python3.13.2 without GIL
-RUN wget https://github.com/python/cpython/archive/refs/tags/v3.13.2.tar.gz && \
-    tar -xzf v3.13.2.tar.gz && \
-    rm v3.13.2.tar.gz && \
-    cd cpython-3.13.2/ && \
+# Install Python without GIL
+RUN wget https://github.com/python/cpython/archive/refs/tags/v${python_patch_version}.tar.gz && \
+    tar -xzf v${python_patch_version}.tar.gz && \
+    rm v${python_patch_version}.tar.gz && \
+    cd cpython-${python_patch_version}/ && \
     ./configure --disable-gil --with-ensurepip && \
     make -j && \
     make install && \
     cd ../ && \
-    rm -rf cpython-3.13.2/
+    rm -rf cpython-${python_patch_version}/
 
 ENV ARROW_PYTHON_VENV /arrow-dev
-RUN python3.13t -m venv ${ARROW_PYTHON_VENV}
+RUN python${python_version}t -m venv ${ARROW_PYTHON_VENV}
 
 ENV PYTHON_GIL 0
 ENV PATH "${ARROW_PYTHON_VENV}/bin:${PATH}"
diff --git a/ci/docker/python-free-threaded-wheel-windows-test-vs2022.dockerfile b/ci/docker/python-free-threaded-wheel-windows-test-vs2022.dockerfile
@@ -26,24 +26,34 @@ FROM ${base}
 
 ARG python=3.13
 
+# hadolint ignore=SC1072
+RUN (if "%python%"=="3.13" setx PYTHON_VERSION "3.13.1") & \
+    (if "%python%"=="3.14" setx PYTHON_VERSION "3.14.0")
+
 SHELL ["powershell", "-NoProfile", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
-RUN $filename = 'python-3.13.1-amd64.exe'; \
-    $url = 'https://www.python.org/ftp/python/3.13.1/' + $filename; \
+RUN $version = $env:PYTHON_VERSION; \
+    $filename = 'python-' + $version + '-amd64.exe'; \
+    $url = 'https://www.python.org/ftp/python/' + $version + '/' + $filename; \
     Invoke-WebRequest -Uri $url -OutFile $filename; \
     Start-Process -FilePath $filename -ArgumentList '/quiet', 'Include_freethreaded=1' -Wait
 
 ENV PYTHON_CMD="py -${python}t"
 
 SHELL ["cmd", "/S", "/C"]
-RUN %PYTHON_CMD% -m pip install -U pip setuptools
-
-COPY python/requirements-wheel-test-3.13t.txt C:/arrow/python/
-# Cython and Pandas wheels for 3.13 free-threaded are not released yet
+RUN %PYTHON_CMD% -m pip install -U pip setuptools & \
+    if "%python%"=="3.13" ( \
+        setx REQUIREMENTS_FILE "requirements-wheel-test-3.13t.txt" \
+    ) else ( \
+        setx REQUIREMENTS_FILE "requirements-wheel-test.txt" \
+    )
+
+COPY python/requirements-wheel-test-3.13t.txt python/requirements-wheel-test.txt C:/arrow/python/
+# Cython and Pandas wheels for free-threaded are not released yet
 RUN %PYTHON_CMD% -m pip install \
     --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple \
     --pre \
     --prefer-binary \
-    -r C:/arrow/python/requirements-wheel-test-3.13t.txt
+    -r C:/arrow/python/%REQUIREMENTS_FILE%
 
 ENV PYTHON="${python}t"
 ENV PYTHON_GIL=0
diff --git a/ci/docker/python-free-threaded-wheel-windows-vs2022.dockerfile b/ci/docker/python-free-threaded-wheel-windows-vs2022.dockerfile
@@ -26,9 +26,13 @@ FROM ${base}
 
 ARG python=3.13
 
+RUN (if "%python%"=="3.13" setx PYTHON_VERSION "3.13.1") & \
+    (if "%python%"=="3.14" setx PYTHON_VERSION "3.14.0")
+
 SHELL ["powershell", "-NoProfile", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
-RUN $filename = 'python-3.13.1-amd64.exe'; \
-    $url = 'https://www.python.org/ftp/python/3.13.1/' + $filename; \
+RUN $version = $env:PYTHON_VERSION; \
+    $filename = 'python-' + $version + '-amd64.exe'; \
+    $url = 'https://www.python.org/ftp/python/' + $version + '/' + $filename; \
     Invoke-WebRequest -Uri $url -OutFile $filename; \
     Start-Process -FilePath $filename -ArgumentList '/quiet', 'Include_freethreaded=1' -Wait
 
diff --git a/ci/docker/python-wheel-musllinux.dockerfile b/ci/docker/python-wheel-musllinux.dockerfile
@@ -82,6 +82,7 @@ RUN --mount=type=secret,id=github_repository_owner \
       export GITHUB_REPOSITORY_OWNER=$(cat /run/secrets/github_repository_owner); \
       export GITHUB_TOKEN=$(cat /run/secrets/github_token); \
       export VCPKG_BINARY_SOURCES=$(cat /run/secrets/vcpkg_binary_sources); \
+      export CMAKE_POLICY_VERSION_MINIMUM=3.5; \
       arrow/ci/scripts/install_vcpkg.sh ${VCPKG_ROOT} ${vcpkg} && \
       vcpkg install \
         --clean-after-build \
@@ -110,10 +111,5 @@ RUN PYTHON_ROOT=$(find /opt/python -name cp${PYTHON_VERSION/./}-${PYTHON_ABI_TAG
 SHELL ["/bin/bash", "-i", "-c", "-l"]
 ENTRYPOINT ["/bin/bash", "-i", "-c", "-l"]
 
-# Remove once there are released Cython wheels for 3.13 free-threaded available
-RUN if [ "${python_abi_tag}" = "cp313t" ]; then \
-      pip install cython --pre --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" --prefer-binary ; \
-    fi
-
 COPY python/requirements-wheel-build.txt /arrow/python/
 RUN pip install -r /arrow/python/requirements-wheel-build.txt
diff --git a/ci/docker/python-wheel-windows-test-vs2022.dockerfile b/ci/docker/python-wheel-windows-test-vs2022.dockerfile
@@ -30,8 +30,9 @@ FROM ${base}
 ARG python=3.10
 RUN (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PYTHON_CMD "py -3.10") & \
     (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.9" && setx PYTHON_CMD "py -3.11") & \
-    (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.8" && setx PYTHON_CMD "py -3.12") & \
-    (if "%python%"=="3.13" setx PYTHON_VERSION "3.13.1" && setx PYTHON_CMD "py -3.13")
+    (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.10" && setx PYTHON_CMD "py -3.12") & \
+    (if "%python%"=="3.13" setx PYTHON_VERSION "3.13.7" && setx PYTHON_CMD "py -3.13") & \
+    (if "%python%"=="3.14" setx PYTHON_VERSION "3.14.0" && setx PYTHON_CMD "py -3.14")
 
 # hadolint ignore=DL3059
 RUN choco install -r -y --pre --no-progress --force python --version=%PYTHON_VERSION%
diff --git a/ci/docker/python-wheel-windows-vs2022.dockerfile b/ci/docker/python-wheel-windows-vs2022.dockerfile
@@ -25,8 +25,9 @@ FROM ${base}
 ARG python=3.10
 RUN (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PYTHON_CMD "py -3.10") & \
     (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.9" && setx PYTHON_CMD "py -3.11") & \
-    (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.8" && setx PYTHON_CMD "py -3.12") & \
-    (if "%python%"=="3.13" setx PYTHON_VERSION "3.13.1" && setx PYTHON_CMD "py -3.13")
+    (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.10" && setx PYTHON_CMD "py -3.12") & \
+    (if "%python%"=="3.13" setx PYTHON_VERSION "3.13.7" && setx PYTHON_CMD "py -3.13") & \
+    (if "%python%"=="3.14" setx PYTHON_VERSION "3.14.0" && setx PYTHON_CMD "py -3.14")
 
 RUN choco install -r -y --pre --no-progress python --version=%PYTHON_VERSION%
 RUN %PYTHON_CMD% -m pip install -U pip setuptools
diff --git a/ci/scripts/install_python.sh b/ci/scripts/install_python.sh
@@ -27,9 +27,11 @@ platforms=([windows]=Windows
 declare -A versions
 versions=([3.10]=3.10.11
           [3.11]=3.11.9
-          [3.12]=3.12.9
-          [3.13]=3.13.2
-          [3.13t]=3.13.2)
+          [3.12]=3.12.10
+          [3.13]=3.13.7
+          [3.13t]=3.13.7
+          [3.14]=3.14.0
+          [3.14t]=3.14.0)
 
 if [ "$#" -ne 2 ]; then
   echo "Usage: $0 <platform> <version>"
@@ -50,7 +52,9 @@ if [ "$platform" = "macOS" ]; then
     wget "https://www.python.org/ftp/python/${full_version}/${fname}"
 
     echo "Installing Python..."
-    if [[ $2 == "3.13t" ]]; then
+    if [[ $2 == "3.13t" ]] || [[ $2 == "3.14t" ]]; then
+        # Extract the base version without 't' suffix
+        base_version="${version%t}"
         # See https://github.com/python/cpython/issues/120098#issuecomment-2151122033 for more info on this.
         cat > ./choicechanges.plist <<EOF
 <?xml version="1.0" encoding="UTF-8"?>
@@ -63,7 +67,7 @@ if [ "$platform" = "macOS" ]; then
                 <key>choiceAttribute</key>
                 <string>selected</string>
                 <key>choiceIdentifier</key>
-                <string>org.python.Python.PythonTFramework-3.13</string>
+                <string>org.python.Python.PythonTFramework-${base_version}</string>
         </dict>
 </array>
 </plist>
@@ -76,8 +80,9 @@ EOF
     rm "$fname"
 
     python="/Library/Frameworks/Python.framework/Versions/${version}/bin/python${version}"
-    if [[ $2 == "3.13t" ]]; then
-        python="/Library/Frameworks/PythonT.framework/Versions/3.13/bin/python3.13t"
+    if [[ $2 == "3.13t" ]] || [[ $2 == "3.14t" ]]; then
+        base_version="${version%t}"
+        python="/Library/Frameworks/PythonT.framework/Versions/${base_version}/bin/python${base_version}t"
     fi
 
     echo "Installing Pip..."
diff --git a/ci/scripts/python_wheel_unix_test.sh b/ci/scripts/python_wheel_unix_test.sh
@@ -31,9 +31,9 @@ source_dir=${1}
 : "${ARROW_AZURE:=ON}"
 : "${ARROW_FLIGHT:=ON}"
 : "${ARROW_GCS:=ON}"
+: "${CHECK_IMPORTS:=ON}"
 : "${ARROW_S3:=ON}"
 : "${ARROW_SUBSTRAIT:=ON}"
-: "${CHECK_IMPORTS:=ON}"
 : "${CHECK_WHEEL_CONTENT:=ON}"
 : "${CHECK_UNITTESTS:=ON}"
 : "${INSTALL_PYARROW:=ON}"
@@ -106,10 +106,10 @@ is_free_threaded() {
 
 if [ "${CHECK_UNITTESTS}" == "ON" ]; then
   # Install testing dependencies
-  if [ "$(is_free_threaded)" = "ON" ]; then
-    echo "Free-threaded Python build detected"
+  if [ "$(is_free_threaded)" = "ON" ] && [[ "${PYTHON:-}" == *"3.13"* ]]; then
+    echo "Free-threaded Python 3.13 build detected"
     python -m pip install -U -r "${source_dir}/python/requirements-wheel-test-3.13t.txt"
-  elif [ "$(is_free_threaded)" = "OFF" ]; then
+  else
     echo "Regular Python build detected"
     python -m pip install -U -r "${source_dir}/python/requirements-wheel-test.txt"
   fi
diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
@@ -852,7 +852,7 @@ test_linux_wheels() {
     local arch="x86_64"
   fi
 
-  local python_versions="${TEST_PYTHON_VERSIONS:-3.10 3.11 3.12 3.13}"
+  local python_versions="${TEST_PYTHON_VERSIONS:-3.10 3.11 3.12 3.13 3.14}"
   local platform_tags="${TEST_WHEEL_PLATFORM_TAGS:-manylinux_2_28_${arch}}"
 
   if [ "${SOURCE_KIND}" != "local" ]; then
@@ -891,11 +891,11 @@ test_macos_wheels() {
 
   # apple silicon processor
   if [ "$(uname -m)" = "arm64" ]; then
-    local python_versions="3.10 3.11 3.12 3.13"
+    local python_versions="3.10 3.11 3.12 3.13 3.14"
     local platform_tags="macosx_12_0_arm64"
     local check_flight=OFF
   else
-    local python_versions="3.10 3.11 3.12 3.13"
+    local python_versions="3.10 3.11 3.12 3.13 3.14"
     local platform_tags="macosx_12_0_x86_64"
   fi
 
diff --git a/dev/tasks/python-wheels/github.linux.yml b/dev/tasks/python-wheels/github.linux.yml
@@ -42,16 +42,25 @@ jobs:
       {% endif %}
       PYTHON: "{{ python_version }}"
       PYTHON_ABI_TAG: "{{ python_abi_tag }}"
+      {% if python_version == "3.14" %}
+      PYTHON_IMAGE_TAG: "3.14"
+      PYTHON_PATCH_VERSION: "3.14.0"
+      {% elif python_version == "3.13" %}
       PYTHON_IMAGE_TAG: "{{ python_version }}"
+      PYTHON_PATCH_VERSION: "3.13.7"
+      {% else %}
+      PYTHON_IMAGE_TAG: "{{ python_version }}"
+      {% endif %}
 
     steps:
       {{ macros.github_checkout_arrow()|indent }}
       {{ macros.github_install_archery()|indent }}
       {{ macros.github_login_dockerhub()|indent }}
 
+      # Detect free-threaded Python builds
       - name: Prepare
         run: |
-          if [ "${PYTHON_ABI_TAG}" = "cp313t" ]; then
+          if [[ "${PYTHON_ABI_TAG}" == *t ]]; then
             test_image_prefix=python-free-threaded
           else
             test_image_prefix=python
diff --git a/dev/tasks/python-wheels/github.osx.yml b/dev/tasks/python-wheels/github.osx.yml
@@ -27,7 +27,7 @@
   PYARROW_VERSION: "{{ arrow.no_rc_version }}"
   PYTHON_VERSION: "{{ python_version }}"
   PYTHON_ABI_TAG: "{{ python_abi_tag }}"
-  {% if python_abi_tag == "cp313t" %}
+  {% if python_abi_tag.endswith('t') %}
   PYTHON: "/Library/Frameworks/PythonT.framework/Versions/{{ python_version }}/bin/python{{ python_version }}t"
   {% else %}
   PYTHON: "/Library/Frameworks/Python.framework/Versions/{{ python_version }}/bin/python{{ python_version }}"
@@ -94,7 +94,7 @@ jobs:
             --x-feature=s3
 
       - name: Install Python {{ python_version }}
-        run: sudo arrow/ci/scripts/install_python.sh macos {{ "3.13t" if python_abi_tag == "cp313t" else python_version }}
+        run: sudo arrow/ci/scripts/install_python.sh macos {{ python_version + "t" if python_abi_tag.endswith('t') else python_version }}
 
       - name: Build Wheel
         run: |
@@ -131,7 +131,7 @@ jobs:
       - name: Test Wheel
         env:
           PYTEST_ADDOPTS: "-k 'not test_cancellation'"
-          PYTHON_GIL: {{ 0 if python_abi_tag == "cp313t" else 1 }}
+          PYTHON_GIL: {{ 0 if python_abi_tag.endswith('t') else 1 }}
         run: |
           $PYTHON -m venv test-env
           source test-env/bin/activate
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
diff --git a/docker-compose.yml b/docker-compose.yml
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
diff --git a/python/pyproject.toml b/python/pyproject.toml
diff --git a/python/requirements-wheel-test.txt b/python/requirements-wheel-test.txt
