GH-43964: [Python] Build macOS and manylinux wheels for free-threading (

#43965)

### Rationale for this change

Building free-threaded wheels is necessary to support the free-threaded build. We probably want to upload these wheels as nightlies somewhere as well, so that downstream users can test the free-threading-related changes.

### What changes are included in this PR?

- Add necessary configuration to build 3.13 free-threading wheels on `manylinux`.
- Do necessary changes to build free-threaded wheels on macOS as well.

### Are these changes tested?

I tested the `manylinux` wheel builds. macOS is still untested, since it's not dockerized.

### Are there any user-facing changes?

No.

Related to #43536.
* GitHub Issue: #43964

Lead-authored-by: Lysandros Nikolaou <lisandrosnik@gmail.com>
Co-authored-by: Antoine Pitrou <antoine@python.org>
Signed-off-by: Antoine Pitrou <antoine@python.org>

.env

@@ -72,6 +72,7 @@ NUMPY=latest
 PANDAS=latest
 PYTHON=3.9
 PYTHON_IMAGE_TAG=3.9
+PYTHON_ABI_TAG=cp39
 R=4.4
 SPARK=master
 TURBODBC=latest

ci/docker/python-free-threaded-wheel-manylinux-test-imports.dockerfile

@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG base
+FROM ${base}
+
+RUN apt-get update -y -q && \
+    apt install -y -q --no-install-recommends software-properties-common gpg-agent && \
+    add-apt-repository -y ppa:deadsnakes/ppa && \
+    apt-get update -y -q && \
+    apt install -y -q --no-install-recommends python3.13-dev python3.13-nogil python3.13-venv && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists*
+
+ENV ARROW_PYTHON_VENV /arrow-dev
+RUN python3.13t -m venv ${ARROW_PYTHON_VENV}
+
+ENV PYTHON_GIL 0
+ENV PATH "${ARROW_PYTHON_VENV}/bin:${PATH}"

ci/docker/python-free-threaded-wheel-manylinux-test-unittests.dockerfile

@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG base
+FROM ${base}
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update -y -q && \
+    apt install -y -q --no-install-recommends software-properties-common gpg-agent && \
+    add-apt-repository -y ppa:deadsnakes/ppa && \
+    apt-get update -y -q && \
+    apt install -y -q --no-install-recommends \
+        build-essential \
+        libffi-dev \
+        python3.13-dev \
+        python3.13-nogil \
+        python3.13-venv && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists*
+
+ENV ARROW_PYTHON_VENV /arrow-dev
+RUN python3.13t -m venv ${ARROW_PYTHON_VENV}
+
+ENV PYTHON_GIL 0
+ENV PATH "${ARROW_PYTHON_VENV}/bin:${PATH}"
+
+# pandas doesn't provide wheels for aarch64 yet, so we have to install nightly Cython
+# along with the rest of pandas' build dependencies and disable build isolation
+COPY python/requirements-wheel-test.txt /arrow/python/
+RUN python -m pip install \
+    --pre \
+    --prefer-binary \
+    --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" \
+    Cython numpy
+RUN python -m pip install "meson-python==0.13.1" "meson==1.2.1" wheel "versioneer[toml]" ninja
+RUN python -m pip install --no-build-isolation -r /arrow/python/requirements-wheel-test.txt

ci/docker/python-wheel-manylinux.dockerfile

@@ -105,8 +105,10 @@ RUN pipx upgrade auditwheel
 
 # Configure Python for applications running in the bash shell of this Dockerfile
 ARG python=3.9
+ARG python_abi_tag=cp39
 ENV PYTHON_VERSION=${python}
-RUN PYTHON_ROOT=$(find /opt/python -name cp${PYTHON_VERSION/./}-cp${PYTHON_VERSION/./}) && \
+ENV PYTHON_ABI_TAG=${python_abi_tag}
+RUN PYTHON_ROOT=$(find /opt/python -name cp${PYTHON_VERSION/./}-${PYTHON_ABI_TAG}) && \
     echo "export PATH=$PYTHON_ROOT/bin:\$PATH" >> /etc/profile.d/python.sh
 
 SHELL ["/bin/bash", "-i", "-c"]

ci/scripts/install_python.sh

@@ -29,7 +29,8 @@ versions=([3.9]=3.9.13
           [3.10]=3.10.11
           [3.11]=3.11.9
           [3.12]=3.12.5
-          [3.13]=3.13.0)
+          [3.13]=3.13.0
+          [3.13t]=3.13.0)
 
 if [ "$#" -ne 2 ]; then
   echo "Usage: $0 <platform> <version>"
@@ -46,9 +47,9 @@ full_version=${versions[$2]}
 if [ $platform = "macOS" ]; then
     echo "Downloading Python installer..."
 
-    if [ "$version" = "3.13" ];
+    if [ "$version" = "3.13" ] || [ "$version" = "3.13t" ];
     then
-        fname="python-${full_version}rc1-macos11.pkg"
+        fname="python-${full_version}rc2-macos11.pkg"
     elif [ "$(uname -m)" = "arm64" ] || \
          [ "$version" = "3.10" ] || \
          [ "$version" = "3.11" ] || \
@@ -61,15 +62,39 @@ if [ $platform = "macOS" ]; then
     wget "https://www.python.org/ftp/python/${full_version}/${fname}"
 
     echo "Installing Python..."
-    installer -pkg $fname -target /
+    if [[ $2 == "3.13t" ]]; then
+        # See https://github.com/python/cpython/issues/120098#issuecomment-2151122033 for more info on this.
+        cat > ./choicechanges.plist <<EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<array>
+        <dict>
+                <key>attributeSetting</key>
+                <integer>1</integer>
+                <key>choiceAttribute</key>
+                <string>selected</string>
+                <key>choiceIdentifier</key>
+                <string>org.python.Python.PythonTFramework-3.13</string>
+        </dict>
+</array>
+</plist>
+EOF
+        installer -pkg $fname -applyChoiceChangesXML ./choicechanges.plist -target /
+        rm ./choicechanges.plist
+    else
+        installer -pkg $fname -target /
+    fi
     rm $fname
 
-    echo "Installing Pip..."
     python="/Library/Frameworks/Python.framework/Versions/${version}/bin/python${version}"
-    pip="${python} -m pip"
+    if [[ $2 == "3.13t" ]]; then
+        python="/Library/Frameworks/PythonT.framework/Versions/3.13/bin/python3.13t"
+    fi
 
+    echo "Installing Pip..."
     $python -m ensurepip
-    $pip install -U pip setuptools
+    $python -m pip install -U pip setuptools
 else
     echo "Unsupported platform: $platform"
     exit 1

ci/scripts/python_wheel_unix_test.sh

@@ -59,7 +59,7 @@ export PARQUET_TEST_DATA=${source_dir}/cpp/submodules/parquet-testing/data
 
 if [ "${INSTALL_PYARROW}" == "ON" ]; then
   # Install the built wheels
-  pip install ${source_dir}/python/repaired_wheels/*.whl
+  python -m pip install ${source_dir}/python/repaired_wheels/*.whl
 fi
 
 if [ "${CHECK_IMPORTS}" == "ON" ]; then
@@ -95,7 +95,7 @@ fi
 
 if [ "${CHECK_UNITTESTS}" == "ON" ]; then
   # Install testing dependencies
-  pip install -U -r ${source_dir}/python/requirements-wheel-test.txt
+  python -m pip install -U -r ${source_dir}/python/requirements-wheel-test.txt
 
   # Execute unittest, test dependencies must be installed
   python -c 'import pyarrow; pyarrow.create_library_symlinks()'

dev/tasks/python-wheels/github.linux.yml

@@ -35,6 +35,7 @@ jobs:
       ARCH: arm64v8
       {% endif %}
       PYTHON: "{{ python_version }}"
+      PYTHON_ABI_TAG: "{{ python_abi_tag }}"
       {% if python_version == "3.13" %}
       PYTHON_IMAGE_TAG: "3.13-rc"
       {% else %}
@@ -57,12 +58,24 @@ jobs:
 
       # TODO(kszucs): auditwheel show
       - name: Test wheel
+        if: |
+          '{{ python_abi_tag }}' != 'cp313t'
         shell: bash
         run: |
           source arrow/ci/scripts/util_enable_core_dumps.sh
           archery docker run python-wheel-manylinux-test-imports
           archery docker run python-wheel-manylinux-test-unittests
 
+      # Free-threaded wheels need to be tested using a different docker-compose service
+      - name: Test free-threaded wheel
+        if: |
+          '{{ python_abi_tag }}' == 'cp313t'
+        shell: bash
+        run: |
+          source arrow/ci/scripts/util_enable_core_dumps.sh
+          archery docker run python-free-threaded-wheel-manylinux-test-imports
+          archery docker run python-free-threaded-wheel-manylinux-test-unittests
+
       - name: Test wheel on AlmaLinux 8
         shell: bash
         if: |

dev/tasks/python-wheels/github.osx.yml

@@ -26,7 +26,12 @@
   PYARROW_BUILD_VERBOSE: 1
   PYARROW_VERSION: "{{ arrow.no_rc_version }}"
   PYTHON_VERSION: "{{ python_version }}"
+  PYTHON_ABI_TAG: "{{ python_abi_tag }}"
+  {% if python_abi_tag == "cp313t" %}
+  PYTHON: "/Library/Frameworks/PythonT.framework/Versions/{{ python_version }}/bin/python{{ python_version }}t"
+  {% else %}
   PYTHON: "/Library/Frameworks/Python.framework/Versions/{{ python_version }}/bin/python{{ python_version }}"
+  {% endif %}
   VCPKG_DEFAULT_TRIPLET: "{{ vcpkg_arch }}-osx-static-release"
   VCPKG_FEATURE_FLAGS: "manifests"
   VCPKG_OVERLAY_TRIPLETS: {{ "${{ github.workspace }}/arrow/ci/vcpkg" }}
@@ -99,7 +104,7 @@ jobs:
             --x-feature=s3
 
       - name: Install Python {{ python_version }}
-        run: sudo arrow/ci/scripts/install_python.sh macos {{ python_version }}
+        run: sudo arrow/ci/scripts/install_python.sh macos {{ "3.13t" if python_abi_tag == "cp313t" else python_version }}
 
       - name: Build Wheel
         run: |
@@ -136,6 +141,7 @@ jobs:
       - name: Test Wheel
         env:
           PYTEST_ADDOPTS: "-k 'not test_cancellation'"
+          PYTHON_GIL: {{ 0 if python_abi_tag == "cp313t" else 1 }}
         run: |
           $PYTHON -m venv test-env
           source test-env/bin/activate

dev/tasks/tasks.yml

@@ -394,7 +394,8 @@ tasks:
                                                ("3.10", "cp310", "cp310"),
                                                ("3.11", "cp311", "cp311"),
                                                ("3.12", "cp312", "cp312"),
-                                               ("3.13", "cp313", "cp313")] %}
+                                               ("3.13", "cp313", "cp313"),
+                                               ("3.13", "cp313", "cp313t")] %}
 
 {############################## Wheel Linux ##################################}
 
@@ -403,12 +404,13 @@ tasks:
                                                  ("amd64", "2-28", "manylinux_2_28_x86_64"),
                                                  ("arm64", "2014", "manylinux_2_17_aarch64.manylinux2014_aarch64"),
                                                  ("arm64", "2-28", "manylinux_2_28_aarch64")] %}
-  wheel-manylinux-{{ manylinux_version }}-{{ python_tag }}-{{ arch }}:
+  wheel-manylinux-{{ manylinux_version }}-{{ python_tag }}-{{ abi_tag }}-{{ arch }}:
     ci: github
     template: python-wheels/github.linux.yml
     params:
       arch: "{{ arch }}"
       python_version: "{{ python_version }}"
+      python_abi_tag: "{{ abi_tag }}"
       manylinux_version: "{{ manylinux_version }}"
       wheel_platform_tag: "{{ platform_tag }}"
     artifacts:
@@ -420,13 +422,14 @@ tasks:
 {% for macos_version, macos_codename in [("12.0", "monterey")] %}
   {% set platform_tag = "macosx_{}_x86_64".format(macos_version.replace('.', '_')) %}
 
-  wheel-macos-{{ macos_codename }}-{{ python_tag }}-amd64:
+  wheel-macos-{{ macos_codename }}-{{ python_tag }}-{{ abi_tag }}-amd64:
     ci: github
     template: python-wheels/github.osx.yml
     params:
       arch: "x86_64"
       arrow_jemalloc: "ON"
       python_version: "{{ python_version }}"
+      python_abi_tag: "{{ abi_tag }}"
       macos_deployment_target: "{{ macos_version }}"
       runs_on: "macos-12"
       vcpkg_arch: "amd64"
@@ -435,21 +438,26 @@ tasks:
 
 {% endfor %}
 
-  wheel-macos-monterey-{{ python_tag }}-arm64:
+  wheel-macos-monterey-{{ python_tag }}-{{ abi_tag }}-arm64:
     ci: github
     template: python-wheels/github.osx.yml
     params:
       arch: "arm64"
       arrow_jemalloc: "OFF"
       python_version: "{{ python_version }}"
+      python_abi_tag: "{{ abi_tag }}"
       macos_deployment_target: "12.0"
       runs_on: "macos-14"
       vcpkg_arch: "arm64"
     artifacts:
-      - pyarrow-{no_rc_version}-{{ python_tag }}-{{ python_tag }}-macosx_12_0_arm64.whl
+      - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-macosx_12_0_arm64.whl
 
 {############################## Wheel Windows ################################}
 
+# TODO: Remove this when there's NumPy wheels for Windows.
+# See https://github.com/numpy/numpy/issues/26157 for more info.
+{% if abi_tag != "cp313t" %}
+
   wheel-windows-{{ python_tag }}-amd64:
     ci: github
     template: python-wheels/github.windows.yml
@@ -458,6 +466,7 @@ tasks:
     artifacts:
       - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-win_amd64.whl
 
+{% endif %}
 {% endfor %}
 
 {############################ Python sdist ####################################}