Merge remote-tracking branch 'origin/main' into vector-placement

ci/docker/conda-python.dockerfile

@@ -28,7 +28,7 @@ COPY ci/conda_env_python.txt \
 RUN mamba install -q -y \
  --file arrow/ci/conda_env_python.txt \
  $([ "$python" == $(gdb --batch --eval-command 'python import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') ] && echo "gdb") \
- "python=${python}.*=*_cpython" \
+ "python=${python}.*=*_cp*" \
  nomkl && \
  mamba clean --all
 

ci/docker/python-wheel-manylinux.dockerfile

@@ -114,5 +114,10 @@ RUN PYTHON_ROOT=$(find /opt/python -name cp${PYTHON_VERSION/./}-${PYTHON_ABI_TAG
 SHELL ["/bin/bash", "-i", "-c"]
 ENTRYPOINT ["/bin/bash", "-i", "-c"]
 
+# Remove once there are released Cython wheels for 3.13 free-threaded available
+RUN if [ "${python_abi_tag}" = "cp313t" ]; then \
+ pip install cython --pre --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" --prefer-binary ; \
+ fi
+
 COPY python/requirements-wheel-build.txt /arrow/python/
 RUN pip install -r /arrow/python/requirements-wheel-build.txt

ci/scripts/python_wheel_macos_build.sh

@@ -49,8 +49,15 @@ fi
 echo "=== (${PYTHON_VERSION}) Install Python build dependencies ==="
 export PIP_SITE_PACKAGES=$(python -c 'import site; print(site.getsitepackages()[0])')
 
+# Remove once there are released Cython wheels for 3.13 free-threaded available
+FREE_THREADED_BUILD="$(python -c"import sysconfig; print(bool(sysconfig.get_config_var('Py_GIL_DISABLED')))")"
+if [[ $FREE_THREADED_BUILD == "True" ]]; then
+ pip install cython --pre --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" --prefer-binary
+fi
+# With Python 3.9, the `--upgrade` flag is required to force full replacement of setuptools' distutils patching
+pip install --upgrade --target $PIP_SITE_PACKAGES "setuptools>=58"
+
 pip install \
- --upgrade \
  --only-binary=:all: \
  --target $PIP_SITE_PACKAGES \
  -r ${source_dir}/python/requirements-wheel-build.txt

dev/release/verify-release-candidate.sh

@@ -775,9 +775,9 @@ test_glib() {
  show_header "Build and test C GLib libraries"
 
  # Build and test C GLib
- # We can remove '==2.80.5' once https://github.com/conda-forge/glib-feedstock/issues/191
- # is fixed.
- maybe_setup_conda glib==2.80.5 gobject-introspection meson ninja ruby
+ # We can remove '==2.80.5' and 'python<3.13' once
+ # https://github.com/conda-forge/glib-feedstock/issues/191 is fixed.
+ maybe_setup_conda glib==2.80.5 gobject-introspection meson ninja 'python<3.13' ruby
  maybe_setup_virtualenv meson
 
  # Install bundler if doesn't exist

dev/tasks/java-jars/github.yml

@@ -117,6 +117,7 @@ jobs:
  for python_package in $(brew list | grep python@); do
  brew install --overwrite ${python_package}
  done
+ brew install --overwrite python
 
  brew bundle --file=arrow/cpp/Brewfile
  # We want to link aws-sdk-cpp statically but Homebrew's

dev/tasks/tasks.yml

@@ -416,24 +416,19 @@ tasks:
 
 {############################## Wheel macOS ####################################}
 
-{% for macos_version, macos_codename in [("13.0", "ventura")] %}
- {% set platform_tag = "macosx_{}_x86_64".format(macos_version.replace('.', '_')) %}
-
- wheel-macos-{{ macos_codename }}-{{ python_tag }}-{{ abi_tag }}-amd64:
+ wheel-macos-monterey-{{ python_tag }}-{{ abi_tag }}-amd64:
  ci: github
  template: python-wheels/github.osx.yml
  params:
  arch: "x86_64"
  arrow_jemalloc: "ON"
  python_version: "{{ python_version }}"
  python_abi_tag: "{{ abi_tag }}"
- macos_deployment_target: "{{ macos_version }}"
+ macos_deployment_target: "12.0"
  runs_on: "macos-13"
  vcpkg_arch: "amd64"
  artifacts:
- - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-{{ platform_tag }}.whl
-
-{% endfor %}
+ - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-macosx_12_0_x86_64.whl
 
  wheel-macos-monterey-{{ python_tag }}-{{ abi_tag }}-arm64:
  ci: github
@@ -1213,7 +1208,7 @@ tasks:
  UBUNTU: 22.04
  image: ubuntu-cpp-emscripten
 
-{% for python_version in ["3.9", "3.10", "3.11", "3.12"] %}
+{% for python_version in ["3.9", "3.10", "3.11", "3.12", "3.13"] %}
  test-conda-python-{{ python_version }}:
  ci: github
  template: docker-tests/github.linux.yml

dev/tasks/verify-rc/github.macos.yml

@@ -66,6 +66,14 @@ jobs:
  TEST_{{ target|upper }}: 1
  {% if use_conda %}
  USE_CONDA: 1
+ {% if target == "integration" %}
+ # JPype doesn't work with Python 3.13.
+ # See also:
+ # * https://discuss.python.org/t/api-for-python-3-13-prevents-use-of-3rd-party-gc-allocators/62709/5
+ # * GH-44386
+ # * GH-44389
+ PYTHON_VERSION: "3.12"
+ {% endif %}
  {% endif %}
  run: |
  arrow/dev/release/verify-release-candidate.sh {{ release|default("") }} {{ rc|default("") }}

docs/source/python/install.rst

@@ -71,12 +71,9 @@ See :ref:`python-development`.
 Dependencies
 ------------
 
-Required dependency
-
-* **NumPy 1.16.6** or higher.
-
 Optional dependencies
 
+* **NumPy 1.16.6** or higher.
 * **pandas 1.0** or higher,
 * **cffi**.
 

python/pyarrow/tests/test_dataset.py

@@ -737,20 +737,20 @@ def test_partitioning_pickling(pickle_module):
 @pytest.mark.parametrize(
  "flavor, expected_defined_partition, expected_undefined_partition",
  [
- (ds.HivePartitioning, (r"foo=A/bar=ant%20bee", ""), ("", "")),
- (ds.DirectoryPartitioning, (r"A/ant bee", ""), ("", "")),
- (ds.FilenamePartitioning, ("", r"A_ant bee_"), ("", "_")),
+ ("HivePartitioning", (r"foo=A/bar=ant%20bee", ""), ("", "")),
+ ("DirectoryPartitioning", (r"A/ant bee", ""), ("", "")),
+ ("FilenamePartitioning", ("", r"A_ant bee_"), ("", "_")),
  ],
 )
 def test_dataset_partitioning_format(
- flavor: "ds.Partitioning",
+ flavor: str,
  expected_defined_partition: tuple,
  expected_undefined_partition: tuple,
 ):
 
  partitioning_schema = pa.schema([("foo", pa.string()), ("bar", pa.string())])
 
- partitioning = flavor(schema=partitioning_schema)
+ partitioning = getattr(ds, flavor)(schema=partitioning_schema)
 
  # test forward transformation (format)
  assert (
@@ -782,7 +782,7 @@ def test_dataset_partitioning_format(
  == expected_undefined_partition
  )
 
- if flavor != ds.HivePartitioning:
+ if flavor != "HivePartitioning":
  # Raises error upon filtering for lower level partition without filtering for
  # higher level partition
  with pytest.raises(

python/pyproject.toml

@@ -35,9 +35,6 @@ build-backend = "setuptools.build_meta"
 name = "pyarrow"
 dynamic = ["version"]
 requires-python = ">=3.9"
-dependencies = [
- "numpy >= 1.16.6"
-]
 description = "Python library for Apache Arrow"
 readme = {file = "README.md", content-type = "text/markdown"}
 license = {text = "Apache Software License"}

python/requirements-wheel-build.txt

@@ -1,8 +1,3 @@
-# Remove pre and extra index url once there's NumPy and Cython wheels for 3.13
-# on PyPI
---pre
---extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple"
-
 cython>=0.29.31
 oldest-supported-numpy>=0.14; python_version<'3.9'
 numpy>=2.0.0; python_version>='3.9'

python/requirements-wheel-test.txt

@@ -1,9 +1,3 @@
-# Remove pre and extra index url once there's NumPy and Cython wheels for 3.13
-# on PyPI
---pre
---prefer-binary
---extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple"
-
 cffi
 cython
 hypothesis