diff --git a/ci/docker/conda-python.dockerfile b/ci/docker/conda-python.dockerfile index 3897a7217d975..c08b69e6ef813 100644 --- a/ci/docker/conda-python.dockerfile +++ b/ci/docker/conda-python.dockerfile @@ -28,7 +28,7 @@ COPY ci/conda_env_python.txt \ RUN mamba install -q -y \ --file arrow/ci/conda_env_python.txt \ $([ "$python" == $(gdb --batch --eval-command 'python import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') ] && echo "gdb") \ - "python=${python}.*=*_cpython" \ + "python=${python}.*=*_cp*" \ nomkl && \ mamba clean --all diff --git a/ci/docker/python-wheel-manylinux.dockerfile b/ci/docker/python-wheel-manylinux.dockerfile index d22a70a2d777b..452809ad09c6b 100644 --- a/ci/docker/python-wheel-manylinux.dockerfile +++ b/ci/docker/python-wheel-manylinux.dockerfile @@ -114,5 +114,10 @@ RUN PYTHON_ROOT=$(find /opt/python -name cp${PYTHON_VERSION/./}-${PYTHON_ABI_TAG SHELL ["/bin/bash", "-i", "-c"] ENTRYPOINT ["/bin/bash", "-i", "-c"] +# Remove once there are released Cython wheels for 3.13 free-threaded available +RUN if [ "${python_abi_tag}" = "cp313t" ]; then \ + pip install cython --pre --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" --prefer-binary ; \ + fi + COPY python/requirements-wheel-build.txt /arrow/python/ RUN pip install -r /arrow/python/requirements-wheel-build.txt diff --git a/ci/scripts/python_wheel_macos_build.sh b/ci/scripts/python_wheel_macos_build.sh index d2c392e6b9db3..91925e7abe8b0 100755 --- a/ci/scripts/python_wheel_macos_build.sh +++ b/ci/scripts/python_wheel_macos_build.sh @@ -49,8 +49,15 @@ fi echo "=== (${PYTHON_VERSION}) Install Python build dependencies ===" export PIP_SITE_PACKAGES=$(python -c 'import site; print(site.getsitepackages()[0])') +# Remove once there are released Cython wheels for 3.13 free-threaded available +FREE_THREADED_BUILD="$(python -c"import sysconfig; print(bool(sysconfig.get_config_var('Py_GIL_DISABLED')))")" +if [[ $FREE_THREADED_BUILD == "True" ]]; then + pip install cython --pre --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" --prefer-binary +fi +# With Python 3.9, the `--upgrade` flag is required to force full replacement of setuptools' distutils patching +pip install --upgrade --target $PIP_SITE_PACKAGES "setuptools>=58" + pip install \ - --upgrade \ --only-binary=:all: \ --target $PIP_SITE_PACKAGES \ -r ${source_dir}/python/requirements-wheel-build.txt diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 4e5593525477e..178d8cd458048 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -775,9 +775,9 @@ test_glib() { show_header "Build and test C GLib libraries" # Build and test C GLib - # We can remove '==2.80.5' once https://github.com/conda-forge/glib-feedstock/issues/191 - # is fixed. - maybe_setup_conda glib==2.80.5 gobject-introspection meson ninja ruby + # We can remove '==2.80.5' and 'python<3.13' once + # https://github.com/conda-forge/glib-feedstock/issues/191 is fixed. + maybe_setup_conda glib==2.80.5 gobject-introspection meson ninja 'python<3.13' ruby maybe_setup_virtualenv meson # Install bundler if doesn't exist diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml index 9910daa21ef37..e83552cc7c47c 100644 --- a/dev/tasks/java-jars/github.yml +++ b/dev/tasks/java-jars/github.yml @@ -117,6 +117,7 @@ jobs: for python_package in $(brew list | grep python@); do brew install --overwrite ${python_package} done + brew install --overwrite python brew bundle --file=arrow/cpp/Brewfile # We want to link aws-sdk-cpp statically but Homebrew's diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index d38b822ffd853..9137181466383 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -416,10 +416,7 @@ tasks: {############################## Wheel macOS ####################################} -{% for macos_version, macos_codename in [("13.0", "ventura")] %} - {% set platform_tag = "macosx_{}_x86_64".format(macos_version.replace('.', '_')) %} - - wheel-macos-{{ macos_codename }}-{{ python_tag }}-{{ abi_tag }}-amd64: + wheel-macos-monterey-{{ python_tag }}-{{ abi_tag }}-amd64: ci: github template: python-wheels/github.osx.yml params: @@ -427,13 +424,11 @@ tasks: arrow_jemalloc: "ON" python_version: "{{ python_version }}" python_abi_tag: "{{ abi_tag }}" - macos_deployment_target: "{{ macos_version }}" + macos_deployment_target: "12.0" runs_on: "macos-13" vcpkg_arch: "amd64" artifacts: - - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-{{ platform_tag }}.whl - -{% endfor %} + - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-macosx_12_0_x86_64.whl wheel-macos-monterey-{{ python_tag }}-{{ abi_tag }}-arm64: ci: github @@ -1213,7 +1208,7 @@ tasks: UBUNTU: 22.04 image: ubuntu-cpp-emscripten -{% for python_version in ["3.9", "3.10", "3.11", "3.12"] %} +{% for python_version in ["3.9", "3.10", "3.11", "3.12", "3.13"] %} test-conda-python-{{ python_version }}: ci: github template: docker-tests/github.linux.yml diff --git a/dev/tasks/verify-rc/github.macos.yml b/dev/tasks/verify-rc/github.macos.yml index e2bc7895c6d05..37a3e6f3fd517 100644 --- a/dev/tasks/verify-rc/github.macos.yml +++ b/dev/tasks/verify-rc/github.macos.yml @@ -66,6 +66,14 @@ jobs: TEST_{{ target|upper }}: 1 {% if use_conda %} USE_CONDA: 1 + {% if target == "integration" %} + # JPype doesn't work with Python 3.13. + # See also: + # * https://discuss.python.org/t/api-for-python-3-13-prevents-use-of-3rd-party-gc-allocators/62709/5 + # * GH-44386 + # * GH-44389 + PYTHON_VERSION: "3.12" + {% endif %} {% endif %} run: | arrow/dev/release/verify-release-candidate.sh {{ release|default("") }} {{ rc|default("") }} diff --git a/docs/source/python/install.rst b/docs/source/python/install.rst index 9727a68f7424b..33355b64159f8 100644 --- a/docs/source/python/install.rst +++ b/docs/source/python/install.rst @@ -71,12 +71,9 @@ See :ref:`python-development`. Dependencies ------------ -Required dependency - -* **NumPy 1.16.6** or higher. - Optional dependencies +* **NumPy 1.16.6** or higher. * **pandas 1.0** or higher, * **cffi**. diff --git a/python/pyarrow/tests/test_dataset.py b/python/pyarrow/tests/test_dataset.py index ea6cb88074ab7..0d3a0fbd3bec7 100644 --- a/python/pyarrow/tests/test_dataset.py +++ b/python/pyarrow/tests/test_dataset.py @@ -737,20 +737,20 @@ def test_partitioning_pickling(pickle_module): @pytest.mark.parametrize( "flavor, expected_defined_partition, expected_undefined_partition", [ - (ds.HivePartitioning, (r"foo=A/bar=ant%20bee", ""), ("", "")), - (ds.DirectoryPartitioning, (r"A/ant bee", ""), ("", "")), - (ds.FilenamePartitioning, ("", r"A_ant bee_"), ("", "_")), + ("HivePartitioning", (r"foo=A/bar=ant%20bee", ""), ("", "")), + ("DirectoryPartitioning", (r"A/ant bee", ""), ("", "")), + ("FilenamePartitioning", ("", r"A_ant bee_"), ("", "_")), ], ) def test_dataset_partitioning_format( - flavor: "ds.Partitioning", + flavor: str, expected_defined_partition: tuple, expected_undefined_partition: tuple, ): partitioning_schema = pa.schema([("foo", pa.string()), ("bar", pa.string())]) - partitioning = flavor(schema=partitioning_schema) + partitioning = getattr(ds, flavor)(schema=partitioning_schema) # test forward transformation (format) assert ( @@ -782,7 +782,7 @@ def test_dataset_partitioning_format( == expected_undefined_partition ) - if flavor != ds.HivePartitioning: + if flavor != "HivePartitioning": # Raises error upon filtering for lower level partition without filtering for # higher level partition with pytest.raises( diff --git a/python/pyproject.toml b/python/pyproject.toml index 932210044399e..8748fe4052be1 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -35,9 +35,6 @@ build-backend = "setuptools.build_meta" name = "pyarrow" dynamic = ["version"] requires-python = ">=3.9" -dependencies = [ - "numpy >= 1.16.6" -] description = "Python library for Apache Arrow" readme = {file = "README.md", content-type = "text/markdown"} license = {text = "Apache Software License"} diff --git a/python/requirements-wheel-build.txt b/python/requirements-wheel-build.txt index 2d448004768ce..faa078d3d7fe7 100644 --- a/python/requirements-wheel-build.txt +++ b/python/requirements-wheel-build.txt @@ -1,8 +1,3 @@ -# Remove pre and extra index url once there's NumPy and Cython wheels for 3.13 -# on PyPI ---pre ---extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" - cython>=0.29.31 oldest-supported-numpy>=0.14; python_version<'3.9' numpy>=2.0.0; python_version>='3.9' diff --git a/python/requirements-wheel-test.txt b/python/requirements-wheel-test.txt index d129ac753ab37..c051efb0aaab1 100644 --- a/python/requirements-wheel-test.txt +++ b/python/requirements-wheel-test.txt @@ -1,9 +1,3 @@ -# Remove pre and extra index url once there's NumPy and Cython wheels for 3.13 -# on PyPI ---pre ---prefer-binary ---extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" - cffi cython hypothesis