diff --git a/.circleci/setup_env.sh b/.circleci/setup_env.sh index 52a8cab1cd2de..dc0ba20c190f5 100755 --- a/.circleci/setup_env.sh +++ b/.circleci/setup_env.sh @@ -54,11 +54,7 @@ if pip list | grep -q ^pandas; then pip uninstall -y pandas || true fi -echo "Build extensions" -# GH 47305: Parallel build can causes flaky ImportError from pandas/_libs/tslibs -python setup.py build_ext -q -j1 - echo "Install pandas" -python -m pip install --no-build-isolation --no-use-pep517 -e . +python -m pip install --no-build-isolation -v . echo "done" diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml index 23bb988ef4d73..5acc5ca572128 100644 --- a/.github/actions/build_pandas/action.yml +++ b/.github/actions/build_pandas/action.yml @@ -11,12 +11,15 @@ runs: shell: bash -el {0} - name: Build Pandas + if : ${{ runner.os != 'Windows' }} run: | - python setup.py build_ext -j $N_JOBS - python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index + pip install . --no-build-isolation -v shell: bash -el {0} - env: - # Cannot use parallel compilation on Windows, see https://github.com/pandas-dev/pandas/issues/30873 - # GH 47305: Parallel build causes flaky ImportError: /home/runner/work/pandas/pandas/pandas/_libs/tslibs/timestamps.cpython-38-x86_64-linux-gnu.so: undefined symbol: pandas_datetime_to_datetimestruct - N_JOBS: 1 - #N_JOBS: ${{ runner.os == 'Windows' && 1 || 2 }} + + - name: Build Pandas (Windows) + if: ${{ runner.os == 'Windows' }} + run: | + call micromamba activate test + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" + python -m pip install . --no-build-isolation -v + shell: cmd /C call {0} diff --git a/.github/workflows/32-bit-linux.yml b/.github/workflows/32-bit-linux.yml index cf8a0fe0da91c..073776e3158df 100644 --- a/.github/workflows/32-bit-linux.yml +++ b/.github/workflows/32-bit-linux.yml @@ -39,11 +39,12 @@ jobs: . ~/virtualenvs/pandas-dev/bin/activate && \ python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \ pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \ - python setup.py build_ext -q -j1 && \ - python -m pip install --no-build-isolation --no-use-pep517 -e . && \ - python -m pip list && \ + pip install "git+https://github.com/mesonbuild/meson.git@master" && \ + pip install "git+https://github.com/mesonbuild/meson-python.git@main" && \ export PANDAS_CI=1 && \ - pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml" + python -m pip install --no-build-isolation -v . && \ + python -m pip list && \ + pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml --import-mode=importlib" - name: Publish test results for Python 3.8-32 bit full Linux uses: actions/upload-artifact@v3 diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml index 48a08d4febbaf..45306c65c36ee 100644 --- a/.github/workflows/docbuild-and-upload.yml +++ b/.github/workflows/docbuild-and-upload.yml @@ -49,10 +49,10 @@ jobs: run: python web/pandas_web.py web/pandas --target-path=web/build - name: Build documentation - run: doc/make.py --warnings-are-errors + run: cd doc && python make.py --warnings-are-errors - name: Build documentation zip - run: doc/make.py zip_html + run: cd doc && python make.py zip_html - name: Build the interactive terminal run: | diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index 7c4b36dab109d..145e5e00d57ec 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -75,18 +75,34 @@ jobs: python -m pip install --upgrade pip setuptools wheel python -m pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy python -m pip install git+https://github.com/nedbat/coveragepy.git - python -m pip install python-dateutil pytz cython hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17 + python -m pip install python-dateutil pytz cython + # TODO: update when upstream releases fixes + python -m pip install "git+https://github.com/mesonbuild/meson.git@master" + python -m pip install "git+https://github.com/FFY00/meson-python.git@main" + python -m pip install hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17 python -m pip list - # GH 47305: Parallel build can cause flaky ImportError from pandas/_libs/tslibs + # Sigh, someone (numpy?) is depending on mingw, which pandas doesn't compile with. + # Also, meson doesn't detect visual c++ unless cl.exe is in path. + # TODO: File a bug with meson about this. - name: Build Pandas + if : ${{ runner.os != 'Windows' }} run: | - python setup.py build_ext -q -j1 - python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index + python3 -m pip install . --no-build-isolation -v + shell: bash -el {0} + + - name: Build Pandas (Windows) + if: ${{ runner.os == 'Windows' }} + run: | + call micromamba activate test + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" + python -m pip install . --no-build-isolation -v + shell: cmd /C call {0} - name: Build Version run: | - python -c "import pandas; pandas.show_versions();" + # Can't import pandas from the source directory + cd .. && python -c "import pandas; pandas.show_versions();" - name: Test uses: ./.github/actions/run-tests diff --git a/.gitignore b/.gitignore index 07b1f056d511b..324e978e01e19 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ *.py[ocd] *.so .build_cache_dir +.mesonpy-native-file.ini MANIFEST # Python files # diff --git a/ci/deps/actions-310-numpydev.yaml b/ci/deps/actions-310-numpydev.yaml index ef20c2aa889b9..4de419b6d961c 100644 --- a/ci/deps/actions-310-numpydev.yaml +++ b/ci/deps/actions-310-numpydev.yaml @@ -17,6 +17,8 @@ dependencies: - pip - pip: - "cython" + - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/mesonbuild/meson-python.git@main" - "--extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple" - "--pre" - "numpy" diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml index 9ebc305a0cb0c..15afa7b340e18 100644 --- a/ci/deps/actions-310.yaml +++ b/ci/deps/actions-310.yaml @@ -52,3 +52,6 @@ dependencies: - xlrd - xlsxwriter - zstandard + - pip: + - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/mesonbuild/meson-python.git@main" diff --git a/ci/deps/actions-38-downstream_compat.yaml b/ci/deps/actions-38-downstream_compat.yaml index 7e127b9dfebc1..7ae162ec1216d 100644 --- a/ci/deps/actions-38-downstream_compat.yaml +++ b/ci/deps/actions-38-downstream_compat.yaml @@ -68,3 +68,6 @@ dependencies: - pyyaml - py - pytorch + - pip: + - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/mesonbuild/meson-python.git@main" diff --git a/ci/deps/actions-38-minimum_versions.yaml b/ci/deps/actions-38-minimum_versions.yaml index 7cf6d777ae607..0ac11296cace8 100644 --- a/ci/deps/actions-38-minimum_versions.yaml +++ b/ci/deps/actions-38-minimum_versions.yaml @@ -54,3 +54,6 @@ dependencies: - xlrd=2.0.1 - xlsxwriter=1.4.3 - zstandard=0.15.2 + - pip: + - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/mesonbuild/meson-python.git@main" diff --git a/ci/deps/actions-38.yaml b/ci/deps/actions-38.yaml index 825b8aeebfc2f..1b64641beb3db 100644 --- a/ci/deps/actions-38.yaml +++ b/ci/deps/actions-38.yaml @@ -51,3 +51,6 @@ dependencies: - xlrd - xlsxwriter - zstandard + - pip: + - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/mesonbuild/meson-python.git@main" diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index 1ee96878dbe34..f8c1c8dd82bb6 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -52,3 +52,6 @@ dependencies: - xlrd - xlsxwriter - zstandard + - pip: + - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/mesonbuild/meson-python.git@main" diff --git a/ci/deps/actions-pypy-38.yaml b/ci/deps/actions-pypy-38.yaml index e06b992acc191..f9c51bca85b95 100644 --- a/ci/deps/actions-pypy-38.yaml +++ b/ci/deps/actions-pypy-38.yaml @@ -19,3 +19,6 @@ dependencies: - numpy - python-dateutil - pytz + - pip: + - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/mesonbuild/meson-python.git@main" diff --git a/ci/deps/circle-38-arm64.yaml b/ci/deps/circle-38-arm64.yaml index ae4a82d016131..3e8f89e6e2417 100644 --- a/ci/deps/circle-38-arm64.yaml +++ b/ci/deps/circle-38-arm64.yaml @@ -52,3 +52,6 @@ dependencies: - xlrd - xlsxwriter - zstandard + - pip: + - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/mesonbuild/meson-python.git@main" diff --git a/ci/run_tests.sh b/ci/run_tests.sh index e6de5caf955fc..e6c9be39c17f9 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -1,5 +1,6 @@ #!/bin/bash -e + # Workaround for pytest-xdist (it collects different tests in the workers if PYTHONHASHSEED is not set) # https://github.com/pytest-dev/pytest/issues/920 # https://github.com/pytest-dev/pytest/issues/1075 @@ -13,7 +14,7 @@ if [[ "not network" == *"$PATTERN"* ]]; then fi if [[ "$COVERAGE" == "true" ]]; then - COVERAGE="-s --cov=pandas --cov-report=xml --cov-append" + COVERAGE="-s --cov=pandas --cov-report=xml --cov-append --cov-config=setup.cfg" else COVERAGE="" # We need to reset this for COVERAGE="false" case fi @@ -24,7 +25,7 @@ if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then XVFB="xvfb-run " fi -PYTEST_CMD="${XVFB}pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET" +PYTEST_CMD="${XVFB}pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile --import-mode=importlib $TEST_ARGS $COVERAGE $PYTEST_TARGET" if [[ "$PATTERN" ]]; then PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\"" @@ -33,7 +34,7 @@ fi echo $PYTEST_CMD sh -c "$PYTEST_CMD" -if [[ "$PANDAS_DATA_MANAGER" != "array" && "$PYTEST_TARGET" == "pandas" ]]; then +if [[ "$PANDAS_DATA_MANAGER" != "array" && "$PYTEST_TARGET" == "pandas/pandas" ]]; then # The ArrayManager tests should have already been run by PYTEST_CMD if PANDAS_DATA_MANAGER was already set to array # If we're targeting specific files, e.g. test_downstream.py, don't run. PYTEST_AM_CMD="PANDAS_DATA_MANAGER=array pytest -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE pandas" diff --git a/doc/make.py b/doc/make.py index f5bf170c6274d..5b1365dffbbcd 100755 --- a/doc/make.py +++ b/doc/make.py @@ -353,8 +353,8 @@ def main(): # external libraries (namely Sphinx) to compile this module and resolve # the import of `python_path` correctly. The latter is used to resolve # the import within the module, injecting it into the global namespace - os.environ["PYTHONPATH"] = args.python_path - sys.path.insert(0, args.python_path) + # os.environ["PYTHONPATH"] = args.python_path + # sys.path.insert(0, args.python_path) globals()["pandas"] = importlib.import_module("pandas") # Set the matplotlib backend to the non-interactive Agg backend for all diff --git a/environment.yml b/environment.yml index f6ef6367800bd..d002cb16d59f5 100644 --- a/environment.yml +++ b/environment.yml @@ -130,3 +130,5 @@ dependencies: - pip: - jupyterlite==0.1.0b12 - sphinx-toggleprompt + - "git+https://github.com/mesonbuild/meson.git@master" + - "git+https://github.com/mesonbuild/meson-python.git@main" diff --git a/generate_pxi.py b/generate_pxi.py new file mode 100644 index 0000000000000..3462b97aefcbf --- /dev/null +++ b/generate_pxi.py @@ -0,0 +1,33 @@ +import argparse +import os + +from Cython import Tempita + + +def process_tempita(pxifile, outfile): + with open(pxifile) as f: + tmpl = f.read() + pyxcontent = Tempita.sub(tmpl) + + with open(outfile, "w") as f: + f.write(pyxcontent) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("infile", type=str, help="Path to the input file") + parser.add_argument("-o", "--outdir", type=str, help="Path to the output directory") + args = parser.parse_args() + + if not args.infile.endswith(".in"): + raise ValueError(f"Unexpected extension: {args.infile}") + + outdir_abs = os.path.join(os.getcwd(), args.outdir) + outfile = os.path.join( + outdir_abs, os.path.splitext(os.path.split(args.infile)[1])[0] + ) + + process_tempita(args.infile, outfile) + + +main() diff --git a/generate_version.py b/generate_version.py new file mode 100644 index 0000000000000..fbc78ab12429a --- /dev/null +++ b/generate_version.py @@ -0,0 +1,34 @@ +import argparse +import os + +import versioneer + + +def write_version_info(path): + if os.environ.get("MESON_DIST_ROOT"): + # raise ValueError("dist root is", os.environ.get("MESON_DIST_ROOT")) + path = os.path.join(os.environ.get("MESON_DIST_ROOT"), path) + with open(path, "w") as file: + file.write(f'__version__="{versioneer.get_version()}"\n') + file.write( + f'__git_version__="{versioneer.get_versions()["full-revisionid"]}"\n' + ) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "-o", "--outfile", type=str, help="Path to write version info to" + ) + args = parser.parse_args() + + if not args.outfile.endswith(".py"): + raise ValueError( + f"Output file must be a Python file. " + f"Got: {args.outfile} as filename instead" + ) + + write_version_info(args.outfile) + + +main() diff --git a/meson.build b/meson.build new file mode 100644 index 0000000000000..43b6621410c7e --- /dev/null +++ b/meson.build @@ -0,0 +1,49 @@ +# This file is adapted from https://github.com/scipy/scipy/blob/main/meson.build +project( + 'pandas', + 'c', 'cpp', 'cython', + version: '2.0.0.dev0', + license: 'BSD-3', + # TODO: bump when meson 0.64.0 comes out, + # we are relying on 0.64.0 features + meson_version: '>=0.63', + default_options: [ + # TODO: investigate, does meson try to compile against debug Python + # when buildtype = debug, this seems to be causing problems on CI + # where provided Python is not compiled in debug mode + 'buildtype=release', + # TODO: turn on werror when ready + #'werror=true', + # TODO: We are using POSIX functions(strdup in ujson), so we can't compile + # with strict C99 :( + #'c_std=c99' + ] +) + +add_global_arguments('-DNPY_NO_DEPRECATED_API=0', language : 'c') +add_global_arguments('-DNPY_NO_DEPRECATED_API=0', language : 'cpp') + +py_mod = import('python') +fs = import('fs') +py = py_mod.find_installation('python') +py_dep = py.dependency() +tempita = files('generate_pxi.py') +versioneer = files('generate_version.py') + +if fs.exists('_version_meson.py') + py.install_sources('_version_meson.py', subdir: 'pandas') +else + custom_target('write_version_file', + output: '_version_meson.py', + command: [ + py, versioneer, '-o', '@OUTPUT@' + ], + build_by_default: true, + build_always_stale: true, + install: true, + install_dir: py.get_install_dir(pure: false) / 'pandas' + ) + meson.add_dist_script(py, versioneer, '-o', '_version_meson.py') +endif + +subdir('pandas') diff --git a/pandas/__init__.py b/pandas/__init__.py index cae768dae722e..c2daf70e05f09 100644 --- a/pandas/__init__.py +++ b/pandas/__init__.py @@ -176,12 +176,18 @@ from pandas.util._tester import test # use the closest tagged version if possible -from pandas._version import get_versions +_built_with_meson = False +try: + from pandas._version_meson import __version__, __git_version__ + + _built_with_meson = True +except ImportError: + from pandas._version import get_versions -v = get_versions() -__version__ = v.get("closest-tag", v["version"]) -__git_version__ = v.get("full-revisionid") -del get_versions, v + v = get_versions() + __version__ = v.get("closest-tag", v["version"]) + __git_version__ = v.get("full-revisionid") + del get_versions, v # GH 27101 __deprecated_num_index_names = ["Float64Index", "Int64Index", "UInt64Index"] diff --git a/pandas/_libs/meson.build b/pandas/_libs/meson.build new file mode 100644 index 0000000000000..d4e3819357da4 --- /dev/null +++ b/pandas/_libs/meson.build @@ -0,0 +1,161 @@ +_algos_take_helper = custom_target('algos_take_helper_pxi', + output: 'algos_take_helper.pxi', + input: 'algos_take_helper.pxi.in', + command: [ + py, tempita, '@INPUT@', '-o', '@OUTDIR@' + ] +) +_algos_common_helper = custom_target('algos_common_helper_pxi', + output: 'algos_common_helper.pxi', + input: 'algos_common_helper.pxi.in', + command: [ + py, tempita, '@INPUT@', '-o', '@OUTDIR@' + ] +) +_khash_primitive_helper = custom_target('khash_primitive_helper_pxi', + output: 'khash_for_primitive_helper.pxi', + input: 'khash_for_primitive_helper.pxi.in', + command: [ + py, tempita, '@INPUT@', '-o', '@OUTDIR@' + ] +) +_hashtable_class_helper = custom_target('hashtable_class_helper_pxi', + output: 'hashtable_class_helper.pxi', + input: 'hashtable_class_helper.pxi.in', + command: [ + py, tempita, '@INPUT@', '-o', '@OUTDIR@' + ] +) +_hashtable_func_helper = custom_target('hashtable_func_helper_pxi', + output: 'hashtable_func_helper.pxi', + input: 'hashtable_func_helper.pxi.in', + command: [ + py, tempita, '@INPUT@', '-o', '@OUTDIR@' + ] +) +_index_class_helper = custom_target('index_class_helper_pxi', + output: 'index_class_helper.pxi', + input: 'index_class_helper.pxi.in', + command: [ + py, tempita, '@INPUT@', '-o', '@OUTDIR@' + ] +) +_sparse_op_helper = custom_target('sparse_op_helper_pxi', + output: 'sparse_op_helper.pxi', + input: 'sparse_op_helper.pxi.in', + command: [ + py, tempita, '@INPUT@', '-o', '@OUTDIR@' + ] +) +_intervaltree_helper = custom_target('intervaltree_helper_pxi', + output: 'intervaltree.pxi', + input: 'intervaltree.pxi.in', + command: [ + py, tempita, '@INPUT@', '-o', '@OUTDIR@' + ] +) +_khash_primitive_helper_dep = declare_dependency(sources: _khash_primitive_helper) +# TODO: can this be removed, I wish meson copied .pyx source to the build dir automatically +# The reason we can't build the pyx files inplace and copy to build dir is because +# the generated pxi files cannot be written to the source directory. +# (Meson only supports out of tree builds) +cython_sources_list = [ + # List of cython sources e.g. .pyx, .pxd & __init__.py + # Does NOT include .pxi.in + '__init__.py', + 'algos.pxd', + 'algos.pyx', + 'arrays.pxd', + 'dtypes.pxd', + 'hashtable.pxd', + 'hashtable.pyx', + 'index.pyx', + 'indexing.pyx', + 'internals.pyx', + 'interval.pyx', + 'join.pyx', + 'khash.pxd', + 'lib.pxd', + 'missing.pxd', + 'parsers.pyx', + 'sparse.pyx', + 'testing.pyx', + 'tslib.pyx', + 'util.pxd', +] +cython_sources = {} +cython_sources_tgts = [] + +foreach source: cython_sources_list + source_pyx = fs.copyfile(source) + cython_sources += {source: source_pyx} + cython_sources_tgts += source_pyx +endforeach + +subdir('tslibs') + +libs_sources = { + # Dict of extension name -> dict of {sources, include_dirs, and deps} + # numpy include dir is implicitly included + 'algos': {'sources': [cython_sources['algos.pyx'], _algos_common_helper, _algos_take_helper, _khash_primitive_helper], + 'deps': declare_dependency(sources: [cython_sources['khash.pxd'], cython_sources['util.pxd'],_algos_common_helper, _algos_take_helper, _khash_primitive_helper]), + 'include_dirs': klib_include}, + 'arrays': {'sources': ['arrays.pyx']}, + 'groupby': {'sources': ['groupby.pyx']}, + 'hashing': {'sources': ['hashing.pyx']}, + 'hashtable': {'sources': [cython_sources['hashtable.pyx'], _khash_primitive_helper, _hashtable_class_helper, _hashtable_func_helper], + 'include_dirs': klib_include}, + 'index': {'sources': [cython_sources['index.pyx'], _index_class_helper], + 'include_dirs': [klib_include, 'tslibs']}, + 'indexing': {'sources': ['indexing.pyx']}, + 'internals': {'sources': ['internals.pyx']}, + 'interval': {'sources': [cython_sources['interval.pyx'], _intervaltree_helper], + 'include_dirs': [klib_include, 'tslibs']}, + 'join': {'sources': [cython_sources['join.pyx'], _khash_primitive_helper], + 'include_dirs': klib_include, + 'deps': _khash_primitive_helper_dep}, + 'lib': {'sources': ['lib.pyx', 'src/parser/tokenizer.c'], + 'include_dirs': [klib_include, inc_datetime]}, + 'missing': {'sources': ['missing.pyx'], + 'include_dirs': [inc_datetime]}, + 'parsers': {'sources': [cython_sources['parsers.pyx'], 'src/parser/tokenizer.c', 'src/parser/io.c'], + 'include_dirs': [klib_include, 'src'], + 'deps': _khash_primitive_helper_dep}, + 'json': {'sources': ['src/ujson/python/ujson.c', + 'src/ujson/python/objToJSON.c', + 'src/ujson/python/date_conversions.c', + 'src/ujson/python/JSONtoObj.c', + 'src/ujson/lib/ultrajsonenc.c', + 'src/ujson/lib/ultrajsondec.c', + 'tslibs/src/datetime/np_datetime.c', + 'tslibs/src/datetime/np_datetime_strings.c'], + 'include_dirs': [inc_datetime, 'src/ujson/lib', 'src/ujson/python']}, + 'reduction': {'sources': ['reduction.pyx']}, + 'ops': {'sources': ['ops.pyx']}, + 'ops_dispatch': {'sources': ['ops_dispatch.pyx']}, + 'properties': {'sources': ['properties.pyx']}, + 'reshape': {'sources': ['reshape.pyx']}, + 'sparse': {'sources': [cython_sources['sparse.pyx'], _sparse_op_helper]}, + 'tslib': {'sources': ['tslib.pyx', 'tslibs/src/datetime/np_datetime.c'], + 'include_dirs': inc_datetime}, + 'testing': {'sources': ['testing.pyx']}, + 'writers': {'sources': ['writers.pyx']} +} + + +foreach ext_name, ext_dict : libs_sources + py.extension_module( + ext_name, + ext_dict.get('sources'), + include_directories: [inc_np] + ext_dict.get('include_dirs', ''), + dependencies: ext_dict.get('deps', ''), + subdir: 'pandas/_libs', + install: true + ) +endforeach + +py.install_sources('__init__.py', + pure: false, + subdir: 'pandas/_libs') + +subdir('window') diff --git a/pandas/_libs/tslibs/meson.build b/pandas/_libs/tslibs/meson.build new file mode 100644 index 0000000000000..cf25d7025a6cd --- /dev/null +++ b/pandas/_libs/tslibs/meson.build @@ -0,0 +1,75 @@ +# TODO: can this be removed, I wish meson copied .pyx source to the build dir automatically +tslibs_pxd_sources_list = [ + # List of cython sources e.g. .pyx, .pxd & __init__.py + # Does NOT include .pxi.in + '__init__.py', + 'base.pxd', + 'ccalendar.pxd', + 'conversion.pxd', + 'dtypes.pxd', + 'nattype.pxd', + 'np_datetime.pxd', + 'offsets.pxd', + 'parsing.pxd', + 'period.pxd', + 'timedeltas.pxd', + 'timestamps.pxd', + 'timezones.pxd', + 'tzconversion.pxd', + 'util.pxd', +] + +foreach source: tslibs_pxd_sources_list + source_pxd = configure_file( + input: source, + output: source, + copy: true + ) +endforeach + +tslibs_sources = { + # Dict of extension name -> dict of {sources, include_dirs, and deps} + # numpy include dir is implicitly included + 'base': {'sources': ['base.pyx']}, + 'ccalendar': {'sources': ['ccalendar.pyx']}, + 'dtypes': {'sources': ['dtypes.pyx']}, + 'conversion': {'sources': ['conversion.pyx', 'src/datetime/np_datetime.c'], + 'include_dirs': inc_datetime}, + 'fields': {'sources': ['fields.pyx', 'src/datetime/np_datetime.c']}, + 'nattype': {'sources': ['nattype.pyx']}, + 'np_datetime': {'sources': ['np_datetime.pyx', 'src/datetime/np_datetime.c', 'src/datetime/np_datetime_strings.c'], + 'include_dirs': inc_datetime}, + 'offsets': {'sources': ['offsets.pyx', 'src/datetime/np_datetime.c'], + 'include_dirs': inc_datetime}, + 'parsing': {'sources': ['parsing.pyx', '../src/parser/tokenizer.c'], + 'include_dirs': klib_include}, + 'period': {'sources': ['period.pyx', 'src/datetime/np_datetime.c'], + 'include_dirs': inc_datetime}, + 'strptime': {'sources': ['strptime.pyx', 'src/datetime/np_datetime.c'], + 'include_dirs': inc_datetime}, + 'timedeltas': {'sources': ['timedeltas.pyx', 'src/datetime/np_datetime.c'], + 'include_dirs': inc_datetime}, + 'timestamps': {'sources': ['timestamps.pyx', 'src/datetime/np_datetime.c'], + 'include_dirs': inc_datetime}, + 'timezones': {'sources': ['timezones.pyx', 'src/datetime/np_datetime.c'], + 'include_dirs': inc_datetime}, + 'tzconversion': {'sources': ['tzconversion.pyx', 'src/datetime/np_datetime.c'], + 'include_dirs': inc_datetime}, + 'vectorized': {'sources': ['vectorized.pyx', 'src/datetime/np_datetime.c'], + 'include_dirs': inc_datetime} +} + +foreach ext_name, ext_dict : tslibs_sources + py.extension_module( + ext_name, + ext_dict.get('sources'), + include_directories: [inc_np] + ext_dict.get('include_dirs', ''), + dependencies: ext_dict.get('deps', ''), + subdir: 'pandas/_libs/tslibs', + install: true + ) +endforeach + +py.install_sources('__init__.py', + pure: false, + subdir: 'pandas/_libs/tslibs') diff --git a/pandas/_libs/window/meson.build b/pandas/_libs/window/meson.build new file mode 100644 index 0000000000000..7d7c34a57c6a6 --- /dev/null +++ b/pandas/_libs/window/meson.build @@ -0,0 +1,18 @@ +py.extension_module( + 'aggregations', + ['aggregations.pyx'], + include_directories: [inc_np, '../src'], + dependencies: [py_dep], + subdir: 'pandas/_libs/window', + override_options : ['cython_language=cpp'], + install: true +) + +py.extension_module( + 'indexers', + ['indexers.pyx'], + include_directories: [inc_np], + dependencies: [py_dep], + subdir: 'pandas/_libs/window', + install: true +) diff --git a/pandas/io/meson.build b/pandas/io/meson.build new file mode 100644 index 0000000000000..04d7585ebc274 --- /dev/null +++ b/pandas/io/meson.build @@ -0,0 +1,35 @@ +subdirs_list = [ + # exclude sas, since it contains extension modules + # and has its own meson.build + 'clipboard', + 'excel', + 'formats', + 'json', + 'parsers' +] +foreach subdir: subdirs_list + install_subdir(subdir, install_dir: py.get_install_dir(pure: false) / 'pandas/io') +endforeach +top_level_py_list = [ + '__init__.py', + 'api.py', + 'clipboards.py', + 'common.py', + 'feather_format.py', + 'gbq.py', + 'html.py', + 'orc.py', + 'parquet.py', + 'pickle.py', + 'pytables.py', + 'spss.py', + 'sql.py', + 'stata.py', + 'xml.py' +] +foreach file: top_level_py_list + py.install_sources(file, + pure: false, + subdir: 'pandas/io') +endforeach +subdir('sas') diff --git a/pandas/io/sas/meson.build b/pandas/io/sas/meson.build new file mode 100644 index 0000000000000..172db6334734f --- /dev/null +++ b/pandas/io/sas/meson.build @@ -0,0 +1,34 @@ +py.extension_module( + '_sas', + ['sas.pyx'], + include_directories: [inc_np], + dependencies: [py_dep], + # The file is named sas.pyx but we want the + # extension module to be named _sas + cython_args: ['--module-name=pandas.io.sas._sas'], + subdir: 'pandas/io/sas', + install: true +) +py.extension_module( + '_byteswap', + ['byteswap.pyx'], + include_directories: [inc_np], + dependencies: [py_dep], + # The file is named byteswap.pyx but we want the + # extension module to be named _byteswap + cython_args: ['--module-name=pandas.io.sas._byteswap'], + subdir: 'pandas/io/sas', + install: true +) +top_level_py_list = [ + '__init__.py', + 'sas7bdat.py', + 'sas_constants.py', + 'sas_xport.py', + 'sasreader.py' +] +foreach file: top_level_py_list + py.install_sources(file, + pure: false, + subdir: 'pandas/io/sas') +endforeach diff --git a/pandas/meson.build b/pandas/meson.build new file mode 100644 index 0000000000000..8ffa524570815 --- /dev/null +++ b/pandas/meson.build @@ -0,0 +1,46 @@ +incdir_numpy = run_command(py, + [ + '-c', + 'import os; os.chdir(".."); import numpy; print(numpy.get_include())' + ], + check: true +).stdout().strip() + +inc_np = include_directories(incdir_numpy) +klib_include = include_directories('_libs/src/klib') +inc_datetime = include_directories('_libs/tslibs') + +fs.copyfile('__init__.py') + +subdir('_libs') +subdir('io') + +subdirs_list = [ + '_config', + '_libs', + '_testing', + 'api', + 'arrays', + 'compat', + 'core', + 'errors', + 'plotting', + 'tests', + 'tseries', + 'util' +] +foreach subdir: subdirs_list + install_subdir(subdir, install_dir: py.get_install_dir(pure: false) / 'pandas') +endforeach +top_level_py_list = [ + '__init__.py', + '_typing.py', + '_version.py', + 'conftest.py', + 'testing.py' +] +foreach file: top_level_py_list + py.install_sources(file, + pure: false, + subdir: 'pandas') +endforeach diff --git a/pandas/tests/api/test_api.py b/pandas/tests/api/test_api.py index b3a60c1fc5d37..4f084c76fa1c7 100644 --- a/pandas/tests/api/test_api.py +++ b/pandas/tests/api/test_api.py @@ -26,7 +26,7 @@ def check(self, namespace, expected, ignored=None): class TestPDApi(Base): # these are optionally imported based on testing # & need to be ignored - ignored = ["tests", "locale", "conftest"] + ignored = ["tests", "locale", "conftest", "_version_meson"] # top-level sub-packages public_lib = [ @@ -40,7 +40,7 @@ class TestPDApi(Base): "io", "tseries", ] - private_lib = ["compat", "core", "pandas", "util"] + private_lib = ["compat", "core", "pandas", "util", "_built_with_meson"] # misc misc = ["IndexSlice", "NaT", "NA"] @@ -186,8 +186,9 @@ class TestPDApi(Base): "_is_numpy_dev", "_testing", "_typing", - "_version", ] + if not pd._built_with_meson: + private_modules.append("_version") def test_api(self): diff --git a/pandas/tests/io/test_compression.py b/pandas/tests/io/test_compression.py index 782753177f245..6b39b167f9232 100644 --- a/pandas/tests/io/test_compression.py +++ b/pandas/tests/io/test_compression.py @@ -201,9 +201,13 @@ def test_gzip_reproducibility_file_object(): assert output == buffer.getvalue() -def test_with_missing_lzma(): +def test_with_missing_lzma(monkeypatch): """Tests if import pandas works when lzma is not present.""" # https://github.com/pandas-dev/pandas/issues/27575 + # Can't import pandas from the test directory since its not + # built inplace with meson + if pd._built_with_meson: + monkeypatch.chdir("..") code = textwrap.dedent( """\ import sys @@ -214,10 +218,14 @@ def test_with_missing_lzma(): subprocess.check_output([sys.executable, "-c", code], stderr=subprocess.PIPE) -def test_with_missing_lzma_runtime(): +def test_with_missing_lzma_runtime(monkeypatch): """Tests if RuntimeError is hit when calling lzma without having the module available. """ + # Can't import pandas from the test directory since its not + # built inplace with meson + if pd._built_with_meson: + monkeypatch.chdir("..") code = textwrap.dedent( """ import sys diff --git a/pandas/tests/plotting/test_converter.py b/pandas/tests/plotting/test_converter.py index 9a6fed1afad1f..017bdb1f4e039 100644 --- a/pandas/tests/plotting/test_converter.py +++ b/pandas/tests/plotting/test_converter.py @@ -18,6 +18,7 @@ PeriodIndex, Series, Timestamp, + _built_with_meson, arrays, date_range, ) @@ -45,8 +46,12 @@ dates = pytest.importorskip("matplotlib.dates") -def test_registry_mpl_resets(): +def test_registry_mpl_resets(monkeypatch): # Check that Matplotlib converters are properly reset (see issue #27481) + # Can't import pandas from the test directory since its not + # built inplace with meson + if _built_with_meson: + monkeypatch.chdir("..") code = ( "import matplotlib.units as units; " "import matplotlib.dates as mdates; " @@ -65,7 +70,11 @@ def test_timtetonum_accepts_unicode(): class TestRegistration: - def test_dont_register_by_default(self): + def test_dont_register_by_default(self, monkeypatch): + # Can't import pandas from the test directory since its not + # built inplace with meson + if _built_with_meson: + monkeypatch.chdir("..") # Run in subprocess to ensure a clean state code = ( "import matplotlib.units; " diff --git a/pandas/tests/test_downstream.py b/pandas/tests/test_downstream.py index cea9484fbbf80..e00fa9a245782 100644 --- a/pandas/tests/test_downstream.py +++ b/pandas/tests/test_downstream.py @@ -133,13 +133,21 @@ def test_xarray_cftimeindex_nearest(): assert result == expected -def test_oo_optimizable(): +def test_oo_optimizable(monkeypatch): # GH 21071 + # Can't import pandas from the test directory since its not + # built inplace with meson + if pd._built_with_meson: + monkeypatch.chdir("..") subprocess.check_call([sys.executable, "-OO", "-c", "import pandas"]) -def test_oo_optimized_datetime_index_unpickle(): +def test_oo_optimized_datetime_index_unpickle(monkeypatch): # GH 42866 + # Can't import pandas from the test directory since its not + # built inplace with meson + if pd._built_with_meson: + monkeypatch.chdir("..") subprocess.check_call( [ sys.executable, @@ -269,7 +277,11 @@ def test_yaml_dump(df): tm.assert_frame_equal(df, loaded2) -def test_missing_required_dependency(): +def test_missing_required_dependency(monkeypatch): + # TODO: This test is basically disabled until we have + # editable installs in meson-python. Re-enable this when + # that happens. + # GH 23868 # To ensure proper isolation, we pass these flags # -S : disable site-packages @@ -282,6 +294,11 @@ def test_missing_required_dependency(): # We skip this test if pandas is installed as a site package. We first # import the package normally and check the path to the module before # executing the test which imports pandas with site packages disabled. + + # Can't import pandas from the test directory since its not + # built inplace with meson + if pd._built_with_meson: + monkeypatch.chdir("..") call = [pyexe, "-c", "import pandas;print(pandas.__file__)"] output = subprocess.check_output(call).decode() if "site-packages" in output: diff --git a/pandas/util/_print_versions.py b/pandas/util/_print_versions.py index 91d518d1ab496..20d63d66a0279 100644 --- a/pandas/util/_print_versions.py +++ b/pandas/util/_print_versions.py @@ -21,10 +21,15 @@ def _get_commit_hash() -> str | None: Use vendored versioneer code to get git hash, which handles git worktree correctly. """ - from pandas._version import get_versions + try: + from pandas._version_meson import __git_version__ - versions = get_versions() - return versions["full-revisionid"] + return __git_version__ + except ImportError: + from pandas._version import get_versions + + versions = get_versions() + return versions["full-revisionid"] def _get_sys_info() -> dict[str, JSONSerializable]: diff --git a/pyproject.toml b/pyproject.toml index 397f74ddab71a..3ec0b7a9efe71 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,14 +2,12 @@ # Minimum requirements for the build system to execute. # See https://github.com/scipy/scipy/pull/12940 for the AIX issue. requires = [ - "setuptools>=51.0.0", + "meson-python", "wheel", "Cython>=0.29.32,<3", # Note: sync with setup.py, environment.yml and asv.conf.json "oldest-supported-numpy>=2022.8.16" ] -# uncomment to enable pep517 after versioneer problem is fixed. -# https://github.com/python-versioneer/python-versioneer/issues/193 -# build-backend = "setuptools.build_meta" +build-backend = "mesonpy" [tool.cibuildwheel] skip = "cp36-* cp37-* pp37-* *-manylinux_i686 *_ppc64le *_s390x *-musllinux*" diff --git a/requirements-dev.txt b/requirements-dev.txt index 5e98113625374..fe464827da419 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -100,4 +100,5 @@ requests jupyterlab >=3.4,<4 jupyterlite==0.1.0b12 sphinx-toggleprompt -setuptools>=51.0.0 +git+https://github.com/mesonbuild/meson.git@master +git+https://github.com/mesonbuild/meson-python.git@main diff --git a/scripts/validate_min_versions_in_sync.py b/scripts/validate_min_versions_in_sync.py index cb6a204094bf5..b48b1eaeb2719 100755 --- a/scripts/validate_min_versions_in_sync.py +++ b/scripts/validate_min_versions_in_sync.py @@ -58,7 +58,11 @@ def get_versions_from_ci(content: list[str]) -> tuple[dict[str, str], dict[str, elif "# optional dependencies" in line: seen_optional = True elif seen_required and line.strip(): - package, version = line.strip().split("=") + try: + package, version = line.strip().split("=") + except ValueError: + # pip dependencies, just skip + continue package = package[2:] if package in EXCLUDE_DEPS: continue