Meson poc

.circleci/setup_env.sh

@@ -54,11 +54,7 @@ if pip list | grep -q ^pandas; then
     pip uninstall -y pandas || true
 fi
 
-echo "Build extensions"
-# GH 47305: Parallel build can causes flaky ImportError from pandas/_libs/tslibs
-python setup.py build_ext -q -j1
-
 echo "Install pandas"
-python -m pip install --no-build-isolation --no-use-pep517 -e .
+python -m pip install --no-build-isolation -v .
 
 echo "done"

.github/actions/build_pandas/action.yml

@@ -11,12 +11,15 @@ runs:
       shell: bash -el {0}
 
     - name: Build Pandas
+      if : ${{ runner.os != 'Windows' }}
       run: |
-        python setup.py build_ext -j $N_JOBS
-        python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index
+        pip install . --no-build-isolation -v
       shell: bash -el {0}
-      env:
-        # Cannot use parallel compilation on Windows, see https://github.com/pandas-dev/pandas/issues/30873
-        # GH 47305: Parallel build causes flaky ImportError: /home/runner/work/pandas/pandas/pandas/_libs/tslibs/timestamps.cpython-38-x86_64-linux-gnu.so: undefined symbol: pandas_datetime_to_datetimestruct
-        N_JOBS: 1
-        #N_JOBS: ${{ runner.os == 'Windows' && 1 || 2 }}
+
+    - name: Build Pandas (Windows)
+      if: ${{ runner.os == 'Windows' }}
+      run: |
+        call micromamba activate test
+        call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
+        python -m pip install . --no-build-isolation -v
+      shell: cmd /C call {0}

.github/workflows/32-bit-linux.yml

@@ -39,11 +39,12 @@ jobs:
           . ~/virtualenvs/pandas-dev/bin/activate && \
           python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \
           pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \
-          python setup.py build_ext -q -j1 && \
-          python -m pip install --no-build-isolation --no-use-pep517 -e . && \
-          python -m pip list && \
+          pip install "git+https://github.com/mesonbuild/meson.git@master" && \
+          pip install "git+https://github.com/mesonbuild/meson-python.git@main" && \
           export PANDAS_CI=1 && \
-          pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml"
+          python -m pip install --no-build-isolation -v . && \
+          python -m pip list && \
+          pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml --import-mode=importlib"
 
       - name: Publish test results for Python 3.8-32 bit full Linux
         uses: actions/upload-artifact@v3

.github/workflows/docbuild-and-upload.yml

@@ -49,10 +49,10 @@ jobs:
       run: python web/pandas_web.py web/pandas --target-path=web/build
 
     - name: Build documentation
-      run: doc/make.py --warnings-are-errors
+      run: cd doc && python make.py --warnings-are-errors
 
     - name: Build documentation zip
-      run: doc/make.py zip_html
+      run: cd doc && python make.py zip_html
 
     - name: Build the interactive terminal
       run: |

.github/workflows/python-dev.yml

@@ -75,18 +75,34 @@ jobs:
         python -m pip install --upgrade pip setuptools wheel
         python -m pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
         python -m pip install git+https://github.com/nedbat/coveragepy.git
-        python -m pip install python-dateutil pytz cython hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17
+        python -m pip install python-dateutil pytz cython
+        # TODO: update when upstream releases fixes
+        python -m pip install "git+https://github.com/mesonbuild/meson.git@master"
+        python -m pip install "git+https://github.com/FFY00/meson-python.git@main"
+        python -m pip install hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17
         python -m pip list
 
-    # GH 47305: Parallel build can cause flaky ImportError from pandas/_libs/tslibs
+      # Sigh, someone (numpy?) is depending on mingw, which pandas doesn't compile with.
+      # Also, meson doesn't detect visual c++ unless cl.exe is in path.
+      # TODO: File a bug with meson about this.
     - name: Build Pandas
+      if : ${{ runner.os != 'Windows' }}
       run: |
-        python setup.py build_ext -q -j1
-        python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index
+        python3 -m pip install . --no-build-isolation -v
+      shell: bash -el {0}
+
+    - name: Build Pandas (Windows)
+      if: ${{ runner.os == 'Windows' }}
+      run: |
+        call micromamba activate test
+        call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
+        python -m pip install . --no-build-isolation -v
+      shell: cmd /C call {0}
 
     - name: Build Version
       run: |
-        python -c "import pandas; pandas.show_versions();"
+        # Can't import pandas from the source directory
+        cd .. && python -c "import pandas; pandas.show_versions();"
 
     - name: Test
       uses: ./.github/actions/run-tests

.gitignore

@@ -36,6 +36,7 @@
 *.py[ocd]
 *.so
 .build_cache_dir
+.mesonpy-native-file.ini
 MANIFEST
 
 # Python files #

ci/deps/actions-310-numpydev.yaml

@@ -17,6 +17,8 @@ dependencies:
   - pip
   - pip:
     - "cython"
+    - "git+https://github.com/mesonbuild/meson.git@master"
+    - "git+https://github.com/mesonbuild/meson-python.git@main"
     - "--extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple"
     - "--pre"
     - "numpy"

ci/deps/actions-310.yaml

@@ -52,3 +52,6 @@ dependencies:
   - xlrd
   - xlsxwriter
   - zstandard
+  - pip:
+      - "git+https://github.com/mesonbuild/meson.git@master"
+      - "git+https://github.com/mesonbuild/meson-python.git@main"

ci/deps/actions-38-downstream_compat.yaml

@@ -68,3 +68,6 @@ dependencies:
   - pyyaml
   - py
   - pytorch
+  - pip:
+      - "git+https://github.com/mesonbuild/meson.git@master"
+      - "git+https://github.com/mesonbuild/meson-python.git@main"

ci/deps/actions-38-minimum_versions.yaml

@@ -54,3 +54,6 @@ dependencies:
   - xlrd=2.0.1
   - xlsxwriter=1.4.3
   - zstandard=0.15.2
+  - pip:
+      - "git+https://github.com/mesonbuild/meson.git@master"
+      - "git+https://github.com/mesonbuild/meson-python.git@main"

ci/deps/actions-38.yaml

@@ -51,3 +51,6 @@ dependencies:
   - xlrd
   - xlsxwriter
   - zstandard
+  - pip:
+      - "git+https://github.com/mesonbuild/meson.git@master"
+      - "git+https://github.com/mesonbuild/meson-python.git@main"

ci/deps/actions-39.yaml

@@ -52,3 +52,6 @@ dependencies:
   - xlrd
   - xlsxwriter
   - zstandard
+  - pip:
+      - "git+https://github.com/mesonbuild/meson.git@master"
+      - "git+https://github.com/mesonbuild/meson-python.git@main"

ci/deps/actions-pypy-38.yaml

@@ -19,3 +19,6 @@ dependencies:
   - numpy
   - python-dateutil
   - pytz
+  - pip:
+      - "git+https://github.com/mesonbuild/meson.git@master"
+      - "git+https://github.com/mesonbuild/meson-python.git@main"

ci/deps/circle-38-arm64.yaml

@@ -52,3 +52,6 @@ dependencies:
   - xlrd
   - xlsxwriter
   - zstandard
+  - pip:
+      - "git+https://github.com/mesonbuild/meson.git@master"
+      - "git+https://github.com/mesonbuild/meson-python.git@main"

ci/run_tests.sh

@@ -1,5 +1,6 @@
 #!/bin/bash -e
 
+
 # Workaround for pytest-xdist (it collects different tests in the workers if PYTHONHASHSEED is not set)
 # https://github.com/pytest-dev/pytest/issues/920
 # https://github.com/pytest-dev/pytest/issues/1075
@@ -13,7 +14,7 @@ if [[ "not network" == *"$PATTERN"* ]]; then
 fi
 
 if [[ "$COVERAGE" == "true" ]]; then
-    COVERAGE="-s --cov=pandas --cov-report=xml --cov-append"
+    COVERAGE="-s --cov=pandas --cov-report=xml --cov-append --cov-config=setup.cfg"
 else
     COVERAGE="" # We need to reset this for COVERAGE="false" case
 fi
@@ -24,7 +25,7 @@ if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then
     XVFB="xvfb-run "
 fi
 
-PYTEST_CMD="${XVFB}pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET"
+PYTEST_CMD="${XVFB}pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile --import-mode=importlib $TEST_ARGS $COVERAGE $PYTEST_TARGET"
 
 if [[ "$PATTERN" ]]; then
   PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\""
@@ -33,7 +34,7 @@ fi
 echo $PYTEST_CMD
 sh -c "$PYTEST_CMD"
 
-if [[ "$PANDAS_DATA_MANAGER" != "array" && "$PYTEST_TARGET" == "pandas" ]]; then
+if [[ "$PANDAS_DATA_MANAGER" != "array" && "$PYTEST_TARGET" == "pandas/pandas" ]]; then
     # The ArrayManager tests should have already been run by PYTEST_CMD if PANDAS_DATA_MANAGER was already set to array
     # If we're targeting specific files, e.g. test_downstream.py, don't run.
     PYTEST_AM_CMD="PANDAS_DATA_MANAGER=array pytest -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE pandas"

doc/make.py

@@ -353,8 +353,8 @@ def main():
     # external libraries (namely Sphinx) to compile this module and resolve
     # the import of `python_path` correctly. The latter is used to resolve
     # the import within the module, injecting it into the global namespace
-    os.environ["PYTHONPATH"] = args.python_path
-    sys.path.insert(0, args.python_path)
+    # os.environ["PYTHONPATH"] = args.python_path
+    # sys.path.insert(0, args.python_path)
     globals()["pandas"] = importlib.import_module("pandas")
 
     # Set the matplotlib backend to the non-interactive Agg backend for all

environment.yml

@@ -130,3 +130,5 @@ dependencies:
   - pip:
       - jupyterlite==0.1.0b12
       - sphinx-toggleprompt
+      - "git+https://github.com/mesonbuild/meson.git@master"
+      - "git+https://github.com/mesonbuild/meson-python.git@main"

generate_pxi.py

@@ -0,0 +1,33 @@
+import argparse
+import os
+
+from Cython import Tempita
+
+
+def process_tempita(pxifile, outfile):
+    with open(pxifile) as f:
+        tmpl = f.read()
+    pyxcontent = Tempita.sub(tmpl)
+
+    with open(outfile, "w") as f:
+        f.write(pyxcontent)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("infile", type=str, help="Path to the input file")
+    parser.add_argument("-o", "--outdir", type=str, help="Path to the output directory")
+    args = parser.parse_args()
+
+    if not args.infile.endswith(".in"):
+        raise ValueError(f"Unexpected extension: {args.infile}")
+
+    outdir_abs = os.path.join(os.getcwd(), args.outdir)
+    outfile = os.path.join(
+        outdir_abs, os.path.splitext(os.path.split(args.infile)[1])[0]
+    )
+
+    process_tempita(args.infile, outfile)
+
+
+main()

generate_version.py

@@ -0,0 +1,34 @@
+import argparse
+import os
+
+import versioneer
+
+
+def write_version_info(path):
+    if os.environ.get("MESON_DIST_ROOT"):
+        # raise ValueError("dist root is", os.environ.get("MESON_DIST_ROOT"))
+        path = os.path.join(os.environ.get("MESON_DIST_ROOT"), path)
+    with open(path, "w") as file:
+        file.write(f'__version__="{versioneer.get_version()}"\n')
+        file.write(
+            f'__git_version__="{versioneer.get_versions()["full-revisionid"]}"\n'
+        )
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-o", "--outfile", type=str, help="Path to write version info to"
+    )
+    args = parser.parse_args()
+
+    if not args.outfile.endswith(".py"):
+        raise ValueError(
+            f"Output file must be a Python file. "
+            f"Got: {args.outfile} as filename instead"
+        )
+
+    write_version_info(args.outfile)
+
+
+main()

meson.build

@@ -0,0 +1,49 @@
+# This file is adapted from https://github.com/scipy/scipy/blob/main/meson.build
+project(
+    'pandas',
+    'c', 'cpp', 'cython',
+    version: '2.0.0.dev0',
+    license: 'BSD-3',
+    # TODO: bump when meson 0.64.0 comes out,
+    # we are relying on 0.64.0 features
+    meson_version: '>=0.63',
+    default_options: [
+        # TODO: investigate, does meson try to compile against debug Python
+        # when buildtype = debug, this seems to be causing problems on CI
+        # where provided Python is not compiled in debug mode
+        'buildtype=release',
+        # TODO: turn on werror when ready
+        #'werror=true',
+        # TODO: We are using POSIX functions(strdup in ujson), so we can't compile
+        # with strict C99 :(
+        #'c_std=c99'
+    ]
+)
+
+add_global_arguments('-DNPY_NO_DEPRECATED_API=0', language : 'c')
+add_global_arguments('-DNPY_NO_DEPRECATED_API=0', language : 'cpp')
+
+py_mod = import('python')
+fs = import('fs')
+py = py_mod.find_installation('python')
+py_dep = py.dependency()
+tempita = files('generate_pxi.py')
+versioneer = files('generate_version.py')
+
+if fs.exists('_version_meson.py')
+    py.install_sources('_version_meson.py', subdir: 'pandas')
+else
+    custom_target('write_version_file',
+        output: '_version_meson.py',
+        command: [
+            py, versioneer, '-o', '@OUTPUT@'
+        ],
+        build_by_default: true,
+        build_always_stale: true,
+        install: true,
+        install_dir: py.get_install_dir(pure: false) / 'pandas'
+    )
+    meson.add_dist_script(py, versioneer, '-o', '_version_meson.py')
+endif
+
+subdir('pandas')

pandas/__init__.py

@@ -176,12 +176,18 @@
 from pandas.util._tester import test
 
 # use the closest tagged version if possible
-from pandas._version import get_versions
+_built_with_meson = False
+try:
+    from pandas._version_meson import __version__, __git_version__
+
+    _built_with_meson = True
+except ImportError:
+    from pandas._version import get_versions
 
-v = get_versions()
-__version__ = v.get("closest-tag", v["version"])
-__git_version__ = v.get("full-revisionid")
-del get_versions, v
+    v = get_versions()
+    __version__ = v.get("closest-tag", v["version"])
+    __git_version__ = v.get("full-revisionid")
+    del get_versions, v
 
 # GH 27101
 __deprecated_num_index_names = ["Float64Index", "Int64Index", "UInt64Index"]