BLD: Setup meson builds (#49115)

Author: lithomas1

.circleci/setup_env.sh

@@ -54,10 +54,7 @@ if pip list | grep -q ^pandas; then
     pip uninstall -y pandas || true
 fi
 
-echo "Build extensions"
-python setup.py build_ext -q -j4
-
 echo "Install pandas"
-python -m pip install --no-build-isolation --no-use-pep517 -e .
+python -m pip install --no-build-isolation -ve .
 
 echo "done"

.github/actions/build_pandas/action.yml

@@ -1,5 +1,9 @@
 name: Build pandas
 description: Rebuilds the C extensions and installs pandas
+inputs:
+  editable:
+    description: Whether to build pandas in editable mode (default true)
+    default: true
 runs:
   using: composite
   steps:
@@ -12,9 +16,9 @@ runs:
 
     - name: Build Pandas
       run: |
-        python setup.py build_ext -j $N_JOBS
-        python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index
+        if [[ ${{ inputs.editable }} == "true" ]]; then
+          pip install -e . --no-build-isolation -v
+        else
+          pip install . --no-build-isolation -v
+        fi
       shell: bash -el {0}
-      env:
-        # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources
-        N_JOBS: ${{ runner.os == 'macOS' && 3 || 2 }}

.github/workflows/code-checks.yml

@@ -63,8 +63,24 @@ jobs:
     - name: Build Pandas
       id: build
       uses: ./.github/actions/build_pandas
+      with:
+        editable: false
 
     # The following checks are independent of each other and should still be run if one fails
+
+    # TODO: The doctests have to be run first right now, since the Cython doctests only work
+    # with pandas installed in non-editable mode
+    # This can be removed once pytest-cython doesn't require C extensions to be installed inplace
+    - name: Run doctests
+      run: cd ci && ./code_checks.sh doctests
+      if: ${{ steps.build.outcome == 'success' && always() }}
+
+    - name: Install pandas in editable mode
+      id: build-editable
+      uses: ./.github/actions/build_pandas
+      with:
+        editable: true
+
     - name: Check for no warnings when building single-page docs
       run: ci/code_checks.sh single-docs
       if: ${{ steps.build.outcome == 'success' && always() }}
@@ -73,10 +89,6 @@ jobs:
       run: ci/code_checks.sh code
       if: ${{ steps.build.outcome == 'success' && always() }}
 
-    - name: Run doctests
-      run: ci/code_checks.sh doctests
-      if: ${{ steps.build.outcome == 'success' && always() }}
-
     - name: Run docstring validation
       run: ci/code_checks.sh docstrings
       if: ${{ steps.build.outcome == 'success' && always() }}

.github/workflows/package-checks.yml

@@ -44,13 +44,10 @@ jobs:
         with:
           python-version: '3.10'
 
-      - name: Install required dependencies
-        run: |
-          python -m pip install --upgrade pip setuptools wheel python-dateutil pytz numpy cython
-          python -m pip install versioneer[toml]
-
       - name: Pip install with extra
-        run: python -m pip install -e .[${{ matrix.extra }}] --no-build-isolation
+        run: |
+          python -m pip install .[${{ matrix.extra }}] -v
+        shell: bash -el {0}
   conda_forge_recipe:
     if: ${{ github.event.label.name == 'Build' || contains(github.event.pull_request.labels.*.name, 'Build') || github.event_name == 'push'}}
     runs-on: ubuntu-22.04

.github/workflows/unit-tests.yml

@@ -229,10 +229,9 @@ jobs:
         run: |
           /opt/python/cp39-cp39/bin/python -m venv ~/virtualenvs/pandas-dev
           . ~/virtualenvs/pandas-dev/bin/activate
-          python -m pip install --no-cache-dir --no-deps -U pip wheel setuptools
+          python -m pip install -U pip wheel setuptools meson[ninja]==1.0.1 meson-python==0.13.1
           python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
-          python setup.py build_ext -q -j$(nproc)
-          python -m pip install --no-cache-dir --no-build-isolation --no-use-pep517 -e .
+          python -m pip install --no-cache-dir --no-build-isolation -e .
           python -m pip list
           export PANDAS_CI=1
           python -m pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml
@@ -268,10 +267,9 @@ jobs:
         run: |
           /opt/python/cp39-cp39/bin/python -m venv ~/virtualenvs/pandas-dev
           . ~/virtualenvs/pandas-dev/bin/activate
-          python -m pip install --no-cache-dir --no-deps -U pip wheel setuptools
+          python -m pip install -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.0.1
           python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
-          python setup.py build_ext -q -j$(nproc)
-          python -m pip install --no-cache-dir --no-build-isolation --no-use-pep517 -e .
+          python -m pip install --no-cache-dir --no-build-isolation -e .
           python -m pip list --no-cache-dir
 
       - name: Run Tests
@@ -347,8 +345,7 @@ jobs:
 
       - name: Build Pandas
         run: |
-          python setup.py build_ext -q -j4
-          python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index
+          python -m pip install -e . --no-build-isolation --no-index
 
       - name: Build Version
         run: |

.github/workflows/wheels.yml

@@ -56,7 +56,6 @@ jobs:
         - [ubuntu-20.04, manylinux_x86_64]
         - [macos-11, macosx_*]
         - [windows-2019, win_amd64]
-        - [windows-2019, win32]
         # TODO: support PyPy?
         python: [["cp39", "3.9"], ["cp310", "3.10"],  ["cp311", "3.11"]]#  "pp39"]
     env:

.gitignore

@@ -36,6 +36,7 @@
 *.py[ocd]
 *.so
 .build_cache_dir
+.mesonpy-native-file.ini
 MANIFEST
 
 # Python files #
@@ -76,6 +77,8 @@ coverage_html_report
 __pycache__
 # pytest-monkeytype
 monkeytype.sqlite3
+# meson editable install folder
+.mesonpy
 
 
 # OS generated files #

asv_bench/asv.conf.json

@@ -41,7 +41,6 @@
     // pip (with all the conda available packages installed first,
     // followed by the pip installed packages).
     "matrix": {
-        "numpy": [],
         "Cython": ["0.29.33"],
         "matplotlib": [],
         "sqlalchemy": [],
@@ -56,6 +55,9 @@
         "xlrd": [],
         "odfpy": [],
         "jinja2": [],
+        "meson": [],
+        "meson-python": [],
+        "python-build": [],
     },
     "conda_channels": ["conda-forge"],
     // Combinations of libraries/python versions can be excluded/included
@@ -125,7 +127,5 @@
     "regression_thresholds": {
     },
     "build_command":
-    ["python -m pip install versioneer[toml]",
-     "python setup.py build -j4",
-     "PIP_NO_BUILD_ISOLATION=false python -mpip wheel --no-deps --no-index -w {build_cache_dir} {build_dir}"],
+    ["python -m build -Cbuilddir=builddir --wheel --outdir {build_cache_dir} {build_dir}"]
 }

ci/code_checks.sh

@@ -65,13 +65,8 @@ fi
 ### DOCTESTS ###
 if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then
 
-    MSG='Doctests' ; echo $MSG
-    # Ignore test_*.py files or else the unit tests will run
-    python -m pytest --doctest-modules --ignore-glob="**/test_*.py" pandas
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
-    MSG='Cython Doctests' ; echo $MSG
-    python -m pytest --doctest-cython pandas/_libs
+    MSG='Python and Cython Doctests' ; echo $MSG
+    python -c 'import pandas as pd; pd.test(run_doctests=True)'
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
 fi

ci/deps/actions-310-numpydev.yaml

@@ -6,11 +6,16 @@ dependencies:
 
   # build dependencies
   - versioneer[toml]
+  - meson[ninja]=1.0.1
+  - meson-python=0.13.1
 
   # test dependencies
   - pytest>=7.0.0
   - pytest-cov
-  - pytest-xdist>=2.2.0
+  # Once pytest-cov > 4 comes out, unpin this
+  # Right now, a DeprecationWarning related to rsyncdir
+  # causes an InternalError within pytest
+  - pytest-xdist>=2.2.0, <3
   - hypothesis>=6.46.1
   - pytest-asyncio>=0.17.0
 

ci/deps/actions-310.yaml

@@ -7,6 +7,8 @@ dependencies:
   # build dependencies
   - versioneer[toml]
   - cython>=0.29.33
+  - meson[ninja]=1.0.1
+  - meson-python=0.13.1
 
   # test dependencies
   - pytest>=7.0.0

ci/deps/actions-311-pyarrownightly.yaml

@@ -6,7 +6,9 @@ dependencies:
 
   # build dependencies
   - versioneer[toml]
+  - meson[ninja]=1.0.1
   - cython>=0.29.33
+  - meson-python=0.13.1
 
   # test dependencies
   - pytest>=7.0.0

ci/deps/actions-311.yaml

@@ -7,6 +7,8 @@ dependencies:
   # build dependencies
   - versioneer[toml]
   - cython>=0.29.33
+  - meson[ninja]=1.0.1
+  - meson-python=0.13.1
 
   # test dependencies
   - pytest>=7.0.0

ci/deps/actions-39-downstream_compat.yaml

@@ -8,6 +8,8 @@ dependencies:
   # build dependencies
   - versioneer[toml]
   - cython>=0.29.33
+  - meson[ninja]=1.0.1
+  - meson-python=0.13.1
 
   # test dependencies
   - pytest>=7.0.0
@@ -69,7 +71,6 @@ dependencies:
   - pandas-datareader
   - pyyaml
   - py
-
   - pip:
     - pyqt5>=5.15.6
     - tzdata>=2022.1

ci/deps/actions-39-minimum_versions.yaml

@@ -9,6 +9,8 @@ dependencies:
   # build dependencies
   - versioneer[toml]
   - cython>=0.29.33
+  - meson[ninja]=1.0.1
+  - meson-python=0.13.1
 
   # test dependencies
   - pytest>=7.0.0

ci/deps/actions-39.yaml

@@ -7,6 +7,8 @@ dependencies:
   # build dependencies
   - versioneer[toml]
   - cython>=0.29.33
+  - meson[ninja]=1.0.1
+  - meson-python=0.13.1
 
   # test dependencies
   - pytest>=7.0.0

ci/deps/actions-pypy-39.yaml

@@ -10,6 +10,8 @@ dependencies:
   # build dependencies
   - versioneer[toml]
   - cython>=0.29.33
+  - meson[ninja]=1.0.1
+  - meson-python=0.13.1
 
   # test dependencies
   - pytest>=7.0.0
@@ -22,6 +24,5 @@ dependencies:
   - numpy
   - python-dateutil
   - pytz
-
   - pip:
     - tzdata>=2022.1

ci/deps/circle-39-arm64.yaml

@@ -7,6 +7,8 @@ dependencies:
   # build dependencies
   - versioneer[toml]
   - cython>=0.29.33
+  - meson[ninja]=1.0.1
+  - meson-python=0.13.1
 
   # test dependencies
   - pytest>=7.0.0

ci/run_tests.sh

@@ -12,15 +12,15 @@ if [[ "not network" == *"$PATTERN"* ]]; then
     export http_proxy=http://1.2.3.4 https_proxy=http://1.2.3.4;
 fi
 
-COVERAGE="-s --cov=pandas --cov-report=xml --cov-append"
+COVERAGE="-s --cov=pandas --cov-report=xml --cov-append --cov-config=pyproject.toml"
 
 # If no X server is found, we use xvfb to emulate it
 if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then
     export DISPLAY=":0"
     XVFB="xvfb-run "
 fi
 
-PYTEST_CMD="${XVFB}pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET"
+PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 ${XVFB}pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET"
 
 if [[ "$PATTERN" ]]; then
   PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\""

doc/source/development/contributing_environment.rst

@@ -207,13 +207,47 @@ for :ref:`building pandas with GitPod <contributing-gitpod>`.
 Step 3: build and install pandas
 --------------------------------
 
-You can now run::
+There are currently two supported ways of building pandas, pip/meson and setuptools(setup.py).
+Historically, pandas has only supported using setuptools to build pandas. However, this method
+requires a lot of convoluted code in setup.py and also has many issues in compiling pandas in parallel
+due to limitations in setuptools.
+
+The newer build system, invokes the meson backend through pip (via a `PEP 517 <https://peps.python.org/pep-0517/>`_ build).
+It automatically uses all available cores on your CPU, and also avoids the need for manual rebuilds by
+rebuilding automatically whenever pandas is imported(with an editable install).
+
+For these reasons, you should compile pandas with meson.
+Because the meson build system is newer, you may find bugs/minor issues as it matures. You can report these bugs
+`here <https://github.com/pandas-dev/pandas/issues/49683>`_.
+
+To compile pandas with meson, run::
 
    # Build and install pandas
-   # The number after -j is the number of compiling jobs run in parallel
-   # Change it according to your machine's hardware spec
-   python setup.py build_ext -j 4
-   python -m pip install -e . --no-build-isolation --no-use-pep517
+   python -m pip install -ve . --no-build-isolation
+
+** Build options **
+
+It is possible to pass options from the pip frontend to the meson backend if you would like to configure your
+install. Occasionally, you'll want to use this to adjust the build directory, and/or toggle debug/optimization levels.
+
+You can pass a build directory to pandas by appending ``--config-settings builddir="your builddir here"`` to your pip command.
+This option allows you to configure where meson stores your built C extensions, and allows for fast rebuilds.
+
+Sometimes, it might be useful to compile pandas with debugging symbols, when debugging C extensions.
+Appending ``--config-settings setup-args="-Ddebug=true"`` will do the trick.
+
+With pip, it is possible to chain together multiple config settings (for example specifying both a build directory
+and building with debug symbols would look like
+``--config-settings builddir="your builddir here" --config-settings=setup-args="-Dbuildtype=debug"``.
+
+**Compiling pandas with setup.py**
+
+.. note::
+   This method of compiling pandas will be deprecated and removed very soon, as the meson backend matures.
+
+To compile pandas with setuptools, run::
+
+   python setup.py develop
 
 .. note::
    You will need to repeat this step each time the C extensions change, for example
@@ -226,5 +260,22 @@ At this point you should be able to import pandas from your locally built versio
    >>> print(pandas.__version__)  # note: the exact output may differ
    2.0.0.dev0+880.g2b9e661fbb.dirty
 
-This will create the new environment, and not touch any of your existing environments,
-nor any existing Python installation.
+When building pandas with meson, importing pandas will automatically trigger a rebuild, even when C/Cython files are modified.
+By default, no output will be produced by this rebuild (the import will just take longer). If you would like to see meson's
+output when importing pandas, you can set the environment variable ``MESONPY_EDTIABLE_VERBOSE``. For example, this would be::
+
+   # On Linux/macOS
+   MESONPY_EDITABLE_VERBOSE=1 python
+
+   # Windows
+   set MESONPY_EDITABLE_VERBOSE=1 # Only need to set this once per session
+   python
+
+If you would like to see this verbose output every time, you can set the ``editable-verbose`` config setting to ``true`` like so::
+
+   python -m pip install -ve . --config-settings editable-verbose=true
+
+.. tip::
+   If you ever find yourself wondering whether setuptools or meson was used to build your pandas,
+   you can check the value of ``pandas._built_with_meson``, which will be true if meson was used
+   to compile pandas.