Move Python build to pyproject.toml, bump Python minver to 3.7, fix…

… macos wheel generation (#1916) * Failing Macos Python wheel builds fixed. * Macos Python wheels now come with dual-arch (x86-64 and arm64) * Moved (nearly) all Python build instructions to `pyproject.toml` * Enables 'build isolation', and need to specify build-deps only once, no need for users or CI scripts to pre-install them. * Enables editable `pip` installs (`pip install -e ./arbor`) * Compatible with 'build frontends' `pip` and `build`. * Passing CMake options actually got shorter * Drop Python 3.6 support.
arbor-sim · Jul 4, 2022 · 8af6bd2 · 8af6bd2
1 parent c38db01
commit 8af6bd2
Show file tree

Hide file tree

Showing 11 changed files with 128 additions and 128 deletions.
diff --git a/.github/workflows/ciwheel.yml b/.github/workflows/ciwheel.yml
@@ -24,38 +24,10 @@ jobs:
         with:
           fetch-depth: 0
           submodules: recursive
-      - name: Update pip
-        run: python -m pip install --upgrade pip
-
-      - name: Build wheels Linux
-        if: ${{ startsWith(matrix.os, 'ubuntu') }}
-        uses: pypa/cibuildwheel@v2.3.0
-        with:
-          output-dir: dist
-        env:
-          CIBW_BEFORE_ALL: yum -y install libxml2-devel
-          CIBW_BEFORE_BUILD: python -m pip install numpy setuptools scikit-build ninja cmake
-          CIBW_BUILD: "cp3*-manylinux_x86_64"
-          CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
-          CIBW_ARCHS_LINUX: x86_64
-          CIBW_REPAIR_WHEEL_COMMAND: 'auditwheel repair -w {dest_dir} {wheel} && python /project/scripts/patchwheel.py {dest_dir}'
-          CIBW_TEST_COMMAND: python -m unittest discover -v -s {project}/python
-
-      - name: Build wheels macos
-        if: ${{ startsWith(matrix.os, 'macos') }}
-        uses: pypa/cibuildwheel@v2.3.0
-        with:
-          output-dir: dist
-        env:
-          MACOSX_DEPLOYMENT_TARGET: "10.15" #needed to undo some CIBW settings
-          CIBW_BEFORE_BUILD: python -m pip install numpy setuptools scikit-build ninja cmake
-          CIBW_BUILD: "cp3*-macosx_x86_64"
-          CIBW_ARCHS_MACOS: x86_64 universal2
-          CIBW_TEST_COMMAND: python -m unittest discover -v -s {project}/python
-
-      # this action runs auditwheel automatically with the following args:
-      # https://cibuildwheel.readthedocs.io/en/stable/options/#repair-wheel-command
-
+      - name: Install cibuildwheel
+        run: python3 -m pip install cibuildwheel
+      - name: Build wheels
+        run: python3 -m cibuildwheel --output-dir dist
       - uses: actions/upload-artifact@v2
         with:
           name: dist
@@ -68,18 +40,16 @@ jobs:
     steps:
       - name: Set up Python
         uses: actions/setup-python@v2
-      - name: Update pip
-        run: python -m pip install --upgrade pip
       - name: Get packages
-        run: python -m pip install numpy setuptools scikit-build ninja cmake
+        run: python3 -m pip install build
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
           submodules: recursive
       - name: Make sdist
-        run:  python setup.py sdist
+        run: python3 -m build -s
       - name: Install sdist
-        run:  python -m pip install dist/arbor*.tar.gz
+        run:  python3 -m pip install dist/arbor*.tar.gz
       - name: Run Python tests
         run: python3 -m unittest discover -v -s python
       - name: Run Python examples

diff --git a/.github/workflows/test-everything.yml b/.github/workflows/test-everything.yml
@@ -18,7 +18,7 @@ jobs:
             os:    "ubuntu-18.04",
             cc:    "gcc-8",
             cxx:   "g++-8",
-            py:    "3.6",
+            py:    "3.7",
             cmake: "3.18.x",
             mpi:   "ON",
             simd:  "OFF"
@@ -28,7 +28,7 @@ jobs:
             os:    "ubuntu-18.04",
             cc:    "clang-8",
             cxx:   "clang++-8",
-            py:    "3.6",
+            py:    "3.7",
             cmake: "3.18.x",
             mpi:   "ON",
             simd:  "OFF"
@@ -38,7 +38,7 @@ jobs:
             os:    "macos-10.15",
             cc:    "clang",
             cxx:   "clang++",
-            py:    "3.6",
+            py:    "3.7",
             cmake: "3.18.x",
             mpi:   "ON",
             simd:  "OFF"
@@ -48,7 +48,7 @@ jobs:
             os:    "ubuntu-20.04",
             cc:    "gcc-10",
             cxx:   "g++-10",
-            py:    "3.9",
+            py:    "3.10",
             cmake: "3.22.x",
             mpi:   "ON",
             simd:  "OFF"
@@ -58,7 +58,7 @@ jobs:
             os:    "ubuntu-20.04",
             cc:    "gcc-10",
             cxx:   "g++-10",
-            py:    "3.9",
+            py:    "3.10",
             cmake: "3.22.x",
             mpi:   "OFF",
             simd:  "ON"
@@ -68,7 +68,7 @@ jobs:
             os:    "ubuntu-20.04",
             cc:    "clang-10",
             cxx:   "clang++-10",
-            py:    "3.9",
+            py:    "3.10",
             cmake: "3.22.x",
             mpi:   "ON",
             simd:  "OFF"
@@ -78,7 +78,7 @@ jobs:
             os:    "macos-11",
             cc:    "clang",
             cxx:   "clang++",
-            py:    "3.9",
+            py:    "3.10",
             cmake: "3.22.x",
             mpi:   "ON",
             simd:  "OFF"
@@ -202,16 +202,12 @@ jobs:
     name: "Pip build test + Python examples test"
     runs-on: ubuntu-latest
     steps:
-      - name: Update pip
-        run: python -m pip install --upgrade pip
-      - name: Install Python packages
-        run:  pip install numpy setuptools scikit-build ninja cmake
       - name: Clone w/ submodules
         uses: actions/checkout@v2
         with:
           submodules: recursive
       - name: Build and install Arbor using pip + build flags
-        run: python3 -m pip install --verbose --install-option="-DARB_VECTORIZE=ON" --install-option="-DARB_ARCH=native" .
+        run: CMAKE_ARGS="-DARB_VECTORIZE=ON -DARB_ARCH=native" python3 -m pip install .
       - name: Check that build flags match
         run: |
           python3 -c "import arbor; print(arbor.config())" | grep -q "'arch': 'native'"

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -302,7 +302,7 @@ endif()
 #----------------------------------------------------------
 
 # The minimum version of Python supported by Arbor.
-set(arb_py_version 3.6.0)
+set(arb_py_version 3.7.0)
 
 if(DEFINED PYTHON_EXECUTABLE)
     set(Python3_EXECUTABLE ${PYTHON_EXECUTABLE})
@@ -311,7 +311,7 @@ endif()
 if(ARB_WITH_PYTHON)
     cmake_dependent_option(ARB_USE_BUNDLED_PYBIND11 "Use bundled pybind11" ON "ARB_WITH_PYTHON;ARB_USE_BUNDLED_LIBS" OFF)
 
-    if(DEFINED ENV{CIBUILDWHEEL})
+    if(DEFINED ENV{CIBUILDWHEEL} AND (UNIX AND NOT APPLE))
         find_package(Python3 ${arb_py_version} COMPONENTS Interpreter Development.Module REQUIRED)
     else()
         find_package(Python3 ${arb_py_version} COMPONENTS Interpreter Development REQUIRED)

diff --git a/doc/dependencies.csv b/doc/dependencies.csv
@@ -19,7 +19,7 @@ bench,Google-benchmark,,submodule ``ext/google-benchmark``,
 --,fmt,,submodule ``ext/fmt``,
 --,tinyopt,,source copy ``ext/tinyopt``,
 ARB_WITH_PYTHON,pybind11,,submodule ``python/pybind11``,
-ARB_WITH_PYTHON,Python,3.6,Python compatiblity is the range between the latest officially released point version and the minimum here specified.,"* it is not more advanced than the version specified by NEP 29.
+ARB_WITH_PYTHON,Python,3.7,Python compatiblity is the range between the latest officially released point version and the minimum here specified.,"* it is not more advanced than the version specified by NEP 29.
 * it is available as a cray-python version on Piz Daint
 * it is available on labs.ebrains.eu
 

diff --git a/doc/install/build_install.rst b/doc/install/build_install.rst
@@ -135,7 +135,7 @@ More information on building with MPI is in the `HPC cluster section <cluster_>`
 Python
 ~~~~~~
 
-Arbor has a Python frontend, for which a minimum of Python 3.6 is required.
+Arbor has a Python frontend, for which a minimum of Python 3.7 is required.
 In addition, `numpy` is a runtime requirement for the Python package.
 In order to use MPI in combination with the python frontend the
 `mpi4py <https://mpi4py.readthedocs.io/en/stable/install.html#>`_

diff --git a/doc/install/python.rst b/doc/install/python.rst
@@ -6,7 +6,7 @@ Python Installation
 Arbor's Python API will be the most convenient interface for most users.
 
 .. note::
-    Arbor requires Python version 3.6 and later. It is advised that you update ``pip`` as well.
+    Arbor requires Python version 3.7 and later. It is advised that you update ``pip`` as well.
     We strongly encourage using ``pip`` to install Arbor.
 
     To get help in case of problems installing with pip, run pip with the ``--verbose`` flag, and attach the output
@@ -42,15 +42,11 @@ You are now ready to use Arbor! You can continue reading these documentation pag
     for any other platforms than listed above, ``pip`` will attempt a build from source and thus require these
     packages as well.
 
-    * Ubuntu/Debian: `git cmake gcc python3-dev python3-pip libxml2-dev`
-    * Fedora/CentOS/OpenSuse: `git cmake gcc-c++ python3-devel python3-pip libxml2-devel`
-    * MacOS: get `brew` `here <https://brew.sh>`_ and run `brew install cmake clang python3 libxml2`
+    * Ubuntu/Debian: ``git cmake gcc python3-dev python3-pip libxml2-dev``
+    * Fedora/CentOS/OpenSuse: ``git cmake gcc-c++ python3-devel python3-pip libxml2-devel``
+    * MacOS: get ``brew`` `here <https://brew.sh>`_ and run ``brew install cmake clang python3 libxml2``
     * Windows: the simplest way is to use `WSL <https://docs.microsoft.com/en-us/windows/wsl/install-win10>`_ and then follow the instructions for Ubuntu.
 
-    In addition, you'll need a few Python packages present:
-
-    ``pip3 install ninja scikit-build wheel setuptools numpy``
-
 .. _in_python_custom:
 
 Customising Arbor
@@ -79,22 +75,18 @@ Every time you make changes to the code, you'll have to repeat the second step.
 Advanced options
 ^^^^^^^^^^^^^^^^
 
-By default Arbor is installed with multi-threading enabled. To enable more
-advanced forms of parallelism and other features, Arbor comes with a few
-compilation options. These are of the form ``-D<KEY>=<VALUE>``, must be appended
-to the ``pip`` invocation via ``--install-option="-D<...>" --install-option="-D<...>" ...`` and can
-be used on both local (``pip3 install ./arbor``) and remote (``pip3 install
-arbor``) copies of Arbor. See the examples below.
+Arbor comes with a few compilation options, some of them related to advanced forms of parallelism and other features.
+The options and flags are the same :ref:`as documented for the CMAKE build <quickstart>`, but they are passed differently.
+To enable more, they must be placed in the ``CMAKE_ARGS`` environment variable.
+The simplest way to do this is by prepending the ``pip`` command with ``CMAKE_ARGS=""``,
+where you place the arguments separated by space inside the quotes.
 
 .. Note::
 
    If you run into build issues while experimenting with build options, be sure
    to remove the ``_skbuild`` directory. If you had Arbor installed already,
    you may need to remove it first before you can (re)compile it with the flags you need.
 
-   Also, make sure to pass each option individually via
-   ``--install-option="..."``.
-
 The following flags can be used to configure the installation:
 
 * ``ARB_WITH_NEUROML=<ON|OFF>``: Enable support for NeuroML2 morphologies,
@@ -119,6 +111,14 @@ The following flags can be used to configure the installation:
    and ``CMake`` under the hood, so all flags and options valid in ``CMake`` can
    be used in this fashion.
 
+   Allthough the
+   `scikit-build documentation <https://scikit-build.readthedocs.io/en/latest/usage.html#environment-variable-configuration>`_
+   mentions that you can also pass the build options with ``--install-option=""``,
+   this will cause ``pip`` to build all dependencies, including all build-dependencies,
+   instead of downloading them from PyPI.
+   ``CMAKE_ARGS=""`` saves you the build time, and also downloading and setting up the dependencies they in turn require to be present.
+   Setting ``CMAKE_ARGS=""`` is in addition compatible with build front-ends like `build <https://pypa-build.readthedocs.io>`_.
+
    Detailed instructions on how to install using CMake are in the :ref:`Python
    configuration <install-python>` section of the :ref:`installation guide
    <in_build_install>`. CMake is recommended if you need more control over
@@ -138,33 +138,33 @@ In the examples below we assume you are installing from a local copy.
 
 .. code-block:: bash
 
-    pip3 install ./arbor --install-option="-DARB_WITH_MPI=ON"
+    CMAKE_ARGS="-DARB_WITH_MPI=ON" pip3 install ./arbor
 
 **Compile with** :ref:`vectorization <install-vectorize>` on a system with a SkyLake
 :ref:`architecture <install-architecture>`:
 
 .. code-block:: bash
 
-    pip3 install ./arbor --install-option="-DARB_VECTORIZE=ON" --install-option="-DARB_ARCH=skylake"
-
+    CMAKE_ARGS="-DARB_VECTORIZE=ON -DARB_ARCH=skylake" pip3 install ./arbor
+    
 **Enable NVIDIA GPUs (compiled with nvcc)**. This requires the :ref:`CUDA toolkit <install-gpu>`:
 
 .. code-block:: bash
 
-    pip3 install ./arbor --install-option="-DARB_GPU=cuda"
+    CMAKE_ARGS="-DARB_GPU=cuda" pip3 install ./arbor
 
 **Enable NVIDIA GPUs (compiled with clang)**. This also requires the :ref:`CUDA toolkit <install-gpu>`:
 
 .. code-block:: bash
 
-    pip3 install ./arbor --install-option="-DARB_GPU=cuda-clang"
+    CMAKE_ARGS="-DARB_GPU=cuda-clang" pip3 install ./arbor
 
 **Enable AMD GPUs (compiled with hipcc)**. This requires setting the ``CC`` and ``CXX``
-:ref:`environment variables <install-gpu>`
+:ref:`environment variables <install-gpu>`:
 
 .. code-block:: bash
 
-    pip3 install ./arbor --install-option="-DARB_GPU=hip"
+    CC=clang CXX=hipcc CMAKE_ARGS="-DARB_GPU=hip" pip3 install ./arbor
 
 Note on performance
 -------------------

diff --git a/pyproject.toml b/pyproject.toml
@@ -0,0 +1,72 @@
+[project]
+name = "arbor"
+dynamic = ["version", "readme"]
+license = {file = "LICENSE"}
+description = "High performance simulation of networks of multicompartment neurons."
+requires-python = ">=3.7"
+keywords = ["simulator", "neuroscience", "morphological detail", "HPC", "GPU", "C++"]
+authors = [
+    {name = "Arbor Dev Team", email = "contact@arbor-sim.org"}
+]
+maintainers = [
+    {name = "Arbor Dev Team", email = "contact@arbor-sim.org"}
+]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: BSD License",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3.7",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: C++"
+]
+dependencies = [
+    "numpy"
+]
+
+[tool.setuptools]
+py-modules = ["arbor"]
+
+[tool.setuptools.dynamic]
+version = {file = ["VERSION"]}
+readme = {file = ["python/readme.md"]}
+
+[project.urls]
+homepage = "https://arbor-sim.org"
+documentation = "https://docs.arbor-sim.org"
+repository = "https://github.com/arbor-sim/arbor"
+changelog = "https://github.com/arbor-sim/arbor/releases"
+
+[build-system]
+requires = [
+    "setuptools",
+    "wheel",
+    "scikit-build",
+    "cmake>=3.18",
+    "ninja",
+]
+build-backend = "setuptools.build_meta"
+
+[tool.cibuildwheel]
+build-frontend = "build"
+build = ["cp3*-manylinux_x86_64","cp3*-macosx_universal2"]#,"cp3*-musllinux_x86_64","cp3*-musllinux_aarch64"]
+skip = "cp36-*"
+test-command = "python3 -m unittest discover -v -s {project}/python"
+
+[tool.cibuildwheel.macos]
+archs = ["x86_64", "universal2", "arm64"]
+
+[tool.cibuildwheel.macos.environment]
+MACOSX_DEPLOYMENT_TARGET = "10.15"
+
+[tool.cibuildwheel.linux]
+archs = ["x86_64"]
+manylinux-x86_64-image = "manylinux2014"
+before-all = "yum -y install libxml2-devel"
+repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel} && python3 /project/scripts/patchwheel.py {dest_dir}"
+
+[[tool.cibuildwheel.overrides]]
+select = "*-musllinux*"
+before-all = "apk add libxml2-dev"
diff --git a/python/example/single_cell_allen.py b/python/example/single_cell_allen.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 from collections import defaultdict
+from dataclasses import dataclass
 import json
 import arbor
 import seaborn
@@ -15,13 +16,12 @@ def load_allen_fit(fit):
         fit = json.load(fd)
 
     # cable parameters convenience class
+    @dataclass
     class parameters:
-        def __init__(self, **kwargs):
-            self.cm = None
-            self.tempK = None
-            self.Vm = None
-            self.rL = None
-            self.__dict__.update(kwargs)
+        cm: float = None
+        tempK: float = None
+        Vm: float = None
+        rL: float = None
 
     param = defaultdict(parameters)
     mechs = defaultdict(dict)