Spack-solvable versions, add tests to Spack workflow

.github/workflows/spack.yml

@@ -12,7 +12,9 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-22.04]
-        python-version: [3.8]
+        python-version: ['3.10']
+        spack-version: ['develop', 'latest_release']
+      fail-fast: false
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -25,18 +27,49 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.spack-cache
-          key: spack-cache-${{ matrix.os }}-${{ github.run_id }}
-          restore-keys: spack-cache-${{ matrix.os }}-
-
-      - name: Build Arbor's Spack package against the develop branch
-        run: arbor/scripts/build_spack_package.sh arbor develop
-
-      # build_spack_package.sh only builds, does not install, therefore can't run Python code.
-      # - name: Run Python examples
-      #   run: |
-      #     cd arbor
-      #     scripts/run_python_examples.sh
-      # - name: Test executables
-      #   run: |
-      #     cd arbor
-      #     scripts/test_executables.sh
+          key: spack-cache-${{ matrix.os }}-${{ matrix.spack-version }}-${{ github.run_id }}
+          restore-keys: spack-cache-${{ matrix.os }}-${{ matrix.spack-version }}-
+      - name: Get Spack (develop)
+        if: ${{ matrix.spack-version == 'develop' }}
+        run: |
+          git clone -c feature.manyFiles=true --depth 1 https://github.com/spack/spack.git
+      - name: Get Spack (latest_release)
+        if: ${{ matrix.spack-version == 'latest_release' }}
+        run: |
+          wget "$(curl -sH "Accept: application/vnd.github.v3+json" https://api.github.com/repos/spack/spack/releases/latest | grep browser_download_url |  cut -d '"' -f 4)"
+          tar xfz spack*.tar.gz
+          mv spack*/ spack
+      - name: Prep
+        run: |
+          mkdir ~/.spack
+          cp arbor/spack/config.yaml ~/.spack
+          source spack/share/spack/setup-env.sh
+          spack repo create custom_repo
+          mkdir -p custom_repo/packages/arbor
+          spack repo add custom_repo
+          spack reindex
+          cp arbor/spack/package.py custom_repo/packages/arbor
+      - name: Build Arbor
+        run: |
+          source spack/share/spack/setup-env.sh
+          cd arbor
+          spack dev-build arbor@develop +python
+      - name: Load Arbor and verify installation, Python Examples.
+        run: |
+          source spack/share/spack/setup-env.sh
+          spack load arbor
+          cd arbor
+          scripts/run_python_examples.sh
+          scripts/test_executables.sh
+      - name: Remove Arbor
+        run: |
+          source spack/share/spack/setup-env.sh
+          spack uninstall -yafR arbor
+      - name: Failure, upload logs
+        uses: actions/upload-artifact@v3
+        if: ${{ failure() }}
+        with:
+          name: spack-log-${{ matrix.spack-version }}
+          path: |
+            arbor/*.txt
+            !arbor/CMakeLists.txt

CMakeLists.txt

@@ -114,7 +114,7 @@ include(ErrorTarget)  # reguired for add_error_target
 
 # Set release as the default build type (CMake default is debug.)
 
-if (NOT CMAKE_BUILD_TYPE)
+if (NOT CMAKE_BUILD_TYPE AND NOT BUILD_TESTING)
     set(CMAKE_BUILD_TYPE release CACHE STRING "Choose the type of build." FORCE)
     # Set the possible values of build type for cmake-gui
     set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "debug" "release")

doc/contrib/release.rst

@@ -45,14 +45,14 @@ Update tags/versions and test
    - Double check that all examples/tutorials/etc are covered by CI
    - Check Python/pip/PyPi metadata and scripts, e.g. ``setup.py``, ``pyproject.toml``
 
-#. Create new temp-branch ending in ``-rc``. E.g. ``v0.6-rc``
+#. Create new temp-branch ending in ``-rc``. E.g. ``v0.9.0-rc``
 #. Bump the ``VERSION`` file:
 
    - For as long as `scikit-build-core` does not support loading fields from external files, also bump in `pyproject.toml`
    - See also :ref:`dev-version`
    - Append ``-rc``. (Make sure there's no ``-dev``)
 
-#. Create a draft PR. Tag and push with ``-rc``. E.g. ``v0.6-rc``
+#. Create a draft PR. Tag and push with ``-rc``. E.g. ``v0.9.0-rc``
 
    - on cmdline: ``git tag -a TAGNAME``
    - ``git push upstream refs/tags/TAGNAME``
@@ -126,7 +126,7 @@ Post Release
 
 #. Update Zenodo with authors and changelog created in previous step and submit.
 
-#. Make a new PR setting ``VERSION`` to the next with a trailing ``-dev``. E.g. if you just release ``3.14``, change ``VERSION`` to ``3.15-dev``
+#. Make a new PR setting ``VERSION`` to the next with a trailing ``-dev``. E.g. if you just released ``3.14.15``, change ``VERSION`` to ``3.15.16-dev``. Make the number portion always consists of a triple. Shorter versions are uninstallable by Spack (``spack install arbor@0.8`` will install v0.8.1, due to anything shorter than a triple being interpreted as a version range). 
 
    - Update ``spack/package.py``. The checksum of the targz is the sha256sum.
    - Include changes such as to ``CITATIONS``, ``doc/index.rst`` in postrel PR. Copy Zenodo BibTex export to ``CITATIONS``.

pyproject.toml

@@ -64,8 +64,7 @@ build-frontend = "build"
 build = ["cp3*-*linux*","cp3*-macosx*"]
 skip = ["cp36*", "pp*", "*musllinux*"]
 test-command = "python -m unittest discover -v -s {project}/python"
-before-build = "python -m pip install pip --upgrade"
-before-test = "python -m pip install pip --upgrade"
+dependency-versions = "latest"
 
 [tool.cibuildwheel.macos]
 archs = ["universal2"]

scripts/arbor-build-catalogue

@@ -15,20 +15,20 @@ config = A.config()
 prefix = Path(config["prefix"])
 CXX = Path(config["CXX"])
 if not CXX.exists():
-    try:
-        # Example <>/lib/python3.10/site-packages/arbor
-        altern = "c++"
-        print(
-            f"Warning: prefix '{CXX}' does not exist, falling back to '{altern}'.",
-            file=sys.stderr,
-        )
-        CXX = altern
-    except:
+    # Example <>/lib/python3.10/site-packages/arbor
+    altern = "c++"
+    from shutil import which
+    if which(altern) == None:
         print(
             f"Error: Neither prefix '{CXX}' nor fallback '{altern}' exist. Please provide a path to a C++ compiler with --cxx",
             file=sys.stderr,
         )
         exit(-1)
+    print(
+        f"Warning: prefix '{CXX}' does not exist, falling back to '{altern}'.",
+        file=sys.stderr,
+    )
+    CXX = altern
 
 
 def parse_arguments():
@@ -312,13 +312,10 @@ with TemporaryDirectory() as tmp:
     cmake_cmd = "cmake .."
     make_cmd = "make"
     if verbose:
-        out, err = (None, None)
         make_cmd += " VERBOSE=1"
-    else:
-        out, err = (sp.PIPE, sp.PIPE)
     try:
-        sp.run(cmake_cmd, shell=True, check=True, stdout=out, stderr=err)
-        sp.run(make_cmd, shell=True, check=True, stdout=out, stderr=err)
+        sp.run(cmake_cmd, shell=True, check=True, capture_output = True, text = True)
+        sp.run(make_cmd, shell=True, check=True, capture_output = True, text = True)
         shutil.copy2(f"{name}-catalogue.so", pwd)
     except sp.CalledProcessError as e:
         import traceback as tb
@@ -327,9 +324,9 @@ with TemporaryDirectory() as tmp:
             # Not in verbose mode, so we have captured the
             # `stdout` and `stderr` and can print it to the user.
             sys.stdout.write("Build log:\n")
-            sys.stdout.write(e.stdout.decode())
+            sys.stdout.write(e.stdout)
             sys.stderr.write(tb.format_exc() + " Error:\n\n")
-            sys.stderr.write(e.stderr.decode())
+            sys.stderr.write(e.stderr)
         else:
             # In verbose mode the outputs weren't captured and
             # have been streamed to `stdout` and `stderr` already.

scripts/build_spack_package.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 # checks out Spack and Arbor and builds it with the package.py from Arbor's repo
 # Spack can be the latest release or the develop branch
+# Also runs unit tests.
 
 set -Eeuo pipefail
 
@@ -61,5 +62,4 @@ spack reindex
 
 cp $ARBOR_DIR/spack/package.py $SPACK_CUSTOM_REPO/packages/arbor
 cd $ARBOR_DIR
-ARBOR_VERSION=$(cat "$ARBOR_DIR/VERSION")
-spack dev-build arbor@${ARBOR_VERSION} +python
+spack dev-build --test root arbor@develop +python

scripts/run_python_examples.sh

@@ -15,6 +15,7 @@ $PREFIX python3 -m pip install -r python/example/example_requirements.txt -U
 runpyex () {
   echo "=== Executing $1 ======================================"
   $PREFIX python3 python/example/$*
+  echo "### Example $1: OK."
 }
 
 runpyex brunel.py -n 400 -m 100 -e 20 -p 0.1 -w 1.2 -d 1 -g 0.5 -l 5 -t 100 -s 1 -G 50 -S 123

scripts/test_executables.sh

@@ -6,7 +6,9 @@ set -Eeuo pipefail
 echo "=== Executing modcc test ======================================"
 modcc python/example/cat/dummy.mod
 test -f "dummy.hpp"
+echo "executable modcc: OK."
 
 echo "=== Executing a-b-c test ======================================"
-arbor-build-catalogue -v cat python/example/cat
+arbor-build-catalogue --cxx=c++ cat python/example/cat
 ./scripts/test-catalogue.py ./cat-catalogue.so
+echo "executable a-b-c: OK."

spack/package.py

@@ -5,12 +5,6 @@
 
 from spack.package import *
 
-try:
-    with open("VERSION", "r") as file:
-        DEVVERSION = file.readline().strip()
-except:
-    DEVVERSION = None
-
 
 class Arbor(CMakePackage, CudaPackage):
     """Arbor is a high-performance library for computational neuroscience
@@ -21,26 +15,25 @@ class Arbor(CMakePackage, CudaPackage):
     url = "https://github.com/arbor-sim/arbor/releases/download/v0.8.1/arbor-v0.8.1-full.tar.gz"
     maintainers = ["thorstenhater", "brenthuisman", "haampie"]
 
-    version("master", branch="master", submodules=True)
-    if DEVVERSION:
-        version(DEVVERSION)
+    version("master", branch="master")
+    version("develop")
     version(
         "0.8.1",
         sha256="caebf96676ace6a9c50436541c420ca4bb53f0639dcab825de6fa370aacf6baa",
         url="https://github.com/arbor-sim/arbor/releases/download/v0.8.1/arbor-v0.8.1-full.tar.gz",
     )
     version(
-        "0.8",
+        "0.8.0",
         sha256="18df5600308841616996a9de93b55a105be0f59692daa5febd3a65aae5bc2c5d",
         url="https://github.com/arbor-sim/arbor/releases/download/v0.8/arbor-v0.8-full.tar.gz",
     )
     version(
-        "0.7",
+        "0.7.0",
         sha256="c3a6b7193946aee882bb85f9c38beac74209842ee94e80840968997ba3b84543",
         url="https://github.com/arbor-sim/arbor/releases/download/v0.7/arbor-v0.7-full.tar.gz",
     )
     version(
-        "0.6",
+        "0.6.0",
         sha256="4cd333b18effc8833428ddc0b99e7dc976804771bc85da90034c272c7019e1e8",
         url="https://github.com/arbor-sim/arbor/releases/download/v0.6/arbor-v0.6-full.tar.gz",
     )
@@ -99,9 +92,9 @@ class Arbor(CMakePackage, CudaPackage):
     depends_on("python@3.7:", when="+python", type=("build", "run"))
     depends_on("py-numpy", when="+python", type=("build", "run"))
     with when("+python"):
-        depends_on("py-pybind11@2.6:", type=("build"))
-        depends_on("py-pybind11@2.8.1:", when="@0.5.3:", type=("build"))
-        depends_on("py-pybind11@2.10.1:", when="@0.7.1:", type=("build"))
+        depends_on("py-pybind11@2.6:", type="build")
+        depends_on("py-pybind11@2.8.1:", when="@0.5.3:", type="build")
+        depends_on("py-pybind11@2.10.1:", when="@0.7.1:", type="build")
 
     # sphinx based documentation
     depends_on("python@3.7:", when="+doc", type="build")
@@ -132,3 +125,8 @@ def cmake_args(self):
         args.append("-DARB_CXX_FLAGS_TARGET=" + opt_flags)
 
         return args
+
+    @run_after("install", when="+python")
+    @on_package_attributes(run_tests=True)
+    def install_test(self):
+        python("-c", "import arbor")