Merge branch 'master' into api

.circleci/config.yml

@@ -54,10 +54,9 @@ jobs:
               310) pyenv_py_ver="3.10" ;;
               311) pyenv_py_ver="3.11" ;;
               312) pyenv_py_ver="3.12" ;;
-              *) echo "Error: pyenv python version not specified!" && exit 1;;
+              *) echo "Error: pyenv python version not specified or not supported." && exit 1;;
             esac
 
-            cd /opt/circleci/.pyenv/plugins/python-build/../.. && git fetch --all && git checkout -B master origin/master && cd -
             env PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install $pyenv_py_ver --force
             pyenv global $pyenv_py_ver
             export PYTHON_EXE=$(which python)

.github/ISSUE_TEMPLATE/release-patch.md

@@ -18,6 +18,8 @@ Pre-release
   - [ ] Update semantic version in `CMakeLists.txt`
   - [ ] Update changelog below and agree on it with everyone; then commit it to `docs/changelog` in the cherrypicks PR (copy structure as-is)
   - [ ] Update `docs/index.rst` accordingly with the new `.pkg` and `.exe` links for `PKG installer` and `Windows Installer`
+- [ ] Activate ReadtheDocs for the cherry-pick branch and ensure the documentation builds (when logged in, go to [the versions page](https://readthedocs.org/projects/nrn/versions/) and set the version for your branch to Active and Hidden)
+- [ ] Run a test wheel build WITHOUT upload on the cherry-pick branch to ensure all the wheels build ([see details](https://nrn.readthedocs.io/en/latest/install/python_wheels.html#publishing-the-wheels-on-pypi-via-azure))
 
 Sanity checks
 ---
@@ -33,7 +35,7 @@ Releasing
 - [ ] Build release wheels but WITHOUT upload ([see details](https://nrn.readthedocs.io/en/latest/install/python_wheels.html#publishing-the-wheels-on-pypi-via-azure))
 - [ ] Create, test and upload manual artifacts
   - [ ] MacOS package installer (manual task, ask Michael)
-  - [ ] arm64 wheels (manual task, check with Alex or Pramod)
+  - [ ] arm64 wheels (manual task, check with Erik, Goran or Pramod)
   - [ ] aarch64 wheels (use existing `release/x.y-aarch64` branch for this, see [guide](https://nrn.readthedocs.io/en/latest/install/python_wheels.html#publishing-the-wheels-on-pypi-via-circleci))
 - [ ] Publish the `x.y.z` wheels on PyPI; see [wheel publishing instructions](https://nrn.readthedocs.io/en/latest/install/python_wheels.html#publishing-the-wheels-on-pypi-via-azure)
 - [ ] Once wheels are published, activate the `x.y.z` tag on ReadTheDocs

.github/ISSUE_TEMPLATE/release.md

@@ -19,6 +19,7 @@ Sanity checks
 - [ ] Create `release/x.y` branch and make sure GitHub, Azure and CircleCI builds pass
 - [ ] Run [nrn-build-ci](https://github.com/neuronsimulator/nrn-build-ci/actions/workflows/build-neuron.yml) for the respective Azure build; see [Azure drop guide](https://github.com/neuronsimulator/nrn-build-ci#azure-wheels-testing---manual-workflow)
 - [ ] Activate ReadTheDocs build for `release/x.y` & make it hidden. Check docs are fine after build is done.
+- [ ] Run a test wheel build WITHOUT upload for `release/x.y` to ensure all the wheels build ([see details](https://nrn.readthedocs.io/en/latest/install/python_wheels.html#publishing-the-wheels-on-pypi-via-azure))
 - [ ] Run BBP Simulation Stack & other relevant tests
 
 
@@ -32,7 +33,7 @@ Releasing
 - [ ] Build release wheels but WITHOUT upload ([see details](https://nrn.readthedocs.io/en/latest/install/python_wheels.html#publishing-the-wheels-on-pypi-via-azure))
 - [ ] Create, test and upload manual artifacts
   - [ ] MacOS package installer (manual task, ask Michael)
-  - [ ] arm64 wheels (manual task, check with Alex or Pramod)
+  - [ ] arm64 wheels (manual task, check with Erik, Goran or Pramod)
   - [ ] aarch64 wheels (create a `release/x.y-aarch64` branch for this, see [guide](https://nrn.readthedocs.io/en/latest/install/python_wheels.html#publishing-the-wheels-on-pypi-via-circleci))
 - [ ] Publish the `x.y.z` wheels on Pypi; see [wheel publishing instructions](https://nrn.readthedocs.io/en/latest/install/python_wheels.html#publishing-the-wheels-on-pypi-via-azure)
 - [ ] Once wheels are published, activate the `x.y.z` tag on ReadTheDocs

.github/workflows/coverage.yml

@@ -1,7 +1,8 @@
 name: NEURON Code Coverage
 
 concurrency:
-  group: ${{ github.workflow }}#${{ github.ref }}
+  # Don't cancel on master, creating a PR when a push workflow is already going will cancel the push workflow in favour of the PR workflow
+  group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/master' && github.run_id || github.event.number && github.head_ref || github.ref_name }}
   cancel-in-progress: true
 
 on:
@@ -73,7 +74,7 @@ jobs:
           sudo apt-get install xvfb
           sudo /usr/bin/Xvfb $DISPLAY -screen 0 1600x1200x24 -noreset -nolock -shmem &  # run in bg
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 2
 
@@ -83,7 +84,7 @@ jobs:
             git submodule update --init --recursive --force --depth 1 -- external/nmodl
 
       - name: Set up Python@${{ env.PY_MIN_VERSION }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PY_MIN_VERSION }}
 
@@ -95,12 +96,12 @@ jobs:
           python -m pip install --upgrade -r ci_requirements.txt
 
       - name: Set up Python@${{ env.PY_MID_VERSION }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PY_MID_VERSION }}
 
       - name: Set up Python@${{ env.PY_MAX_VERSION }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PY_MAX_VERSION }}
 
@@ -110,7 +111,7 @@ jobs:
           python -m pip install --upgrade pip -r nrn_requirements.txt
           python -m pip install --upgrade -r external/nmodl/requirements.txt
           python -m pip install --upgrade -r ci_requirements.txt
-          
+
 
       - name: Build & Test
         id: build-test
@@ -184,7 +185,7 @@ jobs:
         if: failure() && contains(github.event.pull_request.title, 'live-debug-coverage')
         uses: mxschmitt/action-tmate@v3
 
-      - uses: codecov/codecov-action@v3
+      - uses: codecov/codecov-action@v4
         with:
           directory: ./build
           fail_ci_if_error: true

.github/workflows/docs.yml

@@ -1,7 +1,8 @@
 name: NEURON Documentation
 
 concurrency:
-  group: ${{ github.workflow }}#${{ github.ref }}
+  # Don't cancel on master, creating a PR when a push workflow is already going will cancel the push workflow in favour of the PR workflow
+  group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/master' && github.run_id || github.event.number && github.head_ref || github.ref_name }}
   cancel-in-progress: true
 
 on:
@@ -36,11 +37,11 @@ jobs:
         shell: bash
 
       - name: Set up Python@${{ env.DEFAULT_PY_VERSION }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ env.DEFAULT_PY_VERSION }}
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Install Python dependencies
         working-directory: ${{runner.workspace}}/nrn

.github/workflows/external.yml

@@ -1,7 +1,8 @@
 name: External CIs
 
 concurrency:
-  group: ${{ github.workflow }}#${{ github.ref }}
+  # Don't cancel on master, creating a PR when a push workflow is already going will cancel the push workflow in favour of the PR workflow
+  group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/master' && github.run_id || github.event.number && github.head_ref || github.ref_name }}
   cancel-in-progress: true
 
 on:

.github/workflows/formatting.yml

@@ -1,7 +1,8 @@
 name: Check formatting
 
 concurrency:
-  group: ${{ github.workflow }}#${{ github.ref }}
+  # Don't cancel on master, creating a PR when a push workflow is already going will cancel the push workflow in favour of the PR workflow
+  group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/master' && github.run_id || github.event.number && github.head_ref || github.ref_name }}
   cancel-in-progress: true
 
 on:
@@ -20,10 +21,31 @@ jobs:
     runs-on: ubuntu-22.04
     timeout-minutes: 5
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Update submodule
         working-directory: ${{runner.workspace}}/nrn
         run: git submodule update --init external/coding-conventions
       - name: Check formatting
         working-directory: ${{runner.workspace}}/nrn
         run: external/coding-conventions/bin/format -v --dry-run
+
+      # If formatting fails, apply formatting and push changes.
+      # This will trigger another workflow run, which will cancel the current one.
+      - name: Apply formatting
+        working-directory: ${{runner.workspace}}/nrn
+        if: failure() && github.event_name == 'pull_request'
+        run: |
+          # Checkout PR
+          gh pr checkout ${{ github.event.pull_request.number }}
+
+          # Apply formatting
+          external/coding-conventions/bin/format -v
+
+          # Commit & push changes
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git add -u :/
+          git commit -a -m "Fix formatting"
+          git push
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/neuron-ci.yml

@@ -1,7 +1,8 @@
 name: NEURON CI
 
 concurrency:
-  group: ${{ github.workflow }}#${{ github.ref }}
+  # Don't cancel on master, creating a PR when a push workflow is already going will cancel the push workflow in favour of the PR workflow
+  group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/master' && github.run_id || github.event.number && github.head_ref || github.ref_name }}
   cancel-in-progress: true
 
 on:
@@ -89,7 +90,7 @@ jobs:
 
     steps:
       - name: Setup cmake
-        uses: jwlawson/actions-setup-cmake@v1
+        uses: jwlawson/actions-setup-cmake@v2
         with:
           cmake-version : ${{(matrix.config.python_dynamic || matrix.config.build_mode == 'setuptools') && env.DYNAMIC_PYTHON_CMAKE_VERSION || env.DESIRED_CMAKE_VERSION}}
 
@@ -127,7 +128,7 @@ jobs:
           echo CI_OS_NAME=linux >> $GITHUB_ENV
         shell: bash
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 2
 
@@ -138,7 +139,7 @@ jobs:
 
       - name: Set up Python@${{ env.PY_MIN_VERSION }}
         if: ${{matrix.config.python_dynamic == 'ON'}}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PY_MIN_VERSION }}
 
@@ -151,7 +152,7 @@ jobs:
           python -m pip install --upgrade -r ci_requirements.txt
 
       - name: Set up Python@${{ env.PY_MAX_VERSION }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PY_MAX_VERSION }}
 
@@ -196,23 +197,11 @@ jobs:
           cat matrix.json
           echo -----
 
-      # Workaround for https://github.com/actions/cache/issues/92
-      - name: Checkout cache action
-        uses: actions/checkout@v3
-        with:
-          repository: actions/cache
-          ref: v3
-          path: tmp/actions/cache
-
-      - name: Make actions/cache@v3 run even on failure
-        run: |
-          sed -i'.bak' -e '/ post-if: /d' tmp/actions/cache/action.yml
-
       - name: Restore compiler cache
-        uses: ./tmp/actions/cache
+        uses: actions/cache@v4
         with:
-          path: |
-            ${{runner.workspace}}/ccache
+          path: ${{runner.workspace}}/ccache
+          save-always: true
           key: ${{matrix.os}}-${{hashfiles('matrix.json')}}-${{github.ref}}-${{github.sha}}
           restore-keys: |
             ${{matrix.os}}-${{hashfiles('matrix.json')}}-${{github.ref}}-

.github/workflows/release.yml

@@ -25,7 +25,7 @@ jobs:
       release_url: ${{ steps.create_release.outputs.upload_url }}
       rel_tag: ${{ env.REL_TAG }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         name: Checkout branch ${{ env.REL_BRANCH }}
         with:
             ref: ${{ env.REL_BRANCH }}

.github/workflows/windows.yml

@@ -1,7 +1,8 @@
 name: Windows Installer
 
 concurrency:
-  group: ${{ github.workflow }}#${{ github.ref }}
+  # Don't cancel on master, creating a PR when a push workflow is already going will cancel the push workflow in favour of the PR workflow
+  group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/master' && github.run_id || github.event.number && github.head_ref || github.ref_name }}
   cancel-in-progress: true
 
 on:
@@ -35,7 +36,7 @@ jobs:
     timeout-minutes: 45
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         ref: ${{ inputs.tag }}
 
@@ -46,7 +47,7 @@ jobs:
       working-directory: ${{runner.workspace}}\nrn
 
     - name: Set up Python3
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.11'
 
@@ -79,7 +80,7 @@ jobs:
       uses: mxschmitt/action-tmate@v3
 
     - name: Upload build artifact
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: nrn-nightly-AMD64.exe
         path: ${{runner.workspace}}\nrn\nrn-nightly-AMD64.exe

.gitlab-ci.yml

@@ -7,7 +7,6 @@ include:
   - project: hpc/gitlab-upload-logs
     file: enable-upload.yml
 
-
 # see https://gitlab.com/gitlab-org/gitlab/-/issues/263401 for why we specify the flags like this now
 # 130 characters seems to be the point at which jobs refuse to run
 .matrix:
@@ -263,3 +262,60 @@ test:neuron:nmodl:nvhpc:omp:legacy:
 test:neuron:nmodl:nvhpc:omp:
   extends: [.test_neuron, .gpu_node]
   needs: ["build:neuron:nmodl:nvhpc:omp"]
+
+
+# Container building
+mac_m1_container_build:
+  stage: .pre
+  tags:
+    - macos-arm64
+  script:
+    - if [ -z "${ARM64_IMAGE_TAG}" ]; then
+    -   echo "Please set the ARM64_IMAGE_TAG variable"
+    -   exit 1
+    - fi
+    - cd packaging/python
+    - echo "Replacing symlinks with their targets to keep podman happy"
+    - find . -type l -exec cp $(realpath {}) ./TEMP \; -exec rm {} \; -exec mv TEMP {} \;
+    - ls -l
+    - export BUILDAH_FORMAT=docker # enables ONBUILD instructions which are not OCI compatible
+    - machine_status=$(podman machine inspect | awk '/State/ {print $2}' | tr -d '",')
+      # If you start the machine yourself, make sure BUILDAH_FORMAT and the http proxy variables are set in the shell before doing so!
+    - if [[ "${machine_status}" != "running" ]]; then
+    -     echo "Machine is in ${machine_status} status - starting"
+    -     podman machine start
+    - fi
+    - podman build -t neuronsimulator/neuron_wheel:${ARM64_IMAGE_TAG} --build-arg MANYLINUX_IMAGE=manylinux2014_aarch64 -f Dockerfile .
+    - podman login -u ${DOCKER_HUB_USER} -p ${DOCKER_HUB_AUTH_TOKEN} docker.io
+    - podman push neuronsimulator/neuron_wheel:${ARM64_IMAGE_TAG}
+    - podman rmi localhost/neuronsimulator/neuron_wheel:${ARM64_IMAGE_TAG}
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "web"
+  when: manual
+
+x86_64_container_build:
+  stage: .pre
+  image:
+    name: quay.io/buildah/stable
+    entrypoint: [""]
+  variables:
+    KUBERNETES_CPU_LIMIT: 4
+    KUBERNETES_CPU_REQUEST: 2
+    KUBERNETES_MEMORY_LIMIT: 8Gi
+    KUBERNETES_MEMORY_REQUEST: 4Gi
+  tags:
+    - kubernetes
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "web"
+  when: manual
+  script:
+    - if [ -z "${X86_IMAGE_TAG}" ]; then
+    -   echo "Please set the X86_IMAGE_TAG variable"
+    -   exit 1
+    - fi
+    - export STORAGE_DRIVER=vfs    # allows to build inside containers without additional mounts
+    - export BUILDAH_FORMAT=docker # enables ONBUILD instructions which are not OCI compatible
+    - cd packaging/python
+    - buildah bud --iidfile image_id -t neuronsimulator/neuron_wheel:${X86_IMAGE_TAG} -f Dockerfile .
+    - buildah login -u ${DOCKER_HUB_USER} -p ${DOCKER_HUB_AUTH_TOKEN} docker.io
+    - buildah push $(<image_id) docker://docker.io/neuronsimulator/neuron_wheel:${X86_IMAGE_TAG}

.readthedocs.yml

@@ -1,4 +1,9 @@
 version: 2
 
+build:
+  os: "ubuntu-22.04"
+  tools:
+    python: "mambaforge-22.9"
+
 conda:
   environment: docs/conda_environment.yml