Merge pull request #3071 from PrincetonUniversity/devel

Devel
PrincetonUniversity · Nov 12, 2024 · cc0a663 · cc0a663
2 parents 9b68ab8 + f5064d1
commit cc0a663
Show file tree

Hide file tree

Showing 293 changed files with 17,496 additions and 8,811 deletions.
diff --git a/.github/actions/install-pnl/action.yml b/.github/actions/install-pnl/action.yml
@@ -5,6 +5,13 @@ inputs:
     description: 'PsyNeuLink features to install'
     required: true
     default: ''
+outputs:
+  wheel:
+    description: 'Returns path to the built .whl package'
+    value: ${{ steps.dist.outputs.wheel }}
+  sdist:
+    description: 'Returns path to the built .sdist package'
+    value: ${{ steps.dist.outputs.sdist }}
 
 runs:
   using: "composite"
@@ -83,12 +90,21 @@ runs:
           echo "new_package=''" >> $GITHUB_OUTPUT
         fi
 
+    - name: Build dist
+      id: dist
+      shell: bash
+      run: |
+        pip install setuptools wheel
+        python setup.py sdist bdist_wheel
+        echo "wheel=$(ls dist/*.whl)" | tee -a "$GITHUB_OUTPUT"
+        echo "sdist=$(ls dist/*.sdist)" | tee -a "$GITHUB_OUTPUT"
+
     - name: Python dependencies
       shell: bash
       run: |
-        pip install -e .[${{ inputs.features }}] -c env_constraints.txt -c broken_trans_deps.txt
+        pip install ${{ steps.dist.outputs.wheel }}[${{ inputs.features }}] -c env_constraints.txt -c broken_trans_deps.txt
 
-    - name: "Cleanup old wheels"
+    - name: Cleanup old wheels
       shell: bash
       run: |
         pip cache info

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -27,5 +27,5 @@ updates:
       prefix: "requirements"
     labels:
       - "deps"
-    open-pull-requests-limit: 15
+    open-pull-requests-limit: 25
     rebase-strategy: "disabled"
diff --git a/.github/workflows/pnl-ci-docs.yml b/.github/workflows/pnl-ci-docs.yml
@@ -20,19 +20,11 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
-      # Matrix setup is a hacky way to include 'base' build in pull requests
-      # The entire matrix is set up and 'base' builds are pruned based
-      # on event name and final configuration (ubuntu, python3.11).
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
-        python-architecture: ['x64']
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
         os: [ubuntu-latest, macos-latest, windows-latest]
-        event:
-          - ${{ github.event_name }}
-        pnl-version: ['head', 'base']
+        pnl-version: ${{ (github.event_name == 'push') && fromJSON('["head"]') || fromJSON('["head", "base"]') }}
         exclude:
-          - event: 'push'
-            pnl-version: 'base'
           - os: macos-latest
             pnl-version: 'base'
           - os: windows-latest
@@ -45,6 +37,16 @@ jobs:
             pnl-version: 'base'
           - python-version: '3.10'
             pnl-version: 'base'
+          - python-version: '3.12'
+            pnl-version: 'base'
+
+            # Python 3.7 x64 on macos-14 (arm64) images is broken [0]
+            # and arm64 version is not available [1].
+            # Restrict python 3.7 macos runs to macos-13
+            # [0] https://github.com/actions/setup-python/issues/855
+            # [1] https://github.com/actions/setup-python/issues/856
+          - python-version: '3.7'
+            os: macos-latest
 
     outputs:
       on_master: ${{ steps.on_master.outputs.on-branch }}
@@ -81,7 +83,6 @@ jobs:
         # Block python3.7.17 on macos. see:
         # https://github.com/actions/setup-python/issues/682
         python-version: ${{ (matrix.os == 'macos-latest' && matrix.python-version == '3.7') && '3.7.16' || matrix.python-version }}
-        architecture: ${{ matrix.python-architecture }}
 
     - name: Get pip cache location
       shell: bash
@@ -95,11 +96,11 @@ jobs:
       uses: actions/cache@v4
       with:
         path: ${{ steps.pip_cache.outputs.pip_cache_dir }}/wheels
-        key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-${{ hashFiles('requirements.txt', 'doc_requirements.txt') }}-${{ github.sha }}
-        restore-keys: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-${{ hashFiles('requirements.txt', 'doc_requirements.txt') }}
+        key: ${{ runner.os }}-python-${{ matrix.python-version }}-pip-wheels-${{ hashFiles('requirements.txt', 'doc_requirements.txt') }}-${{ github.sha }}
+        restore-keys: ${{ runner.os }}-python-${{ matrix.python-version }}-pip-wheels-${{ hashFiles('requirements.txt', 'doc_requirements.txt') }}
 
     # We need to install all PNL deps since docs config imports psyneulink module
-    - name: Install local, editable PNL package
+    - name: Install PNL package
       uses: ./.github/actions/install-pnl
       with:
         features: 'doc'
@@ -126,7 +127,7 @@ jobs:
     - name: Upload Documentation
       uses: actions/upload-artifact@v4
       with:
-        name: Documentation-${{matrix.pnl-version}}-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.python-architecture }}
+        name: Documentation-${{matrix.pnl-version}}-${{ matrix.os }}-${{ matrix.python-version }}
         retention-days: 1
         path: docs/build/html
 
@@ -172,7 +173,7 @@ jobs:
     - name: Download branch docs
       uses: actions/download-artifact@v4
       with:
-        name: Documentation-head-${{ matrix.os }}-${{ matrix.python-version }}-x64
+        name: Documentation-head-${{ matrix.os }}-${{ matrix.python-version }}
         path: _built_docs/${{ github.ref }}
       if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/devel' || github.ref == 'refs/heads/docs'
 
@@ -189,7 +190,7 @@ jobs:
     - name: Download main docs
       uses: actions/download-artifact@v4
       with:
-        name: Documentation-head-${{ matrix.os }}-${{ matrix.python-version }}-x64
+        name: Documentation-head-${{ matrix.os }}-${{ matrix.python-version }}
         # This overwrites files in current directory
       if: startsWith(github.ref, 'refs/tags/') && contains(needs.*.outputs.on_master, 'master')
 

diff --git a/.github/workflows/pnl-ci.yml b/.github/workflows/pnl-ci.yml
@@ -13,11 +13,6 @@ on:
       - 'v**'
   pull_request:
 
-env:
-  SELF_HOSTED_MACOS: ${{ secrets.SELF_HOSTED_MACOS }}
-  SELF_HOSTED_LINUX: ${{ secrets.SELF_HOSTED_LINUX }}
-  SELF_HOSTED_WINDOWS: ${{ secrets.SELF_HOSTED_WINDOWS }}
-
 # run only the latest instance of this workflow job for the current branch/PR
 # cancel older runs
 # fall back to run id if not available (run id is unique -> no cancellations)
@@ -26,31 +21,19 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  # A job to select self-hosted runner if requested by an env var
-  select-runner:
-    runs-on: ubuntu-latest
-
-    outputs:
-      self_hosted_macos: ${{ steps.is_self_hosted.outputs.macos && 'macos' || '' }}
-      self_hosted_linux: ${{ steps.is_self_hosted.outputs.linux && 'linux' || '' }}
-      self_hosted_windows: ${{ steps.is_self_hosted.outputs.windows && 'windows' || '' }}
-
-    steps:
-    - name: Add macos
-      id: is_self_hosted
-      run: |
-        echo "macos=$SELF_HOSTED_MACOS" | tee -a $GITHUB_OUTPUT
-        echo "linux=$SELF_HOSTED_LINUX" | tee -a $GITHUB_OUTPUT
-        echo "windows=$SELF_HOSTED_WINDOWS" | tee -a $GITHUB_OUTPUT
-
-  # the main build job
+  # The main test job
   build:
-    needs: select-runner
-    runs-on: ${{ (contains(needs.select-runner.outputs.*, matrix.os) && fromJSON(format('[ "self-hosted","{0}", "X64" ]', matrix.os))) || format('{0}-latest', matrix.os) }}
+    runs-on: ${{ (contains(vars.SELF_HOSTED, format(';{0}_{1}_{2}_{3};', matrix.os, matrix.python-version, matrix.python-architecture, matrix.extra-args))
+                && fromJSON(format('[ "self-hosted","{0}", "X64", "enabled" ]', matrix.os == 'ubuntu' && 'Linux' || matrix.os)))
+                || format('{0}-latest', matrix.os) }}
+    env:
+      # Keep DESCRIPTION in sync with the above
+      DESCRIPTION: ${{ format(';{0}_{1}_{2}_{3};', matrix.os, matrix.python-version, matrix.python-architecture, matrix.extra-args) }}
+      SELF_HOSTED: ${{ vars.SELF_HOSTED }}
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.7', '3.11']
+        python-version: ['3.7', '3.11', '3.12']
         python-architecture: ['x64']
         extra-args: ['']
         os: [ubuntu, macos, windows]
@@ -95,8 +78,8 @@ jobs:
 
           # add python 3.8 build on macos since 3.7 is broken
           # https://github.com/actions/virtual-environments/issues/4230
+          # use default python-architecture
           - python-version: '3.8'
-            python-architecture: 'x64'
             os: macos
 
         exclude:
@@ -149,8 +132,9 @@ jobs:
         key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-${{ hashFiles('requirements.txt', 'dev_requirements.txt') }}-${{ github.sha }}
         restore-keys: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ matrix.python-architecture }}-pip-wheels-${{ hashFiles('requirements.txt', 'dev_requirements.txt') }}
 
-    - name: Install local, editable PNL package
+    - name: Install PNL package
       uses: ./.github/actions/install-pnl
+      id: install
       with:
         features: 'dev'
 
@@ -202,15 +186,10 @@ jobs:
           echo "::warning::Not uploading to coveralls.io, token not available!"
         fi
 
-    - name: Build dist
-      run: |
-        pip install setuptools wheel
-        python setup.py sdist bdist_wheel
-
     - name: Upload dist packages
       uses: actions/upload-artifact@v4
       if: matrix.version-restrict == ''
       with:
         name: dist-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.python-architecture }}
-        path: dist/
+        path: ${{ steps.install.outputs.wheel }} ${{ steps.install.outputs.sdist }}
         retention-days: 2
diff --git a/.github/workflows/test-release.yml b/.github/workflows/test-release.yml
@@ -55,7 +55,7 @@ jobs:
         # It's not possible to condition steps on env or secrets,
         # We need an explicit check here
         if [ -n "$TWINE_USERNAME" -a -n "$TWINE_PASSWORD" ]; then
-          twine upload dist/*
+          twine upload --verbose dist/*
         else
           echo "::warning::Not uploading to test PyPI, no credentials available!"
         fi
@@ -69,7 +69,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.7, 3.8, 3.9]
+        python-version: [3.7, 3.8, 3.9, '3.10', 3.11, 3.12]
         os: [ubuntu-latest, macos-latest, windows-latest]
         dist: [wheel, sdist]
 
@@ -157,7 +157,7 @@ jobs:
         # It's not possible to condition steps on env or secrets,
         # We need an explicit check here
         if [ -n "$TWINE_USERNAME" -a -n "$TWINE_PASSWORD" ]; then
-          twine upload dist/*
+          twine upload --verbose dist/*
         else
           echo "::warning::Not uploading to PyPI, no credentials available!"
         fi
@@ -181,53 +181,9 @@ jobs:
         path: dist/
 
     - name: Upload dist files to release
-      uses: actions/github-script@v7
-      with:
-        script: |
-          const fs = require('fs')
-          tag = context.ref.split('/').pop()
-          console.log('running on:' + context.ref);
-          console.log('Looking for release for tag:' + tag);
-
-          var release
-          try {
-            release = await github.rest.repos.getReleaseByTag({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              tag: tag
-            });
-            console.log('Release found at: ' + release.data.html_url); // '
-          } catch (err) {
-            if (err.status == 404) {
-              console.log('Release not found, creating a new one');
-              release = await github.rest.repos.createRelease({
-                  owner: context.repo.owner,
-                  repo: context.repo.repo,
-                  tag_name: tag,
-                  name: 'Release ' + tag,
-                  body: 'New features and fixed bugs'
-              });
-            } else {
-              throw err;
-            }
-          }
-
-          console.log('Using release upload url: ' + release['data']['upload_url']);
-          // Determine content-length for header to upload asset
-          for (asset of ['${{ needs.create-python-dist.outputs.wheel }}', '${{ needs.create-python-dist.outputs.sdist }}']) {
-            const file_path = 'dist/' + asset;
-            const file_size = file_path => fs.statSync(file_path).size;
-            console.log('Uploading: ' + file_path);
-
-            // Setup headers for API call, see Octokit Documentation:
-            // https://octokit.github.io/rest.js/#octokit-routes-repos-upload-release-asset for more information
-            const headers = { 'content-type': 'application/zip', 'content-length': file_size(file_path) };
-
-            // Upload a release asset
-            const uploadAssetResponse = await github.rest.repos.uploadReleaseAsset({
-              url: release.data.upload_url,
-              headers,
-              name: asset,
-              file: fs.readFileSync(file_path)
-            });
-          }
+      shell: bash
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        previous_release=$(gh --repo ${{ github.repository }} release list | head -n 1 | awk '{print $1}')
+        gh --repo ${{ github.repository }} release create --generate-notes --notes-start-tag "$previous_release" ${{ github.ref }} dist/*
diff --git a/README.rst b/README.rst
@@ -146,7 +146,6 @@ Dependencies that are automatically installed (except those noted as optional) i
    * pillow
    * llvmlite
    * mpi4py (optional)
-   * autograd (optional)
 
 Lists of required packages for PsyNeuLink, developing PsyNeuLink, and running the PsyNeuLink tutorial are also
 stored in pip-style `requirements.txt`, `dev_requirements.txt`, and `tutorial_requirements.txt` in the source code.
@@ -173,13 +172,13 @@ To run the tutorial locally, you must run python 3.5 and install additional pack
 
 ::
 
-    pip install psyneulink[tutorial]
+    pip install "psyneulink[tutorial]"
 
 or if you downloaded the source:
 
 ::
 
-    pip install .[tutorial]
+    pip install ".[tutorial]"
 
 
 To access the tutorial, make sure you fulfill the requirements mentioned above, download the tutorial notebook (/tutorial/PsyNeuLink Tutorial.ipynb), then run the terminal command