Devel

.appveyor.yml

@@ -0,0 +1,54 @@
+version: '{build}'
+branches:
+  except:
+  - gh-pages
+skip_tags: true
+skip_branch_with_pr: true
+clone_depth: 15
+image: Ubuntu2004
+environment:
+  PIP_PROGRESS_BAR: off
+  APPVEYOR_CONSOLE_DISABLE_PTY: true
+  OMP_NUM_THREADS: 1
+  OPENBLAS_NUM_THREADS: 1
+  matrix:
+  - PYTHON: python3.8
+
+init:
+- ps: |
+    sudo apt-get update
+    sudo apt-get install -y --no-install-recommends build-essential graphviz git curl $Env:PYTHON-dev $Env:PYTHON-distutils $Env:PYTHON-venv
+    sudo update-alternatives --install /usr/bin/python python /usr/bin/$Env:PYTHON 10
+    sudo update-alternatives --install /usr/bin/python3 python3 /usr/bin/$Env:PYTHON 10
+    sudo update-alternatives --set python /usr/bin/$Env:PYTHON
+    sudo update-alternatives --set python3 /usr/bin/$Env:PYTHON
+
+install:
+- ps: |
+    python -m venv venv
+    venv/bin/Activate.ps1
+    echo "#!/bin/sh" > venv/bin/xdg-open
+    chmod +x venv/bin/xdg-open
+
+    pip --version
+    pip install -U pip wheel cython
+    pip install -U coveralls
+    pip install -U -e .[dev]
+
+build: off
+
+test_script:
+- ps: |
+    if (Test-Path Env:\COVERALLS_REPO_TOKEN ) {
+      $Env:COV="--cov=psyneulink"
+    }
+    python -m pytest --junit-xml=tests_out.xml $Env:COV --verbosity=0 --capture=sys -o console_output_style=count
+
+on_success:
+- ps: |
+    if (Test-Path Env:\COVERALLS_REPO_TOKEN ) {
+      coveralls
+    }
+
+on_finish:
+- ps: curl -X POST -F "file=@tests_out.xml" https://ci.appveyor.com/api/testresults/junit/$Env:APPVEYOR_JOB_ID

.github/actions/install-pnl/action.yml

@@ -18,26 +18,75 @@ runs:
       run: |
         case "$RUNNER_OS" in
           macOS*) brew install graphviz ;;
-          Linux*) sudo apt-get install -y graphviz ;;
+          Linux*) sudo apt-get update && sudo apt-get install -y --no-install-recommends graphviz ;;
           Windows*) choco install --no-progress -y graphviz --version=2.38.0.20190211 ;;
           *) echo "Unsupported OS"; exit 1 ;;
         esac
 
+    - name: Setup Python venv
+      shell: bash
+      run: |
+        case "$RUNNER_OS" in
+          macOS* | Linux*)
+            export VIRTUAL_ENV='${{ runner.temp }}/_venv'; export PYTHON_LOC=$VIRTUAL_ENV/bin/ ;;
+          Windows*)
+            export VIRTUAL_ENV='${{ runner.temp }}\_venv'; export PYTHON_LOC=$VIRTUAL_ENV\\Scripts ;;
+          *) echo "Unsupported OS"; exit 1 ;;
+        esac
+        python -m venv $VIRTUAL_ENV
+        echo "VIRTUAL_ENV=$VIRTUAL_ENV" >> $GITHUB_ENV
+        echo "$PYTHON_LOC" >> $GITHUB_PATH
+
+    - name: Update pip and wheel
+      shell: bash
+      run: python -m pip install --upgrade pip wheel
+
     - name: Drop pytorch on x86
       shell: bash
       run: |
+        echo > env_constraints.txt
         if [ $(python -c 'import struct; print(struct.calcsize("P") * 8)') == 32 ]; then
           sed -i /torch/d requirements.txt
+          sed -i /modeci_mdf/d requirements.txt
           # pywinpty is a transitive dependency and v1.0+ removed support for x86 wheels
-          # terminado >= 0.10.0 pulls in pywinpty >= 1.1.0
-          [[ ${{ runner.os }} = Windows* ]] && pip install "pywinpty<1" "terminado<0.10"
+          echo "pywinpty<1" >> env_constraints.txt
+          # jupyter_sever pulls jupyter_server_terminals which depends on in pywinpty >= 2.0.3
+          echo "jupyter_server<2" >> env_constraints.txt
+          # scipy >=1.9.2 doesn't provide win32 wheel and GA doesn't have working fortran on windows
+          echo "scipy<1.9.2" >> env_constraints.txt
+          # scikit-learn >= 1.1.3 doesn't provide win32 wheel
+          echo "scikit-learn<1.1.3" >> env_constraints.txt
+          # countourpy >=1.1.0 doesn't provide win32 wheel
+          echo "contourpy<1.1.0" >> env_constraints.txt
+          # pillow >= 10.0.0 doesn't provide win32 wheel
+          echo "pillow < 10.0.0" >> env_constraints.txt
+        fi
+
+    - name: Install updated package
+      if: ${{ startsWith(github.head_ref, 'dependabot/pip') && matrix.pnl-version != 'base' }}
+      shell: bash
+      id: new_package
+      run: |
+        # The branch name pattern is: "dependabot/pip/$TARGET/$PACKAGE{-gt-$MINVERSION,,}{-lt-$MAXVERSION,}{-$VERSION,}
+        # The expression below extracts just the $PACKAGE part
+        export NEW_PACKAGE=$(echo '${{ github.head_ref }}' | cut -f 4 -d/ | sed 's/-gt.*//' | sed 's/-lt.*//' | sed 's/-[0-9\.]*$//' )
+        if grep "$NEW_PACKAGE" *requirements.txt; then 
+          echo "new_package=$NEW_PACKAGE" >> $GITHUB_OUTPUT
+          # save a list of all installed packages (including pip, wheel; it's never empty)
+          pip freeze --all > orig
+          pip install "$(echo $NEW_PACKAGE | sed 's/[-_]/./g' | xargs grep *requirements.txt -h -e | head -n1)" -c env_constraints.txt -c broken_trans_deps.txt
+          pip show "$NEW_PACKAGE" | grep 'Version' | tee new_version.deps
+          # uninstall new packages but skip those from previous steps (pywinpty, terminado on windows x86)
+          # the 'orig' list is not empty (includes at least pip, wheel)
+          pip uninstall -y $(pip freeze -r orig | sed '1,/## /d')
+        else
+          echo "new_package=''" >> $GITHUB_OUTPUT
         fi
 
     - name: Python dependencies
       shell: bash
       run: |
-        python -m pip install --upgrade pip wheel
-        pip install -e .[${{ inputs.features }}]
+        pip install -e .[${{ inputs.features }}] -c env_constraints.txt -c broken_trans_deps.txt
 
     - name: "Cleanup old wheels"
       shell: bash
@@ -51,3 +100,13 @@ runs:
             pip cache remove -v $P || true
           fi
         done
+
+    - name: Check updated package
+      if: ${{ startsWith(github.head_ref, 'dependabot/pip') && matrix.pnl-version != 'base' && steps.new_package.outputs.new_package != '' }}
+      shell: bash
+      run: |
+        if [ $(pip list | grep -o ${{ steps.new_package.outputs.new_package }} | wc -l) != "0" ] ; then
+          pip show ${{ steps.new_package.outputs.new_package }} | grep 'Version' | tee installed_version.deps
+          cmp -s new_version.deps installed_version.deps || echo "::error::Package version restricted by dependencies: ${{ steps.new_package.outputs.new_package }}"
+          diff new_version.deps installed_version.deps
+        fi

.github/actions/on-branch/action.yml

@@ -25,4 +25,4 @@ runs:
         git describe --always --tags
         export ON_BRANCH=$(git branch -a --contains ${{ github.ref }} | grep -q '^  remotes/origin/${{ inputs.branch }}$' && echo "${{ inputs.branch }}" || echo "")
         echo "Found out: ${ON_BRANCH}"
-        echo "::set-output name=on_branch::$ON_BRANCH"
+        echo "on_branch=$ON_BRANCH" >> $GITHUB_OUTPUT

.github/workflows/codeql.yml

@@ -0,0 +1,58 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "master", "devel", "codeql" ]
+    paths-ignore:
+      - 'docs/**'
+  pull_request:
+    branches: [ "master", "devel" ]
+  schedule:
+    - cron: "14 21 * * 5"
+
+# run only the latest instance of this workflow job for the current branch/PR
+# cancel older runs
+# fall back to run id if not available (run id is unique -> no cancellations)
+concurrency:
+  group: ci-${{ github.ref || github.run_id }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ python ]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 100
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: ${{ matrix.language }}
+          queries: +security-and-quality
+
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v3
+
+      - name: Cache cleanup
+        shell: bash
+        run: |
+          $CODEQL_PYTHON -m pip cache info
+          $CODEQL_PYTHON -m pip cache purge
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3
+        with:
+          category: "/language:${{ matrix.language }}"

.github/workflows/compare-comment.yml

@@ -11,15 +11,17 @@ jobs:
     runs-on: ubuntu-latest
     if: github.event.workflow_run.event == 'pull_request' &&
         github.event.workflow_run.conclusion == 'success'
+    permissions:
+      actions: read
+      pull-requests: write
 
     steps:
-
     - name: 'Download docs artifacts'
       id: docs-artifacts
-      uses: actions/github-script@v4.0.2
+      uses: actions/github-script@v7
       with:
         script: |
-          var artifacts = await github.actions.listWorkflowRunArtifacts({
+          var artifacts = await github.rest.actions.listWorkflowRunArtifacts({
              owner: context.repo.owner,
              repo: context.repo.repo,
              run_id: ${{ github.event.workflow_run.id }},
@@ -32,16 +34,19 @@ jobs:
           core.setOutput('DOCS_GEN_ENV', docsSuffix);
 
           var docsArtifacts = artifacts.data.artifacts.filter((artifact) => {
-            return artifact.name.endsWith(docsSuffix) && artifact.name.startsWith('Documentation-')
+            return (
+              (artifact.name.endsWith(docsSuffix) && artifact.name.startsWith('Documentation-'))
+              || artifact.name == 'pr_number'
+            )
           });
 
-          // check that we got exactly 2 artifacts
-          console.assert(docsArtifacts.length == 2, docsSuffix, docsArtifacts, artifacts.data.artifacts);
+          // check that we got exactly 3 artifacts
+          console.assert(docsArtifacts.length == 3, docsSuffix, docsArtifacts, artifacts.data.artifacts);
 
           var fs = require('fs');
           for (artifact of docsArtifacts) {
             console.log('Downloading: ' + artifact.name);
-            var download = await github.actions.downloadArtifact({
+            var download = await github.rest.actions.downloadArtifact({
                owner: context.repo.owner,
                repo: context.repo.repo,
                artifact_id: artifact.id,
@@ -55,6 +60,7 @@ jobs:
       run: |
         unzip Documentation-base-*.zip -d docs-base/
         unzip Documentation-head-*.zip -d docs-head/
+        unzip pr_number.zip
 
     - name: Compare
       shell: bash
@@ -64,16 +70,17 @@ jobs:
         (diff -r docs-base docs-head && echo 'No differences!' || true) | tee ./result.diff
 
     - name: Post comment with docs diff
-      uses: actions/github-script@v4.0.2
+      uses: actions/github-script@v7
       with:
         script: |
           var fs = require('fs');
           var text = fs.readFileSync("./result.diff").slice(0,16384);
+          var issue_number = Number(fs.readFileSync('./pr_number.txt'));
 
-          console.log('Posting diff to PR: ${{ github.event.workflow_run.pull_requests[0].number }}')
+          console.log('Posting diff to PR: ' + issue_number);
 
-          github.issues.createComment({
-            issue_number: ${{ github.event.workflow_run.pull_requests[0].number }},
+          github.rest.issues.createComment({
+            issue_number: issue_number,
             owner: context.repo.owner,
             repo: context.repo.repo,
             body: 'This PR causes the following changes to the html docs (${{ steps.docs-artifacts.outputs.DOCS_GEN_ENV }}):\n```\n' + text + '\n...\n```\nSee CI logs for the full diff.'