CI: try parallelizing notebook execution

volkamerlab · Nov 13, 2023 · fd65ca4 · fd65ca4
1 parent ca89d98
commit fd65ca4
Showing 1 changed file with 136 additions and 100 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -8,126 +8,162 @@ on:
     branches:
       - "master"
       - "dev"
+  workflow_dispatch:
   schedule:
     # every two weeks on Monday at 3 am
     - cron: "0 3/336 * * 1"
 
 jobs:
-  test:
-    name: Pytest
-    runs-on: ${{ matrix.cfg.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        cfg:
-          - os: ubuntu-latest
-            python-version: "3.8"
-          - os: ubuntu-latest
-            python-version: "3.9"
-          - os: macos-latest
-            python-version: "3.8"
-          - os: windows-latest
-            python-version: "3.8"
-
-    env:
-      PYVER: ${{ matrix.cfg.python-version }}
-      PACKAGE: teachopencadd
+  setup:
+    runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout Repository
+        uses: actions/checkout@v3
 
+      - name: Set up Conda
       - uses: conda-incubator/setup-miniconda@v2
         with:
-          python-version: ${{ matrix.cfg.python-version }}
           miniforge-variant: Mambaforge
           miniforge-version: latest
           activate-environment: teachopencadd
           environment-file: devtools/test_env.yml
 
-      - name: Additional info about the build
-        shell: bash
-        run: |
-          uname -a
-          df -h
-          ulimit -a
-
-      - name: Install package
-        shell: bash -l {0}
-        run: |
-          pip install . --no-deps -vv
+  test:
+    needs: setup
+    runs-on: ubuntu-latest
 
-      - name: Environment Information
-        shell: bash -l {0}
-        run: |
-          conda info --all
-          conda list
+    strategy:
+      matrix:
+        notebook:
+          - teachopencadd/talktorials/T001_query_chembl/talktorial.ipynb
+          - teachopencadd/talktorials/T002_compound_adme/talktorial.ipynb
+          - teachopencadd/talktorials/T003_compound_unwanted_substructures/talktorial.ipynb
 
-      - name: Test CLI
-        shell: bash -l {0}
-        run: |
-          teachopencadd -h
-          pytest -v --cov=${PACKAGE} --cov-report=xml --color=yes ${PACKAGE}/tests/
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v3
 
       - name: Run tests
         shell: bash -l {0}
         run: |
-          PYTEST_ARGS="--nbval-lax --current-env --dist loadscope --numprocesses 2"
-
-          PYTEST_IGNORE_T001="--ignore=teachopencadd/talktorials/T001_query_chembl/talktorial.ipynb"
-          PYTEST_IGNORE_T018="--ignore=teachopencadd/talktorials/T018_automated_cadd_pipeline/talktorial.ipynb"
-          PYTEST_IGNORE_T019="--ignore=teachopencadd/talktorials/T019_md_simulation/talktorial.ipynb"
-          PYTEST_IGNORE_T020="--ignore=teachopencadd/talktorials/T020_md_analysis/talktorial.ipynb"
-          PYTEST_IGNORE_T035="--ignore=teachopencadd/talktorials/T035_graph_neural_networks/talktorial.ipynb"
-          PYTEST_IGNORE_T036="--ignore=teachopencadd/talktorials/T036_e3_equivariant_gnn/talktorial.ipynb"
-          PYTEST_IGNORE_T038="--ignore=teachopencadd/talktorials/T038_protein_ligand_interaction_prediction/talktorial.ipynb"
-
-          IGNORE="$PYTEST_IGNORE_T001"
-
-          if [ "$RUNNER_OS" == "Windows" ]; then
-            # Ignore T019 under Windows, see https://github.com/volkamerlab/teachopencadd/issues/313
-            IGNORE="$IGNORE $PYTEST_IGNORE_T019"
-            IGNORE="$IGNORE $PYTEST_IGNORE_T035"
-            IGNORE="$IGNORE $PYTEST_IGNORE_T036"
-            IGNORE="$IGNORE $PYTEST_IGNORE_T038"
-          fi
-          
-          if [ "$RUNNER_OS" == "Linux" ]; then
-            IGNORE="$IGNORE $PYTEST_IGNORE_T018" # see 385
-          fi
-          
-          if [ "$RUNNER_OS" == "macOS" ]; then
-            IGNORE="$IGNORE $PYTEST_IGNORE_T018" # see 385
-            IGNORE="$IGNORE $PYTEST_IGNORE_T019"
-            IGNORE="$IGNORE $PYTEST_IGNORE_T035" # may cause kernel dying in pytest
-            IGNORE="$IGNORE $PYTEST_IGNORE_T036" # may cause kernel dying in pytest
-          fi
-
           pytest $PYTEST_ARGS teachopencadd/talktorials/ $IGNORE
 
-  format:
-    name: Black
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v3
-      - name: Install format dependencies
-        run: python -m pip install black-nb shyaml
-      - name: Run black-nb
-        run: |
-          black-nb -l 99 --check teachopencadd/talktorials/T*/talktorial.ipynb
-
-  readmes:
-    name: READMEs
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v3
-      - name: Install nbformat
-        run: python -m pip install nbformat shyaml
-      - name: Check READMEs are up-to-date
-        run: |
-          for path in teachopencadd/talktorials/T*/talktorial.ipynb; do
-            python devtools/regenerate_readmes.py --output README.md $path
-          done
-          # If the script above introduced changes (it shouldn't!), git diff will have output and an error code
-          git diff --exit-code
+  #test:
+    #name: Pytest
+    #runs-on: ${{ matrix.cfg.os }}
+    #strategy:
+      #fail-fast: false
+      #matrix:
+        #cfg:
+          #- os: ubuntu-latest
+            #python-version: "3.8"
+          #- os: ubuntu-latest
+            #python-version: "3.9"
+          #- os: macos-latest
+            #python-version: "3.8"
+          #- os: windows-latest
+            #python-version: "3.8"
+
+    #env:
+      #PYVER: ${{ matrix.cfg.python-version }}
+      #PACKAGE: teachopencadd
+
+    #steps:
+      #- uses: actions/checkout@v3
+
+      #- uses: conda-incubator/setup-miniconda@v2
+        #with:
+          #python-version: ${{ matrix.cfg.python-version }}
+          #miniforge-variant: Mambaforge
+          #miniforge-version: latest
+          #activate-environment: teachopencadd
+          #environment-file: devtools/test_env.yml
+
+      #- name: Additional info about the build
+        #shell: bash
+        #run: |
+          #uname -a
+          #df -h
+          #ulimit -a
+
+      #- name: Install package
+        #shell: bash -l {0}
+        #run: |
+          #pip install . --no-deps -vv
+
+      #- name: Environment Information
+        #shell: bash -l {0}
+        #run: |
+          #conda info --all
+          #conda list
+
+      #- name: Test CLI
+        #shell: bash -l {0}
+        #run: |
+          #teachopencadd -h
+          #pytest -v --cov=${PACKAGE} --cov-report=xml --color=yes ${PACKAGE}/tests/
+
+      #- name: Run tests
+        #shell: bash -l {0}
+        #run: |
+          #PYTEST_ARGS="--nbval-lax --current-env --dist loadscope --numprocesses 2"
+
+          #PYTEST_IGNORE_T001="--ignore=teachopencadd/talktorials/T001_query_chembl/talktorial.ipynb"
+          #PYTEST_IGNORE_T018="--ignore=teachopencadd/talktorials/T018_automated_cadd_pipeline/talktorial.ipynb"
+          #PYTEST_IGNORE_T019="--ignore=teachopencadd/talktorials/T019_md_simulation/talktorial.ipynb"
+          #PYTEST_IGNORE_T020="--ignore=teachopencadd/talktorials/T020_md_analysis/talktorial.ipynb"
+          #PYTEST_IGNORE_T035="--ignore=teachopencadd/talktorials/T035_graph_neural_networks/talktorial.ipynb"
+          #PYTEST_IGNORE_T036="--ignore=teachopencadd/talktorials/T036_e3_equivariant_gnn/talktorial.ipynb"
+          #PYTEST_IGNORE_T038="--ignore=teachopencadd/talktorials/T038_protein_ligand_interaction_prediction/talktorial.ipynb"
+
+          #IGNORE="$PYTEST_IGNORE_T001"
+
+          #if [ "$RUNNER_OS" == "Windows" ]; then
+            ## Ignore T019 under Windows, see https://github.com/volkamerlab/teachopencadd/issues/313
+            #IGNORE="$IGNORE $PYTEST_IGNORE_T019"
+            #IGNORE="$IGNORE $PYTEST_IGNORE_T035"
+            #IGNORE="$IGNORE $PYTEST_IGNORE_T036"
+            #IGNORE="$IGNORE $PYTEST_IGNORE_T038"
+          #fi
+
+          #if [ "$RUNNER_OS" == "Linux" ]; then
+            #IGNORE="$IGNORE $PYTEST_IGNORE_T018" # see 385
+          #fi
+
+          #if [ "$RUNNER_OS" == "macOS" ]; then
+            #IGNORE="$IGNORE $PYTEST_IGNORE_T018" # see 385
+            #IGNORE="$IGNORE $PYTEST_IGNORE_T019"
+            #IGNORE="$IGNORE $PYTEST_IGNORE_T035" # may cause kernel dying in pytest
+            #IGNORE="$IGNORE $PYTEST_IGNORE_T036" # may cause kernel dying in pytest
+          #fi
+
+          #pytest $PYTEST_ARGS teachopencadd/talktorials/ $IGNORE
+
+  #format:
+    #name: Black
+    #runs-on: ubuntu-latest
+    #steps:
+      #- uses: actions/checkout@v3
+      #- uses: actions/setup-python@v3
+      #- name: Install format dependencies
+        #run: python -m pip install black-nb shyaml
+      #- name: Run black-nb
+        #run: |
+          #black-nb -l 99 --check teachopencadd/talktorials/T*/talktorial.ipynb
+
+  #readmes:
+    #name: READMEs
+    #runs-on: ubuntu-latest
+    #steps:
+      #- uses: actions/checkout@v3
+      #- uses: actions/setup-python@v3
+      #- name: Install nbformat
+        #run: python -m pip install nbformat shyaml
+      #- name: Check READMEs are up-to-date
+        #run: |
+          #for path in teachopencadd/talktorials/T*/talktorial.ipynb; do
+            #python devtools/regenerate_readmes.py --output README.md $path
+          #done
+          ## If the script above introduced changes (it shouldn't!), git diff will have output and an error code
+          #git diff --exit-code