Test Fh/notebook job cron

.github/workflows/notebook-test.yml

@@ -23,10 +23,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.11"
-          cache: "pip"
 
       - name: Get all changed files
         id: all-changed-files
@@ -44,35 +40,11 @@ jobs:
             docs/build/circuit-visualization.ipynb
             docs/transpile/transpiler-stages.ipynb
 
-      - name: Install Linux dependencies
-        if: steps.linux-changed-files.outputs.any_changed == 'true'
-        run: |
-          sudo apt-get update
-          sudo apt-get install texlive-pictures texlive-latex-extra poppler-utils graphviz
-
-      - name: Install Python packages
-        # This is to save our account in the next step. Note that the
-        # package will be re-installed during the "Run tox" step.
-        run: pip install qiskit-ibm-runtime tox
-
-      - name: Save IBM Quantum account
-        if: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
-        shell: python
-        run: |
-          # This saves the result for qiskit-ibm-provider too
-          from qiskit_ibm_runtime import QiskitRuntimeService
-          QiskitRuntimeService.save_account(
-              channel="ibm_quantum",
-              instance="ibm-q/open/main",
-              token="${{ secrets.IBM_QUANTUM_TEST_TOKEN }}",
-              set_as_default=True
-          )
-
-      - name: Cache tox environment
-        uses: actions/cache@v3
+      - name: Setup environment
+        uses: ./.github/workflows/reusable/setup-notebook-testing
         with:
-          path: ".tox"
-          key: ${{ hashFiles('scripts/nb-tester/requirements.txt') }}
+          install-linux-deps: ${{ steps.linux-changed-files.outputs.any_changed }}
+          ibm-quantum-token: ${{ secrets.IBM_QUANTUM_TEST_TOKEN }}
 
       - name: Check lint
         shell: python

.github/workflows/notebook-weekly-test.yml

@@ -0,0 +1,54 @@
+# This code is a Qiskit project.
+#
+# (C) Copyright IBM 2024.
+#
+# This code is licensed under the Apache License, Version 2.0. You may
+# obtain a copy of this license in the LICENSE file in the root directory
+# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Any modifications or derivative works of this code must retain this
+# copyright notice, and modified files need to carry a notice indicating
+# that they have been altered from the originals.
+
+name: Test notebooks that submit jobs
+on:
+  schedule:
+    - cron: "0 0 1,15 * *" # At 00:00 on day-of-month 1 and 15.
+  workflow_dispatch:
+jobs:
+  execute:
+    name: Execute notebooks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup environment
+        uses: ./.github/workflows/reusable/setup-notebook-testing
+        with:
+          ibm-quantum-token: ${{ secrets.IBM_QUANTUM_TEST_TOKEN }}
+
+      - name: Execute notebooks
+        run: tox -- --only-submit-jobs
+
+  make_issue:
+    name: Make issue on failure
+    needs: [execute]
+    if: ${{ failure() && github.event_name == 'schedule' }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Post issue
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const message = `Today's scheduled test of notebooks that submit jobs failed.
+            Please [check the logs](https://github.com/Qiskit/documentation/actions/runs/${{ github.run_id }}/).
+            > [!NOTE]
+            > This issue was created by a GitHub action.
+            `
+            github.rest.issues.create({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              title: "Weekly notebook test failed",
+              body: message,
+              assignees: ["frankharkins", "javabster", "kevinsung",]
+            })

.github/workflows/reusable/setup-notebook-testing/action.yml

@@ -0,0 +1,56 @@
+# This code is a Qiskit project.
+#
+# (C) Copyright IBM 2024.
+#
+# This code is licensed under the Apache License, Version 2.0. You may
+# obtain a copy of this license in the LICENSE file in the root directory
+# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Any modifications or derivative works of this code must retain this
+# copyright notice, and modified files need to carry a notice indicating
+# that they have been altered from the originals.
+
+name: Notebook testing setup
+description: Set up Python, tox, and IBM Quantum account
+inputs:
+  install-linux-deps:
+    description: Whether to install extra dependencies
+    default: 'false'
+  ibm-quantum-token:
+    required: true
+runs:
+  using: "composite"
+  steps:
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+          cache: "pip"
+
+      - name: Install Python packages
+        shell: bash
+        run: pip install tox qiskit-ibm-runtime
+
+      - name: Cache tox environment
+        uses: actions/cache@v3
+        with:
+          path: ".tox"
+          key: ${{ hashFiles('scripts/nb-tester/requirements.txt') }}
+
+      - name: Save IBM Quantum account
+        shell: python
+        run: |
+          from qiskit_ibm_runtime import QiskitRuntimeService
+          QiskitRuntimeService.save_account(
+              channel="ibm_quantum",
+              instance="ibm-q/open/main",
+              token="${{ inputs.ibm-quantum-token }}",
+              set_as_default=True
+          )
+
+      - name: Install Linux dependencies
+        if: ${{ inputs.install-linux-deps == 'true' }}
+        shell: bash
+        run: |
+          sudo apt-get update
+          sudo apt-get install texlive-pictures texlive-latex-extra poppler-utils graphviz
+

scripts/nb-tester/test-notebook.py

@@ -37,31 +37,39 @@
 ]
 
 
-def filter_paths(paths: list[Path], submit_jobs: bool) -> Iterator[Path]:
+def matches(path: Path, glob_list: list[str]) -> bool:
+    return any(path.match(glob) for glob in glob_list)
+
+def filter_paths(paths: list[Path], submit_jobs: bool, only_submit_jobs: bool) -> Iterator[Path]:
     """
     Filter out any paths we don't want to run, printing messages.
     """
+    if only_submit_jobs:
+        submit_jobs = True
     for path in paths:
         if path.suffix != ".ipynb":
             print(f"ℹ️ Skipping {path}; file is not `.ipynb` format.")
             continue
 
-        if any(path.match(glob) for glob in NOTEBOOKS_EXCLUDE):
+        if matches(path, NOTEBOOKS_EXCLUDE):
             this_file = Path(__file__).resolve()
             print(
                 f"ℹ️ Skipping {path}; to run it, edit `NOTEBOOKS_EXCLUDE` in {this_file}."
             )
             continue
 
-        if (
-            not submit_jobs
-            and any(path.match(glob) for glob in NOTEBOOKS_THAT_SUBMIT_JOBS)
-        ):
+        if not submit_jobs and matches(path, NOTEBOOKS_THAT_SUBMIT_JOBS):
             print(
                 f"ℹ️ Skipping {path} as it submits jobs; use the --submit-jobs flag to run it."
             )
             continue
 
+        if only_submit_jobs and not matches(path, NOTEBOOKS_THAT_SUBMIT_JOBS):
+            print(
+                f"ℹ️ Skipping {path} as it does not submit jobs and the --only-submit-jobs flag is set."
+            )
+            continue
+
         yield path
 
 
@@ -165,19 +173,17 @@ def _execute_notebook(filepath: Path, options: ExecuteOptions) -> nbformat.Noteb
     return nb
 
 
-def find_notebooks(*, submit_jobs: bool = False) -> list[Path]:
+def find_notebooks() -> list[Path]:
     """
     Get paths to all notebooks in NOTEBOOKS_GLOB that are not excluded by
     NOTEBOOKS_EXCLUDE
     """
     all_notebooks = Path(".").rglob(NOTEBOOKS_GLOB)
     excluded_notebooks = NOTEBOOKS_EXCLUDE
-    if not submit_jobs:
-        excluded_notebooks += NOTEBOOKS_THAT_SUBMIT_JOBS
     return [
         path
         for path in all_notebooks
-        if not any(path.match(glob) for glob in excluded_notebooks)
+        if not matches(path, excluded_notebooks)
     ]
 
 
@@ -238,14 +244,18 @@ def create_argument_parser() -> argparse.ArgumentParser:
             "quantum resources! Only use this argument occasionally and intentionally." 
         ),
     )
+    parser.add_argument(
+        "--only-submit-jobs",
+        action="store_true",
+        help="Same as --submit-jobs, but also skips notebooks that do not submit jobs to IBM Quantum",
+    )
     return parser
 
 
 if __name__ == "__main__":
     args = create_argument_parser().parse_args()
-
-    paths = map(Path, args.filenames or find_notebooks(submit_jobs=args.submit_jobs))
-    filtered_paths = filter_paths(paths, submit_jobs=args.submit_jobs)
+    paths = map(Path, args.filenames or find_notebooks())
+    filtered_paths = filter_paths(paths, submit_jobs=args.submit_jobs, only_submit_jobs=args.only_submit_jobs)
 
     # Execute notebooks
     start_time = datetime.now()