Merge 6ac9245 into c29b025

.github/workflows/notebook-test.yml

@@ -0,0 +1,54 @@
+# This code is a Qiskit project.
+#
+# (C) Copyright IBM 2023.
+#
+# This code is licensed under the Apache License, Version 2.0. You may
+# obtain a copy of this license in the LICENSE file in the root directory
+# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Any modifications or derivative works of this code must retain this
+# copyright notice, and modified files need to carry a notice indicating
+# that they have been altered from the originals.
+
+name: Test notebooks
+on:
+  pull_request:
+    paths:
+      - "docs/**/*.ipynb"
+      - "!docs/api/**/*"
+  workflow_dispatch:
+jobs:
+  execute:
+    name: Execute notebooks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: Install LaTeX dependencies
+        run: |
+          sudo apt-get install texlive-pictures poppler-utils
+
+      - name: Install Qiskit IBM Runtime to save account
+        # To save account; this is re-installed during the "Run tox" step
+        run: pip install qiskit-ibm-runtime
+
+      - name: Save IBM Quantum account
+        if: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
+        shell: python
+        run: |
+          from qiskit_ibm_runtime import QiskitRuntimeService
+          QiskitRuntimeService.save_account(
+              channel="ibm_quantum",
+              instance="ibm-q/open/main",
+              token="${{ secrets.IBM_QUANTUM_TEST_TOKEN }}",
+              set_as_default=True
+          )
+
+      - name: Run tox
+        uses: lsst-sqre/run-tox@97818256d9fa3c72d0c12f31660718adb495a1cb
+        with:
+          python-version: "3.11"
+          tox-envs: "py311"

README.md

@@ -121,6 +121,11 @@ pipx install tox
   tox -- optional/paths/to/notebooks.ipynb --write
   ```
 
+> [!NOTE]
+> If your notebook submits hardware jobs to IBM Quantum, you must add it to the
+> ignore list in `scripts/nb-tester/test-notebooks.py`. Retrieving information
+> is ok.
+
 ## Check for broken links
 
 CI will check for broken links. You can also check locally:

docs/build/circuit-visualization.ipynb

@@ -11,7 +11,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "A visualization is useful while working with quantum circuits. Below find options in Qiskit for drawing circuits, plotting data from executed jobs, seeing the state of a quantum computer, and more."
+    "A visualization is useful while working with quantum circuits. Below find options in Qiskit for drawing circuits, plotting data from executed jobs, seeing the state of a quantum computer, and more. "
    ]
   },
   {

scripts/nb-tester/test-notebook.py

@@ -14,78 +14,145 @@
 from pathlib import Path
 import nbconvert
 import nbformat
+import nbclient
+from datetime import datetime
+from qiskit_ibm_runtime import QiskitRuntimeService
+import argparse
 
-WRITE_FLAG = "--write"
 NOTEBOOKS_GLOB = "docs/**/*.ipynb"
 NOTEBOOKS_EXCLUDE = [
     "docs/api/**",
     "**/.ipynb_checkpoints/**",
-    # Following notebooks have code errors
+    # Following notebooks are broken
     "docs/transpile/transpiler-stages.ipynb",
-    # Following notebooks make requests so can't be tested yet
     "docs/run/get-backend-information.ipynb",
+]
+NOTEBOOKS_THAT_SUBMIT_JOBS = [
     "docs/start/hello-world.ipynb",
 ]
 
 
-def execute_notebook(path: Path, write=False) -> bool:
+class ExecuteOptions:
+    write: bool
+    submit_jobs: bool
+
+
+def execute_notebook(path: Path, options: ExecuteOptions) -> bool:
     """
     Wrapper function for `_execute_notebook` to print status
     """
     print(f"▶️  {path}", end="", flush=True)
+    possible_exceptions = (
+        nbconvert.preprocessors.CellExecutionError,
+        nbclient.exceptions.CellTimeoutError,
+    )
     try:
-        _execute_notebook(path, write)
-    except nbconvert.preprocessors.CellExecutionError as err:
+        _execute_notebook(path, options)
+    except possible_exceptions as err:
         print("\r❌\n")
         print(err)
+        with open("latex_error.log") as f:
+            print(f.read())
         return False
     print("\r✅")
     return True
 
 
-def _execute_notebook(filepath: Path, write=False) -> None:
+def _execute_notebook(filepath: Path, options: ExecuteOptions) -> None:
     """
     Use nbconvert to execute notebook
     """
     nb = nbformat.read(filepath, as_version=4)
 
     processor = nbconvert.preprocessors.ExecutePreprocessor(
-        timeout=100,
+        timeout=-1 if options.submit_jobs else 100,
         kernel_name="python3",
     )
 
     processor.preprocess(nb)
 
-    if not write:
+    if not options.write:
         return
 
     for cell in nb.cells:
+        # To avoid noisy diffs
         cell.metadata.pop("execution", None)
     nbformat.write(nb, filepath)
 
 
-def find_notebooks() -> list[Path]:
+def find_notebooks(submit_jobs=False) -> list[Path]:
     """
     Get paths to all notebooks in NOTEBOOKS_GLOB that are not excluded by
     NOTEBOOKS_EXCLUDE
     """
     all_notebooks = Path(".").rglob(NOTEBOOKS_GLOB)
+    excluded_notebooks = NOTEBOOKS_EXCLUDE
+    if not submit_jobs:
+        excluded_notebooks += NOTEBOOKS_THAT_SUBMIT_JOBS
     return [
         path
         for path in all_notebooks
-        if not any(path.match(glob) for glob in NOTEBOOKS_EXCLUDE)
+        if not any(path.match(glob) for glob in excluded_notebooks)
     ]
 
 
-if __name__ == "__main__":
-    args = sys.argv[1:]
-    write = WRITE_FLAG in args
-    if write:
-        args.remove(WRITE_FLAG)
+def cancel_trailing_jobs(start_time: datetime) -> bool:
+    """
+    Cancel any runtime jobs created after `start_time`.
+    Return True if non exist, False otherwise.
+    """
+    service = QiskitRuntimeService()
+    jobs = [j for j in service.jobs(created_after=start_time) if not j.in_final_state()]
+    if not jobs:
+        return True
+
+    print(
+        f"⚠️ Cancelling {len(jobs)} job(s) created after {start_time}.\n"
+        "Add any notebooks that submit jobs to NOTEBOOKS_EXCLUDE in "
+        "`scripts/nb-tester/test-notebook.py`."
+    )
+    for job in jobs:
+        job.cancel()
+    return False
 
-    notebook_paths = args or find_notebooks()
+
+parser = argparse.ArgumentParser(
+    prog="Notebook executor",
+    description="For testing notebooks and updating their outputs",
+    epilog="For help, contact Frank Harkins",
+)
+parser.add_argument(
+    "filenames",
+    help=(
+        "Paths to notebooks. If not provided, the script will search for "
+        "notebooks in `docs/`. To exclude a notebook from this process, add it "
+        "to `NOTEBOOKS_EXCLUDE` in the script."
+    ),
+    nargs="*",
+)
+parser.add_argument(
+    "-w",
+    "--write",
+    action="store_true",
+    help="overwrite notebooks with execution outputs",
+)
+parser.add_argument(
+    "--submit-jobs",
+    action="store_true",
+    help=(
+        "run notebooks that submit jobs to IBM Quantum and wait indefinitely "
+        "for jobs to complete"
+    ),
+)
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+    notebook_paths = args.filenames or find_notebooks(args.submit_jobs)
+    start_time = datetime.now()
     print("Executing notebooks:")
-    results = [execute_notebook(path, write) for path in notebook_paths]
+    results = [execute_notebook(path, args) for path in notebook_paths]
+    print("Checking for trailing jobs...")
+    results.append(cancel_trailing_jobs(start_time))
     if not all(results):
         sys.exit(1)
     sys.exit(0)