Mock hardware when testing notebooks

.github/workflows/notebook-test-cron.yml

@@ -30,7 +30,7 @@ jobs:
 
       - name: Execute notebooks
         timeout-minutes: 350
-        run: tox -- --write --only-submit-jobs
+        run: tox -- --write --only-unmockable
 
       - name: Detect changed notebooks
         id: changed-notebooks

README.md

@@ -116,9 +116,11 @@ You also need to install a few system dependencies: TeX, Poppler, and graphviz.
   ```
 
 > [!NOTE]
-> If your notebook submits hardware jobs to IBM Quantum, you must add it to the
-> ignore list in `scripts/nb-tester/test-notebooks.py`. This is not needed if
-> you only retrieve information.
+> When testing notebooks, we avoid sending jobs to IBM Quantum by patching
+> `least_busy` to return a fake backend. Try to make sure your notebook works
+> with this patch. If your notebook can't be tested with this patch, you must
+> add it to the list of `NOTEBOOKS_THAT_CANT_BE_MOCKED` in
+> `scripts/nb-tester/test-notebooks.py`.
 >
 > If your notebook uses the latex circuit drawer (`qc.draw("latex")`), you must
 > add it to the "Check for notebooks that require LaTeX" step in

scripts/nb-tester/test-notebook.py

@@ -30,18 +30,21 @@
     "docs/api/**",
     "**/.ipynb_checkpoints/**",
 ]
-NOTEBOOKS_THAT_SUBMIT_JOBS = [
-    "docs/start/hello-world.ipynb",
-    "docs/analyze/saving-and-retrieving.ipynb",
-    "tutorials/build-repitition-codes/notebook.ipynb",
-    "tutorials/chsh-inequality/notebook.ipynb",
-    "tutorials/grovers-algorithm/notebook.ipynb",
-    "tutorials/quantum-approximate-optimization-algorithm/notebook.ipynb",
-    "tutorials/repeat-until-success/notebook.ipynb",
-    "tutorials/submitting-transpiled-circuits/notebook.ipynb",
-    "tutorials/variational-quantum-eigensolver/notebook.ipynb",
+NOTEBOOKS_THAT_CANT_BE_MOCKED = [
+    # Notebooks that don't work with the fake backend
+    # (e.g. due to large circuits)
 ]
 
+# If not submitting jobs, we mock the real backend by prepending this to each notebook
+MOCKING_CODE = """\
+from qiskit_ibm_runtime import QiskitRuntimeService
+from qiskit_ibm_runtime.fake_provider import FakeManilaV2
+
+def patched_least_busy(self, *args, **kwarg):
+  return FakeManilaV2
+
+QiskitRuntimeService.least_busy = patched_least_busy
+"""
 
 def matches(path: Path, glob_list: list[str]) -> bool:
     return any(path.match(glob) for glob in glob_list)
@@ -50,7 +53,7 @@ def filter_paths(paths: list[Path], args: argparse.Namespace) -> Iterator[Path]:
     """
     Filter out any paths we don't want to run, printing messages.
     """
-    submit_jobs = args.submit_jobs or args.only_submit_jobs
+    submit_jobs = args.submit_jobs or args.only_unmockable
     for path in paths:
         if path.suffix != ".ipynb":
             print(f"ℹ️ Skipping {path}; file is not `.ipynb` format.")
@@ -63,15 +66,15 @@ def filter_paths(paths: list[Path], args: argparse.Namespace) -> Iterator[Path]:
             )
             continue
 
-        if not submit_jobs and matches(path, NOTEBOOKS_THAT_SUBMIT_JOBS):
+        if not submit_jobs and matches(path, NOTEBOOKS_THAT_CANT_BE_MOCKED):
             print(
-                f"ℹ️ Skipping {path} as it submits jobs; use the --submit-jobs flag to run it."
+                f"ℹ️ Skipping {path} as it doesn't work with mock hardware; use the --submit-jobs flag to run it."
             )
             continue
 
-        if args.only_submit_jobs and not matches(path, NOTEBOOKS_THAT_SUBMIT_JOBS):
+        if args.only_unmockable and not matches(path, NOTEBOOKS_THAT_CANT_BE_MOCKED):
             print(
-                f"ℹ️ Skipping {path} as it does not submit jobs and the --only-submit-jobs flag is set."
+                f"ℹ️ Skipping {path} as it can be tested with a mock backend and the --only-unmockable flag is set."
             )
             continue
 
@@ -148,13 +151,31 @@ def execute_notebook(path: Path, args: argparse.Namespace) -> bool:
     print("\r✅")
     return True
 
+def remove_mocking_cell(nb: nbformat.NotebookNode) -> None:
+    """
+    Remove the MOCKING_CODE cell inserted at the start of the notebook and any
+    side effects it caused.
+    """
+    # Remove MOCKING_CODE cell
+    nb.cells.pop(0)
+
+    # Reset execution counts (offset by the MOCKING_CODE cell)
+    for cell in nb.cells:
+        if not hasattr(cell, "execution_count"):
+            continue
+        cell.execution_count -= 1
+        for output in cell.outputs:
+            output.execution_count -= 1
 
 def _execute_notebook(filepath: Path, args: argparse.Namespace) -> nbformat.NotebookNode:
     """
     Use nbconvert to execute notebook
     """
-    submit_jobs = args.submit_jobs or args.only_submit_jobs
+    submit_jobs = args.submit_jobs or args.only_unmockable
     nb = nbformat.read(filepath, as_version=4)
+    if not submit_jobs:
+        # Add code to patch least_busy
+        nb.cells.insert(0, nbformat.v4.new_code_cell(source=MOCKING_CODE))
 
     processor = nbconvert.preprocessors.ExecutePreprocessor(
         # If submitting jobs, we want to wait forever (-1 means no timeout)
@@ -168,6 +189,9 @@ def _execute_notebook(filepath: Path, args: argparse.Namespace) -> nbformat.Note
     if not args.write:
         return nb
 
+    if not submit_jobs:
+        remove_mocking_cell(nb)
+
     for cell in nb.cells:
         # Remove execution metadata to avoid noisy diffs.
         cell.metadata.pop("execution", None)
@@ -254,9 +278,9 @@ def create_argument_parser() -> argparse.ArgumentParser:
         ),
     )
     parser.add_argument(
-        "--only-submit-jobs",
+        "--only-unmockable",
         action="store_true",
-        help="Same as --submit-jobs, but also skips notebooks that do not submit jobs to IBM Quantum",
+        help="Same as --submit-jobs, but only runs notebooks that can't be tested with the fake backend.",
     )
     return parser
 
@@ -269,6 +293,8 @@ def create_argument_parser() -> argparse.ArgumentParser:
     # Execute notebooks
     start_time = datetime.now()
     print("Executing notebooks:")
+    if not args.submit_jobs and not args.only_unmockable:
+        print("ℹ️  Note: Using patched qiskit-ibm-runtime; least_busy will return FakeManilaV2")
     results = [execute_notebook(path, args) for path in filtered_paths]
     print("Checking for trailing jobs...")
     results.append(cancel_trailing_jobs(start_time))