Make sure all notebooks are classified in notebooks.toml (#1359)

Closes #1319.

* The script no longer uses `all-notebooks` as a glob pattern to search,
instead it just iterates through the list of filepaths
* Added a check to detect unclassified notebooks

---------

Co-authored-by: Eric Arellano <14852634+Eric-Arellano@users.noreply.github.com>

scripts/nb-tester/notebooks.toml

@@ -1,7 +1,36 @@
-# Used to find all notebooks in the repo
-all_notebooks = "[!.]*/**/*.ipynb"
+# For notebooks to be tested "normally" (no mocking)
+notebooks_normal_test = [
+    "docs/build/circuit-construction.ipynb",
+    "docs/build/circuit-library.ipynb",
+    "docs/build/circuit-visualization.ipynb",
+    "docs/build/classical-feedforward-and-control-flow.ipynb",
+    "docs/build/operators-overview.ipynb",
+    "docs/build/pulse.ipynb",
+    "docs/build/save-circuits.ipynb",
+    "docs/run/dynamic-circuits-considerations.ipynb",
+    "docs/run/get-backend-information.ipynb",
+    "docs/run/save-jobs.ipynb",
+    "docs/run/visualize-results.ipynb",
+    "docs/transpile/common-parameters.ipynb",
+    "docs/transpile/create-a-transpiler-plugin.ipynb",
+    "docs/transpile/custom-backend.ipynb",
+    "docs/transpile/custom-transpiler-pass.ipynb",
+    "docs/transpile/defaults-and-configuration-options.ipynb",
+    "docs/transpile/dynamical-decoupling-pass-manager.ipynb",
+    "docs/transpile/representing_quantum_computers.ipynb",
+    "docs/transpile/set-optimization.ipynb",
+    "docs/transpile/transpile-with-pass-managers.ipynb",
+    "docs/transpile/transpiler-plugins.ipynb",
+    "docs/transpile/transpiler-stages.ipynb",
+    "docs/verify/building_noise_models.ipynb",
+    "docs/verify/local-testing-mode.ipynb",
+    "docs/verify/plot-quantum-states.ipynb",
+    "docs/verify/simulate-with-qiskit-aer.ipynb",
+    "docs/verify/stabilizer-circuit-simulation.ipynb",
+    "tutorials/explore-composer/explore-composer.ipynb",
+]
 
-# Always exclude notebooks matching the following patterns
+# Don't test the following notebooks (this section can include glob patterns)
 notebooks_exclude = [
     "scripts/ibm-quantum-learning-uploader/test/template.ipynb",
     "**/.ipynb_checkpoints/**",
@@ -20,6 +49,8 @@ notebooks_that_submit_jobs = [
 ]
 
 # The following notebooks submit jobs that are too big to mock with a simulator
+# A job is "too big" if a cell can't run in under 5 mins, or we run out of
+# memory on a reasonable device.
 notebooks_no_mock = [
     "docs/start/hello-world.ipynb",
 ]

scripts/nb-tester/qiskit_docs_notebook_tester/__init__.py

@@ -49,7 +49,7 @@ def patched_least_busy(self, *args, **kwarg):
 @dataclass
 class Config:
     args: argparse.Namespace
-    all_notebooks: str
+    notebooks_normal_test: list[str]
     notebooks_exclude: list[str]
     notebooks_that_submit_jobs: list[str]
     notebooks_no_mock: list[str]
@@ -58,6 +58,21 @@ class Config:
     def all_job_submitting_notebooks(self) -> list[str]:
        return [*self.notebooks_that_submit_jobs, *self.notebooks_no_mock]
 
+    @property
+    def all_notebooks_to_test(self) -> list[str]:
+        return [
+            *self.notebooks_normal_test,
+            *self.notebooks_that_submit_jobs,
+            *self.notebooks_no_mock,
+        ]
+
+    @property
+    def all_notebooks(self) -> list[str]:
+        return [
+            *self.all_notebooks_to_test,
+            *self.notebooks_exclude,
+        ]
+
     @classmethod
     def from_args(cls, args: argparse.Namespace) -> Config:
         """
@@ -76,7 +91,7 @@ def notebooks_to_execute(self) -> Iterator[Path]:
         """
         Yield notebooks to be executed, printing messages for any skipped files.
         """
-        paths = map(Path, self.args.filenames or Path(".").glob(self.all_notebooks))
+        paths = map(Path, self.args.filenames or self.all_notebooks_to_test)
         for path in paths:
             if path.suffix != ".ipynb":
                 print(f"ℹ️ Skipping {path}; file is not `.ipynb` format.")
@@ -102,6 +117,19 @@ def notebooks_to_execute(self) -> Iterator[Path]:
 
             yield path
 
+    def check_all_notebooks_are_classified(self) -> None:
+        unclassified = [
+            path for path in Path(".").glob("[!.]*/**/*.ipynb")
+            if not matches(path, self.all_notebooks)
+        ]
+        if unclassified == []:
+            return
+        raise SystemExit(
+            f"\nThe following notebooks are not classified in {self.args.config_path}:\n  "
+            + "\n  ".join(map(str, unclassified))
+            + "\nAdd them to the appropriate group so we know how to test them.\n"
+        )
+
     def should_patch(self, path: Path) -> bool:
         if self.args.submit_jobs:
             return False
@@ -306,6 +334,7 @@ def get_args() -> argparse.Namespace:
 
 async def _main() -> None:
     config = Config.from_args(get_args())
+    config.check_all_notebooks_are_classified()
     paths = config.notebooks_to_execute()
 
     # Execute notebooks