Pull notebook classification out into separate test (#1397)

We want to run the notebook classification check on every PR. This PR
pulls that test out into a separate pytest suite and runs it as part of
the standard CI job. I'd like to also add some more tests to this suite
as the `nb-tester` has grown in complexity and there were some issues
with it recently.

.github/actions/set-up-notebook-testing/action.yml

@@ -17,7 +17,7 @@ inputs:
     description: Whether to install extra dependencies
     default: "false"
   ibm-quantum-token:
-    required: true
+    default: ""
   instance:
     # We usually only want the same access as an open user, but occasionally we
     # need higher priority to test notebooks that submit jobs in reasonable
@@ -42,6 +42,7 @@ runs:
         key: ${{ hashFiles('scripts/nb-tester/requirements.txt') }}
 
     - name: Save IBM Quantum account
+      if: ${{ inputs.ibm-quantum-token != '' }}
       shell: python
       run: |
         from qiskit_ibm_runtime import QiskitRuntimeService

.github/workflows/main.yml

@@ -59,3 +59,8 @@ jobs:
         run: |
           echo "${{ steps.changed-docs-files.outputs.all_changed_files }}" > changed.txt
           npm run check:pages-render -- --from-file changed.txt
+
+      - name: Setup Python environment
+        uses: ./.github/actions/set-up-notebook-testing
+      - name: nb-tester tests
+        run: tox -e nb-tester

scripts/nb-tester/pyproject.toml

@@ -9,6 +9,7 @@ description = "Tool for the Qiskit docs team to test their notebooks"
 requires-python = ">=3.9"
 license = "Apache-2.0"
 dependencies = [
+  "qiskit-ibm-runtime",
   "nbclient~=0.10.0",
   "nbformat~=5.10.4",
   "ipykernel~=6.29.2",

scripts/nb-tester/qiskit_docs_notebook_tester/__init__.py

@@ -124,19 +124,6 @@ def notebooks_to_execute(self) -> Iterator[Path]:
 
             yield path
 
-    def check_all_notebooks_are_classified(self) -> None:
-        unclassified = [
-            path for path in Path(".").glob("[!.]*/**/*.ipynb")
-            if not matches(path, self.all_notebooks)
-        ]
-        if unclassified == []:
-            return
-        raise SystemExit(
-            f"\nThe following notebooks are not classified in {self.args.config_path}:\n  "
-            + "\n  ".join(map(str, unclassified))
-            + "\nAdd them to the appropriate group so we know how to test them.\n"
-        )
-
     def should_patch(self, path: Path) -> bool:
         if self.args.submit_jobs:
             return False
@@ -348,7 +335,6 @@ def get_args() -> argparse.Namespace:
 
 async def _main() -> None:
     config = Config.from_args(get_args())
-    config.check_all_notebooks_are_classified()
     paths = config.notebooks_to_execute()
 
     # Execute notebooks

scripts/nb-tester/test/test_notebook_classification.py

@@ -0,0 +1,29 @@
+from types import SimpleNamespace
+from pathlib import Path
+
+from qiskit_docs_notebook_tester import Config, matches
+
+
+def test_all_notebooks_are_classified():
+    args = SimpleNamespace(
+        filenames=None,
+        write=False,
+        submit_jobs=True,
+        config_path="scripts/nb-tester/notebooks.toml"
+    )
+
+    config = Config.from_args(args)
+
+
+    unclassified = [
+        path for path in Path(".").glob("[!.]*/**/*.ipynb")
+        if not matches(path, config.all_notebooks)
+    ]
+
+    message = (
+        f"\nThe following notebooks are not classified in {args.config_path}:\n  "
+        + "\n  ".join(map(str, unclassified))
+        + "\nAdd them to the appropriate group so we know how to test them.\n"
+    )
+
+    assert unclassified == [], message

tox.ini

@@ -15,3 +15,9 @@ deps = squeaky==0.7.0
 commands = 
   lint: squeaky --check --no-advice {posargs:docs tutorials}
   fix:  squeaky {posargs:docs tutorials}
+
+[testenv:nb-tester]
+deps =
+  -e scripts/nb-tester
+  pytest
+commands = pytest scripts/nb-tester