From ad1d85eba7c001c823f69ebc2a87043b2bb552bc Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Wed, 6 Dec 2023 13:33:44 +0000 Subject: [PATCH 01/23] Add action --- .github/workflows/notebook-test.yml | 49 +++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/workflows/notebook-test.yml diff --git a/.github/workflows/notebook-test.yml b/.github/workflows/notebook-test.yml new file mode 100644 index 00000000000..24aff14f2c3 --- /dev/null +++ b/.github/workflows/notebook-test.yml @@ -0,0 +1,49 @@ +# This code is a Qiskit project. +# +# (C) Copyright IBM 2023. +# +# This code is licensed under the Apache License, Version 2.0. You may +# obtain a copy of this license in the LICENSE file in the root directory +# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0. +# +# Any modifications or derivative works of this code must retain this +# copyright notice, and modified files need to carry a notice indicating +# that they have been altered from the originals. + +name: Test notebooks +on: + pull_request: + paths: + - "docs/**/*.ipynb" + - "!docs/api/**/*" + workflow_dispatch: +jobs: + execute: + name: Execute notebooks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Install Qiskit IBM Runtime to save account + # To save account; this is re-installed during the "Run tox" step + run: pip install qiskit-ibm-runtime + + - name: Save IBM Quantum account + if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + shell: python + run: | + from qiskit_ibm_runtime import QiskitRuntimeService + QiskitRuntimeService.save_account( + channel="ibm_quantum", + token="${{ secrets.IBM_QUANTUM_TEST_TOKEN }}", + set_as_default=True + ) + + - name: Run tox + uses: lsst-sqre/run-tox@97818256d9fa3c72d0c12f31660718adb495a1cb + with: + python-version: "3.11" + tox-envs: "py311" From d6ebd9a999265108ba4e7688367c995b2d1dd0c4 Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Wed, 6 Dec 2023 14:01:34 +0000 Subject: [PATCH 02/23] Handle submitting jobs to IBM Quantum --- .github/workflows/notebook-test.yml | 1 + README.md | 5 ++ scripts/nb-tester/test-notebook.py | 105 ++++++++++++++++++++++------ 3 files changed, 88 insertions(+), 23 deletions(-) diff --git a/.github/workflows/notebook-test.yml b/.github/workflows/notebook-test.yml index 24aff14f2c3..6d3d95919f9 100644 --- a/.github/workflows/notebook-test.yml +++ b/.github/workflows/notebook-test.yml @@ -38,6 +38,7 @@ jobs: from qiskit_ibm_runtime import QiskitRuntimeService QiskitRuntimeService.save_account( channel="ibm_quantum", + instance="ibm-q/open/main", token="${{ secrets.IBM_QUANTUM_TEST_TOKEN }}", set_as_default=True ) diff --git a/README.md b/README.md index 83db65dd828..19fa6228b2b 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,11 @@ pipx install tox tox -- optional/paths/to/notebooks.ipynb --write ``` +> [!NOTE] +> If your notebook submits hardware jobs to IBM Quantum, you must add it to the +> ignore list in `scripts/nb-tester/test-notebooks.py`. Retrieving information +> is ok. + ## Check for broken links CI will check for broken links. You can also check locally: diff --git a/scripts/nb-tester/test-notebook.py b/scripts/nb-tester/test-notebook.py index 057dbfe4041..5292be3f477 100644 --- a/scripts/nb-tester/test-notebook.py +++ b/scripts/nb-tester/test-notebook.py @@ -14,33 +14,40 @@ from pathlib import Path import nbconvert import nbformat +import nbclient +from datetime import datetime +from qiskit_ibm_runtime import QiskitRuntimeService +import argparse -WRITE_FLAG = "--write" NOTEBOOKS_GLOB = "docs/**/*.ipynb" NOTEBOOKS_EXCLUDE = [ "docs/api/**", "**/.ipynb_checkpoints/**", # Following notebooks have code errors - "docs/build/circuit-construction.ipynb", - "docs/build/pulse.ipynb", - "docs/transpile/custom-transpiler-pass.ipynb", "docs/transpile/transpiler-stages.ipynb", - # Following notebook has some non-python dependency I can't figure out - "docs/build/circuit-visualization.ipynb", - # Following notebooks make requests so can't be tested yet - "docs/run/get-backend-information.ipynb", +] +NOTEBOOKS_THAT_SUBMIT_JOBS = [ "docs/start/hello-world.ipynb", ] -def execute_notebook(path: Path, write=False) -> bool: +class ExecuteOptions: + write: bool + submit_jobs: bool + + +def execute_notebook(path: Path, options: ExecuteOptions) -> bool: """ Wrapper function for `_execute_notebook` to print status """ print(f"▶️ {path}", end="", flush=True) + possible_exceptions = ( + nbconvert.preprocessors.CellExecutionError, + nbclient.exceptions.CellTimeoutError, + ) try: - _execute_notebook(path, write) - except nbconvert.preprocessors.CellExecutionError as err: + _execute_notebook(path, options) + except possible_exceptions as err: print("\r❌\n") print(err) return False @@ -48,49 +55,101 @@ def execute_notebook(path: Path, write=False) -> bool: return True -def _execute_notebook(filepath: Path, write=False) -> None: +def _execute_notebook(filepath: Path, options: ExecuteOptions) -> None: """ Use nbconvert to execute notebook """ nb = nbformat.read(filepath, as_version=4) processor = nbconvert.preprocessors.ExecutePreprocessor( - timeout=100, + timeout=-1 if options.submit_jobs else 100, kernel_name="python3", ) processor.preprocess(nb) - if not write: + if not options.write: return for cell in nb.cells: + # To avoid noisy diffs cell.metadata.pop("execution", None) nbformat.write(nb, filepath) -def find_notebooks() -> list[Path]: +def find_notebooks(submit_jobs=False) -> list[Path]: """ Get paths to all notebooks in NOTEBOOKS_GLOB that are not excluded by NOTEBOOKS_EXCLUDE """ all_notebooks = Path(".").rglob(NOTEBOOKS_GLOB) + excluded_notebooks = NOTEBOOKS_EXCLUDE + if not submit_jobs: + excluded_notebooks += NOTEBOOKS_THAT_SUBMIT_JOBS return [ path for path in all_notebooks - if not any(path.match(glob) for glob in NOTEBOOKS_EXCLUDE) + if not any(path.match(glob) for glob in excluded_notebooks) ] -if __name__ == "__main__": - args = sys.argv[1:] - write = WRITE_FLAG in args - if write: - args.remove(WRITE_FLAG) +def cancel_trailing_jobs(start_time: datetime) -> bool: + """ + Cancel any runtime jobs created after `start_time`. + Return True if non exist, False otherwise. + """ + service = QiskitRuntimeService() + jobs = [j for j in service.jobs(created_after=start_time) if not j.in_final_state()] + if not jobs: + return True + + print( + f"⚠️ Cancelling {len(jobs)} job(s) created after {start_time}.\n" + "Add any notebooks that submit jobs to NOTEBOOKS_EXCLUDE in " + "`scripts/nb-tester/test-notebook.py`." + ) + for job in jobs: + job.cancel() + return False - notebook_paths = args or find_notebooks() + +parser = argparse.ArgumentParser( + prog="Notebook executor", + description="For testing notebooks and updating their outputs", + epilog="For help, contact Frank Harkins", +) +parser.add_argument( + "filenames", + help=( + "Paths to notebooks. If not provided, the script will search for " + "notebooks in `docs/`. To exclude a notebook from this process, add it " + "to `NOTEBOOKS_EXCLUDE` in the script." + ), + nargs="*", +) +parser.add_argument( + "-w", + "--write", + action="store_true", + help="overwrite notebooks with execution outputs", +) +parser.add_argument( + "--submit-jobs", + action="store_true", + help=( + "run notebooks that submit jobs to IBM Quantum and wait indefinitely " + "for jobs to complete" + ), +) + +if __name__ == "__main__": + args = parser.parse_args() + notebook_paths = args.filenames or find_notebooks(args.submit_jobs) + start_time = datetime.now() print("Executing notebooks:") - results = [execute_notebook(path, write) for path in notebook_paths] + results = [execute_notebook(path, args) for path in notebook_paths] + print("Checking for trailing jobs...") + results.append(cancel_trailing_jobs(start_time)) if not all(results): sys.exit(1) sys.exit(0) From ffbbfeda90268362cac58b78d48ced7484138033 Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Mon, 11 Dec 2023 10:39:14 +0000 Subject: [PATCH 03/23] Ignore broken notebook --- scripts/nb-tester/test-notebook.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/nb-tester/test-notebook.py b/scripts/nb-tester/test-notebook.py index 207f40ccdcd..20507476d59 100644 --- a/scripts/nb-tester/test-notebook.py +++ b/scripts/nb-tester/test-notebook.py @@ -23,6 +23,8 @@ NOTEBOOKS_EXCLUDE = [ "docs/api/**", "**/.ipynb_checkpoints/**", + # Following notebooks are broken + "docs/transpile/transpiler-stages.ipynb", ] NOTEBOOKS_THAT_SUBMIT_JOBS = [ "docs/start/hello-world.ipynb", From cb56394dd9e839d7989872153e433876a9b3675e Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Mon, 11 Dec 2023 10:49:37 +0000 Subject: [PATCH 04/23] Ignore broken notebook --- scripts/nb-tester/test-notebook.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/nb-tester/test-notebook.py b/scripts/nb-tester/test-notebook.py index 20507476d59..00d7df784e6 100644 --- a/scripts/nb-tester/test-notebook.py +++ b/scripts/nb-tester/test-notebook.py @@ -25,6 +25,7 @@ "**/.ipynb_checkpoints/**", # Following notebooks are broken "docs/transpile/transpiler-stages.ipynb", + "docs/run/get-backend-information.ipynb", ] NOTEBOOKS_THAT_SUBMIT_JOBS = [ "docs/start/hello-world.ipynb", From 68772037e51c36e6ad287e441ee57a4b10da7786 Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Mon, 11 Dec 2023 10:39:02 +0000 Subject: [PATCH 05/23] Add LaTeX dependency --- .github/workflows/notebook-test.yml | 16 ++++++++++++++++ README.md | 7 +++++-- scripts/nb-tester/test-notebook.py | 8 +++++++- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/.github/workflows/notebook-test.yml b/.github/workflows/notebook-test.yml index 6d3d95919f9..bc3ee833e6c 100644 --- a/.github/workflows/notebook-test.yml +++ b/.github/workflows/notebook-test.yml @@ -27,6 +27,21 @@ jobs: with: python-version: "3.11" + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@af2816c65436325c50621100d67f6e853cd1b0f1 + + - name: Install LaTeX dependencies + run: | + # Add your notebook to this list if it needs latex to run + NEEDS_LATEX="docs/build/circuit-visualization.ipynb" + for FILE in $NEEDS_LATEX; do + if [[ "${{ steps.changed-files.outputs.all_changed_files }}" = *"$FILE"* ]]; then + sudo apt-get install texlive-pictures texlive-latex-extra poppler-utils + break + fi + done + - name: Install Qiskit IBM Runtime to save account # To save account; this is re-installed during the "Run tox" step run: pip install qiskit-ibm-runtime @@ -48,3 +63,4 @@ jobs: with: python-version: "3.11" tox-envs: "py311" + tox-posargs: ${{ steps.changed-files.outputs.all_changed_files }} diff --git a/README.md b/README.md index 74914bc52d9..955d189b68c 100644 --- a/README.md +++ b/README.md @@ -123,8 +123,11 @@ pipx install tox > [!NOTE] > If your notebook submits hardware jobs to IBM Quantum, you must add it to the -> ignore list in `scripts/nb-tester/test-notebooks.py`. Retrieving information -> is ok. +> ignore list in `scripts/nb-tester/test-notebooks.py`. This is not needed if +> you only retrieve information. +> +> If your notebook uses the latex circuit drawer (`qc.draw("latex")`), you must +> add it to the `NEEDS_LATEX` list in `.github/workflows/notebook-test.yml`. ## Check for broken links diff --git a/scripts/nb-tester/test-notebook.py b/scripts/nb-tester/test-notebook.py index 00d7df784e6..b6b79e41cc4 100644 --- a/scripts/nb-tester/test-notebook.py +++ b/scripts/nb-tester/test-notebook.py @@ -37,10 +37,14 @@ class ExecuteOptions: submit_jobs: bool -def execute_notebook(path: Path, options: ExecuteOptions) -> bool: +def execute_notebook(path: str, options: ExecuteOptions) -> bool: """ Wrapper function for `_execute_notebook` to print status """ + path = Path(path) + if path.suffix != ".ipynb": + print(f"⏭️ {path} is not a notebook; skipping") + return True print(f"▶️ {path}", end="", flush=True) possible_exceptions = ( nbconvert.preprocessors.CellExecutionError, @@ -51,6 +55,8 @@ def execute_notebook(path: Path, options: ExecuteOptions) -> bool: except possible_exceptions as err: print("\r❌\n") print(err) + with open("latex_error.log") as f: + print(f.read()) return False print("\r✅") return True From de85125c716a34ec41c8cbf53131ebaad3867612 Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Mon, 11 Dec 2023 13:10:26 +0000 Subject: [PATCH 06/23] Cache pip --- .github/workflows/notebook-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/notebook-test.yml b/.github/workflows/notebook-test.yml index bc3ee833e6c..2ab07b71d6d 100644 --- a/.github/workflows/notebook-test.yml +++ b/.github/workflows/notebook-test.yml @@ -26,6 +26,7 @@ jobs: - uses: actions/setup-python@v4 with: python-version: "3.11" + cache: 'pip' - name: Get changed files id: changed-files From 1d6872fc77eb27a36c0c0e320fc5066c9356989d Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Mon, 11 Dec 2023 13:12:14 +0000 Subject: [PATCH 07/23] Demo: Meaningless change to show CI working --- docs/build/circuit-library.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/build/circuit-library.ipynb b/docs/build/circuit-library.ipynb index 79f6c992e5d..70d753f1a78 100644 --- a/docs/build/circuit-library.ipynb +++ b/docs/build/circuit-library.ipynb @@ -26,7 +26,7 @@ "source": [ "## N-local circuits\n", "\n", - "These circuits alternate layers of single-qubit rotation gates with layers of multi-qubit entangling gates.\n", + "These circuits alternate layers of single-qubit rotation gates with layers of multi-qubit entangling gates. \n", "\n", "This family of circuits is popular in variational quantum algorithms because they can produce a wide range of quantum states. Variational algorithms adjust the gate parameters to find states that have certain properties (such as states that represent a good solution to an optimization problem). For this purpose, many circuits in the library are **parameterized**, which means you can define them without fixed values.\n", "\n", From a05817b15059048e1a67513b19a2ec72b7d97608 Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Mon, 11 Dec 2023 13:14:28 +0000 Subject: [PATCH 08/23] lint --- .github/workflows/notebook-test.yml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/notebook-test.yml b/.github/workflows/notebook-test.yml index 2ab07b71d6d..8e31acaba63 100644 --- a/.github/workflows/notebook-test.yml +++ b/.github/workflows/notebook-test.yml @@ -26,7 +26,7 @@ jobs: - uses: actions/setup-python@v4 with: python-version: "3.11" - cache: 'pip' + cache: "pip" - name: Get changed files id: changed-files diff --git a/README.md b/README.md index 955d189b68c..1181d23494d 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ pipx install tox > If your notebook submits hardware jobs to IBM Quantum, you must add it to the > ignore list in `scripts/nb-tester/test-notebooks.py`. This is not needed if > you only retrieve information. -> +> > If your notebook uses the latex circuit drawer (`qc.draw("latex")`), you must > add it to the `NEEDS_LATEX` list in `.github/workflows/notebook-test.yml`. From 0f386392e62cd21c7a42519efb8e690f9cbed867 Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Mon, 11 Dec 2023 13:16:31 +0000 Subject: [PATCH 09/23] Demo: Revert last change and change notebook that requires LaTeX --- docs/build/circuit-library.ipynb | 2 +- docs/build/circuit-visualization.ipynb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/build/circuit-library.ipynb b/docs/build/circuit-library.ipynb index 70d753f1a78..79f6c992e5d 100644 --- a/docs/build/circuit-library.ipynb +++ b/docs/build/circuit-library.ipynb @@ -26,7 +26,7 @@ "source": [ "## N-local circuits\n", "\n", - "These circuits alternate layers of single-qubit rotation gates with layers of multi-qubit entangling gates. \n", + "These circuits alternate layers of single-qubit rotation gates with layers of multi-qubit entangling gates.\n", "\n", "This family of circuits is popular in variational quantum algorithms because they can produce a wide range of quantum states. Variational algorithms adjust the gate parameters to find states that have certain properties (such as states that represent a good solution to an optimization problem). For this purpose, many circuits in the library are **parameterized**, which means you can define them without fixed values.\n", "\n", diff --git a/docs/build/circuit-visualization.ipynb b/docs/build/circuit-visualization.ipynb index b84b8f04b3f..eab3a881c54 100644 --- a/docs/build/circuit-visualization.ipynb +++ b/docs/build/circuit-visualization.ipynb @@ -11,7 +11,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "A visualization is useful while working with quantum circuits. Below find options in Qiskit for drawing circuits, plotting data from executed jobs, seeing the state of a quantum computer, and more." + "A visualization is useful while working with quantum circuits. Below find options in Qiskit for drawing circuits, plotting data from executed jobs, seeing the state of a quantum computer, and more. " ] }, { From 94cb0158dcf2042185c7b09be8b968272b585015 Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Mon, 11 Dec 2023 13:28:17 +0000 Subject: [PATCH 10/23] Revert meaningless change --- docs/build/circuit-visualization.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/build/circuit-visualization.ipynb b/docs/build/circuit-visualization.ipynb index eab3a881c54..b84b8f04b3f 100644 --- a/docs/build/circuit-visualization.ipynb +++ b/docs/build/circuit-visualization.ipynb @@ -11,7 +11,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "A visualization is useful while working with quantum circuits. Below find options in Qiskit for drawing circuits, plotting data from executed jobs, seeing the state of a quantum computer, and more. " + "A visualization is useful while working with quantum circuits. Below find options in Qiskit for drawing circuits, plotting data from executed jobs, seeing the state of a quantum computer, and more." ] }, { From 4511074af309fe6428586b6097fccff548f68075 Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Tue, 12 Dec 2023 17:28:17 +0000 Subject: [PATCH 11/23] Apply suggestions from code review Co-authored-by: Eric Arellano <14852634+Eric-Arellano@users.noreply.github.com> --- .github/workflows/notebook-test.yml | 3 ++- scripts/nb-tester/test-notebook.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/notebook-test.yml b/.github/workflows/notebook-test.yml index 8e31acaba63..ed7f5952c0e 100644 --- a/.github/workflows/notebook-test.yml +++ b/.github/workflows/notebook-test.yml @@ -44,7 +44,8 @@ jobs: done - name: Install Qiskit IBM Runtime to save account - # To save account; this is re-installed during the "Run tox" step + # This is to save our account in the next step. Note that the + # package will be re-installed during the "Run tox" step. run: pip install qiskit-ibm-runtime - name: Save IBM Quantum account diff --git a/scripts/nb-tester/test-notebook.py b/scripts/nb-tester/test-notebook.py index b6b79e41cc4..5b809ed3fb0 100644 --- a/scripts/nb-tester/test-notebook.py +++ b/scripts/nb-tester/test-notebook.py @@ -32,6 +32,7 @@ ] +@dataclass(frozen=True) class ExecuteOptions: write: bool submit_jobs: bool @@ -79,7 +80,7 @@ def _execute_notebook(filepath: Path, options: ExecuteOptions) -> None: return for cell in nb.cells: - # To avoid noisy diffs + # Remove execution metadata to avoid noisy diffs. cell.metadata.pop("execution", None) nbformat.write(nb, filepath) From e14aafc76a23e1ac16679670cd5503dedf4643f8 Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Wed, 13 Dec 2023 18:57:31 +0000 Subject: [PATCH 12/23] Import order --- scripts/nb-tester/test-notebook.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/nb-tester/test-notebook.py b/scripts/nb-tester/test-notebook.py index 5b809ed3fb0..1a52bb80b3c 100644 --- a/scripts/nb-tester/test-notebook.py +++ b/scripts/nb-tester/test-notebook.py @@ -10,14 +10,16 @@ # notice, and modified files need to carry a notice indicating that they have # been altered from the originals. +import argparse import sys +from dataclasses import dataclass +from datetime import datetime from pathlib import Path + +import nbclient import nbconvert import nbformat -import nbclient -from datetime import datetime from qiskit_ibm_runtime import QiskitRuntimeService -import argparse NOTEBOOKS_GLOB = "docs/**/*.ipynb" NOTEBOOKS_EXCLUDE = [ From 65b861d3844747f20a9277c2759ade8f30444f24 Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Wed, 13 Dec 2023 19:00:08 +0000 Subject: [PATCH 13/23] Create argparser in function and filter paths in main --- scripts/nb-tester/test-notebook.py | 74 ++++++++++++++++-------------- 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/scripts/nb-tester/test-notebook.py b/scripts/nb-tester/test-notebook.py index 1a52bb80b3c..92c8a6ebe17 100644 --- a/scripts/nb-tester/test-notebook.py +++ b/scripts/nb-tester/test-notebook.py @@ -44,10 +44,6 @@ def execute_notebook(path: str, options: ExecuteOptions) -> bool: """ Wrapper function for `_execute_notebook` to print status """ - path = Path(path) - if path.suffix != ".ipynb": - print(f"⏭️ {path} is not a notebook; skipping") - return True print(f"▶️ {path}", end="", flush=True) possible_exceptions = ( nbconvert.preprocessors.CellExecutionError, @@ -123,41 +119,49 @@ def cancel_trailing_jobs(start_time: datetime) -> bool: return False -parser = argparse.ArgumentParser( - prog="Notebook executor", - description="For testing notebooks and updating their outputs", - epilog="For help, contact Frank Harkins", -) -parser.add_argument( - "filenames", - help=( - "Paths to notebooks. If not provided, the script will search for " - "notebooks in `docs/`. To exclude a notebook from this process, add it " - "to `NOTEBOOKS_EXCLUDE` in the script." - ), - nargs="*", -) -parser.add_argument( - "-w", - "--write", - action="store_true", - help="overwrite notebooks with execution outputs", -) -parser.add_argument( - "--submit-jobs", - action="store_true", - help=( - "run notebooks that submit jobs to IBM Quantum and wait indefinitely " - "for jobs to complete" - ), -) +def create_argument_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="Notebook executor", + description="For testing notebooks and updating their outputs", + epilog="For help, contact Frank Harkins", + ) + parser.add_argument( + "filenames", + help=( + "Paths to notebooks. If not provided, the script will search for " + "notebooks in `docs/`. To exclude a notebook from this process, add it " + "to `NOTEBOOKS_EXCLUDE` in the script." + ), + nargs="*", + ) + parser.add_argument( + "-w", + "--write", + action="store_true", + help="overwrite notebooks with execution outputs", + ) + parser.add_argument( + "--submit-jobs", + action="store_true", + help=( + "run notebooks that submit jobs to IBM Quantum and wait indefinitely " + "for jobs to complete" + ), + ) + return parser + if __name__ == "__main__": - args = parser.parse_args() - notebook_paths = args.filenames or find_notebooks(args.submit_jobs) + args = create_argument_parser().parse_args() + + paths = map(Path, args.filenames or find_notebooks(args.submit_jobs)) + + # Execute notebooks start_time = datetime.now() print("Executing notebooks:") - results = [execute_notebook(path, args) for path in notebook_paths] + results = [ + execute_notebook(path, args) for path in paths if path.suffix == ".ipynb" + ] print("Checking for trailing jobs...") results.append(cancel_trailing_jobs(start_time)) if not all(results): From cef4b3514233bede2041540c37208d611227f11f Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Wed, 13 Dec 2023 19:00:33 +0000 Subject: [PATCH 14/23] Improve documentation --- scripts/nb-tester/test-notebook.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/nb-tester/test-notebook.py b/scripts/nb-tester/test-notebook.py index 92c8a6ebe17..da57f388940 100644 --- a/scripts/nb-tester/test-notebook.py +++ b/scripts/nb-tester/test-notebook.py @@ -68,6 +68,7 @@ def _execute_notebook(filepath: Path, options: ExecuteOptions) -> None: nb = nbformat.read(filepath, as_version=4) processor = nbconvert.preprocessors.ExecutePreprocessor( + # If submitting jobs, we want to wait forever (-1 means no timeout) timeout=-1 if options.submit_jobs else 100, kernel_name="python3", ) @@ -103,6 +104,13 @@ def cancel_trailing_jobs(start_time: datetime) -> bool: """ Cancel any runtime jobs created after `start_time`. Return True if non exist, False otherwise. + + Notebooks should not submit jobs during a normal test run. If they do, the + cell will time out and this function will cancel the job to avoid wasting + device time. + + If a notebook submits a job but does not wait for the result, this check + will also catch it and cancel the job. """ service = QiskitRuntimeService() jobs = [j for j in service.jobs(created_after=start_time) if not j.in_final_state()] From 64b8cbf191588b28a2deb23273b9285cfa70d892 Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Wed, 13 Dec 2023 19:00:42 +0000 Subject: [PATCH 15/23] Remove print statement --- scripts/nb-tester/test-notebook.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/nb-tester/test-notebook.py b/scripts/nb-tester/test-notebook.py index da57f388940..9296f279988 100644 --- a/scripts/nb-tester/test-notebook.py +++ b/scripts/nb-tester/test-notebook.py @@ -54,8 +54,6 @@ def execute_notebook(path: str, options: ExecuteOptions) -> bool: except possible_exceptions as err: print("\r❌\n") print(err) - with open("latex_error.log") as f: - print(f.read()) return False print("\r✅") return True From edca8d41975af6b89a6f0eb452dcddb75c7c6741 Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Wed, 13 Dec 2023 19:12:41 +0000 Subject: [PATCH 16/23] Remove bash --- .github/workflows/notebook-test.yml | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/.github/workflows/notebook-test.yml b/.github/workflows/notebook-test.yml index ed7f5952c0e..ca0a412909a 100644 --- a/.github/workflows/notebook-test.yml +++ b/.github/workflows/notebook-test.yml @@ -28,22 +28,25 @@ jobs: python-version: "3.11" cache: "pip" - - name: Get changed files - id: changed-files + - name: Get all changed files + id: all-changed-files uses: tj-actions/changed-files@af2816c65436325c50621100d67f6e853cd1b0f1 + - name: Check for notebooks that require LaTeX + id: latex-changed-files + uses: tj-actions/changed-files@af2816c65436325c50621100d67f6e853cd1b0f1 + with: + # Add your notebook to this list if it needs latex to run + files: | + docs/build/circuit-visualization.ipynb + - name: Install LaTeX dependencies + if: steps.latex-changed-files.outputs.any_changed == 'true' run: | - # Add your notebook to this list if it needs latex to run - NEEDS_LATEX="docs/build/circuit-visualization.ipynb" - for FILE in $NEEDS_LATEX; do - if [[ "${{ steps.changed-files.outputs.all_changed_files }}" = *"$FILE"* ]]; then - sudo apt-get install texlive-pictures texlive-latex-extra poppler-utils - break - fi - done + sudo apt-get update + sudo apt-get install texlive-pictures texlive-latex-extra poppler-utils - - name: Install Qiskit IBM Runtime to save account + - name: Install Qiskit IBM Runtime (to save account) # This is to save our account in the next step. Note that the # package will be re-installed during the "Run tox" step. run: pip install qiskit-ibm-runtime @@ -65,4 +68,4 @@ jobs: with: python-version: "3.11" tox-envs: "py311" - tox-posargs: ${{ steps.changed-files.outputs.all_changed_files }} + tox-posargs: ${{ steps.all-changed-files.outputs.all_changed_files }} From 358eb2fd4624c765cb1ad7a72e47c3f316881014 Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Wed, 13 Dec 2023 19:55:31 +0000 Subject: [PATCH 17/23] Test CI (no latex) --- docs/build/circuit-library.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/build/circuit-library.ipynb b/docs/build/circuit-library.ipynb index 79f6c992e5d..6ad2a7e757b 100644 --- a/docs/build/circuit-library.ipynb +++ b/docs/build/circuit-library.ipynb @@ -5,7 +5,7 @@ "id": "aea62c4c-1e42-4472-9f1d-e52783e81fc6", "metadata": {}, "source": [ - "# Circuit library" + "# Circuit library " ] }, { From 1ff5fed7ff212d2cd23b50b6741c666b9b765e50 Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Wed, 13 Dec 2023 19:56:25 +0000 Subject: [PATCH 18/23] Test CI (with latex) --- docs/build/circuit-library.ipynb | 2 +- docs/build/circuit-visualization.ipynb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/build/circuit-library.ipynb b/docs/build/circuit-library.ipynb index 6ad2a7e757b..79f6c992e5d 100644 --- a/docs/build/circuit-library.ipynb +++ b/docs/build/circuit-library.ipynb @@ -5,7 +5,7 @@ "id": "aea62c4c-1e42-4472-9f1d-e52783e81fc6", "metadata": {}, "source": [ - "# Circuit library " + "# Circuit library" ] }, { diff --git a/docs/build/circuit-visualization.ipynb b/docs/build/circuit-visualization.ipynb index b84b8f04b3f..58fa1b8840f 100644 --- a/docs/build/circuit-visualization.ipynb +++ b/docs/build/circuit-visualization.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Visualize circuits" + "# Visualize circuits " ] }, { From caec53a8c28326bad19c6eb783843b185c872d56 Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Wed, 13 Dec 2023 19:57:53 +0000 Subject: [PATCH 19/23] Revert CI-testing changes --- docs/build/circuit-visualization.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/build/circuit-visualization.ipynb b/docs/build/circuit-visualization.ipynb index 58fa1b8840f..b84b8f04b3f 100644 --- a/docs/build/circuit-visualization.ipynb +++ b/docs/build/circuit-visualization.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Visualize circuits " + "# Visualize circuits" ] }, { From 9b58ed6a2a0e8f25f1fab73ce7e87a578cf7b585 Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Wed, 13 Dec 2023 20:04:15 +0000 Subject: [PATCH 20/23] Remove run-tox action --- .github/workflows/notebook-test.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/notebook-test.yml b/.github/workflows/notebook-test.yml index ca0a412909a..f5f008b9402 100644 --- a/.github/workflows/notebook-test.yml +++ b/.github/workflows/notebook-test.yml @@ -46,10 +46,10 @@ jobs: sudo apt-get update sudo apt-get install texlive-pictures texlive-latex-extra poppler-utils - - name: Install Qiskit IBM Runtime (to save account) + - name: Install Python packages # This is to save our account in the next step. Note that the # package will be re-installed during the "Run tox" step. - run: pip install qiskit-ibm-runtime + run: pip install qiskit-ibm-runtime tox - name: Save IBM Quantum account if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} @@ -63,9 +63,11 @@ jobs: set_as_default=True ) - - name: Run tox - uses: lsst-sqre/run-tox@97818256d9fa3c72d0c12f31660718adb495a1cb + - name: Cache tox environment + uses: actions/cache@v3 with: - python-version: "3.11" - tox-envs: "py311" - tox-posargs: ${{ steps.all-changed-files.outputs.all_changed_files }} + path: ".tox" + key: ${{ hashFiles('scripts/nb-tester/requirements.txt') }} + + - name: Run tox + run: tox -- ${{ steps.all-changed-files.outputs.all_changed_files }} From 607d428e20f8c927870a2279e78dddad44c3316d Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Wed, 13 Dec 2023 22:03:14 +0000 Subject: [PATCH 21/23] Apply suggestions from code review Co-authored-by: Eric Arellano <14852634+Eric-Arellano@users.noreply.github.com> --- scripts/nb-tester/test-notebook.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/nb-tester/test-notebook.py b/scripts/nb-tester/test-notebook.py index 9296f279988..ed46baf5904 100644 --- a/scripts/nb-tester/test-notebook.py +++ b/scripts/nb-tester/test-notebook.py @@ -40,7 +40,7 @@ class ExecuteOptions: submit_jobs: bool -def execute_notebook(path: str, options: ExecuteOptions) -> bool: +def execute_notebook(path: Path, options: ExecuteOptions) -> bool: """ Wrapper function for `_execute_notebook` to print status """ @@ -82,7 +82,7 @@ def _execute_notebook(filepath: Path, options: ExecuteOptions) -> None: nbformat.write(nb, filepath) -def find_notebooks(submit_jobs=False) -> list[Path]: +def find_notebooks(*, submit_jobs: bool = False) -> list[Path]: """ Get paths to all notebooks in NOTEBOOKS_GLOB that are not excluded by NOTEBOOKS_EXCLUDE @@ -129,7 +129,6 @@ def create_argument_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="Notebook executor", description="For testing notebooks and updating their outputs", - epilog="For help, contact Frank Harkins", ) parser.add_argument( "filenames", @@ -144,14 +143,15 @@ def create_argument_parser() -> argparse.ArgumentParser: "-w", "--write", action="store_true", - help="overwrite notebooks with execution outputs", + help="Overwrite notebooks with the results of this script's execution.", ) parser.add_argument( "--submit-jobs", action="store_true", help=( - "run notebooks that submit jobs to IBM Quantum and wait indefinitely " - "for jobs to complete" + "Run notebooks that submit jobs to IBM Quantum and wait indefinitely " + "for jobs to complete. Warning: this has a real cost because it uses " + "quantum resources! Only use this argument occasionally and intentionally." ), ) return parser From dd2aee7a0b436bc4b263c3c844cfd3dba7427e33 Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Wed, 13 Dec 2023 23:03:51 +0000 Subject: [PATCH 22/23] Update docs (forgot to push this earlier) --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1181d23494d..06a274cefbf 100644 --- a/README.md +++ b/README.md @@ -127,7 +127,8 @@ pipx install tox > you only retrieve information. > > If your notebook uses the latex circuit drawer (`qc.draw("latex")`), you must -> add it to the `NEEDS_LATEX` list in `.github/workflows/notebook-test.yml`. +> add it to the "Check for notebooks that require LaTeX" step in +> `.github/workflows/notebook-test.yml`. ## Check for broken links From 8a45b5c04f82c0251464f3292f08474cd061c698 Mon Sep 17 00:00:00 2001 From: Frank Harkins Date: Thu, 14 Dec 2023 14:57:50 +0000 Subject: [PATCH 23/23] Update scripts/nb-tester/test-notebook.py Co-authored-by: Arnau Casau <47946624+arnaucasau@users.noreply.github.com> --- scripts/nb-tester/test-notebook.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/nb-tester/test-notebook.py b/scripts/nb-tester/test-notebook.py index ed46baf5904..fec43aaf81c 100644 --- a/scripts/nb-tester/test-notebook.py +++ b/scripts/nb-tester/test-notebook.py @@ -160,7 +160,7 @@ def create_argument_parser() -> argparse.ArgumentParser: if __name__ == "__main__": args = create_argument_parser().parse_args() - paths = map(Path, args.filenames or find_notebooks(args.submit_jobs)) + paths = map(Path, args.filenames or find_notebooks(submit_jobs=args.submit_jobs)) # Execute notebooks start_time = datetime.now()