diff --git a/.github/workflows/notebook-test.yml b/.github/workflows/notebook-test.yml index f2fa3da0a78..7f9f59ff4f7 100644 --- a/.github/workflows/notebook-test.yml +++ b/.github/workflows/notebook-test.yml @@ -23,10 +23,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 - with: - python-version: "3.11" - cache: "pip" - name: Get all changed files id: all-changed-files @@ -44,35 +40,11 @@ jobs: docs/build/circuit-visualization.ipynb docs/transpile/transpiler-stages.ipynb - - name: Install Linux dependencies - if: steps.linux-changed-files.outputs.any_changed == 'true' - run: | - sudo apt-get update - sudo apt-get install texlive-pictures texlive-latex-extra poppler-utils graphviz - - - name: Install Python packages - # This is to save our account in the next step. Note that the - # package will be re-installed during the "Run tox" step. - run: pip install qiskit-ibm-runtime tox - - - name: Save IBM Quantum account - if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} - shell: python - run: | - # This saves the result for qiskit-ibm-provider too - from qiskit_ibm_runtime import QiskitRuntimeService - QiskitRuntimeService.save_account( - channel="ibm_quantum", - instance="ibm-q/open/main", - token="${{ secrets.IBM_QUANTUM_TEST_TOKEN }}", - set_as_default=True - ) - - - name: Cache tox environment - uses: actions/cache@v3 + - name: Setup environment + uses: ./.github/workflows/reusable/setup-notebook-testing with: - path: ".tox" - key: ${{ hashFiles('scripts/nb-tester/requirements.txt') }} + install-linux-deps: ${{ steps.linux-changed-files.outputs.any_changed }} + ibm-quantum-token: ${{ secrets.IBM_QUANTUM_TEST_TOKEN }} - name: Check lint shell: python diff --git a/.github/workflows/notebook-weekly-test.yml b/.github/workflows/notebook-weekly-test.yml new file mode 100644 index 00000000000..1b58711ff0e --- /dev/null +++ b/.github/workflows/notebook-weekly-test.yml @@ -0,0 +1,54 @@ +# This code is a Qiskit project. +# +# (C) Copyright IBM 2024. +# +# This code is licensed under the Apache License, Version 2.0. You may +# obtain a copy of this license in the LICENSE file in the root directory +# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0. +# +# Any modifications or derivative works of this code must retain this +# copyright notice, and modified files need to carry a notice indicating +# that they have been altered from the originals. + +name: Test notebooks that submit jobs +on: + schedule: + - cron: "0 0 1,15 * *" # At 00:00 on day-of-month 1 and 15. + workflow_dispatch: +jobs: + execute: + name: Execute notebooks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup environment + uses: ./.github/workflows/reusable/setup-notebook-testing + with: + ibm-quantum-token: ${{ secrets.IBM_QUANTUM_TEST_TOKEN }} + + - name: Execute notebooks + run: tox -- --only-submit-jobs + + make_issue: + name: Make issue on failure + needs: [execute] + if: ${{ failure() && github.event_name == 'schedule' }} + runs-on: ubuntu-latest + steps: + - name: Post issue + uses: actions/github-script@v7 + with: + script: | + const message = `Today's scheduled test of notebooks that submit jobs failed. + Please [check the logs](https://github.com/Qiskit/documentation/actions/runs/${{ github.run_id }}/). + > [!NOTE] + > This issue was created by a GitHub action. + ` + github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: "Weekly notebook test failed", + body: message, + assignees: ["frankharkins", "javabster", "kevinsung",] + }) diff --git a/.github/workflows/reusable/setup-notebook-testing/action.yml b/.github/workflows/reusable/setup-notebook-testing/action.yml new file mode 100644 index 00000000000..67199a1e68f --- /dev/null +++ b/.github/workflows/reusable/setup-notebook-testing/action.yml @@ -0,0 +1,56 @@ +# This code is a Qiskit project. +# +# (C) Copyright IBM 2024. +# +# This code is licensed under the Apache License, Version 2.0. You may +# obtain a copy of this license in the LICENSE file in the root directory +# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0. +# +# Any modifications or derivative works of this code must retain this +# copyright notice, and modified files need to carry a notice indicating +# that they have been altered from the originals. + +name: Notebook testing setup +description: Set up Python, tox, and IBM Quantum account +inputs: + install-linux-deps: + description: Whether to install extra dependencies + default: 'false' + ibm-quantum-token: + required: true +runs: + using: "composite" + steps: + - uses: actions/setup-python@v4 + with: + python-version: "3.11" + cache: "pip" + + - name: Install Python packages + shell: bash + run: pip install tox qiskit-ibm-runtime + + - name: Cache tox environment + uses: actions/cache@v3 + with: + path: ".tox" + key: ${{ hashFiles('scripts/nb-tester/requirements.txt') }} + + - name: Save IBM Quantum account + shell: python + run: | + from qiskit_ibm_runtime import QiskitRuntimeService + QiskitRuntimeService.save_account( + channel="ibm_quantum", + instance="ibm-q/open/main", + token="${{ inputs.ibm-quantum-token }}", + set_as_default=True + ) + + - name: Install Linux dependencies + if: ${{ inputs.install-linux-deps == 'true' }} + shell: bash + run: | + sudo apt-get update + sudo apt-get install texlive-pictures texlive-latex-extra poppler-utils graphviz + diff --git a/scripts/nb-tester/test-notebook.py b/scripts/nb-tester/test-notebook.py index 4501c507651..a1a54754390 100644 --- a/scripts/nb-tester/test-notebook.py +++ b/scripts/nb-tester/test-notebook.py @@ -37,31 +37,39 @@ ] -def filter_paths(paths: list[Path], submit_jobs: bool) -> Iterator[Path]: +def matches(path: Path, glob_list: list[str]) -> bool: + return any(path.match(glob) for glob in glob_list) + +def filter_paths(paths: list[Path], submit_jobs: bool, only_submit_jobs: bool) -> Iterator[Path]: """ Filter out any paths we don't want to run, printing messages. """ + if only_submit_jobs: + submit_jobs = True for path in paths: if path.suffix != ".ipynb": print(f"ℹ️ Skipping {path}; file is not `.ipynb` format.") continue - if any(path.match(glob) for glob in NOTEBOOKS_EXCLUDE): + if matches(path, NOTEBOOKS_EXCLUDE): this_file = Path(__file__).resolve() print( f"ℹ️ Skipping {path}; to run it, edit `NOTEBOOKS_EXCLUDE` in {this_file}." ) continue - if ( - not submit_jobs - and any(path.match(glob) for glob in NOTEBOOKS_THAT_SUBMIT_JOBS) - ): + if not submit_jobs and matches(path, NOTEBOOKS_THAT_SUBMIT_JOBS): print( f"ℹ️ Skipping {path} as it submits jobs; use the --submit-jobs flag to run it." ) continue + if only_submit_jobs and not matches(path, NOTEBOOKS_THAT_SUBMIT_JOBS): + print( + f"ℹ️ Skipping {path} as it does not submit jobs and the --only-submit-jobs flag is set." + ) + continue + yield path @@ -165,19 +173,17 @@ def _execute_notebook(filepath: Path, options: ExecuteOptions) -> nbformat.Noteb return nb -def find_notebooks(*, submit_jobs: bool = False) -> list[Path]: +def find_notebooks() -> list[Path]: """ Get paths to all notebooks in NOTEBOOKS_GLOB that are not excluded by NOTEBOOKS_EXCLUDE """ all_notebooks = Path(".").rglob(NOTEBOOKS_GLOB) excluded_notebooks = NOTEBOOKS_EXCLUDE - if not submit_jobs: - excluded_notebooks += NOTEBOOKS_THAT_SUBMIT_JOBS return [ path for path in all_notebooks - if not any(path.match(glob) for glob in excluded_notebooks) + if not matches(path, excluded_notebooks) ] @@ -238,14 +244,18 @@ def create_argument_parser() -> argparse.ArgumentParser: "quantum resources! Only use this argument occasionally and intentionally." ), ) + parser.add_argument( + "--only-submit-jobs", + action="store_true", + help="Same as --submit-jobs, but also skips notebooks that do not submit jobs to IBM Quantum", + ) return parser if __name__ == "__main__": args = create_argument_parser().parse_args() - - paths = map(Path, args.filenames or find_notebooks(submit_jobs=args.submit_jobs)) - filtered_paths = filter_paths(paths, submit_jobs=args.submit_jobs) + paths = map(Path, args.filenames or find_notebooks()) + filtered_paths = filter_paths(paths, submit_jobs=args.submit_jobs, only_submit_jobs=args.only_submit_jobs) # Execute notebooks start_time = datetime.now()