Update .ipynb notebooks from .py in pre-commit hook locally

.github/workflows/docs.yml

@@ -14,74 +14,26 @@ defaults:
     shell: bash
 
 jobs:
-  convert_scripts:
-    name: Translate scripts to notebooks
-    runs-on: ubuntu-latest
-    permissions:
-      pull-requests: write
-      contents: write
-    outputs:
-      commit_hash: ${{ steps.add-commit-push.outputs.commit_hash }}
-    container:
-      image: ghcr.io/ptb-mr/mrpro_py311:latest
-      options: --user runner
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v4
-        with:
-          token: ${{ secrets.commit }}
-          ref: ${{ github.event.pull_request.head.ref }}
-          fetch-depth: 0
-
-      - name: Install mrpro and dependencies
-        run: pip install --upgrade --upgrade-strategy "eager" .[notebook]
-
-      - name: Translate scripts to notebooks
-        run: |
-          scripts=$(ls ./examples/*.py)
-          for script in $scripts
-            do jupytext --set-kernel "python3" --update --output ${script//.py/.ipynb} $script
-          done
-
-      - name: Check if any notebooks have been changed
-        uses: tj-actions/verify-changed-files@v20
-        id: verify-changed-notebooks
-        with:
-          files: ./examples/*.ipynb
-
-      - name: Commit notebooks
-        if: steps.verify-changed-notebooks.outputs.files_changed == 'true'
-        uses: actions4git/add-commit-push@v1
-        with:
-          commit-message: Notebooks updated
-
-      - name: Get hash of last commit
-        id: add-commit-push
-        run: echo "commit_hash=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
-
   get_notebooks:
     name: Get list of notebooks
-    needs: convert_scripts
     runs-on: ubuntu-latest
     steps:
       - name: Checkout mrpro repo
         uses: actions/checkout@v4
-        with:
-          ref: ${{ needs.convert_scripts.outputs.commit_hash }}
 
       - id: set-matrix
         run: |
-          echo "notebooks=$(find examples -type f -name '*.ipynb' | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
+          echo "notebook_paths=$(find examples/notebooks -type f -name '*.ipynb' | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
 
       - name: Notebook overview
         run: |
-          echo "jupyter-notebooks: ${{ steps.set-matrix.outputs.notebooks }}"
+          echo "jupyter-notebooks: ${{ steps.set-matrix.outputs.notebook_paths }}"
     outputs:
-      notebooks: ${{ steps.set-matrix.outputs.notebooks }}
+      notebook_paths: ${{ steps.set-matrix.outputs.notebook_paths }}
 
   run_notebook:
     name: Run notebook
-    needs: [convert_scripts, get_notebooks]
+    needs: get_notebooks
     runs-on: ubuntu-latest
     permissions:
       pull-requests: write
@@ -92,23 +44,21 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        notebook: ${{ fromJson(needs.get_notebooks.outputs.notebooks) }}
+        notebook_path: ${{ fromJson(needs.get_notebooks.outputs.notebook_paths) }}
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
-        with:
-          ref: ${{ needs.convert_scripts.outputs.commit_hash }}
 
       - name: Install mrpro and dependencies
         run: pip install --upgrade --upgrade-strategy "eager" .[notebook]
 
       - name: Notebook name
         run: |
-          echo "current jupyter-notebook: ${{ matrix.notebook }}"
+          echo "current jupyter-notebook: ${{ matrix.notebook_path }}"
 
       - name: Add nb-myst download badge
         run: |
-          notebook=${{ matrix.notebook }}
+          notebook=${{ matrix.notebook_path }}
           notebook_name=$(basename $notebook)
           download_badge_md="[![Download notebook](https://img.shields.io/badge/Download-notebook-blue?logo=jupyter)](path:$notebook_name)"
           python_command="import nbformat as nbf\n\
@@ -129,12 +79,12 @@ jobs:
         env:
           RUNNER: ${{ toJson(runner) }}
         with:
-          notebook: ./${{ matrix.notebook }}
+          notebook: ${{ matrix.notebook_path }}
 
       - name: Get artifact names
         id: artifact_names
         run: |
-          notebook=${{ matrix.notebook }}
+          notebook=${{ matrix.notebook_path }}
           echo "ARTIFACT_NAME=$(basename ${notebook/.ipynb})" >> $GITHUB_OUTPUT
           echo "IPYNB_EXECUTED=$(basename $notebook)" >> $GITHUB_OUTPUT
 
@@ -149,7 +99,7 @@ jobs:
 
   create_documentation:
     name: Build and deploy documentation
-    needs: [convert_scripts, run_notebook]
+    needs: run_notebook
     runs-on: ubuntu-latest
     container:
       image: ghcr.io/ptb-mr/mrpro_py311:latest
@@ -160,8 +110,6 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-        with:
-          ref: ${{ needs.convert_scripts.outputs.commit_hash }}
 
       - name: Install mrpro and dependencies
         run: pip install --upgrade --upgrade-strategy "eager" .[docs]

.gitignore

@@ -183,6 +183,7 @@ cython_debug/
 
 # Documentation
 **/_autosummary
+**/_notebooks
 
 # Ignore Conda environment directories:
 .conda

.pre-commit-config.yaml

@@ -32,6 +32,42 @@ repos:
         args: [--double-quotes, --fix]
         exclude: ^tests/
 
+  - repo: https://github.com/kynan/nbstripout
+    rev: 0.8.0
+    hooks:
+    # cleans the .ipynbs (removes outputs, resets all cell-ids to 0..N, cleans steps)
+    # also clean any kernel information left after execution
+    - id: nbstripout
+      name: clean .ipynb output
+      args: [--extra-keys, "metadata.language_info"]
+      files: examples/notebooks
+
+  - repo: local
+    hooks:
+     - id: jupytext
+       name: convert .py to .ipynb
+       language: python
+
+       additional_dependencies:
+        - jupytext
+       entry: >
+        jupytext
+        --update
+        --pipe
+        "python .precommit/add_notebook_preamble.py {}"
+        --to
+        "../notebooks//ipynb"
+        --update-metadata
+        '{
+          "accelerator": "GPU",
+          "colab": {"gpuType": "T4","provenance": []},
+          "kernelspec": {"display_name": "Python 3 (ipykernel)","language": "python","name": "python3"}
+        }'
+       always_run: true
+       pass_filenames: true
+       files: ^examples/scripts/.*py
+       types_or: [python]
+
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v1.14.1
     hooks:

.precommit/add_notebook_preamble.py

@@ -0,0 +1,36 @@
+"""Add a colab badge and pip install to notebooks."""
+
+import sys
+from pathlib import Path
+
+# the filename is the name of temp file created by jupytext, not an original notebook
+file = Path(sys.argv[1])
+# the temp filename for "iteratitive_sense_reconstruction.py" is like "iterative_sense_reconstruction-42_5f4kv.py"
+basename = file.stem.rpartition('-')[0]
+
+badge_svg = 'https://colab.research.google.com/assets/colab-badge.svg'
+ipynb_link = f'https://colab.research.google.com/github/PTB-MR/mrpro/blob/main/examples/notebooks/{basename}.ipynb'
+badge_markdown = f'[![Open In Colab]({badge_svg})]({ipynb_link})'
+badge_pyprocent = f'# %% [markdown]\n# {badge_markdown}\n'
+import_python = """# %% tags=["remove-cell"]
+import importlib
+
+if not importlib.util.find_spec('mrpro'):
+    %pip install mrpro[notebook]
+"""
+
+# the temp files of jupytext have the header which looks like:
+# ---
+# jupyter:
+#   jupytext:
+# multiple lines...
+# ---
+# we need to insert the #markdown cell after the header
+# insert the badge_pyprocent string after the second occurrence of '# ---'
+split_sequence = '# ---\n'
+old = file.read_text()
+split_text = old.split(split_sequence)
+new = ''.join(
+    [split_text[0], split_sequence, split_text[1], split_sequence, badge_pyprocent, import_python, *split_text[2:]]
+)
+file.write_text(new)

docs/Makefile

@@ -18,6 +18,7 @@ help:
 # be removed when https://github.com/sphinx-doc/sphinx/issues/1999 is fixed.
 clean:
 	rm -rfv "$(SOURCEDIR)/_autosummary"
+	rm -rfv "$(SOURCEDIR)/_notebooks"
 	$(SPHINXBUILD)  -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
 # Catch-all target: route all unknown targets to Sphinx using the new

docs/source/conf.py

@@ -10,7 +10,11 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 
 import os
+import shutil
 import sys
+from pathlib import Path
+
+import nbformat
 from sphinx_pyproject import SphinxConfig
 
 from mrpro  import __version__ as project_version
@@ -79,6 +83,31 @@
     ],
 }
 
+
+def sync_notebooks(source_folder, dest_folder):
+    """Sync notebooks from source to destination folder.
+
+    Copy only new or updated files.
+    Set execution mode to 'force' for all copied files and 'off' for all existing files.
+    """
+    dest = Path(dest_folder)
+    dest.mkdir(parents=True, exist_ok=True)
+    for src_file in Path(source_folder).iterdir():
+        if src_file.is_file():
+            dest_file = dest / src_file.name
+            if not dest_file.exists() or src_file.stat().st_mtime > dest_file.stat().st_mtime:
+                shutil.copy2(src_file, dest_file)
+                print(f'Copied {src_file} to {dest_file}. Setting execution mode to "force".')
+                mode = 'force'
+            else:
+                print(f'Existing {dest_file}. Skipping execution.')
+                mode = 'off'
+            content = nbformat.read(dest_file, as_version=nbformat.NO_CONVERT)
+            content.metadata['mystnb'] = {'execution_mode': mode}
+            nbformat.write(content, dest_file)
+
 def setup(app):
     # forces mathjax on all pages
     app.set_html_assets_policy('always')
+    sync_notebooks(app.srcdir.parent.parent/'examples'/'notebooks', app.srcdir/'_notebooks')
+

examples/ruff.toml → examples/.ruff.toml

@@ -1,9 +1,10 @@
 extend = "../pyproject.toml"
-
+exclude = ["notebooks/*.ipynb"]
 lint.extend-ignore = [
     "D100",   #undocumented-public-module
     "T20",    #print
     "E402",   #module-import-not-at-top-of-file
     "S101",   #assert
     "SIM115", #context manager for opening files
+    "ERA001", #commented out code
 ]