Use new myst build in pullrequest and push

.github/workflows/build-book-pullrequest.yaml

@@ -24,6 +24,9 @@ on:
         default: 'true'
         type: string  # had a lot of trouble with boolean types, see https://github.com/actions/runner/issues/1483
 
+permissions:
+  contents: read
+
 jobs:
   build-book:
     runs-on: ubuntu-latest
@@ -33,9 +36,21 @@ jobs:
     steps:
       - name: checkout files in repo
         uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+      - name: Check Changes
+        run: |
+          # check if binder-folder has changed
+          # if changed, then a docker rebuild is needed
+          echo "REBUILD=$(git diff --quiet ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }} -- binder || echo changed)" >> $GITHUB_ENV
       - name: remove Dockerfile
         run: |
-          rm binder/Dockerfile
+          if [[ ${{ env.REBUILD }} == "changed" ]]; then
+            echo "binder-folder HAS changed - NEEDS rebuild"
+            rm binder/Dockerfile
+          else
+            echo "binder-folder NOT changed - NO rebuild"
+          fi
       - name: update jupyter dependencies with repo2docker
         uses: jupyterhub/repo2docker-action@master
         with:
@@ -55,18 +70,18 @@ jobs:
         uses: addnab/docker-run-action@v3
         with:
           image: ghcr.io/openradar/erad2024:latest
-          options: --user root -v ${{ github.workspace }}:/work
+          options: --user root -v ${{ github.workspace }}:/work -e BASE_URL=/${{ github.event.repository.name }}/_preview/${{ github.event.pull_request.number }}
           shell: bash -l {0}
           run: |
-            # copy baltrad notebooks to book dir
-            # re add when baltrad issues are sorted out
-            # cp -rp /home/jovyan/notebooks/baltrad* /work/notebooks/.
-            # cp -rp /home/jovyan/notebooks/pyart2baltrad* /work/notebooks/.
-            # /work/install/nbstripout_notebooks.sh
             source /srv/conda/etc/profile.d/conda.sh
             conda activate notebook
             mamba list
-            jupyter-book build /work/${{ inputs.path_to_notebooks }}
+            echo env
+            # run notebooks
+            pytest -n auto --verbose --durations=15 --pyargs notebooks
+            # build the book and copy to outside /work-folder
+            myst build --ci --html
+            cp -rp /home/jovyan/_build /work/.
       - name: Zip the book
         run: |
           set -x

.github/workflows/build-book.yaml

@@ -34,9 +34,21 @@ jobs:
     steps:
       - name: checkout files in repo
         uses: actions/checkout@v4
+      - name: Check Changes
+        with:
+          fetch-depth: 2
+        run: |
+          # check if binder-folder has changed
+          # if changed, then a docker rebuild is needed
+          echo "REBUILD=$(git diff --quiet ${{ github.event.pull_request.before }}..${{ github.event.after }} -- binder || echo changed)" >> $GITHUB_ENV
       - name: remove Dockerfile
         run: |
-          rm binder/Dockerfile
+          if [[ ${{ env.REBUILD }} == "changed" ]]; then
+            echo "binder-folder HAS changed - NEEDS rebuild"
+            rm binder/Dockerfile
+          else
+            echo "binder-folder NOT changed - NO rebuild"
+          fi
       - name: update jupyter dependencies with repo2docker
         uses: jupyterhub/repo2docker-action@master
         with:
@@ -53,6 +65,8 @@ jobs:
     container:
       image: ghcr.io/openradar/erad2024:latest
       options: --user root
+      env:
+        BASE_URL: /${{ github.event.repository.name }}
     needs: [build-container]
     defaults:
       run:
@@ -66,7 +80,10 @@ jobs:
           # cp -rp /home/jovyan/notebooks/baltrad* notebooks/.
           # cp -rp /home/jovyan/notebooks/pyart2baltrad* notebooks/.
           # install/nbstripout_notebooks.sh
-          jupyter-book build ${{ inputs.path_to_notebooks }}
+          # run notebooks
+          pytest -n auto --verbose --durations=15 --pyargs notebooks
+          # build the book and copy to outside /work-folder
+          myst build --ci --html
       - name: Zip the book
         run: |
           set -x

binder/appendix.txt

@@ -2,7 +2,7 @@ USER root
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt update && \
     wget -q -O /tmp/lrose.deb https://github.com/NCAR/lrose-core/releases/download/lrose-core-20240525/lrose-core-20240525.ubuntu_22.04.amd64.deb && \
-    apt-get -y install /tmp/lrose.deb && \
+    apt-get -y install /tmp/lrose.deb nodejs && \
     apt-get clean && \
     rm -rf /tmp/lrose.deb
 

binder/environment.yml

@@ -33,3 +33,8 @@ dependencies:
   # debugging
   - sphinx>=7.4.7
   - sqlite>=3.46.0
+  # mystmd
+  - mystmd
+  - ipykernel
+  - pytest
+  - pytest-xdist

conftest.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+# Copyright (c) 2020-2024, wradlib developers.
+# Copyright (c) 2024, openradar developers.
+# Distributed under the MIT License. See LICENSE.txt for more info.
+
+import os
+import sys
+
+import nbformat
+import pytest
+from nbconvert.preprocessors import ExecutePreprocessor
+from nbconvert.preprocessors.execute import CellExecutionError
+from packaging.version import Version
+
+
+def pytest_collect_file(parent, file_path):
+    if file_path.suffix == ".ipynb" and "-Copy" not in file_path.name:
+        return NotebookFile.from_parent(parent, path=file_path)
+
+
+class NotebookFile(pytest.File):
+    if Version(pytest.__version__) < Version("5.4.0"):
+
+        @classmethod
+        def from_parent(cls, parent, path):
+            return cls(parent=parent, path=path)
+
+    def collect(self):
+        for f in [self.path]:
+            yield NotebookItem.from_parent(self, name=os.path.basename(f))
+
+    def setup(self):
+        kernel = "python%d" % sys.version_info[0]
+        self.exproc = ExecutePreprocessor(kernel_name=kernel, timeout=600)
+
+
+class NotebookItem(pytest.Item):
+    def __init__(self, name, parent):
+        super().__init__(name, parent)
+
+    if Version(pytest.__version__) < Version("5.4.0"):
+
+        @classmethod
+        def from_parent(cls, parent, name):
+            return cls(parent=parent, name=name)
+
+    def runtest(self):
+        cur_dir = os.path.dirname(self.path)
+
+        # See https://bugs.python.org/issue37373
+        if (
+            sys.version_info[0] == 3
+            and sys.version_info[1] >= 8
+            and sys.platform.startswith("win")
+        ):
+            import asyncio
+
+            asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+
+        with self.path.open() as f:
+            nb = nbformat.read(f, as_version=4)
+            try:
+                self.parent.exproc.preprocess(nb, {"metadata": {"path": cur_dir}})
+            except CellExecutionError as e:
+                raise NotebookException(e)
+
+        with open(self.path, "w", encoding="utf-8") as f:
+            nbformat.write(nb, f)
+
+    def repr_failure(self, excinfo):
+        if isinstance(excinfo.value, NotebookException):
+            return excinfo.exconly()
+
+        return super().repr_failure(excinfo)
+
+    def reportinfo(self):
+        return self.path, 0, "TestCase: %s" % self.name
+
+
+class NotebookException(Exception):
+    pass

install/baltrad/install_baltrad.sh

@@ -26,4 +26,5 @@ ${CONDA_PREFIX}/bin/bash -l $BALTRAD_INSTALL_ROOT/install/baltrad/install_baltra
 ${CONDA_PREFIX}/bin/bash -l $BALTRAD_INSTALL_ROOT/install/baltrad/install_baltrad_wrwp.sh 2>&1 |tee $BALTRAD_INSTALL_ROOT/tmp/install_baltrad_wrwp.log
 ${CONDA_PREFIX}/bin/bash -l $BALTRAD_INSTALL_ROOT/install/baltrad/install_baltrad_drqc.sh 2>&1 |tee $BALTRAD_INSTALL_ROOT/tmp/install_baltrad_drqc.log
 ${CONDA_PREFIX}/bin/bash -l $BALTRAD_INSTALL_ROOT/install/baltrad/install_baltrad_rave_gmap.sh 2>&1 |tee $BALTRAD_INSTALL_ROOT/tmp/install_baltrad_rave_gmap.log
-${CONDA_PREFIX}/bin/bash -l $BALTRAD_INSTALL_ROOT/install/baltrad/install_baltrad_short_course.sh 2>&1 |tee $BALTRAD_INSTALL_ROOT/tmp/install_baltrad_short_course.log
+# do not install notebooks, see https://github.com/openradar/erad2024/issues/60
+# ${CONDA_PREFIX}/bin/bash -l $BALTRAD_INSTALL_ROOT/install/baltrad/install_baltrad_short_course.sh 2>&1 |tee $BALTRAD_INSTALL_ROOT/tmp/install_baltrad_short_course.log