diff --git a/.github/workflows/build-book-pullrequest.yaml b/.github/workflows/build-book-pullrequest.yaml index 232c03cc9..f4f6c9bec 100644 --- a/.github/workflows/build-book-pullrequest.yaml +++ b/.github/workflows/build-book-pullrequest.yaml @@ -24,6 +24,9 @@ on: default: 'true' type: string # had a lot of trouble with boolean types, see https://github.com/actions/runner/issues/1483 +permissions: + contents: read + jobs: build-book: runs-on: ubuntu-latest @@ -33,9 +36,21 @@ jobs: steps: - name: checkout files in repo uses: actions/checkout@v4 + with: + fetch-depth: 2 + - name: Check Changes + run: | + # check if binder-folder has changed + # if changed, then a docker rebuild is needed + echo "REBUILD=$(git diff --quiet ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }} -- binder || echo changed)" >> $GITHUB_ENV - name: remove Dockerfile run: | - rm binder/Dockerfile + if [[ ${{ env.REBUILD }} == "changed" ]]; then + echo "binder-folder HAS changed - NEEDS rebuild" + rm binder/Dockerfile + else + echo "binder-folder NOT changed - NO rebuild" + fi - name: update jupyter dependencies with repo2docker uses: jupyterhub/repo2docker-action@master with: @@ -55,18 +70,18 @@ jobs: uses: addnab/docker-run-action@v3 with: image: ghcr.io/openradar/erad2024:latest - options: --user root -v ${{ github.workspace }}:/work + options: --user root -v ${{ github.workspace }}:/work -e BASE_URL=/${{ github.event.repository.name }}/_preview/${{ github.event.pull_request.number }} shell: bash -l {0} run: | - # copy baltrad notebooks to book dir - # re add when baltrad issues are sorted out - # cp -rp /home/jovyan/notebooks/baltrad* /work/notebooks/. - # cp -rp /home/jovyan/notebooks/pyart2baltrad* /work/notebooks/. - # /work/install/nbstripout_notebooks.sh source /srv/conda/etc/profile.d/conda.sh conda activate notebook mamba list - jupyter-book build /work/${{ inputs.path_to_notebooks }} + echo env + # run notebooks + pytest -n auto --verbose --durations=15 --pyargs notebooks + # build the book and copy to outside /work-folder + myst build --ci --html + cp -rp /home/jovyan/_build /work/. - name: Zip the book run: | set -x diff --git a/.github/workflows/build-book.yaml b/.github/workflows/build-book.yaml index b437a0fdd..46a99b784 100644 --- a/.github/workflows/build-book.yaml +++ b/.github/workflows/build-book.yaml @@ -34,9 +34,21 @@ jobs: steps: - name: checkout files in repo uses: actions/checkout@v4 + - name: Check Changes + with: + fetch-depth: 2 + run: | + # check if binder-folder has changed + # if changed, then a docker rebuild is needed + echo "REBUILD=$(git diff --quiet ${{ github.event.pull_request.before }}..${{ github.event.after }} -- binder || echo changed)" >> $GITHUB_ENV - name: remove Dockerfile run: | - rm binder/Dockerfile + if [[ ${{ env.REBUILD }} == "changed" ]]; then + echo "binder-folder HAS changed - NEEDS rebuild" + rm binder/Dockerfile + else + echo "binder-folder NOT changed - NO rebuild" + fi - name: update jupyter dependencies with repo2docker uses: jupyterhub/repo2docker-action@master with: @@ -53,6 +65,8 @@ jobs: container: image: ghcr.io/openradar/erad2024:latest options: --user root + env: + BASE_URL: /${{ github.event.repository.name }} needs: [build-container] defaults: run: @@ -66,7 +80,10 @@ jobs: # cp -rp /home/jovyan/notebooks/baltrad* notebooks/. # cp -rp /home/jovyan/notebooks/pyart2baltrad* notebooks/. # install/nbstripout_notebooks.sh - jupyter-book build ${{ inputs.path_to_notebooks }} + # run notebooks + pytest -n auto --verbose --durations=15 --pyargs notebooks + # build the book and copy to outside /work-folder + myst build --ci --html - name: Zip the book run: | set -x diff --git a/binder/appendix.txt b/binder/appendix.txt index 1bfb74c8a..1355decd2 100644 --- a/binder/appendix.txt +++ b/binder/appendix.txt @@ -2,7 +2,7 @@ USER root ENV DEBIAN_FRONTEND=noninteractive RUN apt update && \ wget -q -O /tmp/lrose.deb https://github.com/NCAR/lrose-core/releases/download/lrose-core-20240525/lrose-core-20240525.ubuntu_22.04.amd64.deb && \ - apt-get -y install /tmp/lrose.deb && \ + apt-get -y install /tmp/lrose.deb nodejs && \ apt-get clean && \ rm -rf /tmp/lrose.deb diff --git a/binder/environment.yml b/binder/environment.yml index f44cfd1b6..e22d0f8fc 100644 --- a/binder/environment.yml +++ b/binder/environment.yml @@ -33,3 +33,8 @@ dependencies: # debugging - sphinx>=7.4.7 - sqlite>=3.46.0 + # mystmd + - mystmd + - ipykernel + - pytest + - pytest-xdist diff --git a/conftest.py b/conftest.py new file mode 100644 index 000000000..ef504de07 --- /dev/null +++ b/conftest.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# Copyright (c) 2020-2024, wradlib developers. +# Copyright (c) 2024, openradar developers. +# Distributed under the MIT License. See LICENSE.txt for more info. + +import os +import sys + +import nbformat +import pytest +from nbconvert.preprocessors import ExecutePreprocessor +from nbconvert.preprocessors.execute import CellExecutionError +from packaging.version import Version + + +def pytest_collect_file(parent, file_path): + if file_path.suffix == ".ipynb" and "-Copy" not in file_path.name: + return NotebookFile.from_parent(parent, path=file_path) + + +class NotebookFile(pytest.File): + if Version(pytest.__version__) < Version("5.4.0"): + + @classmethod + def from_parent(cls, parent, path): + return cls(parent=parent, path=path) + + def collect(self): + for f in [self.path]: + yield NotebookItem.from_parent(self, name=os.path.basename(f)) + + def setup(self): + kernel = "python%d" % sys.version_info[0] + self.exproc = ExecutePreprocessor(kernel_name=kernel, timeout=600) + + +class NotebookItem(pytest.Item): + def __init__(self, name, parent): + super().__init__(name, parent) + + if Version(pytest.__version__) < Version("5.4.0"): + + @classmethod + def from_parent(cls, parent, name): + return cls(parent=parent, name=name) + + def runtest(self): + cur_dir = os.path.dirname(self.path) + + # See https://bugs.python.org/issue37373 + if ( + sys.version_info[0] == 3 + and sys.version_info[1] >= 8 + and sys.platform.startswith("win") + ): + import asyncio + + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + + with self.path.open() as f: + nb = nbformat.read(f, as_version=4) + try: + self.parent.exproc.preprocess(nb, {"metadata": {"path": cur_dir}}) + except CellExecutionError as e: + raise NotebookException(e) + + with open(self.path, "w", encoding="utf-8") as f: + nbformat.write(nb, f) + + def repr_failure(self, excinfo): + if isinstance(excinfo.value, NotebookException): + return excinfo.exconly() + + return super().repr_failure(excinfo) + + def reportinfo(self): + return self.path, 0, "TestCase: %s" % self.name + + +class NotebookException(Exception): + pass diff --git a/install/baltrad/install_baltrad.sh b/install/baltrad/install_baltrad.sh index 6f2260f51..51a4b27b8 100755 --- a/install/baltrad/install_baltrad.sh +++ b/install/baltrad/install_baltrad.sh @@ -26,4 +26,5 @@ ${CONDA_PREFIX}/bin/bash -l $BALTRAD_INSTALL_ROOT/install/baltrad/install_baltra ${CONDA_PREFIX}/bin/bash -l $BALTRAD_INSTALL_ROOT/install/baltrad/install_baltrad_wrwp.sh 2>&1 |tee $BALTRAD_INSTALL_ROOT/tmp/install_baltrad_wrwp.log ${CONDA_PREFIX}/bin/bash -l $BALTRAD_INSTALL_ROOT/install/baltrad/install_baltrad_drqc.sh 2>&1 |tee $BALTRAD_INSTALL_ROOT/tmp/install_baltrad_drqc.log ${CONDA_PREFIX}/bin/bash -l $BALTRAD_INSTALL_ROOT/install/baltrad/install_baltrad_rave_gmap.sh 2>&1 |tee $BALTRAD_INSTALL_ROOT/tmp/install_baltrad_rave_gmap.log -${CONDA_PREFIX}/bin/bash -l $BALTRAD_INSTALL_ROOT/install/baltrad/install_baltrad_short_course.sh 2>&1 |tee $BALTRAD_INSTALL_ROOT/tmp/install_baltrad_short_course.log \ No newline at end of file +# do not install notebooks, see https://github.com/openradar/erad2024/issues/60 +# ${CONDA_PREFIX}/bin/bash -l $BALTRAD_INSTALL_ROOT/install/baltrad/install_baltrad_short_course.sh 2>&1 |tee $BALTRAD_INSTALL_ROOT/tmp/install_baltrad_short_course.log \ No newline at end of file