Add CI workflow to test Docker image with notebooks

.github/workflows/run-tutorial-notebook.yml

@@ -0,0 +1,103 @@
+# TODO: Move it to the scripts directory when the following issue is resolved:
+# https://github.com/orgs/community/discussions/10773#discussioncomment-2107255
+
+# NOTE: this is a reusable workflow intended to be called from the "Test notebooks" workflow.
+name: Run tutorial notebook in Docker
+
+on:
+  workflow_call:
+    inputs:
+      working-directory:
+        description: Working directory
+        required: false
+        default: .
+        type: string
+      notebook:
+        description: Notebook file to be executed
+        required: true
+        type: string
+      outputs:
+        description: Output files to be uploaded
+        required: false
+        default: ''
+        type: string
+      timeout:
+        description: Timeout per cell in seconds
+        required: false
+        default: 3600
+        type: number
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout the repository
+      uses: actions/checkout@v4
+
+    - name: Download the image
+      uses: actions/download-artifact@v4
+      with:
+        name: image
+        path: /tmp
+
+    - name: Load the image
+      run: |
+        docker load --input /tmp/image.tar
+        docker image ls -a madminer-jupyter-env
+
+    - name: Prepare the shared volume
+      run: |
+        mkdir madminer_shared
+        git -C madminer_shared clone --depth=1 https://github.com/madminer-tool/madminer.git
+
+    - name: Download output files uploaded by previous jobs
+      uses: actions/download-artifact@v4
+      with:
+        pattern: output-*
+        merge-multiple: true
+
+    - name: Run the notebook in a container
+      id: run-notebook
+      continue-on-error: true
+      run: >
+        docker run
+        --rm
+        -v $(pwd)/madminer_shared:/home/shared
+        -v $(pwd)/.github/workflows/scripts/run-tutorial-notebook.sh:/tmp/run-tutorial-notebook.sh:ro
+        madminer-jupyter-env:latest
+        /tmp/run-tutorial-notebook.sh /home/shared/madminer/${{ inputs.working-directory }} ${{ inputs.notebook }} ${{ inputs.timeout }}
+
+    - name: Upload notebook and logs
+      uses: actions/upload-artifact@v4
+      with:
+        name: log-${{ inputs.notebook }}
+        path: |
+          *madminer_shared/madminer/${{ inputs.working-directory }}/${{ inputs.notebook }}
+          *madminer_shared/madminer/${{ inputs.working-directory }}/**/*.log
+        if-no-files-found: ignore
+        overwrite: true
+
+    # Abort before saving output files, if the notebook execution failed.
+    - name: Error handling
+      if: ${{ steps.run-notebook.outcome == 'failure' }}
+      run: exit 1
+
+    # Prepend the prefix to each path in the given list.
+    # The wildcard at the beginning is necessary to preserve the directory structure.
+    - name: Construct artifact path
+      id: artifact-path
+      run: |
+        eof=EOF$(openssl rand -hex 8)
+        echo "value<<$eof" >>$GITHUB_OUTPUT
+        for f in ${{ inputs.outputs }}; do
+          echo '*madminer_shared/madminer/${{ inputs.working-directory }}'/"$f" >>$GITHUB_OUTPUT
+        done
+        echo "$eof" >>$GITHUB_OUTPUT
+
+    - name: Upload output files as artifacts
+      uses: actions/upload-artifact@v4
+      if: inputs.outputs != ''
+      with:
+        name: output-${{ inputs.notebook }}
+        path: ${{ steps.artifact-path.outputs.value }}
+        if-no-files-found: error

.github/workflows/scripts/run-tutorial-notebook.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+# NOTE: this script is intended to be executed within a container.
+set -eu
+
+working_directory=$1
+notebook_file=$2
+timeout=$3
+
+if [ ! -f /.dockerenv ]; then
+  echo 'error: this script must be executed within a container' >&2
+  exit 1
+fi
+
+echo "::group::Install Papermill"
+time {
+  apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+    python3-pip \
+    && rm -rf /var/lib/apt/lists/*
+  # papermill 2.4.0+ is not compatible with jupyter-client 8.0.0+.
+  # https://github.com/scikit-hep/pyhf/issues/2104
+  python3 -m pip install --no-cache-dir papermill==2.6.0 jupyter-client==7.4.9
+}
+echo "::endgroup::"
+
+cd "$working_directory"
+
+# Mitigate the risk of the DataLoader freezing during training.
+# https://github.com/pytorch/pytorch/issues/15808#issuecomment-1291514752
+export OMP_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+
+# We may set a timeout (for each cell) to ensure in-progress notebooks and logs are saved,
+# even if the workflow hits the total 6-hour limit in GitHub Actions.
+if [ "$timeout" -ne 0 ]; then
+  papermill --execution-timeout "$timeout" "$notebook_file" "$notebook_file"
+else
+  papermill "$notebook_file" "$notebook_file"
+fi

.github/workflows/test-notebooks.yml

@@ -0,0 +1,191 @@
+name: Test notebooks
+
+on:
+  pull_request:
+  push:
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and export
+        uses: docker/build-push-action@v6
+        with:
+          tags: madminer-jupyter-env:latest
+          outputs: type=docker,dest=/tmp/image.tar
+          cache-from: type=gha
+          cache-to: type=gha
+
+      - name: Upload the image as an artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: image
+          path: /tmp/image.tar
+
+  tutorial_particle_physics_1:
+    needs: build
+    if: ${{ !cancelled() && !failure() }}
+    uses: ./.github/workflows/run-tutorial-notebook.yml
+    with:
+      working-directory: examples/tutorial_particle_physics
+      notebook: 1_setup.ipynb
+      outputs: data/setup.h5
+
+  tutorial_particle_physics_2a:
+    needs: tutorial_particle_physics_1
+    if: ${{ !cancelled() && !failure() }}
+    uses: ./.github/workflows/run-tutorial-notebook.yml
+    with:
+      working-directory: examples/tutorial_particle_physics
+      notebook: 2a_parton_level_analysis.ipynb
+      outputs: >-
+        data/lhe_data_shuffled.h5
+        data/lhe_data.h5
+
+  tutorial_particle_physics_2b:
+    needs: tutorial_particle_physics_1
+    if: ${{ !cancelled() && !failure() }}
+    uses: ./.github/workflows/run-tutorial-notebook.yml
+    with:
+      working-directory: examples/tutorial_particle_physics
+      notebook: 2b_delphes_level_analysis.ipynb
+      outputs: >-
+        data/delphes_data_shuffled.h5
+        data/delphes_data.h5
+
+  tutorial_particle_physics_3a:
+    needs: tutorial_particle_physics_2a
+    if: ${{ !cancelled() && !failure() }}
+    uses: ./.github/workflows/run-tutorial-notebook.yml
+    with:
+      working-directory: examples/tutorial_particle_physics
+      notebook: 3a_likelihood_ratio.ipynb
+      outputs: >-
+        models/alices_pt_settings.json
+        models/alices_pt_state_dict.pt
+        models/alices_pt_theta_means.npy
+        models/alices_pt_theta_stds.npy
+        models/alices_pt_x_means.npy
+        models/alices_pt_x_stds.npy
+        models/alices_settings.json
+        models/alices_state_dict.pt
+        models/alices_theta_means.npy
+        models/alices_theta_stds.npy
+        models/alices_x_means.npy
+        models/alices_x_stds.npy
+
+  tutorial_particle_physics_3b:
+    needs: tutorial_particle_physics_2a
+    if: ${{ !cancelled() && !failure() }}
+    uses: ./.github/workflows/run-tutorial-notebook.yml
+    with:
+      working-directory: examples/tutorial_particle_physics
+      notebook: 3b_score.ipynb
+      outputs: >-
+        models/sally_settings.json
+        models/sally_state_dict.pt
+        models/sally_x_means.npy
+        models/sally_x_stds.npy
+
+  tutorial_particle_physics_3c:
+    needs: tutorial_particle_physics_2a
+    if: ${{ !cancelled() && !failure() }}
+    uses: ./.github/workflows/run-tutorial-notebook.yml
+    with:
+      working-directory: examples/tutorial_particle_physics
+      notebook: 3c_likelihood.ipynb
+      outputs: >-
+        models/scandal_settings.json
+        models/scandal_state_dict.pt
+        models/scandal_theta_means.npy
+        models/scandal_theta_stds.npy
+        models/scandal_x_means.npy
+        models/scandal_x_stds.npy
+
+  tutorial_particle_physics_4a:
+    # "no SCANDAL" (w/o 3c) as in the output of the current notebook.
+    needs: [tutorial_particle_physics_3a, tutorial_particle_physics_3b]
+    if: ${{ !cancelled() && !failure() }}
+    uses: ./.github/workflows/run-tutorial-notebook.yml
+    with:
+      working-directory: examples/tutorial_particle_physics
+      notebook: 4a_limits.ipynb
+      outputs: limits/limits.npy
+
+  tutorial_particle_physics_4b:
+    needs: [tutorial_particle_physics_3a, tutorial_particle_physics_3b]
+    if: ${{ !cancelled() && !failure() }}
+    uses: ./.github/workflows/run-tutorial-notebook.yml
+    with:
+      working-directory: examples/tutorial_particle_physics
+      notebook: 4b_fisher_information.ipynb
+
+  tutorial_particle_physics_4c:
+    needs: tutorial_particle_physics_4a
+    if: ${{ !cancelled() && !failure() }}
+    uses: ./.github/workflows/run-tutorial-notebook.yml
+    with:
+      working-directory: examples/tutorial_particle_physics
+      notebook: 4c_information_geometry.ipynb
+
+  tutorial_particle_physics_a1:
+    needs: tutorial_particle_physics_1
+    if: ${{ !cancelled() && !failure() }}
+    uses: ./.github/workflows/run-tutorial-notebook.yml
+    with:
+      working-directory: examples/tutorial_particle_physics
+      notebook: A1_systematic_uncertainties.ipynb
+      outputs: >-
+        data/lhe_data_systematics.h5
+        data/setup_systematics.h5
+
+  tutorial_particle_physics_a2:
+    needs: build
+    if: ${{ !cancelled() && !failure() }}
+    uses: ./.github/workflows/run-tutorial-notebook.yml
+    with:
+      working-directory: examples/tutorial_particle_physics
+      notebook: A2_ensemble_methods.ipynb
+
+  tutorial_particle_physics_a3:
+    needs: tutorial_particle_physics_1
+    if: ${{ !cancelled() && !failure() }}
+    uses: ./.github/workflows/run-tutorial-notebook.yml
+    with:
+      working-directory: examples/tutorial_particle_physics
+      notebook: A3_reweighting_existing_samples.ipynb
+      outputs: data/setup_with_extra_benchmark.h5
+
+  tutorial_particle_physics_a4:
+    needs: tutorial_particle_physics_3a
+    if: ${{ !cancelled() && !failure() }}
+    uses: ./.github/workflows/run-tutorial-notebook.yml
+    with:
+      working-directory: examples/tutorial_particle_physics
+      notebook: A4_lh_nosyst.ipynb
+
+  tutorial_particle_physics_a5:
+    needs: tutorial_particle_physics_a1
+    if: ${{ !cancelled() && !failure() }}
+    uses: ./.github/workflows/run-tutorial-notebook.yml
+    with:
+      working-directory: examples/tutorial_particle_physics
+      notebook: A5_test_new_likelihood_module.ipynb
+
+  tutorial_particle_physics_a6:
+    needs: build
+    if: ${{ !cancelled() && !failure() }}
+    uses: ./.github/workflows/run-tutorial-notebook.yml
+    with:
+      working-directory: examples/tutorial_particle_physics
+      notebook: A6_finite_differences.ipynb
+      outputs: >-
+        data/lhe_data_fd.h5
+        data/setup_fd.h5