Skip to content

Commit

Permalink
Merge branch 'main' into linearize
Browse files Browse the repository at this point in the history
  • Loading branch information
fzimmermann89 authored Nov 25, 2024
2 parents 6688633 + de79063 commit 9b84d18
Show file tree
Hide file tree
Showing 157 changed files with 6,232 additions and 1,343 deletions.
6 changes: 6 additions & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
.github
.vscode
binder
examples
docs
tests
59 changes: 39 additions & 20 deletions .github/workflows/deployment.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Publish to PyPI
name: Build and Deploy Package

on:
push:
Expand All @@ -7,21 +7,31 @@ on:

jobs:
build-testpypi-package:
name: Modify version and build package for TestPyPI
name: Build Package for TestPyPI
runs-on: ubuntu-latest
outputs:
version: ${{ steps.set_suffix.outputs.version }}
suffix: ${{ steps.set_suffix.outputs.suffix }}
version_changed: ${{ steps.changes.outputs.version_changed }}
steps:
- name: Checkout code
uses: actions/checkout@v4

- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.x"

- name: Set environment variable for version suffix
- name: Checkout Code
uses: actions/checkout@v4

- name: Check if VERSION file is modified compared to main
uses: dorny/paths-filter@v3
id: changes
with:
base: main
filters: |
version_changed:
- 'src/mrpro/VERSION'
- name: Set Version Suffix
id: set_suffix
run: |
VERSION=$(cat src/mrpro/VERSION)
Expand All @@ -30,12 +40,12 @@ jobs:
echo "suffix=$SUFFIX" >> $GITHUB_OUTPUT
echo "version=$VERSION" >> $GITHUB_OUTPUT
- name: Build package with version suffix
- name: Build Package
run: |
python -m pip install --upgrade build
python -m build
- name: Store TestPyPi distribution
- name: Upload TestPyPI Distribution Artifact
uses: actions/upload-artifact@v4
with:
name: testpypi-package-distribution
Expand All @@ -46,6 +56,7 @@ jobs:
needs:
- build-testpypi-package
runs-on: ubuntu-latest
if: needs.build-testpypi-package.outputs.version_changed == 'true'

environment:
name: testpypi
Expand All @@ -55,7 +66,7 @@ jobs:
id-token: write

steps:
- name: Download TestPyPi distribution
- name: Download TestPyPI Distribution
uses: actions/download-artifact@v4
with:
name: testpypi-package-distribution
Expand All @@ -68,7 +79,7 @@ jobs:
verbose: true

test-install-from-testpypi:
name: Test installation from TestPyPI
name: Test Installation from TestPyPI
needs:
- testpypi-deployment
- build-testpypi-package
Expand All @@ -83,40 +94,48 @@ jobs:
run: |
VERSION=${{ needs.build-testpypi-package.outputs.version }}
SUFFIX=${{ needs.build-testpypi-package.outputs.suffix }}
python -m pip install mrpro==$VERSION$SUFFIX --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/
for i in {1..3}; do
if python -m pip install mrpro==$VERSION$SUFFIX --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/; then
echo "Package installed successfully."
break
else
echo "Attempt $i failed. Retrying in 10 seconds..."
sleep 10
fi
done
build-pypi-package:
name: Build package for PyPI
name: Build Package for PyPI
runs-on: ubuntu-latest
needs:
- test-install-from-testpypi
outputs:
version: ${{ steps.get_version.outputs.version }}
steps:
- name: Checkout code
- name: Checkout Code
uses: actions/checkout@v4

- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.x"

- name: Install setuptools_git_versioning
- name: Install Automatic Versioning Tool
run: |
python -m pip install setuptools-git-versioning
- name: Get current version
- name: Get Current Version
id: get_version
run: |
VERSION=$(python -m setuptools_git_versioning)
echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
- name: Build package
- name: Build Package
run: |
python -m pip install --upgrade build
python -m build
- name: Store PyPi distribution
- name: Store PyPI Distribution
uses: actions/upload-artifact@v4
with:
name: pypi-package-distribution
Expand All @@ -138,13 +157,13 @@ jobs:
id-token: write

steps:
- name: Download PyPi distribution
- name: Download PyPI Distribution
uses: actions/download-artifact@v4
with:
name: pypi-package-distribution
path: dist/

- name: Create tag
- name: Create Tag
uses: actions/github-script@v7
with:
script: |
Expand All @@ -155,7 +174,7 @@ jobs:
sha: context.sha
})
- name: Create release
- name: Create Release
uses: actions/github-script@v7
with:
github-token: "${{ secrets.GITHUB_TOKEN }}"
Expand Down
5 changes: 4 additions & 1 deletion .github/workflows/docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ jobs:
permissions:
packages: write
strategy:
fail-fast: false
matrix:
dockerfile: ${{ fromJson(needs.get_dockerfiles.outputs.dockerfiles) }}
steps:
Expand All @@ -87,7 +88,9 @@ jobs:
- name: Build and push Docker image
uses: docker/build-push-action@v6
with:
context: ./docker
context: .
cache-from: type=gha,scope=${{ matrix.dockerfile }}
cache-to: type=gha,mode=max,scope=${{ matrix.dockerfile }}
file: ./docker/${{ matrix.dockerfile }}
push: true
tags: ${{ steps.image_name.outputs.image_name }}:test
Expand Down
41 changes: 7 additions & 34 deletions .github/workflows/docs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ jobs:
image: ghcr.io/ptb-mr/mrpro_py311:latest
options: --user root
strategy:
fail-fast: false
matrix:
notebook: ${{ fromJson(needs.get_notebooks.outputs.notebooks) }}
steps:
Expand Down Expand Up @@ -119,14 +120,14 @@ jobs:
run: |
notebook=${{ matrix.notebook }}
echo "ARTIFACT_NAME=${notebook/.ipynb/}" >> $GITHUB_OUTPUT
echo "HTML_RESULT=${notebook/.ipynb/.html}" >> $GITHUB_OUTPUT
echo "IPYNB_EXECUTED=${notebook}" >> $GITHUB_OUTPUT
- name: Upload notebook
uses: actions/upload-artifact@v4
if: always()
with:
name: ${{ steps.artifact_names.outputs.ARTIFACT_NAME }}
path: ${{ github.workspace }}/nb-runner.out/${{ steps.artifact_names.outputs.HTML_RESULT }}
path: ${{ github.workspace }}/nb-runner.out/${{ steps.artifact_names.outputs.IPYNB_EXECUTED }}
env:
RUNNER: ${{ toJson(runner) }}

Expand All @@ -149,39 +150,11 @@ jobs:
- name: Install mrpro and dependencies
run: pip install --upgrade --upgrade-strategy "eager" .[docs]

- name: Download notebook html files
- name: Download executed notebook ipynb files
id: download
uses: actions/download-artifact@v4
with:
path: ./docs/source/notebook_artifact/

- name: Copy notebook html files
run: |
mkdir ./docs/source/_notebooks
cd ./docs/source/notebook_artifact/
notebooks=$(grep -rl --include='*' './')
for nb in $notebooks
do
echo "current jupyter-notebook: $nb"
cp ./$nb ../_notebooks/
done
- name: List of notebooks
run: |
cd ./docs/source/_notebooks/
notebooks=$(grep -rl --include='*.html' './')
cd ../
echo "" >> examples.rst
for nb in $notebooks
do
echo " notebook_${nb/.html/.rst}" >> examples.rst
notebook_description=$(grep '<h1 id=' ./_notebooks/$nb | sed 's/.*">\(.*\)<a class=.*/\1/')
echo $notebook_description
echo $notebook_description > "notebook_${nb/.html/.rst}"
echo "========" >> "notebook_${nb/.html/.rst}"
echo ".. raw:: html" >> "notebook_${nb/.html/.rst}"
echo " :file: ./_notebooks/$nb" >> "notebook_${nb/.html/.rst}"
done
path: ./docs/source/_notebooks/

- name: Build docs
run: |
Expand All @@ -194,7 +167,7 @@ jobs:
with:
name: Documentation
path: docs/build/html/

- run: echo 'Artifact url ${{ steps.save_docu.outputs.artifact-url }}'

- run: echo 'Event number ${{ github.event.number }}'
Expand Down Expand Up @@ -224,7 +197,7 @@ jobs:
deploy:
if: github.ref == 'refs/heads/main'
permissions:
pages: write
pages: write
id-token: write
environment:
name: github-pages
Expand Down
47 changes: 25 additions & 22 deletions .github/workflows/pytest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,21 @@ name: PyTest

on:
pull_request:
push:
branches:
- main

jobs:
get_dockerfiles:
name: Get list of dockerfiles for different containers
name: Get List of Dockerfiles for Containers
runs-on: ubuntu-latest
permissions:
packages: read
outputs:
imagenames: ${{ steps.set-matrix.outputs.imagenames }}
steps:
- id: set-matrix
- name: Retrieve Docker Image Names
id: set-matrix
env:
GH_TOKEN: ${{ secrets.GHCR_TOKEN }}
run: |
Expand All @@ -35,57 +39,56 @@ jobs:
echo "image names with tag latest: $imagenames_latest"
echo "imagenames=$imagenames_latest" >> $GITHUB_OUTPUT
- name: Dockerfile overview
- name: Dockerfile Overview
run: |
echo "final list of images with tag latest: ${{ steps.set-matrix.outputs.imagenames }}"
echo "Final list of images with tag latest: ${{ steps.set-matrix.outputs.imagenames }}"
test:
name: Run tests and get coverage report
name: Run Tests and Coverage Report
needs: get_dockerfiles
runs-on: ubuntu-latest
permissions:
pull-requests: write
contents: write
strategy:
fail-fast: false
matrix:
imagename: ${{ fromJson(needs.get_dockerfiles.outputs.imagenames) }}
# runs within Docker container
container:
image: ghcr.io/ptb-mr/${{ matrix.imagename }}:latest
options: --user runner

steps:
- name: Checkout repo
- name: Checkout Repository
uses: actions/checkout@v4

- name: Install mrpro and dependencies
run: pip install --upgrade --upgrade-strategy "eager" .[test]
- name: Install MRpro and Dependencies
run: pip install --upgrade --upgrade-strategy eager .[test]

- name: Install pytest-github-actions-annotate-failures plugin
- name: Install PyTest GitHub Annotation Plugin
run: pip install pytest-github-actions-annotate-failures

- name: Run PyTest
- name: Run PyTest and Generate Coverage Report
run: |
pytest -n 4 -m "not cuda" --junitxml=pytest.xml --cov-report=term-missing:skip-covered --cov=mrpro | tee pytest-coverage.txt
pytest -n 4 -m "not cuda" --junitxml=pytest.xml \
--cov-report=term-missing:skip-covered --cov=mrpro | tee pytest-coverage.txt
- name: Check for pytest.xml
- name: Verify PyTest XML Output
run: |
if [ -f pytest.xml ]; then
echo "pytest.xml file found. Continuing..."
else
echo "pytest.xml file not found. Please check previous 'Run PyTest' section for errors."
if [ ! -f pytest.xml ]; then
echo "PyTest XML report not found. Please check the previous 'Run PyTest' step for errors."
exit 1
fi
- name: Pytest coverage comment
- name: Post PyTest Coverage Comment
id: coverageComment
uses: MishaKav/pytest-coverage-comment@v1.1.53
with:
pytest-coverage-path: ./pytest-coverage.txt
junitxml-path: ./pytest.xml

- name: Create the Badge
- name: Create Coverage Badge on Main Branch Push
uses: schneegans/dynamic-badges-action@v1.7.0
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
with:
auth: ${{ secrets.GIST_SECRET }}
gistID: 48e334a10caf60e6708d7c712e56d241
Expand All @@ -95,12 +98,12 @@ jobs:
color: ${{ steps.coverageComment.outputs.color }}
namedLogo: python

- name: Set pipeline status
- name: Set Pipeline Status Based on Test Results
if: steps.coverageComment.outputs.errors != 0 || steps.coverageComment.outputs.failures != 0
uses: actions/github-script@v7
with:
script: |
core.setFailed('PyTest workflow failed with ${{ steps.coverageComment.outputs.errors }} errors and ${{ steps.coverageComment.outputs.failures }} failures.')
core.setFailed("PyTest workflow failed with ${{ steps.coverageComment.outputs.errors }} errors and ${{ steps.coverageComment.outputs.failures }} failures.")
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ instance/

# Sphinx documentation
docs/_build/
docs/source/_notebooks/*

# PyBuilder
.pybuilder/
Expand Down
Loading

0 comments on commit 9b84d18

Please sign in to comment.