Skip to content

Commit

Permalink
Merge pull request #289 from ecmwf/release/0.5.0
Browse files Browse the repository at this point in the history
Release/0.5.0
  • Loading branch information
sandorkertesz authored Jan 26, 2024
2 parents 17d4f72 + 749a624 commit 826778f
Show file tree
Hide file tree
Showing 140 changed files with 9,406 additions and 1,423 deletions.
9 changes: 9 additions & 0 deletions .github/ci-config.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
dependencies: |
ecmwf/ecbuild
MathisRosenhauer/libaec@master
ecmwf/eccodes
ecmwf/eckit
ecmwf/odc
dependency_branch: develop
parallelism_factor: 8
self_build: false # Only for python packages
24 changes: 24 additions & 0 deletions .github/ci-hpc-config.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
build:
modules:
- ninja
dependencies:
- ecmwf/ecbuild@develop
- MathisRosenhauer/libaec@master
- ecmwf/eccodes@develop
- ecmwf/eckit@develop
- ecmwf/odc@develop
python_dependencies:
- ecmwf/eccodes-python@develop
- ecmwf/cfgrib@master
- ecmwf/pdbufr@master
- ecmwf/pyodc@develop
env:
- ECCODES_SAMPLES_PATH=$ECCODES_DIR/share/eccodes/samples
- ECCODES_DEFINITION_PATH=$ECCODES_DIR/share/eccodes/definitions
parallel: 64
requirements: tests/downstream-ci-requirements.txt
pytest_cmd: |
python -m pytest -vv -m 'not notebook and not no_cache_init' --cov=. --cov-report=xml
python -m pytest -v -m 'notebook'
python -m pytest --forked -vv -m 'no_cache_init'
python -m coverage report
234 changes: 27 additions & 207 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -1,221 +1,41 @@
name: ci

on:
# Trigger the workflow on push to master or develop, except tag creation
push:
branches:
- main
- develop
tags:
- "*"
- 'main'
- 'develop'
tags-ignore:
- '**'

# Trigger the workflow on pull request
pull_request:
branches:
- main
- develop
pull_request_target:
types: [labeled]
workflow_dispatch:

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
# Trigger the workflow manually
workflow_dispatch:

defaults:
run:
shell: bash -l {0}
# Trigger after public PR approved for CI
pull_request_target:
types: [labeled]

jobs:
pre-commit:
if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
ref: ${{ github.event.pull_request.head.sha || github.ref }}
- uses: actions/setup-python@v4
with:
python-version: 3.x
- uses: pre-commit/action@v3.0.0

unit-tests:
name: unit-tests (3.10)
if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v3
with:
ref: ${{ github.event.pull_request.head.sha || github.ref }}
- name: Install Conda environment with Micromamba
uses: mamba-org/provision-with-micromamba@v14
with:
environment-file: tests/environment-unit-tests.yml
environment-name: DEVELOP
channels: conda-forge
cache-env: true
cache-env-key: ubuntu-latest-3.10
extra-specs: |
python=3.10
- name: Install package
run: |
python -m pip install --no-deps -e .
- name: Run tests
run: |
make unit-tests
unit-tests-no-eccodes:
name: unit-tests (3.10)
# Run CI including downstream packages on self-hosted runners
downstream-ci:
name: downstream-ci
if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
runs-on: ubuntu-latest
uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main
with:
earthkit-data: ecmwf/earthkit-data@${{ github.event.pull_request.head.sha || github.sha }}
codecov_upload: true
secrets: inherit

steps:
- uses: actions/checkout@v3
with:
ref: ${{ github.event.pull_request.head.sha || github.ref }}
- name: Install Conda environment with Micromamba
uses: mamba-org/provision-with-micromamba@v14
with:
environment-file: tests/environment-unit-tests.yml
environment-name: DEVELOP
channels: conda-forge
cache-env: true
cache-env-key: ubuntu-latest-3.10-no-eccodes
extra-specs: |
python=3.10
- name: Install package
run: |
python -m pip install --no-deps -e .
micromamba remove eccodes
- name: Run tests without eccodes
run: |
python -m pytest -v -m 'no_eccodes'

type-check:
needs: [unit-tests]
if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
runs-on: ubuntu-latest
defaults:
run:
shell: bash -l {0}

steps:
- uses: actions/checkout@v3
with:
ref: ${{ github.event.pull_request.head.sha || github.ref }}
- name: Install Conda environment with Micromamba
uses: mamba-org/provision-with-micromamba@v12
with:
environment-file: environment.yml
environment-name: DEVELOP
channels: conda-forge
cache-env: true
cache-env-key: ubuntu-latest-3.10
extra-specs: |
python=3.10
- name: Install package
run: |
python -m pip install --no-deps -e .
- name: Run code quality checks
run: |
echo type-check not used
documentation:
needs: [unit-tests]
if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
runs-on: ubuntu-latest
defaults:
run:
shell: bash -l {0}

steps:
- uses: actions/checkout@v3
with:
ref: ${{ github.event.pull_request.head.sha || github.ref }}
- name: Install Conda environment with Micromamba
uses: mamba-org/provision-with-micromamba@v12
with:
environment-file: environment.yml
environment-name: DEVELOP
channels: conda-forge
cache-env: true
cache-env-key: ubuntu-latest-3.10
extra-specs: |
python=3.10
- name: Install package
run: |
python -m pip install --no-deps -e .
- name: Build documentation
run: |
make docs-build
integration-tests:
needs: [unit-tests]
# Build downstream packages on HPC
downstream-ci-hpc:
name: downstream-ci-hpc
if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
runs-on: ubuntu-latest
defaults:
run:
shell: bash -l {0}

strategy:
matrix:
include:
- python-version: "3.10"
# extra: -minver # This will need to be uncommented and environment-minver.yml updated if we want to publish on conda

steps:
- uses: actions/checkout@v3
with:
ref: ${{ github.event.pull_request.head.sha || github.ref }}
- name: Install Conda environment with Micromamba
uses: mamba-org/provision-with-micromamba@v12
with:
environment-file: tests/environment-unit-tests${{ matrix.extra }}.yml
environment-name: DEVELOP${{ matrix.extra }}
channels: conda-forge
cache-env: true
cache-env-key: ubuntu-latest-${{ matrix.python-version }}${{ matrix.extra }}.
extra-specs: |
python=${{matrix.python-version }}
- name: Install package
run: |
python -m pip install --no-deps -e .
- name: Run tests
run: |
make unit-tests
distribution:
needs: [integration-tests, type-check, documentation]
if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v3
with:
ref: ${{ github.event.pull_request.head.sha || github.ref }}
- name: Build distributions
run: |
$CONDA/bin/python -m pip install build
$CONDA/bin/python -m build
- name: Publish a Python distribution to PyPI
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
uses: pypa/gh-action-pypi-publish@release/v1
with:
user: __token__
password: ${{ secrets.PYPI_API_TOKEN }}

notify:
if: always() && ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
needs:
- pre-commit
- unit-tests
- type-check
- documentation
- integration-tests
- distribution
runs-on: ubuntu-latest
steps:
- name: Trigger Teams notification
uses: ecmwf-actions/notify-teams@v1
with:
incoming_webhook: ${{ secrets.MS_TEAMS_INCOMING_WEBHOOK }}
needs_context: ${{ toJSON(needs) }}
uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main
with:
earthkit-data: ecmwf/earthkit-data@${{ github.event.pull_request.head.sha || github.sha }}
secrets: inherit
Loading

0 comments on commit 826778f

Please sign in to comment.