Skip to content

Commit

Permalink
Merge branch 'develop' of github.com:ecmwf/climetlab into develop
Browse files Browse the repository at this point in the history
  • Loading branch information
b8raoult committed Jun 1, 2024
2 parents 85aad15 + 54c94fd commit 4880ab5
Show file tree
Hide file tree
Showing 378 changed files with 853 additions and 2,522 deletions.
16 changes: 7 additions & 9 deletions .github/workflows/qa.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,12 @@ jobs:
name: Code QA
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- run: pip install black flake8 isort
- run: black --version
- run: isort --version
- run: flake8 --version
- run: isort --check .
- run: black --check .
- run: flake8 .
- run: sudo apt-get install -y pandoc # Needed by sphinx for notebooks
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: 3.x
- uses: pre-commit/action@v3.0.1

short-tests:
name: Short tests
Expand All @@ -44,4 +42,4 @@ jobs:
- name: Install climetlab
run: pip install -e .
- run: pip install pytest
- run: pytest -vv -E short
- run: pytest -vv -E short
120 changes: 39 additions & 81 deletions .github/workflows/test-and-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,34 +35,12 @@ jobs:
name: Code QA
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- run: pip install black flake8 isort
- run: black --version
- run: isort --version
- run: flake8 --version
- run: isort --check .
- run: black --check .
- run: flake8 .

# test_build_doc:
# name: Test building documentation
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v3

# - name: Install packages needed to build the documentation
# run: |
# pip install sphinx
# pip install -r docs/requirements.txt
# sudo apt-get install pandoc
# pip freeze

# - name: Documentation
# run: |
# make clean
# make html
# # make linkcheck (don't run yet)
# working-directory: docs
- run: sudo apt-get install -y pandoc # Needed by sphinx for notebooks
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: 3.x
- uses: pre-commit/action@v3.0.1

download-test-data:
name: Download test data
Expand All @@ -71,7 +49,7 @@ jobs:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
python-version: 3.x

- name: Tokens
shell: python
Expand Down Expand Up @@ -114,7 +92,7 @@ jobs:
with:
python-version: ${{ matrix.python-version }}

- name: Tokens
- name: Tokens
shell: python
env:
ECMWFAPIRC: ${{ secrets.ECMWFAPIRC }}
Expand Down Expand Up @@ -151,7 +129,7 @@ jobs:
with:
python-version: ${{ matrix.python-version }}

- name: Tokens
- name: Tokens
shell: python
env:
ECMWFAPIRC: ${{ secrets.ECMWFAPIRC }}
Expand All @@ -172,19 +150,16 @@ jobs:
pip install pytest
pip freeze
- name: Tests notebooks
run: pytest tests/documentation/test_notebooks.py
run: pytest tests/documentation/test_notebooks.py

short-tests:
# if: (github.event_name == 'release' && github.event.action == 'created') || github.ref == 'refs/heads/main'
# if: github.event_name == 'release' && github.event.action == 'created'
# if: github.ref == 'refs/heads/main'

needs: [download-test-data]
strategy:
matrix:
platform: ["ubuntu-latest", "macos-latest", "windows-latest"]
python-version: ["3.8", "3.9", "3.10"]
# platform: ["ubuntu-latest"]
# python-version: ["3.10"]
python-version: ["3.9", "3.10"]

name: Short tests Python ${{ matrix.python-version }} ${{ matrix.platform }}
runs-on: ${{ matrix.platform }}
steps:
Expand All @@ -199,7 +174,8 @@ jobs:
name: test-data
- run: tar xvf test-data.tar
- run: cat test-data/dataA.txt
- name: Tokens

- name: Tokens
# (this should be removed when tests are refactored)
shell: python
env:
Expand All @@ -213,8 +189,12 @@ jobs:
if os.environ[n]:
with open(m, "w") as f:
print(os.environ[n], file=f)
- run: pip install --upgrade pip

- name: Install climetlab
run: pip install -e .

- run: climetlab versions
- name: Install test tools
run: |
Expand All @@ -231,51 +211,29 @@ jobs:
# notebooks need more dependencies
# pytest --durations=0 -E release -k 'not test_notebooks'
pytest --durations=10 -vv -E short
# - name: Install climetlab full
# run: |
# pip install .[interactive,tensorflow,zarr]
# pip freeze
# - name: Tests with dependencies
# run: |
# pytest --durations=10 -vv -E short
check-version-tag:
if: github.event_name == 'release' && github.event.action == 'created'
name: Check versions and tags
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: "3.8"
- name: Check that tag version matches package version
run: |
tag=${GITHUB_REF#refs/tags/}
version=$(python setup.py --version)
echo 'tag='$tag
echo "version file="$version
test "$tag" == "$version"
deploy:
name: Upload to Pypi and release
needs: [check-version-tag, short-tests, quality, tests-with-external-download, tests-notebooks]
# needs: [check-version-tag, short-tests, quality, test_build_doc, tests-with-external-download, tests-notebooks]
needs: [short-tests, quality, tests-with-external-download, tests-notebooks]
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: "3.9"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install setuptools wheel twine
- name: Build pip package
run: python setup.py sdist
- name: Publish
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
run: twine upload --verbose dist/*
- uses: actions/checkout@v4

- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: 3.x

- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install build wheel twine
- name: Build and publish
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
run: |
python -m build
twine upload dist/*
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -171,3 +171,4 @@ tempCodeRunnerFile.py
*.dot
*.zarr/
testdata/
_version.py
60 changes: 39 additions & 21 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,22 +1,40 @@
exclude: 'experiments'
repos:
- repo: https://github.com/ambv/black
rev: 24.1.1
hooks:
- id: black
language_version: python3.10
- repo: https://github.com/pycqa/isort
rev: 5.13.0
hooks:
- id: isort
name: isort (python)
- id: isort
name: isort (cython)
types: [cython]
- id: isort
name: isort (pyi)
types: [pyi]
- repo: https://github.com/pycqa/flake8
rev: 7.0.0
hooks:
- id: flake8


- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.4.0
hooks:
- id: check-yaml # Check YAML files for syntax errors only
args: [--unsafe, --allow-multiple-documents]
- id: debug-statements # Check for debugger imports and py37+ breakpoint()
- id: end-of-file-fixer # Ensure files end in a newline
- id: trailing-whitespace # Trailing whitespace checker
- id: no-commit-to-branch # Prevent committing to main / master
- id: check-added-large-files # Check for large files added to git
- id: check-merge-conflict # Check for files that contain merge conflict

- repo: https://github.com/psf/black-pre-commit-mirror
rev: 24.1.1
hooks:
- id: black
args: [--line-length=120]

- repo: https://github.com/pycqa/isort
rev: 5.13.2
hooks:
- id: isort
args:
- -l 120
- --force-single-line-imports
- --profile black


- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.3.0
hooks:
- id: ruff
args:
- --line-length=120
- --fix
- --exit-non-zero-on-fix
- --preview
4 changes: 1 addition & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,5 @@ The documentation can be found at <https://climetlab.readthedocs.io/>.
See https://climetlab.readthedocs.io/en/latest/guide/pluginlist.html.

### License
[Apache License 2.0](LICENSE) In applying this licence, ECMWF does not waive the privileges and immunities
[Apache License 2.0](LICENSE) In applying this licence, ECMWF does not waive the privileges and immunities
granted to it by virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.


8 changes: 4 additions & 4 deletions climetlab-and-ecmwflibs-install.md
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
# How to run climetlab without ecmwflibs (using conda)

This workaround should address some of the issues related to ecmwflibs version not found or not installing correctly occuring during the installation of the climetlab package.
This workaround should address some of the issues related to ecmwflibs version not found or not installing correctly occuring during the installation of the climetlab package.
This is how to install eccodes manually and use an environment variable to avoid installing ecmwflibs.

```
```
# install eccodes manually
conda install -c conda-forge eccodes
pip install eccodes
Expand All @@ -15,7 +15,7 @@ CLIMETLAB_DO_NOT_INSTALL_ECMWFLIBS=1 pip install climetlab
# Limitations

Obviously installing climetlab without one of its dependencies has some impact on its features.
And installing eccodes is not enough to provide them all : more packages are bundled in ecmwflibs (other than eccodes).
And installing eccodes is not enough to provide them all : more packages are bundled in ecmwflibs (other than eccodes).
Depending on the required functinalities, there may be other missing packages to install manually.

| Feature | default (ecmwflibs) | With workaround above |
Expand All @@ -29,7 +29,7 @@ All CliMetLab features related to **reading (or writing) GRIB** data be supporte
# Debugging

## Check eccodes installation
```
```
import eccodes
print(eccodes.__file__)
```
Expand Down
1 change: 0 additions & 1 deletion climetlab/version

This file was deleted.

22 changes: 0 additions & 22 deletions climetlab/version.py

This file was deleted.

2 changes: 1 addition & 1 deletion dev/dev.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -94,4 +94,4 @@
},
"nbformat": 4,
"nbformat_minor": 2
}
}
2 changes: 1 addition & 1 deletion dev/in-progress.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -1162,4 +1162,4 @@
},
"nbformat": 4,
"nbformat_minor": 2
}
}
24 changes: 8 additions & 16 deletions dev/tf.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,11 @@
import keras.layers as layers
import numpy as np
import tensorflow as tf
import tensorflow.keras as keras
from tensorflow.keras.callbacks import EarlyStopping, TensorBoard
from tensorflow.keras.layers import (
Conv2D,
Dense,
Dropout,
Flatten,
Input,
Lambda,
Reshape,
)
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.layers import Conv2D
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import Dropout
from tensorflow.keras.layers import Flatten
from tensorflow.keras.layers import Input
from tensorflow.keras.layers import Reshape
from tensorflow.keras.models import Sequential

import climetlab as cml
Expand Down Expand Up @@ -46,9 +40,7 @@ def __init__(self, *args, **kwargs):
def call(self, inputs):
w, h = self.kernel_size
inputs = tf.concat([inputs, inputs[:, :, :w, :]], axis=2)
inputs = tf.pad(
inputs, [[0, 0], [h // 2, h // 2], [0, 0], [0, 0]], constant_values=0
)
inputs = tf.pad(inputs, [[0, 0], [h // 2, h // 2], [0, 0], [0, 0]], constant_values=0)
return super().call(inputs)


Expand Down
Loading

0 comments on commit 4880ab5

Please sign in to comment.