Merge branch 'develop' into cross-calibrate-impact-functions

.github/ISSUE_TEMPLATE/bug_report.md

@@ -12,7 +12,7 @@ A clear and concise description of what the bug is.
 
 **To Reproduce**
 Steps to reproduce the behavior/error:
-1. 
+1.
 
 Code example:
 ```python
@@ -29,7 +29,7 @@ If applicable, add screenshots to help explain your problem.
 
 **System Information (please complete the following information):**
  - Operating system and version: [e.g. Ubuntu 22.04, macOS 14.3.1, Windows 10]
- - Python version: [e.g. 3.10] 
+ - Python version: [e.g. 3.10]
  (to obtain this information execute > import sys  >print(sys.version))
 
 **Additional context**

.github/pull_request_template.md

@@ -1,6 +1,6 @@
 Changes proposed in this PR:
-- 
-- 
+-
+-
 
 This PR fixes #
 

.github/scripts/make_release.py

@@ -13,9 +13,9 @@
 def get_version() -> str:
     """Return the current version number, based on the _version.py file."""
     [version_file] = glob.glob("climada*/_version.py")
-    with open(version_file, 'r', encoding="UTF-8") as vfp:
+    with open(version_file, "r", encoding="UTF-8") as vfp:
         content = vfp.read()
-    regex = r'^__version__\s*=\s*[\'\"](.*)[\'\"]\s*$'
+    regex = r"^__version__\s*=\s*[\'\"](.*)[\'\"]\s*$"
     mtch = re.match(regex, content)
     return mtch.group(1)
 

.github/scripts/prepare_release.py

@@ -5,7 +5,7 @@
 
 - update version numbers in _version.py and setup.py
 - purge the "Unreleased" section of CHANGELOG.md and rename it to the new version number
-- copy the README.md file to doc/misc/README.md, 
+- copy the README.md file to doc/misc/README.md,
   but without the badges as they interfere with the sphinx doc builder
 
 All changes are immediately commited to the repository.
@@ -38,28 +38,28 @@ def bump_version_number(version_number: str, level: str) -> str:
     """Return a copy of `version_number` with one level number incremented."""
     major, minor, patch = version_number.split(".")
     if level == "major":
-        major = str(int(major)+1)
+        major = str(int(major) + 1)
         minor = "0"
         patch = "0"
     elif level == "minor":
-        minor = str(int(minor)+1)
+        minor = str(int(minor) + 1)
         patch = "0"
     elif level == "patch":
-        patch = str(int(patch)+1)
+        patch = str(int(patch) + 1)
     else:
         raise ValueError(f"level should be 'major', 'minor' or 'patch', not {level}")
     return ".".join([major, minor, patch])
 
 
 def update_readme(_nvn):
-    """align doc/misc/README.md with ./README.md but remove the non-markdown header lines from """
-    with open("README.md", 'r', encoding="UTF-8") as rmin:
-        lines = [line for line in rmin.readlines() if not line.startswith('[![')]
+    """align doc/misc/README.md with ./README.md but remove the non-markdown header lines from"""
+    with open("README.md", "r", encoding="UTF-8") as rmin:
+        lines = [line for line in rmin.readlines() if not line.startswith("[![")]
     while not lines[0].strip():
         lines = lines[1:]
-    with open("doc/misc/README.md", 'w', encoding="UTF-8") as rmout:
+    with open("doc/misc/README.md", "w", encoding="UTF-8") as rmout:
         rmout.writelines(lines)
-    return GitFile('doc/misc/README.md')
+    return GitFile("doc/misc/README.md")
 
 
 def update_changelog(nvn):
@@ -70,16 +70,16 @@ def update_changelog(nvn):
     release = []
     section_name = None
     section = []
-    with open("CHANGELOG.md", 'r', encoding="UTF-8") as changelog:
+    with open("CHANGELOG.md", "r", encoding="UTF-8") as changelog:
         for line in changelog.readlines():
-            if line.startswith('#'):
-                if line.startswith('### '):
+            if line.startswith("#"):
+                if line.startswith("### "):
                     if section:
                         release.append((section_name, section))
                     section_name = line[4:].strip()
                     section = []
-                    #print("tag:", section_name)
-                elif line.startswith('## '):
+                    # print("tag:", section_name)
+                elif line.startswith("## "):
                     if section:
                         release.append((section_name, section))
                     if release:
@@ -88,15 +88,15 @@ def update_changelog(nvn):
                     release = []
                     section_name = None
                     section = []
-                    #print("release:", release_name)
+                    # print("release:", release_name)
             else:
                 section.append(line)
         if section:
             release.append((section_name, section))
         if release:
             releases.append((release_name, release))
 
-    with open("CHANGELOG.md", 'w', encoding="UTF-8") as changelog:
+    with open("CHANGELOG.md", "w", encoding="UTF-8") as changelog:
         changelog.write("# Changelog\n\n")
         for release_name, release in releases:
             if release_name:
@@ -107,7 +107,11 @@ def update_changelog(nvn):
                 if any(ln.strip() for ln in section):
                     if section_name:
                         changelog.write(f"### {section_name}\n")
-                    lines = [ln.strip() for ln in section if "code freeze date: " not in ln.lower()]
+                    lines = [
+                        ln.strip()
+                        for ln in section
+                        if "code freeze date: " not in ln.lower()
+                    ]
                     if not section_name and release_name.lower() == nvn:
                         print("setting date")
                         for i, line in enumerate(lines):
@@ -116,26 +120,26 @@ def update_changelog(nvn):
                                 lines[i] = f"Release date: {today}"
                     changelog.write(re.sub("\n+$", "\n", "\n".join(lines)))
                     changelog.write("\n")
-    return GitFile('CHANGELOG.md')
+    return GitFile("CHANGELOG.md")
 
 
 def update_version(nvn):
     """Update the _version.py file"""
     [file_with_version] = glob.glob("climada*/_version.py")
-    regex = r'(^__version__\s*=\s*[\'\"]).*([\'\"]\s*$)'
+    regex = r"(^__version__\s*=\s*[\'\"]).*([\'\"]\s*$)"
     return update_file(file_with_version, regex, nvn)
 
 
 def update_setup(new_version_number):
     """Update the setup.py file"""
     file_with_version = "setup.py"
-    regex = r'(^\s+version\s*=\s*[\'\"]).*([\'\"]\s*,\s*$)'
+    regex = r"(^\s+version\s*=\s*[\'\"]).*([\'\"]\s*,\s*$)"
     return update_file(file_with_version, regex, new_version_number)
 
 
 def update_file(file_with_version, regex, new_version_number):
     """Replace the version number(s) in a file, based on a rgular expression."""
-    with open(file_with_version, 'r', encoding="UTF-8") as curf:
+    with open(file_with_version, "r", encoding="UTF-8") as curf:
         lines = curf.readlines()
     successfully_updated = False
     for i, line in enumerate(lines):
@@ -145,14 +149,15 @@ def update_file(file_with_version, regex, new_version_number):
             successfully_updated = True
     if not successfully_updated:
         raise RuntimeError(f"cannot determine version of {file_with_version}")
-    with open(file_with_version, 'w', encoding="UTF-8") as newf:
+    with open(file_with_version, "w", encoding="UTF-8") as newf:
         for line in lines:
             newf.write(line)
     return GitFile(file_with_version)
 
 
-class GitFile():
+class GitFile:
     """Helper class for `git add`."""
+
     def __init__(self, path):
         self.path = path
 
@@ -166,8 +171,9 @@ def gitadd(self):
         ).stdout.decode("utf8")
 
 
-class Git():
+class Git:
     """Helper class for `git commit`."""
+
     def __init__(self):
         _gitname = subprocess.run(
             ["git", "config", "--global", "user.name", "'climada'"],
@@ -228,6 +234,7 @@ def prepare_new_release(level):
 
 if __name__ == "__main__":
     from sys import argv
+
     try:
         LEVEL = argv[1]
     except IndexError:

.github/scripts/setup_devbranch.py

@@ -33,14 +33,15 @@ def get_last_version() -> str:
 
 def update_changelog():
     """Insert a vanilla "Unreleased" section on top."""
-    with open("CHANGELOG.md", 'r', encoding="UTF-8") as changelog:
+    with open("CHANGELOG.md", "r", encoding="UTF-8") as changelog:
         lines = changelog.readlines()
 
     if "## Unreleased" in lines:
         return
 
-    with open("CHANGELOG.md", 'w', encoding="UTF-8") as changelog:
-        changelog.write("""# Changelog
+    with open("CHANGELOG.md", "w", encoding="UTF-8") as changelog:
+        changelog.write(
+            """# Changelog
 
 ## Unreleased
 
@@ -62,27 +63,28 @@ def update_changelog():
 
 ### Removed
 
-""")
+"""
+        )
         changelog.writelines(lines[2:])
 
 
 def update_version(nvn):
     """Update the _version.py file"""
     [file_with_version] = glob.glob("climada*/_version.py")
-    regex = r'(^__version__\s*=\s*[\'\"]).*([\'\"]\s*$)'
+    regex = r"(^__version__\s*=\s*[\'\"]).*([\'\"]\s*$)"
     return update_file(file_with_version, regex, nvn)
 
 
 def update_setup(new_version_number):
     """Update the setup.py file"""
     file_with_version = "setup.py"
-    regex = r'(^\s+version\s*=\s*[\'\"]).*([\'\"]\s*,\s*$)'
+    regex = r"(^\s+version\s*=\s*[\'\"]).*([\'\"]\s*,\s*$)"
     return update_file(file_with_version, regex, new_version_number)
 
 
 def update_file(file_with_version, regex, new_version_number):
     """Replace the version number(s) in a file, based on a rgular expression."""
-    with open(file_with_version, 'r', encoding="UTF-8") as curf:
+    with open(file_with_version, "r", encoding="UTF-8") as curf:
         lines = curf.readlines()
     successfully_updated = False
     for i, line in enumerate(lines):
@@ -92,18 +94,18 @@ def update_file(file_with_version, regex, new_version_number):
             successfully_updated = True
     if not successfully_updated:
         raise RuntimeError(f"cannot determine version of {file_with_version}")
-    with open(file_with_version, 'w', encoding="UTF-8") as newf:
+    with open(file_with_version, "w", encoding="UTF-8") as newf:
         for line in lines:
             newf.write(line)
 
 
 def setup_devbranch():
     """Adjust files after a release was published, i.e.,
     apply the canonical deviations from main in develop.
-    
+
     Just changes files, all `git` commands are in the setup_devbranch.sh file.
     """
-    main_version = get_last_version().strip('v')
+    main_version = get_last_version().strip("v")
     semver = main_version.split(".")
     semver[-1] = f"{int(semver[-1]) + 1}-dev"
     dev_version = ".".join(semver)

.github/workflows/ci.yml

@@ -12,6 +12,7 @@ jobs:
   build-and-test:
     name: 'Core / Unit Test Pipeline'
     runs-on: ubuntu-latest
+    timeout-minutes: 20
     permissions:
       # For publishing results
       checks: write

.github/workflows/pull-request.yml

@@ -0,0 +1,50 @@
+name: GitHub CI
+
+# Execute this for every pull request (opened, reopened, and synchronized)
+on: [pull_request]
+
+jobs:
+  pre-commit-checks:
+    name: 'Core / Pre-Commit Checks'
+    runs-on: ubuntu-latest
+
+    steps:
+      -
+        name: Checkout Repo
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      -
+        name: Checkout target commit
+        run: git -c protocol.version=2 fetch --no-tags --prune --no-recurse-submodules --depth=1 origin ${{ github.event.pull_request.base.ref }}
+      -
+        name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      -
+        # Store the current date to use it as cache key
+        name: Get current date
+        id: date
+        run: echo "date=$(date +%Y-%m-%d)" >> "${GITHUB_OUTPUT}"
+      -
+        name: Cache dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/pip
+            ~/.cache/pre-commit
+          key: ${{ github.event.pull_request.head.ref }}-${{ steps.date.outputs.date }}
+          restore-keys: |
+            ${{ github.event.pull_request.head.ref }}
+            ${{ github.event.pull_request.base.ref }}
+            develop
+            main
+      -
+        name: Install pre-commit and hooks
+        run: |
+          pip install pre-commit
+          pre-commit install --install-hooks
+      -
+        name: Run pre-commit checks
+        run: pre-commit run --show-diff-on-failure --from-ref ${{ github.event.pull_request.base.sha }} --to-ref ${{ github.event.pull_request.head.sha }}

.gitignore

@@ -176,3 +176,10 @@ data/ISIMIP_crop/
 
 # climada data results folder:
 results/
+
+# Hidden files we want to track
+!.gitignore
+!.pre-commit-config.yaml
+!.pylintrc
+!.readthedocs.yml
+!.github

.pre-commit-config.yaml

@@ -0,0 +1,23 @@
+# See https://pre-commit.com for more information
+default_language_version:
+  python: python3
+
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.2.0
+    hooks:
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+
+  - repo: https://github.com/pycqa/isort
+    rev: '5.13.2'
+    hooks:
+      - id: isort
+        args: ["--profile", "black", "--filter-files"]
+
+  # Using this mirror lets us use mypyc-compiled black, which is about 2x faster
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: '24.4.2'
+    hooks:
+      - id: black-jupyter