From 205e38f4f3181fa9293b357f46b06436ffec6ef3 Mon Sep 17 00:00:00 2001 From: Brewster Malevich Date: Thu, 24 Mar 2022 18:01:49 -0700 Subject: [PATCH 1/3] Drop DtrRun, add types, add mypy type checking in CI --- .github/workflows/test.yaml | 3 ++ CHANGELOG.md | 4 +++ pyproject.toml | 8 ++++++ requirements-dev.txt | 1 + src/dearprudence/core.py | 10 +------ src/dearprudence/io.py | 57 ++++++++++++++++++++++--------------- src/dearprudence/utils.py | 5 ++-- 7 files changed, 54 insertions(+), 34 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 4f2026c..1e95b89 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -37,6 +37,9 @@ jobs: - name: Format check with flake8 run: | flake8 + - name: Type check with mypy + run: | + mypy src/dearprudence - name: Test with pytest run: | pytest -v --cov=./src/dearprudence --cov-report term-missing diff --git a/CHANGELOG.md b/CHANGELOG.md index d94e3d6..9739998 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,3 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- Type annotations and mypy testing in CI. +### Removed +- `DtrRun` has been removed. Support for DTR files has been dropped. diff --git a/pyproject.toml b/pyproject.toml index 574a531..4aac22c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,3 +10,11 @@ build-backend = "setuptools.build_meta" [tool.setuptools_scm] fallback_version = "999" write_to = "src/dearprudence/_version.py" + +[tool.mypy] +python_version = "3.10" +warn_unused_configs = true + +[[tool.mypy.overrides]] +module = "intake" +ignore_missing_imports = true diff --git a/requirements-dev.txt b/requirements-dev.txt index 50df77b..4b971f7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,6 +2,7 @@ black build flake8 intake_esm +mypy pytest pytest-cov twine diff --git a/src/dearprudence/core.py b/src/dearprudence/core.py index 019b603..8583ac6 100644 --- a/src/dearprudence/core.py +++ b/src/dearprudence/core.py @@ -1,7 +1,7 @@ from dataclasses import dataclass -__all__ = ["Cmip6Record", "SimpleRun", "DtrRun"] +__all__ = ["Cmip6Record", "SimpleRun"] @dataclass @@ -23,11 +23,3 @@ class SimpleRun: variable_id: str historical: Cmip6Record ssp: Cmip6Record - - -@dataclass -class DtrRun: - target: str - variable_id: str - tasmin: SimpleRun - tasmax: SimpleRun diff --git a/src/dearprudence/io.py b/src/dearprudence/io.py index 1aff059..cacea1c 100644 --- a/src/dearprudence/io.py +++ b/src/dearprudence/io.py @@ -1,13 +1,35 @@ import dataclasses import json +from os import PathLike +from typing import Union, TextIO, BinaryIO, Any, TypedDict, Sequence -from dearprudence.core import Cmip6Record, DtrRun, SimpleRun +from dearprudence.core import Cmip6Record, SimpleRun __all__ = ["read_params", "write_params"] -def _load_paramfile(urlpath): +# TypedDict mappings representing intermediate dicts loaded from JSON. +class Cmip6RecordMapping(TypedDict): + activity_id: str + experiment_id: str + table_id: str + variable_id: str + source_id: str + institution_id: str + member_id: str + grid_label: str + version: str + + +class SimpleRunMapping(TypedDict): + variable_id: str + target: str + historical: Cmip6RecordMapping + ssp: Cmip6RecordMapping + + +def _load_paramfile(urlpath: Union[str, Union[TextIO, BinaryIO]]) -> Sequence[SimpleRunMapping]: # First readline() to pop-off and discard the first yaml bit of # the file, then load as JSON str. Keeps us from depending # on pyyaml. @@ -22,7 +44,7 @@ def _load_paramfile(urlpath): return json.load(urlpath) -def _unpack_simplerun(p): +def _unpack_simplerun(p: SimpleRunMapping) -> SimpleRun: return SimpleRun( target=p["target"], variable_id=p["variable_id"], @@ -31,37 +53,26 @@ def _unpack_simplerun(p): ) -def read_params(urlpath): +def read_params(urlpath: Union[str, Union[TextIO, BinaryIO]]) -> list[SimpleRun]: """Read run parameters form yaml file""" - payload = _load_paramfile(urlpath) - - out = [] - for entry in payload: - if entry["variable_id"].lower() == "dtr": - out.append( - DtrRun( - target=entry["target"], - variable_id=entry["variable_id"], - tasmin=_unpack_simplerun(entry["tasmin"]), - tasmax=_unpack_simplerun(entry["tasmax"]), - ) - ) - else: - out.append(_unpack_simplerun(entry)) - - return out + return [_unpack_simplerun(x) for x in _load_paramfile(urlpath)] class _DataclassJSONEncoder(json.JSONEncoder): """Encoder to dump dataclasses to JSON""" - def default(self, o): + def default(self, o: Any) -> Any: if dataclasses.is_dataclass(o): return dataclasses.asdict(o) return super().default(o) -def write_params(urlpath, runlist, mode="w", pretty=True): +def write_params( + urlpath: Union[Union[str, bytes, PathLike[str], PathLike[bytes]], int], + runlist: Sequence[SimpleRun], + mode: str = "w", + pretty: bool = True, +) -> None: """Write runs parameters to parameter file""" runlist = list(runlist) diff --git a/src/dearprudence/utils.py b/src/dearprudence/utils.py index 5d2c615..e13a465 100644 --- a/src/dearprudence/utils.py +++ b/src/dearprudence/utils.py @@ -1,5 +1,6 @@ from functools import cache +from dearprudence.core import Cmip6Record from dearprudence.errors import ( Cmip6CatalogNoEntriesError, Cmip6CatalogMultipleEntriesError, @@ -11,7 +12,7 @@ @cache def esm_datastore( - json_url="https://storage.googleapis.com/cmip6/pangeo-cmip6-noQC.json", + json_url: str = "https://storage.googleapis.com/cmip6/pangeo-cmip6-noQC.json", ): """ Sugar to create an intake_esm.core.esm_datastore to pass into `in_cmip6_catalog` @@ -31,7 +32,7 @@ def esm_datastore( return intake.open_esm_datastore(json_url) -def cmip6_catalog_has(x, datastore=None): +def cmip6_catalog_has(x: Cmip6Record, datastore=None) -> bool: """Check that Cmip6Record has an entry in CMIP6-In-The-Cloud catalog This requires the ``intake-esm`` package to be installed. From 9a50dce756801f739e38a2d8fefef6b8f9d32208 Mon Sep 17 00:00:00 2001 From: Brewster Malevich Date: Thu, 24 Mar 2022 18:04:18 -0700 Subject: [PATCH 2/3] Format cleanup --- src/dearprudence/io.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/dearprudence/io.py b/src/dearprudence/io.py index cacea1c..e8c9c1c 100644 --- a/src/dearprudence/io.py +++ b/src/dearprudence/io.py @@ -29,7 +29,9 @@ class SimpleRunMapping(TypedDict): ssp: Cmip6RecordMapping -def _load_paramfile(urlpath: Union[str, Union[TextIO, BinaryIO]]) -> Sequence[SimpleRunMapping]: +def _load_paramfile( + urlpath: Union[str, Union[TextIO, BinaryIO]] +) -> Sequence[SimpleRunMapping]: # First readline() to pop-off and discard the first yaml bit of # the file, then load as JSON str. Keeps us from depending # on pyyaml. From 6b86c33663e7abe146ecb411f3a1de470efaea0e Mon Sep 17 00:00:00 2001 From: Brewster Malevich Date: Thu, 24 Mar 2022 18:10:09 -0700 Subject: [PATCH 3/3] Minor correction to bad CI job name --- .github/workflows/test.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1e95b89..0c18662 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -2,12 +2,14 @@ name: Test on: push: - branches: "main" + branches: + - "main" pull_request: - branches: "main" + branches: + - "main" jobs: - build: + test: runs-on: ${{ matrix.os }} strategy: matrix: