Refactor significant portions of the codebase and refactor some land-atm handlers

.github/workflows/build_workflow.yml

@@ -23,7 +23,7 @@ jobs:
  shell: bash -l {0}
  strategy:
  matrix:
- python-version: ["3.9", "3.10"]
+ python-version: ["3.9", "3.10", "3.11"]
  steps:
  - id: skip_check
  uses: fkirc/skip-duplicate-actions@master
@@ -62,28 +62,28 @@ jobs:
  name: Cache Conda env
  uses: actions/cache@v3
  with:
-  path: ${{ env.CONDA }}/envs
-  key:
-  conda-${{ runner.os }}-${{ runner.arch }}-${{ matrix.python-version }}-${{
-  steps.get-date.outputs.today }}-${{hashFiles('conda-env/ci.yml') }}-${{ env.CACHE_NUMBER}}
+ path: ${{ env.CONDA }}/envs
+ key:
+ conda-${{ runner.os }}-${{ runner.arch }}-${{ matrix.python-version }}-${{
+ steps.get-date.outputs.today }}-${{hashFiles('conda-env/ci.yml') }}-${{ env.CACHE_NUMBER}}
  env:
-  # Increase this value to reset cache if conda-env/ci.yml has not changed in the workflow
-  CACHE_NUMBER: 0
+ # Increase this value to reset cache if conda-env/ci.yml has not changed in the workflow
+ CACHE_NUMBER: 0
 
  - if: |
-  steps.skip_check.outputs.should_skip == 'false' &&
-  steps.cache.outputs.cache-hit == 'false'
+ steps.skip_check.outputs.should_skip == 'false' &&
+ steps.cache.outputs.cache-hit == 'false'
  name: Update environment
  run: |
-  mamba env update -n e3sm_to_cmip_ci -f conda-env/ci.yml
-  # Make sure the Python version in the env matches the current matrix version.
-  mamba install -c conda-forge python=${{ matrix.python-version }}
+ mamba env update -n e3sm_to_cmip_ci -f conda-env/ci.yml
+ # Make sure the Python version in the env matches the current matrix version.
+ mamba install -c conda-forge python=${{ matrix.python-version }}
 
  - if: ${{ steps.skip_check.outputs.should_skip == 'false' }}
  name: Install e3sm_to_cmip
  # Source: https://github.com/conda/conda-build/issues/4251#issuecomment-1053460542
  run: |
-  python -m pip install --no-build-isolation --no-deps -e .
+ python -m pip install --no-build-isolation --no-deps -e .
 
  - if: ${{ steps.skip_check.outputs.should_skip != 'true' }}
  name: Check conda env

.pre-commit-config.yaml

@@ -35,4 +35,11 @@ repos:
  - id: mypy
  args: ["--config=pyproject.toml"]
  additional_dependencies:
- [dask, numpy>=1.23.0, xarray>=2023.3.0, types-pytz==2023.3.0.0, types-PyYAML==6.0.12, types-tqdm]
+ [
+ dask,
+ numpy>=1.23.0,
+ xarray>=2023.3.0,
+ types-pytz,
+ types-PyYAML,
+ types-tqdm,
+ ]

.vscode/e3sm_to_cmip.code-workspace

@@ -2,52 +2,71 @@
 // Simply open up this file in VS Code and the editor will be automatically configured using this file.
 // Workspace settings take precedence over your user settings.
 {
- "folders": [
- {
- "path": ".."
- }
- ],
- "settings": {
- // ===================
- // Editor settings
- // ===================
- "editor.formatOnSave": true,
- // ===================
- // Python settings
- // ===================
- "[python]": {
- // editor.rulers: [comments, max line length, wrap line length],
- // Black does not wrap comments.
- "editor.rulers": [80, 88, 120],
- "editor.wordWrap": "wordWrapColumn",
- "editor.wordWrapColumn": 120,
- "editor.defaultFormatter": "ms-python.black-formatter"
- },
- "black-formatter.importStrategy": "fromEnvironment",
- // Code Formatting and Linting
- // ---------------------------
- "flake8.args": ["--config=setup.cfg"],
- "flake8.importStrategy": "fromEnvironment",
- // Type checking
- // ---------------------------
- "mypy-type-checker.args": ["--config=pyproject.toml"],
- "mypy-type-checker.importStrategy": "fromEnvironment",
- // Testing
- // ---------------------------
- "python.testing.unittestEnabled": false,
- "python.testing.pytestEnabled": true,
- // NOTE: Debugger doesn't work if pytest-cov is enabled, so set "--no-cov"
- // https://github.com/microsoft/vscode-python/issues/693
- "python.testing.pytestArgs": ["--no-cov"],
- // ===================
- // Extension settings
- // ===================
- "jupyter.notebookFileRoot": "${workspaceFolder}",
- "autoDocstring.docstringFormat": "numpy",
- "[restructuredtext]": {
- "editor.rulers": [88, 120],
- "editor.wordWrap": "wordWrapColumn",
- "editor.wordWrapColumn": 120
- }
+ "folders": [
+ {
+ "path": ".."
  }
+ ],
+ "settings": {
+ // ===================
+ // Editor settings
+ // ===================
+ "editor.formatOnSave": true,
+ // ===================
+ // Python settings
+ // ===================
+ "[python]": {
+ // editor.rulers: [comments, max line length, wrap line length],
+ // Black does not wrap comments.
+ "editor.rulers": [80, 88, 120],
+ "editor.wordWrap": "wordWrapColumn",
+ "editor.wordWrapColumn": 120,
+ "editor.defaultFormatter": "ms-python.black-formatter"
+ },
+ "black-formatter.importStrategy": "fromEnvironment",
+ // Code Formatting and Linting
+ // ---------------------------
+ "flake8.args": ["--config=setup.cfg"],
+ "flake8.importStrategy": "fromEnvironment",
+ // Type checking
+ // ---------------------------
+ "mypy-type-checker.args": ["--config=pyproject.toml"],
+ "mypy-type-checker.importStrategy": "fromEnvironment",
+ // Testing
+ // ---------------------------
+ "python.testing.unittestEnabled": false,
+ "python.testing.pytestEnabled": true,
+ // NOTE: Debugger doesn't work if pytest-cov is enabled, so set "--no-cov"
+ // https://github.com/microsoft/vscode-python/issues/693
+ "python.testing.pytestArgs": ["--no-cov"],
+ // ===================
+ // Extension settings
+ // ===================
+ "jupyter.notebookFileRoot": "${workspaceFolder}",
+ "autoDocstring.docstringFormat": "numpy",
+ "[restructuredtext]": {
+ "editor.rulers": [88, 120],
+ "editor.wordWrap": "wordWrapColumn",
+ "editor.wordWrapColumn": 120
+ }
+ },
+ // =====================================
+ // VS Code Python Debugger Configuration
+ // =====================================
+ "launch": {
+ "version": "0.2.0",
+ "configurations": [
+ {
+ "name": "Python: Current File",
+ "type": "python",
+ "request": "launch",
+ "program": "${file}",
+ "console": "integratedTerminal",
+ "justMyCode": true,
+ "env": {
+ "PYTHONPATH": "${workspaceFolder}"
+ }
+ }
+ ]
+ }
 }

Makefile

@@ -0,0 +1,87 @@
+.PHONY: clean clean-test clean-pyc clean-build docs help
+.DEFAULT_GOAL := help
+
+define BROWSER_PYSCRIPT
+import os, webbrowser, sys
+
+from urllib.request import pathname2url
+
+webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1])))
+endef
+export BROWSER_PYSCRIPT
+
+define PRINT_HELP_PYSCRIPT
+import re, sys
+
+for line in sys.stdin:
+ match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line)
+ if match:
+ target, help = match.groups()
+ print("%-20s %s" % (target, help))
+endef
+export PRINT_HELP_PYSCRIPT
+
+BROWSER := python -c "$$BROWSER_PYSCRIPT"
+
+# To run these commands: make <COMMAND>
+# ==================================================
+
+help:
+ @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST)
+
+# Clean local repository
+# ----------------------
+clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts
+
+clean-build: ## remove build artifacts
+ rm -fr build/
+ rm -fr conda-build/
+ rm -fr dist/
+ rm -fr .eggs/
+ find . -name '*.egg-info' -exec rm -fr {} +
+ find . -name '*.egg' -exec rm -f {} +
+
+clean-pyc: ## remove Python file artifacts
+ find . -name '*.pyc' -exec rm -f {} +
+ find . -name '*.pyo' -exec rm -f {} +
+ find . -name '*~' -exec rm -f {} +
+ find . -name '__pycache__' -exec rm -fr {} +
+
+clean-test: ## remove test and coverage artifacts
+ rm -fr tests_coverage_reports/
+ rm -f .coverage
+ rm -fr htmlcov/
+ rm -f coverage.xml
+ rm -fr .pytest_cache
+ rm -rf .mypy_cache
+
+clean-logs:
+ rm cmor_logs/*
+ rm logs/*
+
+
+# Quality Assurance
+# ----------------------
+pre-commit: # run pre-commit quality assurance checks
+ pre-commit run --all-files
+
+lint: ## check style with flake8
+ flake8 e3sm_to_cmip tests
+
+test: ## run tests quickly with the default Python and produces code coverage report
+ pytest
+ $(BROWSER) tests_coverage_reports/htmlcov/index.html
+
+# Documentation
+# ----------------------
+docs: ## generate Sphinx HTML documentation, including API docs
+ rm -rf docs/generated
+ cd docs && make html
+ $(MAKE) -C docs clean
+ $(MAKE) -C docs html
+ $(BROWSER) docs/_build/html/index.html
+
+# Build
+# ----------------------
+install: clean ## install the package to the active Python's site-packages
+ python -m pip install .

conda-env/ci.yml

@@ -7,8 +7,6 @@ dependencies:
  # Base - required for building the repository as an Anaconda package
  # ====================
  - python >=3.9
- - cdms2=3.1.5
- - cdutil=8.2.1
  - cmor >=3.7.0
  - dask
  - nco >=5.1.4

conda-env/dev.yml

@@ -7,8 +7,6 @@ dependencies:
  # Base - required for building the repository as an Anaconda package
  # ====================
  - python >=3.9
- - cdms2=3.1.5
- - cdutil=8.2.1
  - cmor >=3.7.0
  - dask
  - nco >=5.1.4