diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 07e49a05..ca28a09b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,12 +13,20 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.9", "3.10"] - numpy_ver: [latest] + python-version: ["3.10", "3.11"] + numpy_ver: ["latest"] + test_config: ["latest"] include: - - python-version: "3.8" - numpy_ver: "1.20.3" - os: "ubuntu-latest" + # NEP29 compliance settings + - python-version: "3.9" + numpy_ver: "1.21" + os: ubuntu-latest + test_config: "NEP29" + # Operational compliance settings + - python-version: "3.6.8" + numpy_ver: "1.19.5" + os: "ubuntu-20.04" + test_config: "Ops" name: Python ${{ matrix.python-version }} on ${{ matrix.os }} with numpy ${{ matrix.numpy_ver }} runs-on: ${{ matrix.os }} @@ -29,15 +37,29 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install standard dependencies + - name: Install Operational dependencies + if: ${{ matrix.test_config == 'Ops'}} run: | - pip install -r test_requirements.txt + pip install numpy==${{ matrix.numpy_ver }} + pip install "cdflib<1.0" pip install -r requirements.txt + pip install -r test_requirements.txt + pip install . - name: Install NEP29 dependencies - if: ${{ matrix.numpy_ver != 'latest'}} + if: ${{ matrix.test_config == 'NEP29'}} run: | pip install numpy==${{ matrix.numpy_ver }} + pip install --upgrade-strategy only-if-needed -r requirements.txt + pip install --upgrade-strategy only-if-needed -r test_requirements.txt + pip install . + + - name: Install standard dependencies + if: ${{ matrix.test_config == 'latest'}} + run: | + pip install -r test_requirements.txt + pip install -r requirements.txt + pip install . - name: Set up pysat run: | @@ -50,6 +72,14 @@ jobs: - name: Evaluate complexity run: flake8 . --count --exit-zero --max-complexity=10 --statistics + - name: Install Operational dependencies success + if: ${{ matrix.test_config == 'Ops'}} + run: | + pip install pysatNASA==0.0.5 + which python + python --version + python -c "import pysatNASA; print(pysatNASA.__version__)" + - name: Run unit and integration tests run: pytest --cov=pysatModels/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 92683bb5..479dfd15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Added manual test for pysat and pysatNASA Release Candidates * Added manual test for pysatModels RC pip install * Updated tests to new pysat and pytest standards - * Added a cap for pysatNASA + * Updated model Instruments to include the new `_clean_warn` attribute * Removed backwards-support for pysat pre-3.0.4 functions * Documentation * Added badges and instructions for PyPi and Zenodo diff --git a/README.md b/README.md index 22cc372a..560cfc77 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ the Space Physics community. This module officially supports Python 3.6+. | ------------------ | ------------------ | | numpy | pyForecastTools | | pandas >= 1.4.0 | pysat >= 3.0.4 | -| requests | pysatNASA <= 0.0.4 | +| requests | pysatNASA | | scipy | | | xarray | | diff --git a/pysatModels/models/methods/general.py b/pysatModels/models/methods/general.py index 987bf4a9..bdc6c9fa 100644 --- a/pysatModels/models/methods/general.py +++ b/pysatModels/models/methods/general.py @@ -7,6 +7,10 @@ import pysatModels +clean_warn = {clean_level: [('logger', 'INFO', 'Cleaning not supported', + clean_level)] + for clean_level in ['clean', 'dusty', 'dirty']} + def clean(inst): """Raise a low-level log message about lack of cleaning.""" diff --git a/pysatModels/models/pydineof_dineof.py b/pysatModels/models/pydineof_dineof.py index 23abad97..6d7fc5b9 100644 --- a/pysatModels/models/pydineof_dineof.py +++ b/pysatModels/models/pydineof_dineof.py @@ -69,6 +69,7 @@ _test_dates = {'': {tag: dt.datetime(2009, 1, 1) for tag in tags.keys()}} _test_download = {'': {'': False, 'test': True}} +_clean_warn = {'': {tag: general.clean_warn for tag in tags.keys()}} # ---------------------------------------------------------------------------- # Instrument methods diff --git a/pysatModels/models/sami2py_sami2.py b/pysatModels/models/sami2py_sami2.py index 8072dbed..d7a02392 100644 --- a/pysatModels/models/sami2py_sami2.py +++ b/pysatModels/models/sami2py_sami2.py @@ -49,6 +49,7 @@ _test_dates = {'': {tag: dt.datetime(2019, 1, 1) for tag in tags.keys()}} _test_download = {'': {'': False, 'test': True}} +_clean_warn = {'': {tag: general.clean_warn for tag in tags.keys()}} # ---------------------------------------------------------------------------- # Instrument methods diff --git a/pysatModels/models/ucar_tiegcm.py b/pysatModels/models/ucar_tiegcm.py index 7d4f1653..1bb827eb 100644 --- a/pysatModels/models/ucar_tiegcm.py +++ b/pysatModels/models/ucar_tiegcm.py @@ -54,6 +54,7 @@ _test_dates = {'': {'': dt.datetime(2019, 1, 1), 'icon': dt.datetime(2020, 1, 10)}} _test_download = {'': {'': False, 'icon': True}} +_clean_warn = {'': {tag: general.clean_warn for tag in tags.keys()}} # ---------------------------------------------------------------------------- # Instrument methods @@ -248,7 +249,7 @@ def download(date_array, tag, inst_id, data_path=None, **kwargs): temp_dir = tempfile.TemporaryDirectory() # Download using NASA CDAWeb methods in pysatNASA - cdw.download(date_array, tag, inst_id, data_path=temp_dir.name, + cdw.download(date_array, temp_dir.name, tag=tag, inst_id=inst_id, supported_tags=download_tags) # Get a list of files in `temp_dir` diff --git a/pysatModels/utils/match.py b/pysatModels/utils/match.py index 2c0534c9..b3542d31 100644 --- a/pysatModels/utils/match.py +++ b/pysatModels/utils/match.py @@ -162,8 +162,13 @@ def collect_inst_model_pairs(start, stop, tinc, inst, inst_download_kwargs=None, # Download the instrument data, if needed and wanted if not skip_download and (stop - start).days != len(inst.files[start:stop]): - missing_times = [tt for tt in pds.date_range(start, stop, freq='1D', - inclusive='left') + try: + missing_date_range = pds.date_range(start, stop, freq='1D', + inclusive='left') + except TypeError: + missing_date_range = pds.date_range(start, stop, freq='1D') + + missing_times = [tt for tt in missing_date_range if tt not in inst.files[start:stop].index] for tt in missing_times: inst.download(start=tt, stop=tt + pds.DateOffset(days=1), diff --git a/requirements.txt b/requirements.txt index 0f66a313..540b4e62 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ numpy packaging -pandas >= 1.4.0 +pandas pyForecastTools pysat >= 3.0.4 -pysatNASA <= 0.0.4 +pysatNASA requests scipy xarray diff --git a/setup.cfg b/setup.cfg index 98728f11..df9c3d20 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,16 +14,17 @@ classifiers = License :: Freely Distributable Natural Language :: English Programming Language :: Python :: 3 - Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 Operating System :: MacOS :: MacOS X Operating System :: POSIX :: Linux Operating System :: Microsoft :: Windows license_file = LICENSE [options] -python_requires = >= 3.5 +python_requires = >= 3.6 setup_requires = setuptools >= 38.6; pip >= 10 packages = find: include_package_data = True