Merge pull request #65 from euroargodev/xarray-016

Fix argopy broken by last xarray, fsspec and erddapy releases

.github/workflows/pythonFREEtests.yml

@@ -0,0 +1,53 @@
+on:
+  pull_request:
+    types: [ready_for_review]
+
+jobs:
+  free_env:
+    name: unit-test FREE env
+
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.experimental }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.7, 3.8]
+        os: [ubuntu-latest, macos-latest]
+        experimental: [true]
+
+    steps:
+      - uses: actions/checkout@master
+
+      - name: Set up environment with py-${{ matrix.python-version }}
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          activate-environment: argopy-tests
+          environment-file: ci/requirements/py${{matrix.python-version}}-free.yml
+          auto-update-conda: false
+          python-version: ${{ matrix.python-version }}
+          auto-activate-base: false
+
+      - shell: bash -l {0}
+        run: |
+          conda info
+          conda list
+
+      - name: Lint with flake8
+        shell: bash -l {0}
+        run: |
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
+      - name: Test with pytest
+        shell: bash -l {0}
+        run: |
+          pytest -s --verbosity=3 --cov=./ --cov-config=.coveragerc --cov-report xml:cov.xml --cov-report term-missing
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: ./cov.xml
+          flags: unittests
+          name: codecov-github
+          fail_ci_if_error: true

.github/workflows/pythontests.yml

@@ -1,64 +1,53 @@
-# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
-
 name: build
 
-on: [push, pull_request]
+on: [push]
 
 jobs:
   unit-testing:
 
     runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.experimental }}
     strategy:
+      max-parallel: 12
+      fail-fast: false
       matrix:
         python-version: [3.6, 3.7, 3.8]
         os: [ubuntu-latest, macos-latest]
+        experimental: [false]
 
     steps:
       - uses: actions/checkout@master
 
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@master
+      - name: Set up environment with py-${{ matrix.python-version }}
+        uses: conda-incubator/setup-miniconda@v2
         with:
+          activate-environment: argopy-tests
+          environment-file: ci/requirements/py${{matrix.python-version}}-dev.yml
+          auto-update-conda: false
           python-version: ${{ matrix.python-version }}
+          auto-activate-base: false
 
-      #    - name: Set up Conda for py-${{ matrix.python-version }}
-      #      uses: goanpeca/setup-miniconda@v1
-      #      with:
-      #        auto-update-conda: true
-      #        python-version: ${{ matrix.python-version }}
-      #        auto-activate-base: false
-
-      - name: Install dependencies with pip
+      - shell: bash -l {0}
         run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
-          pip install distributed aiohttp ipywidgets
-
-      #    - name: Install dependencies with conda
-      #      shell: bash -l {0}
-      #      run: |
-      #        conda info
-      #        conda install matplotlib seaborn cartopy
+          conda info
+          conda list
 
       - name: Lint with flake8
+        shell: bash -l {0}
         run: |
-          pip install flake8
-          # stop the build if there are Python syntax errors or undefined names
           flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
 
       - name: Test with pytest
+        shell: bash -l {0}
         run: |
-          pip install pytest pytest-cov
           pytest -s --verbosity=3 --cov=./ --cov-config=.coveragerc --cov-report xml:cov.xml --cov-report term-missing
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v1
         with:
-          token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
-          file: ./cov.xml # optional
-          flags: unittests # optional
-          name: codecov-github # optional
-          fail_ci_if_error: true # optional (default = false)
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: ./cov.xml
+          flags: unittests
+          name: codecov-github
+          fail_ci_if_error: true

.gitignore

@@ -188,4 +188,5 @@ fabric.properties
 #pytest quai20
 .vscode/
 .pytest_cache
-knotebooks/
+knotebooks/
+argopy/tests/cov.xml

argopy/data_fetchers/argovis_data.py

@@ -15,7 +15,6 @@
 from .proto import ArgoDataFetcherProto
 from abc import abstractmethod
 import warnings
-import distributed
 
 from argopy.stores import httpstore
 from argopy.options import OPTIONS
@@ -30,6 +29,7 @@
     api_server + "/selection/overview"
 )  # URL to check if the API is alive
 
+
 class ArgovisDataFetcher(ArgoDataFetcherProto):
     ###
     # Methods to be customised for a specific Argovis request
@@ -337,7 +337,7 @@ def dashboard(self, **kw):
 
 
 class Fetch_box(ArgovisDataFetcher):
-	
+
     def init(self, box: list):
         """ Create Argo data loader
 
@@ -388,8 +388,8 @@ def get_url_shape(self):
 
     def get_url_rect(self):
         """ Return the URL used to download data """
-        strCorner = lambda b, i: str([b[i[0]],b[i[1]]]).replace(" ", "")
-        strDate = lambda b, i: pd.to_datetime(b[i]).strftime("%Y-%m-%dT%H:%M:%SZ")
+        def strCorner(b, i): return str([b[i[0]], b[i[1]]]).replace(" ", "")
+        def strDate(b, i): return pd.to_datetime(b[i]).strftime("%Y-%m-%dT%H:%M:%SZ")
         url = self.server + "/selection/box/profiles"
         url += "?startDate={}".format(strDate(self.BOX, 6))
         url += "&endDate={}".format(strDate(self.BOX, 7))

argopy/data_fetchers/erddap_data.py

@@ -25,13 +25,16 @@
 from argopy.stores import httpstore
 from argopy.plotters import open_dashboard
 
-# Dirty fix before https://github.com/ioos/erddapy/issues/140
-if isconnected():
+
+# Load erddapy according to available version (breaking changes in v0.8.0)
+try:
     from erddapy import ERDDAP
     from erddapy.utilities import parse_dates, quote_string_constraints
-else:
-    from unittest.mock import MagicMock
-    sys.modules['ERDDAP'] = MagicMock()
+except:
+    # >= v0.8.0
+    from erddapy.erddapy import ERDDAP
+    from erddapy.erddapy import _quote_string_constraints as quote_string_constraints
+    from erddapy.erddapy import parse_dates
 
 
 access_points = ['wmo', 'box']

argopy/data_fetchers/erddap_index.py

@@ -19,10 +19,17 @@
 from argopy.utilities import load_dict, mapp_dict, isconnected, format_oneline
 from argopy.stores import httpstore
 
-# Dirty fix before https://github.com/ioos/erddapy/issues/140
-if isconnected():
+
+# Load erddapy according to available version (breaking changes in v0.8.0)
+try:
     from erddapy import ERDDAP
     from erddapy.utilities import parse_dates, quote_string_constraints
+except:
+    # >= v0.8.0
+    from erddapy.erddapy import ERDDAP
+    from erddapy.erddapy import _quote_string_constraints as quote_string_constraints
+    from erddapy.erddapy import parse_dates
+
 
 access_points = ['wmo', 'box']
 exit_formats = ['xarray', 'dataframe']

argopy/data_fetchers/localftp_index.py

@@ -28,22 +28,12 @@
 
 """
 import os
-from glob import glob
 import numpy as np
-import pandas as pd
-import xarray as xr
 from abc import ABC, abstractmethod
-import warnings
-import getpass
 
-import multiprocessing as mp
-import distributed
-
-from .proto import ArgoDataFetcherProto
-from argopy.errors import NetCDF4FileNotFoundError
-from argopy.utilities import list_standard_variables, load_dict, mapp_dict, check_localftp, format_oneline
+from argopy.utilities import load_dict, mapp_dict, check_localftp, format_oneline
 from argopy.options import OPTIONS
-from argopy.stores import filestore, indexstore, indexfilter_wmo, indexfilter_box
+from argopy.stores import indexstore, indexfilter_wmo, indexfilter_box
 
 access_points = ['wmo', 'box']
 exit_formats = ['xarray', 'dataframe']
@@ -180,7 +170,7 @@ def init(self, box: list = [-180, 180, -90, 90, '1900-01-01', '2100-12-31']):
                 The box domain to load all Argo data for:
                 box = [lon_min, lon_max, lat_min, lat_max, pres_min, pres_max, datim_min, datim_max]
         """
-        if len(box) != 4 and len(box) !=6 :
+        if len(box) != 4 and len(box) != 6:
             raise ValueError('Box must be 4 or 6 length')
         self.BOX = box
         self.fcls = indexfilter_box(self.BOX)

argopy/errors.py

@@ -4,10 +4,12 @@
 A bunch of custom errors used in argopy.
 """
 
+
 class DataNotFound(ValueError):
     """ Raise when a data selection returns nothing """
     pass
 
+
 class FtpPathError(ValueError):
     """ Raise when the ftp path is not appropriate """
     pass

argopy/plotters.py

@@ -68,7 +68,7 @@ def open_dashboard(wmo=None, cyc=None, width="100%", height=1000, url=None, type
                 url = "https://fleetmonitoring.euro-argo.eu"
             else:
                 url = "https://fleetmonitoring.euro-argo.eu/float/{}".format(str(wmo))
-        elif type == 'coriolis': # Open Coriolis dashboard
+        elif type == 'coriolis':  # Open Coriolis dashboard
             if wmo is not None:
                 url = ("https://co-insitucharts.ifremer.fr/platform/{}/charts").format(str(wmo))
 

argopy/xarray.py

@@ -214,13 +214,34 @@ def filter_data_mode(self, keep_error: bool = True, errors: str = 'raise'):
         #########
         # Sub-functions
         #########
+        def safe_where_eq(xds, key, value):
+            # xds.where(xds[key] == value, drop=True) is not safe to empty time variables, cf issue #64
+            try:
+                return xds.where(xds[key] == value, drop=True)
+            except ValueError as v:
+                if v.args[0] == ("zero-size array to reduction operation "
+                                 "minimum which has no identity"):
+                    # A bug in xarray will cause a ValueError if trying to
+                    # decode the times in a NetCDF file with length 0.
+                    # See:
+                    # https://github.com/pydata/xarray/issues/1329
+                    # https://github.com/euroargodev/argopy/issues/64
+                    # Here, we just need to return an empty array
+                    TIME = xds['TIME']
+                    xds = xds.drop_vars('TIME')
+                    xds = xds.where(xds[key] == value, drop=True)
+                    xds['TIME'] = xr.DataArray(np.arange(len(xds['N_POINTS'])), dims='N_POINTS',
+                                               attrs=TIME.attrs).astype(np.datetime64)
+                    xds = xds.set_coords('TIME')
+                    return xds
+
         def ds_split_datamode(xds):
             """ Create one dataset for each of the data_mode
 
                 Split full dataset into 3 datasets
             """
             # Real-time:
-            argo_r = xds.where(xds['DATA_MODE'] == 'R', drop=True)
+            argo_r = safe_where_eq(xds, 'DATA_MODE', 'R')
             for v in plist:
                 vname = v.upper() + '_ADJUSTED'
                 if vname in argo_r:
@@ -232,7 +253,7 @@ def ds_split_datamode(xds):
                 if vname in argo_r:
                     argo_r = argo_r.drop_vars(vname)
             # Real-time adjusted:
-            argo_a = xds.where(xds['DATA_MODE'] == 'A', drop=True)
+            argo_a = safe_where_eq(xds, 'DATA_MODE', 'A')
             for v in plist:
                 vname = v.upper()
                 if vname in argo_a:
@@ -241,7 +262,8 @@ def ds_split_datamode(xds):
                 if vname in argo_a:
                     argo_a = argo_a.drop_vars(vname)
             # Delayed mode:
-            argo_d = xds.where(xds['DATA_MODE'] == 'D', drop=True)
+            argo_d = safe_where_eq(xds, 'DATA_MODE', 'D')
+
             return argo_r, argo_a, argo_d
 
         def fill_adjusted_nan(ds, vname):
@@ -295,7 +317,9 @@ def new_arrays(argo_r, argo_a, argo_d, vname):
                 return ds
 
         # Define variables to filter:
-        possible_list = ['PRES', 'TEMP', 'PSAL',
+        possible_list = ['PRES',
+                         'TEMP',
+                         'PSAL',
                          'DOXY',
                          'CHLA',
                          'BBP532',