Add optional fail span to climatology test

Make climatology config time and depth ranges inclusive of low endpoint

Add climatology tests for inclusion of span endpoints

Make climatology depth parameter optional

ioos_qc/qartod.py

@@ -205,6 +205,7 @@ class ClimatologyConfig(object):
         tspan: 2-tuple range.
                 If period is defined, then this is a numeric range.
                 If period is not defined, then its a date range.
+        fspan: (optional) 2-tuple range of valid values. This is passed in as the fail_span to the gross_range test.
         vspan: 2-tuple range of valid values. This is passed in as the suspect_span to the gross_range test.
         zspan: (optional) Vertical (depth) range, in meters positive down
         period: (optional) The unit the tspan argument is in. Defaults to datetime object
@@ -218,6 +219,7 @@ class ClimatologyConfig(object):
     """
     mem = namedtuple('window', [
         'tspan',
+        'fspan',
         'vspan',
         'zspan',
         'period'
@@ -259,7 +261,7 @@ def values(self, tind, zind=None):
                     span = m.vspan
         return span
 
-    def add(self, tspan, vspan, zspan = None, period = None):
+    def add(self, tspan, vspan, fspan = None, zspan = None, period = None):
 
         assert isfixedlength(tspan, 2)
         # If period is defined, tspan is a numeric
@@ -275,6 +277,10 @@ def add(self, tspan, vspan, zspan = None, period = None):
         assert isfixedlength(vspan, 2)
         vspan = span(*sorted(vspan))
 
+        if fspan is not None:
+            assert isfixedlength(fspan, 2)
+            fspan = span(*sorted(fspan))
+
         if zspan is not None:
             assert isfixedlength(zspan, 2)
             zspan = span(*sorted(zspan))
@@ -289,14 +295,14 @@ def add(self, tspan, vspan, zspan = None, period = None):
         self._members.append(
             self.mem(
                 tspan,
+                fspan,
                 vspan,
                 zspan,
                 period
             )
         )
 
     def check(self, tinp, inp, zinp):
-
         # Start with everything as UNKNOWN (1)
         flag_arr = np.ma.empty(inp.size, dtype='uint8')
         flag_arr.fill(QartodFlags.UNKNOWN)
@@ -320,33 +326,39 @@ def check(self, tinp, inp, zinp):
                 tinp_copy = tinp
 
             # If a zspan is defined but we don't have z input (zinp), skip this member
-            # Note: `zinp.any()` can return `np.ma.masked` so we also check using isnan
-            if not isnan(m.zspan) and (not zinp.any() or isnan(zinp.any())):
+            # Note: `zinp.count()` can return `np.ma.masked` so we also check using isnan
+            if not isnan(m.zspan) and (not zinp.count() or isnan(zinp.any())):
                 continue
 
             # Indexes that align with the T
-            t_idx = (tinp_copy > m.tspan.minv) & (tinp_copy <= m.tspan.maxv)
+            t_idx = (tinp_copy >= m.tspan.minv) & (tinp_copy <= m.tspan.maxv)
 
             # Indexes that align with the Z
             if not isnan(m.zspan):
                 # Only test non-masked values between the min and max
-                z_idx = (~zinp.mask) & (zinp > m.zspan.minv) & (zinp <= m.zspan.maxv)
+                z_idx = (~zinp.mask) & (zinp >= m.zspan.minv) & (zinp <= m.zspan.maxv)
             else:
-                # Only test the values with masked Z, ie values with no Z
-                z_idx = zinp.mask
+                # If there is no z data in the config, don't try to filter by depth!
+                # Set z_idx to all True to prevent filtering
+                z_idx = np.ones(inp.size, dtype=bool)
 
             # Combine the T and Z indexes
             values_idx = (t_idx & z_idx)
 
-            # Suspect data for this value span. Combined with the values_idx it
-            # represents the subset ofdata that should be suspect for this member.
-            # We split it into two indexes so we can also set all values outside of the
-            # suspect range to GOOD by taking the inverse of the suspect_idx
+            # Failed and suspect data for this value span. Combining fail_idx or
+            # suspect_idx with values_idx represents the subsets of data that should be
+            # fail and suspect respectively.
+            if not isnan(m.fspan):
+                fail_idx = (inp < m.fspan.minv) | (inp > m.fspan.maxv)
+            else:
+                fail_idx = np.zeros(inp.size, dtype=bool)
+
             suspect_idx = (inp < m.vspan.minv) | (inp > m.vspan.maxv)
 
             with np.errstate(invalid='ignore'):
-                flag_arr[(values_idx & suspect_idx)] = QartodFlags.SUSPECT
-                flag_arr[(values_idx & ~suspect_idx)] = QartodFlags.GOOD
+                flag_arr[(values_idx & fail_idx)] = QartodFlags.FAIL
+                flag_arr[(values_idx & ~fail_idx & suspect_idx)] = QartodFlags.SUSPECT
+                flag_arr[(values_idx & ~fail_idx & ~suspect_idx)] = QartodFlags.GOOD
 
         return flag_arr
 
@@ -362,7 +374,7 @@ def convert(config):
         return c
 
 
-def climatology_test(config, inp, tinp, zinp):
+def climatology_test(config, inp, tinp, zinp=None):
     """Checks that values are within reasonable range bounds and flags as SUSPECT.
 
     Data for which no ClimatologyConfig member exists is marked as UNKNOWN.
@@ -386,6 +398,7 @@ def climatology_test(config, inp, tinp, zinp):
     config = ClimatologyConfig.convert(config)
 
     tinp = mapdates(tinp)
+
     with warnings.catch_warnings():
         warnings.simplefilter("ignore")
         inp = np.ma.masked_invalid(np.array(inp).astype(np.floating))

pytest.ini

@@ -1,6 +1,6 @@
 [pytest]
 
-addopts = -s -rxs -v
+addopts = -rxs -v
 
 flake8-max-line-length = 100
 flake8-ignore =

tests/test_qartod.py

@@ -444,6 +444,215 @@ def test_weekofyear_periods(self):
         self._run_test(cc)
 
 
+class QartodClimatologyInclusiveRangesTest(unittest.TestCase):
+    # Test that the various configuration spans (tspan, vspan, fspan, zspan) are
+    # inclusive of both endpoints.
+    def setUp(self):
+        self.cc = qartod.ClimatologyConfig()
+        self.cc.add(
+            tspan=(np.datetime64('2019-11-01'), np.datetime64('2020-02-04')),
+            fspan=(40, 70),
+            vspan=(50, 60),
+            zspan=(0, 10)
+        )
+
+    def _run_test(self, test_inputs, expected_result):
+        times, values, depths = zip(*test_inputs)
+        inputs = [
+            values,
+            np.asarray(values, dtype=np.floating),
+            dask_arr(np.asarray(values, dtype=np.floating))
+        ]
+
+        for i in inputs:
+            results = qartod.climatology_test(
+                config=self.cc,
+                tinp=times,
+                inp=i,
+                zinp=depths
+            )
+            npt.assert_array_equal(
+                results,
+                np.ma.array(expected_result)
+            )
+
+    def test_tspan_out_of_range_low(self):
+        test_inputs = [
+            (
+                np.datetime64('2019-10-31'),
+                55,
+                5
+            )
+        ]
+        expected_result=[2]
+        self._run_test(test_inputs, expected_result)
+
+    def test_tspan_minimum(self):
+        test_inputs = [
+            (
+                np.datetime64('2019-11-01'),
+                55,
+                5
+            )
+        ]
+        expected_result=[1]
+        self._run_test(test_inputs, expected_result)
+
+    def test_tspan_maximum(self):
+        test_inputs = [
+            (
+                np.datetime64('2020-02-04'),
+                55,
+                5
+            )
+        ]
+        expected_result=[1]
+        self._run_test(test_inputs, expected_result)
+
+    def test_tspan_out_of_range_high(self):
+        test_inputs = [
+            (
+                np.datetime64('2020-02-05'),
+                55,
+                5
+            )
+        ]
+        expected_result=[2]
+        self._run_test(test_inputs, expected_result)
+
+    def test_vspan_out_of_range_low(self):
+        test_inputs = [
+            (
+                np.datetime64('2020-01-01'),
+                49,
+                5
+            )
+        ]
+        expected_result=[3]
+        self._run_test(test_inputs, expected_result)
+
+    def test_vspan_minimum(self):
+        test_inputs = [
+            (
+                np.datetime64('2020-01-01'),
+                50,
+                5
+            )
+        ]
+        expected_result=[1]
+        self._run_test(test_inputs, expected_result)
+
+    def test_vspan_maximum(self):
+        test_inputs = [
+            (
+                np.datetime64('2020-01-01'),
+                60,
+                5
+            )
+        ]
+        expected_result=[1]
+        self._run_test(test_inputs, expected_result)
+
+    def test_vspan_out_of_range_high(self):
+        test_inputs = [
+            (
+                np.datetime64('2020-01-01'),
+                61,
+                5
+            )
+        ]
+        expected_result=[3]
+        self._run_test(test_inputs, expected_result)
+
+    def test_fspan_out_of_range_low(self):
+        test_inputs = [
+            (
+                np.datetime64('2020-01-01'),
+                30,
+                5
+            )
+        ]
+        expected_result=[4]
+        self._run_test(test_inputs, expected_result)
+
+    def test_fspan_minimum(self):
+        test_inputs = [
+            (
+                np.datetime64('2020-01-01'),
+                40,
+                5
+            )
+        ]
+        expected_result=[3]
+        self._run_test(test_inputs, expected_result)
+
+    def test_fspan_maximum(self):
+        test_inputs = [
+            (
+                np.datetime64('2020-01-01'),
+                70,
+                5
+            )
+        ]
+        expected_result=[3]
+        self._run_test(test_inputs, expected_result)
+
+    def test_fspan_out_of_range_high(self):
+        test_inputs = [
+            (
+                np.datetime64('2020-01-01'),
+                71,
+                5
+            )
+        ]
+        expected_result=[4]
+        self._run_test(test_inputs, expected_result)
+
+    def test_zspan_out_of_range_low(self):
+        test_inputs = [
+            (
+                np.datetime64('2020-01-01'),
+                55,
+                -1
+            )
+        ]
+        expected_result=[2]
+        self._run_test(test_inputs, expected_result)
+
+    def test_zspan_minimum(self):
+        test_inputs = [
+            (
+                np.datetime64('2020-01-01'),
+                55,
+                0
+            )
+        ]
+        expected_result=[1]
+        self._run_test(test_inputs, expected_result)
+
+    def test_zspan_maximum(self):
+        test_inputs = [
+            (
+                np.datetime64('2020-01-01'),
+                55,
+                10
+            )
+        ]
+        expected_result=[1]
+        self._run_test(test_inputs, expected_result)
+
+    def test_zspan_out_of_range_high(self):
+        test_inputs = [
+            (
+                np.datetime64('2020-01-01'),
+                55,
+                11
+            )
+        ]
+        expected_result=[2]
+        self._run_test(test_inputs, expected_result)
+
+
 class QartodClimatologyDepthTest(unittest.TestCase):
 
     def setUp(self):