From c12de6c059270a3adc333ca3721ae2de9b361058 Mon Sep 17 00:00:00 2001 From: Seth Foote Date: Tue, 4 Feb 2020 15:53:57 -0500 Subject: [PATCH] Add optional fail span to climatology test Make climatology config time and depth ranges inclusive of low endpoint Add climatology tests for inclusion of span endpoints Make climatology depth parameter optional --- ioos_qc/qartod.py | 43 +++++---- pytest.ini | 2 +- tests/test_qartod.py | 209 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 238 insertions(+), 16 deletions(-) diff --git a/ioos_qc/qartod.py b/ioos_qc/qartod.py index 4af2863..7647070 100644 --- a/ioos_qc/qartod.py +++ b/ioos_qc/qartod.py @@ -205,6 +205,7 @@ class ClimatologyConfig(object): tspan: 2-tuple range. If period is defined, then this is a numeric range. If period is not defined, then its a date range. + fspan: (optional) 2-tuple range of valid values. This is passed in as the fail_span to the gross_range test. vspan: 2-tuple range of valid values. This is passed in as the suspect_span to the gross_range test. zspan: (optional) Vertical (depth) range, in meters positive down period: (optional) The unit the tspan argument is in. Defaults to datetime object @@ -218,6 +219,7 @@ class ClimatologyConfig(object): """ mem = namedtuple('window', [ 'tspan', + 'fspan', 'vspan', 'zspan', 'period' @@ -259,7 +261,7 @@ def values(self, tind, zind=None): span = m.vspan return span - def add(self, tspan, vspan, zspan = None, period = None): + def add(self, tspan, vspan, fspan = None, zspan = None, period = None): assert isfixedlength(tspan, 2) # If period is defined, tspan is a numeric @@ -275,6 +277,10 @@ def add(self, tspan, vspan, zspan = None, period = None): assert isfixedlength(vspan, 2) vspan = span(*sorted(vspan)) + if fspan is not None: + assert isfixedlength(fspan, 2) + fspan = span(*sorted(fspan)) + if zspan is not None: assert isfixedlength(zspan, 2) zspan = span(*sorted(zspan)) @@ -289,6 +295,7 @@ def add(self, tspan, vspan, zspan = None, period = None): self._members.append( self.mem( tspan, + fspan, vspan, zspan, period @@ -296,7 +303,6 @@ def add(self, tspan, vspan, zspan = None, period = None): ) def check(self, tinp, inp, zinp): - # Start with everything as UNKNOWN (1) flag_arr = np.ma.empty(inp.size, dtype='uint8') flag_arr.fill(QartodFlags.UNKNOWN) @@ -320,33 +326,39 @@ def check(self, tinp, inp, zinp): tinp_copy = tinp # If a zspan is defined but we don't have z input (zinp), skip this member - # Note: `zinp.any()` can return `np.ma.masked` so we also check using isnan - if not isnan(m.zspan) and (not zinp.any() or isnan(zinp.any())): + # Note: `zinp.count()` can return `np.ma.masked` so we also check using isnan + if not isnan(m.zspan) and (not zinp.count() or isnan(zinp.any())): continue # Indexes that align with the T - t_idx = (tinp_copy > m.tspan.minv) & (tinp_copy <= m.tspan.maxv) + t_idx = (tinp_copy >= m.tspan.minv) & (tinp_copy <= m.tspan.maxv) # Indexes that align with the Z if not isnan(m.zspan): # Only test non-masked values between the min and max - z_idx = (~zinp.mask) & (zinp > m.zspan.minv) & (zinp <= m.zspan.maxv) + z_idx = (~zinp.mask) & (zinp >= m.zspan.minv) & (zinp <= m.zspan.maxv) else: - # Only test the values with masked Z, ie values with no Z - z_idx = zinp.mask + # If there is no z data in the config, don't try to filter by depth! + # Set z_idx to all True to prevent filtering + z_idx = np.ones(inp.size, dtype=bool) # Combine the T and Z indexes values_idx = (t_idx & z_idx) - # Suspect data for this value span. Combined with the values_idx it - # represents the subset ofdata that should be suspect for this member. - # We split it into two indexes so we can also set all values outside of the - # suspect range to GOOD by taking the inverse of the suspect_idx + # Failed and suspect data for this value span. Combining fail_idx or + # suspect_idx with values_idx represents the subsets of data that should be + # fail and suspect respectively. + if not isnan(m.fspan): + fail_idx = (inp < m.fspan.minv) | (inp > m.fspan.maxv) + else: + fail_idx = np.zeros(inp.size, dtype=bool) + suspect_idx = (inp < m.vspan.minv) | (inp > m.vspan.maxv) with np.errstate(invalid='ignore'): - flag_arr[(values_idx & suspect_idx)] = QartodFlags.SUSPECT - flag_arr[(values_idx & ~suspect_idx)] = QartodFlags.GOOD + flag_arr[(values_idx & fail_idx)] = QartodFlags.FAIL + flag_arr[(values_idx & ~fail_idx & suspect_idx)] = QartodFlags.SUSPECT + flag_arr[(values_idx & ~fail_idx & ~suspect_idx)] = QartodFlags.GOOD return flag_arr @@ -362,7 +374,7 @@ def convert(config): return c -def climatology_test(config, inp, tinp, zinp): +def climatology_test(config, inp, tinp, zinp=None): """Checks that values are within reasonable range bounds and flags as SUSPECT. Data for which no ClimatologyConfig member exists is marked as UNKNOWN. @@ -386,6 +398,7 @@ def climatology_test(config, inp, tinp, zinp): config = ClimatologyConfig.convert(config) tinp = mapdates(tinp) + with warnings.catch_warnings(): warnings.simplefilter("ignore") inp = np.ma.masked_invalid(np.array(inp).astype(np.floating)) diff --git a/pytest.ini b/pytest.ini index 0e19dbd..acb27f3 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,6 @@ [pytest] -addopts = -s -rxs -v +addopts = -rxs -v flake8-max-line-length = 100 flake8-ignore = diff --git a/tests/test_qartod.py b/tests/test_qartod.py index b6c7a11..b424d72 100644 --- a/tests/test_qartod.py +++ b/tests/test_qartod.py @@ -444,6 +444,215 @@ def test_weekofyear_periods(self): self._run_test(cc) +class QartodClimatologyInclusiveRangesTest(unittest.TestCase): + # Test that the various configuration spans (tspan, vspan, fspan, zspan) are + # inclusive of both endpoints. + def setUp(self): + self.cc = qartod.ClimatologyConfig() + self.cc.add( + tspan=(np.datetime64('2019-11-01'), np.datetime64('2020-02-04')), + fspan=(40, 70), + vspan=(50, 60), + zspan=(0, 10) + ) + + def _run_test(self, test_inputs, expected_result): + times, values, depths = zip(*test_inputs) + inputs = [ + values, + np.asarray(values, dtype=np.floating), + dask_arr(np.asarray(values, dtype=np.floating)) + ] + + for i in inputs: + results = qartod.climatology_test( + config=self.cc, + tinp=times, + inp=i, + zinp=depths + ) + npt.assert_array_equal( + results, + np.ma.array(expected_result) + ) + + def test_tspan_out_of_range_low(self): + test_inputs = [ + ( + np.datetime64('2019-10-31'), + 55, + 5 + ) + ] + expected_result=[2] + self._run_test(test_inputs, expected_result) + + def test_tspan_minimum(self): + test_inputs = [ + ( + np.datetime64('2019-11-01'), + 55, + 5 + ) + ] + expected_result=[1] + self._run_test(test_inputs, expected_result) + + def test_tspan_maximum(self): + test_inputs = [ + ( + np.datetime64('2020-02-04'), + 55, + 5 + ) + ] + expected_result=[1] + self._run_test(test_inputs, expected_result) + + def test_tspan_out_of_range_high(self): + test_inputs = [ + ( + np.datetime64('2020-02-05'), + 55, + 5 + ) + ] + expected_result=[2] + self._run_test(test_inputs, expected_result) + + def test_vspan_out_of_range_low(self): + test_inputs = [ + ( + np.datetime64('2020-01-01'), + 49, + 5 + ) + ] + expected_result=[3] + self._run_test(test_inputs, expected_result) + + def test_vspan_minimum(self): + test_inputs = [ + ( + np.datetime64('2020-01-01'), + 50, + 5 + ) + ] + expected_result=[1] + self._run_test(test_inputs, expected_result) + + def test_vspan_maximum(self): + test_inputs = [ + ( + np.datetime64('2020-01-01'), + 60, + 5 + ) + ] + expected_result=[1] + self._run_test(test_inputs, expected_result) + + def test_vspan_out_of_range_high(self): + test_inputs = [ + ( + np.datetime64('2020-01-01'), + 61, + 5 + ) + ] + expected_result=[3] + self._run_test(test_inputs, expected_result) + + def test_fspan_out_of_range_low(self): + test_inputs = [ + ( + np.datetime64('2020-01-01'), + 30, + 5 + ) + ] + expected_result=[4] + self._run_test(test_inputs, expected_result) + + def test_fspan_minimum(self): + test_inputs = [ + ( + np.datetime64('2020-01-01'), + 40, + 5 + ) + ] + expected_result=[3] + self._run_test(test_inputs, expected_result) + + def test_fspan_maximum(self): + test_inputs = [ + ( + np.datetime64('2020-01-01'), + 70, + 5 + ) + ] + expected_result=[3] + self._run_test(test_inputs, expected_result) + + def test_fspan_out_of_range_high(self): + test_inputs = [ + ( + np.datetime64('2020-01-01'), + 71, + 5 + ) + ] + expected_result=[4] + self._run_test(test_inputs, expected_result) + + def test_zspan_out_of_range_low(self): + test_inputs = [ + ( + np.datetime64('2020-01-01'), + 55, + -1 + ) + ] + expected_result=[2] + self._run_test(test_inputs, expected_result) + + def test_zspan_minimum(self): + test_inputs = [ + ( + np.datetime64('2020-01-01'), + 55, + 0 + ) + ] + expected_result=[1] + self._run_test(test_inputs, expected_result) + + def test_zspan_maximum(self): + test_inputs = [ + ( + np.datetime64('2020-01-01'), + 55, + 10 + ) + ] + expected_result=[1] + self._run_test(test_inputs, expected_result) + + def test_zspan_out_of_range_high(self): + test_inputs = [ + ( + np.datetime64('2020-01-01'), + 55, + 11 + ) + ] + expected_result=[2] + self._run_test(test_inputs, expected_result) + + class QartodClimatologyDepthTest(unittest.TestCase): def setUp(self):