From ce6766b2c935165f4ae0b59a65d6ee235d18654f Mon Sep 17 00:00:00 2001 From: gavinevans Date: Mon, 24 Jan 2022 15:43:43 +0000 Subject: [PATCH 1/3] Consolidate scale parameter usage across EMOS and ECC (#1642) * Modifications to code, documentation and test, particularly to improve/clarify the usage of the scale parameter. * Update docstrings following review comments. --- .../ensemble_calibration.rst | 39 ++++-- improver/calibration/ensemble_calibration.py | 9 +- .../ensemble_copula_coupling.py | 27 ++--- ...alibratedForecastDistributionParameters.py | 16 +-- ...LocationAndScaleParametersToPercentiles.py | 113 ++++++++---------- ...cationAndScaleParametersToProbabilities.py | 12 +- 6 files changed, 116 insertions(+), 100 deletions(-) diff --git a/doc/source/extended_documentation/calibration/ensemble_calibration/ensemble_calibration.rst b/doc/source/extended_documentation/calibration/ensemble_calibration/ensemble_calibration.rst index 55a01d59ab..292a7597a1 100644 --- a/doc/source/extended_documentation/calibration/ensemble_calibration/ensemble_calibration.rst +++ b/doc/source/extended_documentation/calibration/ensemble_calibration/ensemble_calibration.rst @@ -56,16 +56,16 @@ A normal (Gaussian) distribution is often represented using the syntax: where :math:`\mu` is mean and :math:`\sigma^{2}` is the variance. The normal distribution is a special case, where :math:`\mu` can be interpreted as both the mean and the location parameter and :math:`\sigma^{2}` can be interpreted -as both the variance and the scale parameter. For an alternative distribution, -such as a truncated normal distribution that has been truncated to lie within -0 and infinity, the distribution can be represented as: +as both the variance and the square of the scale parameter. For an alternative +distribution, such as a truncated normal distribution that has been truncated +to lie within 0 and infinity, the distribution can be represented as: .. math:: \mathcal{N^0}(\mu,\,\sigma^{2}) In this case, the :math:`\mu` is strictly interpreted as the location parameter -and :math:`\sigma^{2}` is strictly interpreted as the scale parameter. +and :math:`\sigma^{2}` is strictly interpreted as the square of the scale parameter. =============================== What is the location parameter? @@ -82,6 +82,29 @@ The scale parameter indicates the width in the distribution. If the scale parameter is large, then the distribution will be broader. If the scale is smaller, then the distribution will be narrower. +**************************************************** +Implementation details +**************************************************** + +In this implementation, we will choose to define the distributions +using the scale parameter (as this matches scipy's expectation), +rather than the square of the scale parameter: + +.. math:: + + \mathcal{N}(\mu,\,\sigma) + +The full equation when estimating the EMOS coefficients using +the ensemble mean is therefore: + +.. math:: + + \mathcal{N}(a + b\bar{X}, \sqrt{c + dS^{2}}) + +This matches the equations noted in `Allen et al., 2021`_. + +.. _Allen et al., 2021: https://doi.org/10.1002/qj.3983 + **************************************************** Estimating EMOS coefficients using the ensemble mean **************************************************** @@ -91,7 +114,7 @@ If the predictor is the ensemble mean, coefficients are estimated as .. math:: - \mathcal{N}(a + \bar{X}, c + dS^{2}) + \mathcal{N}(a + b\bar{X}, \sqrt{c + dS^{2}}) where N is a chosen distribution and values of a, b, c and d are solved in the format of :math:`\alpha, \beta, \gamma` and :math:`\delta`, see the equations @@ -121,7 +144,7 @@ If the predictor is the ensemble realizations, coefficients are estimated for .. math:: - \mathcal{N}(a + b_1X_1 + ... + b_mX_m, c + dS^{2}) + \mathcal{N}(a + b_1X_1 + ... + b_mX_m, \sqrt{c + dS^{2}}) where N is a chosen distribution, the values of a, b, c and d relate to alpha, beta, gamma and delta through the equations above with @@ -140,14 +163,14 @@ The EMOS coefficients represent adjustments to the ensemble mean and ensemble variance, in order to generate the location and scale parameters that, for the chosen distribution, minimise the CRPS. The coefficients can therefore be used to construct the location parameter, :math:`\mu`, and scale parameter, -:math:`\sigma^{2}`, for the calibrated forecast from today's ensemble mean, or +:math:`\sigma`, for the calibrated forecast from today's ensemble mean, or ensemble realizations, and the ensemble variance. .. math:: \mu = a + b\bar{X} - \sigma^{2} = c + dS^{2} + \sigma = \sqrt{c + dS^{2}} Note here that this procedure holds whether the distribution is normal, i.e. where the application of the EMOS coefficients to the raw ensemble mean results diff --git a/improver/calibration/ensemble_calibration.py b/improver/calibration/ensemble_calibration.py index 38b1931743..994a3f1939 100644 --- a/improver/calibration/ensemble_calibration.py +++ b/improver/calibration/ensemble_calibration.py @@ -1581,9 +1581,10 @@ def _calculate_scale_parameter(self) -> ndarray: ) # Calculating the scale parameter, based on the raw variance S^2, - # where predicted variance = c + dS^2, where c = (gamma)^2 and - # d = (delta)^2 - scale_parameter = ( + # where predicted scale parameter (or equivalently standard deviation + # for a normal distribution) = sqrt(c + dS^2), where c = (gamma)^2 and + # d = (delta)^2. + scale_parameter = np.sqrt( self.coefficients_cubelist.extract_cube("emos_coefficient_gamma").data * self.coefficients_cubelist.extract_cube("emos_coefficient_gamma").data + self.coefficients_cubelist.extract_cube("emos_coefficient_delta").data @@ -1622,7 +1623,7 @@ def _create_output_cubes( ) scale_parameter_cube = create_new_diagnostic_cube( "scale_parameter", - f"({template_cube.units})^2", + template_cube.units, template_cube, template_cube.attributes, data=scale_parameter, diff --git a/improver/ensemble_copula_coupling/ensemble_copula_coupling.py b/improver/ensemble_copula_coupling/ensemble_copula_coupling.py index ec0bacbf6a..d8dfc463ef 100644 --- a/improver/ensemble_copula_coupling/ensemble_copula_coupling.py +++ b/improver/ensemble_copula_coupling/ensemble_copula_coupling.py @@ -871,10 +871,10 @@ def _location_and_scale_parameters_to_percentiles( (len(percentiles_as_fractions), location_data.shape[0]), dtype=np.float32 ) - self._rescale_shape_parameters(location_data, np.sqrt(scale_data)) + self._rescale_shape_parameters(location_data, scale_data) percentile_method = self.distribution( - *self.shape_parameters, loc=location_data, scale=np.sqrt(scale_data) + *self.shape_parameters, loc=location_data, scale=scale_data ) # Loop over percentiles, and use the distribution as the @@ -885,8 +885,9 @@ def _location_and_scale_parameters_to_percentiles( result[index, :] = percentile_method.ppf(percentile_list) # If percent point function (PPF) returns NaNs, fill in # mean instead of NaN values. NaN will only be generated if the - # variance is zero. Therefore, if the variance is zero, the mean - # value is used for all gridpoints with a NaN. + # scale parameter (standard deviation) is zero. Therefore, if the + # scale parameter (standard deviation) is zero, the mean value is + # used for all gridpoints with a NaN. if np.any(scale_data == 0): nan_index = np.argwhere(np.isnan(result[index, :])) result[index, nan_index] = location_data[nan_index] @@ -1039,11 +1040,9 @@ def _check_unit_compatibility( ) -> None: """ The location parameter, scale parameters, and threshold values come - from three different cubes. They should all be in the same base unit, - with the units of the scale parameter being the squared units of the - location parameter and threshold values. This is a sanity check to - ensure the units are as expected, converting units of the location - parameter and scale parameter if possible. + from three different cubes. This is a sanity check to ensure the units + are as expected, converting units of the location parameter and + scale parameter if possible. Args: location_parameter: @@ -1060,11 +1059,11 @@ def _check_unit_compatibility( try: location_parameter.convert_units(threshold_units) - scale_parameter.convert_units(threshold_units ** 2) + scale_parameter.convert_units(threshold_units) except ValueError as err: msg = ( - "Error: {} This is likely because the mean " - "variance and template cube threshold units are " + "Error: {} This is likely because the location parameter, " + "scale parameter and template cube threshold units are " "not equivalent/compatible.".format(err) ) raise ValueError(msg) @@ -1107,7 +1106,7 @@ def _location_and_scale_parameters_to_probabilities( relative_to_threshold = probability_is_above_or_below(probability_cube_template) self._rescale_shape_parameters( - location_parameter.data.flatten(), np.sqrt(scale_parameter.data).flatten() + location_parameter.data.flatten(), scale_parameter.data.flatten() ) # Loop over thresholds, and use the specified distribution with the @@ -1118,7 +1117,7 @@ def _location_and_scale_parameters_to_probabilities( distribution = self.distribution( *self.shape_parameters, loc=location_parameter.data.flatten(), - scale=np.sqrt(scale_parameter.data.flatten()), + scale=scale_parameter.data.flatten(), ) probability_method = distribution.cdf diff --git a/improver_tests/calibration/ensemble_calibration/test_CalibratedForecastDistributionParameters.py b/improver_tests/calibration/ensemble_calibration/test_CalibratedForecastDistributionParameters.py index 22075aae32..7529aac227 100644 --- a/improver_tests/calibration/ensemble_calibration/test_CalibratedForecastDistributionParameters.py +++ b/improver_tests/calibration/ensemble_calibration/test_CalibratedForecastDistributionParameters.py @@ -169,9 +169,9 @@ def setUp(self): ) self.expected_scale_param_mean = np.array( [ - [0.2316, 0.2342, 0.0168], - [0.0271, 0.0237, 0.0168], - [0.0634, 0.1151, 0.0116], + [0.4813, 0.4840, 0.1295], + [0.1647, 0.1538, 0.1295], + [0.2517, 0.3393, 0.1076], ], dtype=np.float32, ) @@ -188,7 +188,7 @@ def setUp(self): ) self.expected_scale_param_realizations_sites = np.array( - [0, 0, 0, 0], dtype=np.float32 + [0.0005, 0.0005, 0.0005, 0.0005], dtype=np.float32 ) self.expected_loc_param_mean_alt = np.array( @@ -202,9 +202,9 @@ def setUp(self): self.expected_scale_param_mean_alt = np.array( [ - [0.4347, 0.4396, 0.0308], - [0.0503, 0.0438, 0.0308], - [0.1184, 0.2157, 0.0211], + [0.6593, 0.663, 0.1756], + [0.2242, 0.2093, 0.1756], + [0.3441, 0.4645, 0.1452], ], dtype=np.float32, ) @@ -219,7 +219,7 @@ def setUp(self): self.expected_scale_param_mean_cube = set_up_variable_cube( self.expected_scale_param_mean, name="scale_parameter", - units="Kelvin^2", + units="K", attributes=MANDATORY_ATTRIBUTE_DEFAULTS, ) diff --git a/improver_tests/ensemble_copula_coupling/test_ConvertLocationAndScaleParametersToPercentiles.py b/improver_tests/ensemble_copula_coupling/test_ConvertLocationAndScaleParametersToPercentiles.py index 9d0a2eccd1..d370046428 100644 --- a/improver_tests/ensemble_copula_coupling/test_ConvertLocationAndScaleParametersToPercentiles.py +++ b/improver_tests/ensemble_copula_coupling/test_ConvertLocationAndScaleParametersToPercentiles.py @@ -72,19 +72,19 @@ def setUp(self): self.data = np.array( [ [ - [225.568115, 236.818115, 248.068115], - [259.318115, 270.568115, 281.818115], - [293.068115, 304.318115, 315.568115], + [225.5681, 236.8181, 248.0681], + [259.3181, 270.5681, 281.8181], + [293.0681, 304.3181, 315.5681], ], [ - [229.483322, 240.733322, 251.983322], - [263.233307, 274.483307, 285.733307], - [296.983307, 308.233307, 319.483307], + [229.4833, 240.7333, 251.9833], + [263.2333, 274.4833, 285.7333], + [296.9833, 308.2333, 319.4833], ], [ - [233.398529, 244.648529, 255.898529], - [267.148499, 278.398499, 289.648499], - [300.898499, 312.148499, 323.398499], + [233.3985, 244.6485, 255.8985], + [267.1485, 278.3985, 289.6485], + [300.8985, 312.1485, 323.3985], ], ], dtype=np.float32, @@ -94,7 +94,7 @@ def setUp(self): "realization", iris.analysis.MEAN ) self.scale_parameter = self.temperature_cube.collapsed( - "realization", iris.analysis.VARIANCE + "realization", iris.analysis.STD_DEV, ) self.percentiles = [10, 50, 90] @@ -104,8 +104,8 @@ def test_check_data(self): Test that the plugin returns an Iris.cube.Cube matching the expected data values when a cubes containing location and scale parameters are passed in, which are equivalent to the ensemble mean and ensemble - variance. The resulting data values are the percentiles, which have - been generated. + standard deviation. The resulting data values are the percentiles, which + have been generated. """ result = Plugin()._location_and_scale_parameters_to_percentiles( self.location_parameter, @@ -185,10 +185,10 @@ def test_simple_data_truncnorm_distribution(self): """ Test that the plugin returns an iris.cube.Cube matching the expected data values when cubes containing the location parameter and scale - parameter are passed in. In this test, the ensemble mean and variance - is used as a proxy for the location and scale parameter. The resulting - data values are the percentiles, which have been generated using a - truncated normal distribution. + parameter are passed in. In this test, the ensemble mean and standard + deviation is used as a proxy for the location and scale parameter. + The resulting data values are the percentiles, which have been + generated using a truncated normal distribution. """ data = np.array( [ @@ -202,19 +202,19 @@ def test_simple_data_truncnorm_distribution(self): expected_data = np.array( [ [ - [1.3042759, 1.3042759, 1.3042759], - [1.3042759, 1.3042759, 1.3042759], - [1.3042759, 1.3042759, 1.3042759], + [1.0121, 1.0121, 1.0121], + [1.0121, 1.0121, 1.0121], + [1.0121, 1.0121, 1.0121], ], [ - [3.0300407, 3.0300407, 3.0300407], - [3.0300407, 3.0300407, 3.0300407], - [3.0300407, 3.0300407, 3.0300407], + [3.1677, 3.1677, 3.1677], + [3.1677, 3.1677, 3.1677], + [3.1677, 3.1677, 3.1677], ], [ - [4.8261294, 4.8261294, 4.8261294], - [4.8261294, 4.8261294, 4.8261294], - [4.8261294, 4.8261294, 4.8261294], + [5.6412, 5.6412, 5.6412], + [5.6412, 5.6412, 5.6412], + [5.6412, 5.6412, 5.6412], ], ] ) @@ -225,31 +225,31 @@ def test_simple_data_truncnorm_distribution(self): "realization", iris.analysis.MEAN ) current_forecast_predictor.data = current_forecast_predictor.data + 1 - # Use an adjusted version of the ensemble variance as a proxy for the + # Use an adjusted version of the ensemble standard deviation as a proxy for the # scale parameter for the truncated normal distribution. - current_forecast_variance = self.temperature_cube.collapsed( - "realization", iris.analysis.VARIANCE + current_forecast_stddev = self.temperature_cube.collapsed( + "realization", iris.analysis.STD_DEV, ) - current_forecast_variance.data = current_forecast_variance.data + 1 + current_forecast_stddev.data = current_forecast_stddev.data + 1 plugin = Plugin( distribution="truncnorm", shape_parameters=np.array([0, np.inf], dtype=np.float32), ) result = plugin._location_and_scale_parameters_to_percentiles( current_forecast_predictor, - current_forecast_variance, + current_forecast_stddev, self.temperature_cube, self.percentiles, ) self.assertIsInstance(result, Cube) - self.assertArrayAlmostEqual(result.data, expected_data) + np.testing.assert_allclose(result.data, expected_data, rtol=1.0e-4) @ManageWarnings(ignored_messages=["Collapsing a non-contiguous coordinate."]) def test_simple_data(self): """ Test that the plugin returns the expected values for the generated percentiles when an idealised set of data values between 1 and 3 - is used to create the mean (location parameter) and the variance + is used to create the mean (location parameter) and the standard deviation (scale parameter). """ data = np.array( @@ -280,16 +280,16 @@ def test_simple_data(self): current_forecast_predictor = self.temperature_cube.collapsed( "realization", iris.analysis.MEAN ) - current_forecast_variance = self.temperature_cube.collapsed( - "realization", iris.analysis.VARIANCE + current_forecast_stddev = self.temperature_cube.collapsed( + "realization", iris.analysis.STD_DEV ) result = Plugin()._location_and_scale_parameters_to_percentiles( current_forecast_predictor, - current_forecast_variance, + current_forecast_stddev, self.temperature_cube, self.percentiles, ) - self.assertArrayAlmostEqual(result.data, expected_data) + np.testing.assert_allclose(result.data, expected_data, rtol=1.0e-4) @ManageWarnings( ignored_messages=[ @@ -322,16 +322,16 @@ def test_if_identical_data(self): current_forecast_predictor = self.temperature_cube.collapsed( "realization", iris.analysis.MEAN ) - current_forecast_variance = self.temperature_cube.collapsed( - "realization", iris.analysis.VARIANCE + current_forecast_stddev = self.temperature_cube.collapsed( + "realization", iris.analysis.STD_DEV ) result = Plugin()._location_and_scale_parameters_to_percentiles( current_forecast_predictor, - current_forecast_variance, + current_forecast_stddev, self.temperature_cube, self.percentiles, ) - self.assertArrayAlmostEqual(result.data, expected_data) + np.testing.assert_allclose(result.data, expected_data, rtol=1.0e-4) @ManageWarnings( ignored_messages=[ @@ -368,16 +368,16 @@ def test_if_nearly_identical_data(self): current_forecast_predictor = self.temperature_cube.collapsed( "realization", iris.analysis.MEAN ) - current_forecast_variance = self.temperature_cube.collapsed( - "realization", iris.analysis.VARIANCE + current_forecast_stddev = self.temperature_cube.collapsed( + "realization", iris.analysis.STD_DEV ) result = Plugin()._location_and_scale_parameters_to_percentiles( current_forecast_predictor, - current_forecast_variance, + current_forecast_stddev, self.temperature_cube, self.percentiles, ) - self.assertArrayAlmostEqual(result.data, expected_data) + np.testing.assert_allclose(result.data, expected_data, rtol=1.0e-4) @ManageWarnings(ignored_messages=["Collapsing a non-contiguous coordinate."]) def test_many_percentiles(self): @@ -415,24 +415,19 @@ def test_spot_forecasts_check_data(self): """ Test that the plugin returns an Iris.cube.Cube matching the expected data values when a cube containing mean (location parameter) and - variance (scale parameter) is passed in. The resulting data values are + standard deviation (scale parameter) is passed in. The resulting data values are the percentiles, which have been generated for a spot forecast. """ data = np.reshape(self.data, (3, 9)) cube = set_up_spot_test_cube() current_forecast_predictor = cube.collapsed("realization", iris.analysis.MEAN) - current_forecast_variance = cube.collapsed( - "realization", iris.analysis.VARIANCE - ) + current_forecast_stddev = cube.collapsed("realization", iris.analysis.STD_DEV) result = Plugin()._location_and_scale_parameters_to_percentiles( - current_forecast_predictor, - current_forecast_variance, - cube, - self.percentiles, + current_forecast_predictor, current_forecast_stddev, cube, self.percentiles, ) self.assertIsInstance(result, Cube) - self.assertArrayAlmostEqual(result.data, data) + np.testing.assert_allclose(result.data, data, rtol=1.0e-4) @ManageWarnings(ignored_messages=["Collapsing a non-contiguous coordinate."]) def test_scalar_realisation_percentile(self): @@ -458,9 +453,7 @@ def setUp(self): """Set up temperature cube.""" self.cube = set_up_variable_cube(ECC_TEMPERATURE_REALIZATIONS) self.forecast_predictor = self.cube.collapsed("realization", iris.analysis.MEAN) - self.forecast_variance = self.cube.collapsed( - "realization", iris.analysis.VARIANCE - ) + self.forecast_stddev = self.cube.collapsed("realization", iris.analysis.STD_DEV) self.no_of_percentiles = len(self.cube.coord("realization").points) @ManageWarnings( @@ -473,7 +466,7 @@ def test_basic(self): """Test that the plugin returns an Iris.cube.Cube.""" result = Plugin().process( self.forecast_predictor, - self.forecast_variance, + self.forecast_stddev, self.cube, no_of_percentiles=self.no_of_percentiles, ) @@ -512,7 +505,7 @@ def test_number_of_percentiles(self): result = Plugin().process( self.forecast_predictor, - self.forecast_variance, + self.forecast_stddev, self.cube, no_of_percentiles=self.no_of_percentiles, ) @@ -554,7 +547,7 @@ def test_list_of_percentiles(self): result = Plugin().process( self.forecast_predictor, - self.forecast_variance, + self.forecast_stddev, self.cube, percentiles=percentiles, ) @@ -579,7 +572,7 @@ def test_multiple_keyword_arguments_error(self): with self.assertRaisesRegex(ValueError, msg): Plugin().process( self.forecast_predictor, - self.forecast_variance, + self.forecast_stddev, self.cube, no_of_percentiles=self.no_of_percentiles, percentiles=percentiles, diff --git a/improver_tests/ensemble_copula_coupling/test_ConvertLocationAndScaleParametersToProbabilities.py b/improver_tests/ensemble_copula_coupling/test_ConvertLocationAndScaleParametersToProbabilities.py index 08e4cff2fd..39d41364a6 100644 --- a/improver_tests/ensemble_copula_coupling/test_ConvertLocationAndScaleParametersToProbabilities.py +++ b/improver_tests/ensemble_copula_coupling/test_ConvertLocationAndScaleParametersToProbabilities.py @@ -119,7 +119,7 @@ def setUp(self): self.location_parameters = self.template_cube[0, :, :].copy() self.location_parameters.units = "Celsius" self.scale_parameters = self.template_cube[0, :, :].copy() - self.scale_parameters.units = "Celsius2" + self.scale_parameters.units = "Celsius" def test_compatible_units(self): """Pass in compatible cubes that should not raise an exception. No @@ -133,7 +133,7 @@ def test_convertible_units(self): """Pass in cubes with units that can be made equivalent by modification to match the threshold units.""" self.location_parameters.units = "Fahrenheit" - self.scale_parameters.units = "Fahrenheit2" + self.scale_parameters.units = "Fahrenheit" Plugin()._check_unit_compatibility( self.location_parameters, self.scale_parameters, self.template_cube ) @@ -143,7 +143,7 @@ def test_incompatible_units(self): """Pass in cubes of incompatible units that should raise an exception.""" self.scale_parameters.units = "m s-1" - msg = "This is likely because the mean" + msg = "This is likely because the location" with self.assertRaisesRegex(ValueError, msg): Plugin()._check_unit_compatibility( self.location_parameters, self.scale_parameters, self.template_cube @@ -163,7 +163,7 @@ def setUp(self): ) location_parameter_values = np.ones((3, 3)) * 2 - scale_parameter_values = np.ones((3, 3)) * 4 + scale_parameter_values = np.ones((3, 3)) * 2 self.expected = (np.ones((3, 3, 3)) * [0.75, 0.5, 0.25]).T self.location_parameter_values = self.template_cube[0, :, :].copy( data=location_parameter_values @@ -172,7 +172,7 @@ def setUp(self): self.scale_parameter_values = self.template_cube[0, :, :].copy( data=scale_parameter_values ) - self.scale_parameter_values.units = "Celsius2" + self.scale_parameter_values.units = "Celsius" def test_threshold_above_cube(self): """Test that the expected probabilities are returned for a cube in @@ -288,7 +288,7 @@ def setUp(self): self.scale_parameter_values = self.template_cube[0, :, :].copy( data=scale_parameter_values ) - self.scale_parameter_values.units = "Celsius2" + self.scale_parameter_values.units = "Celsius" def test_metadata_matches_template(self): """Test that the returned cube's metadata matches the template cube.""" From a631b32215ed192db7696a31aa210e6cf4b34b47 Mon Sep 17 00:00:00 2001 From: bayliffe Date: Tue, 25 Jan 2022 10:24:53 +0000 Subject: [PATCH 2/3] MOBT-154: Reunification of wx decision trees (#1639) * Method for reintegration chosen. Method implemented, but changes to tests only just begun. * Further tests and tweaks. * Formatting changes. * Updated checksums. * Added another check in the code. Added an associated test, and fixed some old tests that were implemented incorrectly. * Flake 8 fixes. * Bracket bug fix. * Additional test for exception. * Documentation update. * Review tweaks. * Added args entry. --- .../wxcode/build_a_decision_tree.rst | 14 +++- improver/cli/wxcode.py | 14 +++- improver/wxcode/utilities.py | 49 +++++++++-- improver/wxcode/weather_symbols.py | 42 +++++++--- improver_tests/acceptance/SHA256SUMS | 3 +- improver_tests/acceptance/test_wxcode.py | 20 +++-- improver_tests/wxcode/wxcode/__init__.py | 21 ++++- .../wxcode/wxcode/test_WeatherSymbols.py | 50 +++++++---- .../wxcode/wxcode/test_utilities.py | 84 ++++++++++++++++++- 9 files changed, 243 insertions(+), 54 deletions(-) diff --git a/doc/source/extended_documentation/wxcode/build_a_decision_tree.rst b/doc/source/extended_documentation/wxcode/build_a_decision_tree.rst index d2a86a0232..51c2f39d00 100644 --- a/doc/source/extended_documentation/wxcode/build_a_decision_tree.rst +++ b/doc/source/extended_documentation/wxcode/build_a_decision_tree.rst @@ -77,10 +77,16 @@ accessed with this key contains the essentials that make the node function. condition_combination. Alternatively, if they are being manipulated within the node (e.g. added together), they must be separated by the desired operators (e.g. 'diagnostic1', '+', 'diagnostic2'). - - **diagnostic_thresholds** (List(List(float, str)): The diagnostic threshold - value and units being used in the test. A threshold [value, units] pair must - be provided for each diagnostic field with the same nested list structure; as - the basic unit is a list of value and unit, the overall nested structure is + - **diagnostic_thresholds** (List(List(float, str, Optional(int))): The + diagnostic threshold value and units being used in the test. An optional + third value provides a period in seconds that is associated with the + threshold value. For example, a precipitation accumulation threshold might + be given for a 1-hour period (3600 seconds). If instead 3-hour symbols are + being produced using 3-hour precipitation accumulations then the threshold + value will be scaled up by a factor of 3. Only thresholds with an + associated period will be scaled in this way. A threshold [value, units] pair + must be provided for each diagnostic field with the same nested list structure; + as the basic unit is a list of value and unit, the overall nested structure is one list deeper. - **diagnostic_conditions** (as diagnostic_fields): The expected inequality that has been used to construct the input probability field. This is checked diff --git a/improver/cli/wxcode.py b/improver/cli/wxcode.py index 6fe63f51c0..22baa7beb5 100755 --- a/improver/cli/wxcode.py +++ b/improver/cli/wxcode.py @@ -40,6 +40,7 @@ def process( *cubes: cli.inputcube, wxtree: cli.inputjson = None, model_id_attr: str = None, + target_period: int = None, check_tree: bool = False, ): """ Processes cube for Weather symbols. @@ -54,6 +55,13 @@ def process( Name of attribute recording source models that should be inherited by the output cube. The source models are expected as a space-separated string. + target_period: + The period in seconds that the weather symbol being produced should + represent. This should correspond with any period diagnostics, e.g. + precipitation accumulation, being used as input. This is used to scale + any threshold values that are defined with an associated period in + the decision tree. It will only be used if the decision tree + provided has threshold values defined with an associated period. check_tree (bool): If set the decision tree will be checked to see if it conforms to the expected format; the only other argument required is the path @@ -68,7 +76,7 @@ def process( if check_tree: from improver.wxcode.utilities import check_tree - return check_tree(wxtree) + return check_tree(wxtree, target_period=target_period) from iris.cube import CubeList @@ -77,4 +85,6 @@ def process( if not cubes: raise RuntimeError("Not enough input arguments. See help for more information.") - return WeatherSymbols(wxtree, model_id_attr=model_id_attr)(CubeList(cubes)) + return WeatherSymbols( + wxtree, model_id_attr=model_id_attr, target_period=target_period + )(CubeList(cubes)) diff --git a/improver/wxcode/utilities.py b/improver/wxcode/utilities.py index 18a6e7aafd..6968c72a5a 100644 --- a/improver/wxcode/utilities.py +++ b/improver/wxcode/utilities.py @@ -31,7 +31,7 @@ """This module defines the utilities required for wxcode plugin """ from collections import OrderedDict -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional import iris from iris.cube import Cube @@ -102,22 +102,49 @@ } -def update_tree_units(tree: Dict[str, Dict[str, Any]]) -> Dict[str, Dict[str, Any]]: +def update_tree_thresholds( + tree: Dict[str, Dict[str, Any]], target_period: Optional[int] = None +) -> Dict[str, Dict[str, Any]]: """ Replaces value / unit pairs from tree definition with an Iris AuxCoord - that encodes the same information. + that encodes the same information. Also scales any threshold values that + have an associated period (e.g. accumulation in 3600 seconds) by a factor + to reflect the target period (e.g. a 3-hour, 10800 second, weather symbol). Args: tree: Weather symbols decision tree. + target_period: + The period in seconds that the weather symbol being produced should + represent. This should correspond with any period diagnostics, e.g. + precipitation accumulation, being used as input. This is used to scale + any threshold values that are defined with an associated period in + the decision tree. Returns: - The tree now containing AuxCoords instead of value / unit pairs. + The tree now containing AuxCoords instead of value / unit pairs, with + period diagnostic threshold values scaled appropriately. + + Raises: + ValueError: If thresholds are defined with an associated period and no + target_period is provided. """ def _make_thresholds_with_units(items): if isinstance(items[0], list): return [_make_thresholds_with_units(item) for item in items] - values, units = items + try: + values, units, period = items + except ValueError: + values, units = items + else: + if not target_period: + raise ValueError( + "The decision tree contains thresholds defined for a particular " + "time period, e.g. the accumulation over 3600 seconds. To " + "use such a decision tree a target_period must be provided " + "such that the threshold values can be scaled appropriately." + ) + values = values * target_period / period return iris.coords.AuxCoord(values, units=units) for query in tree.values(): @@ -338,13 +365,21 @@ def _checker(lists): return _checker(query) -def check_tree(wxtree: Dict[str, Dict[str, Any]]) -> str: +def check_tree( + wxtree: Dict[str, Dict[str, Any]], target_period: Optional[int] = None +) -> str: """Perform some checks to ensure the provided decision tree is valid. Args: wxtree: Weather symbols decision tree definition, provided as a dictionary. + target_period: + The period in seconds that the weather symbol being produced should + represent. This should correspond with any period diagnostics, e.g. + precipitation accumulation, being used as input. This is used to scale + any threshold values that are defined with an associated period in + the decision tree. Returns: A list of problems found in the decision tree, or if none are found, the @@ -358,7 +393,7 @@ def check_tree(wxtree: Dict[str, Dict[str, Any]]) -> str: raise ValueError("Decision tree is not a dictionary") issues = [] - wxtree = update_tree_units(wxtree) + wxtree = update_tree_thresholds(wxtree, target_period) valid_codes = list(WX_DICT.keys()) all_key_words = REQUIRED_KEY_WORDS + OPTIONAL_KEY_WORDS diff --git a/improver/wxcode/weather_symbols.py b/improver/wxcode/weather_symbols.py index 54a18cdcc6..5da55c21b2 100644 --- a/improver/wxcode/weather_symbols.py +++ b/improver/wxcode/weather_symbols.py @@ -58,7 +58,7 @@ get_parameter_names, is_variable, update_daynight, - update_tree_units, + update_tree_thresholds, weather_code_attributes, ) @@ -94,7 +94,12 @@ class WeatherSymbols(BasePlugin): .. include:: extended_documentation/wxcode/build_a_decision_tree.rst """ - def __init__(self, wxtree: dict, model_id_attr: Optional[str] = None) -> None: + def __init__( + self, + wxtree: dict, + model_id_attr: Optional[str] = None, + target_period: Optional[int] = None, + ) -> None: """ Define a decision tree for determining weather symbols based upon the input diagnostics. Use this decision tree to allocate a weather @@ -108,6 +113,13 @@ def __init__(self, wxtree: dict, model_id_attr: Optional[str] = None) -> None: Name of attribute recording source models that should be inherited by the output cube. The source models are expected as a space-separated string. + target_period: + The period in seconds that the weather symbol being produced should + represent. This should correspond with any period diagnostics, e.g. + precipitation accumulation, being used as input. This is used to scale + any threshold values that are defined with an associated period in + the decision tree. It will only be used if the decision tree + provided has threshold values defined with an associated period. float_tolerance defines the tolerance when matching thresholds to allow for the difficulty of float comparisons. @@ -116,15 +128,10 @@ def __init__(self, wxtree: dict, model_id_attr: Optional[str] = None) -> None: or snowfall rate could not trigger it. """ - def make_thresholds_with_units(items): - if isinstance(items[0], list): - return [make_thresholds_with_units(item) for item in items] - values, units = items - return iris.coords.AuxCoord(values, units=units) - self.model_id_attr = model_id_attr self.start_node = list(wxtree.keys())[0] - self.queries = update_tree_units(wxtree) + self.target_period = target_period + self.queries = update_tree_thresholds(wxtree, target_period) self.float_tolerance = 0.01 self.float_abs_tolerance = 1e-12 # flag to indicate whether to expect "threshold" as a coordinate name @@ -259,6 +266,7 @@ def check_coincidence(self, cubes: Union[List[Cube], CubeList]) -> Cube: Raises: ValueError: If validity times differ for diagnostics. ValueError: If period diagnostics have different periods. + ValueError: If period diagnostics do not match target_period. """ times = [] bounds = [] @@ -293,6 +301,14 @@ def check_coincidence(self, cubes: Union[List[Cube], CubeList]) -> Cube: "all describe the same period to be used together." f"\n{diagnostic_bounds}" ) + # Check that if time bounded diagnostics are used they match the + # user specified target_period. + if bounds and self.target_period: + if not np.diff(bounds[0])[0] == self.target_period: + raise ValueError( + f"Diagnostic periods ({np.diff(bounds[0])[0]}) do not match " + f"the user specified target_period ({self.target_period})." + ) @staticmethod def _invert_comparator(comparator: str) -> str: @@ -567,8 +583,8 @@ def compare_array_to_threshold( return arr >= threshold else: raise ValueError( - "Invalid comparator: {}. ".format(comparator), - "Comparator must be one of '<', '>', '<=', '>='.", + f"Invalid comparator: {comparator}. " + "Comparator must be one of '<', '>', '<=', '>='." ) def evaluate_extract_expression( @@ -699,8 +715,8 @@ def is_chain(item): res = res | new_res else: msg = ( - "Invalid condition chain found. First element has length > 1 ", - "but second element is not 'AND' or 'OR'.", + "Invalid condition chain found. First element has length > 1 " + "but second element is not 'AND' or 'OR'." ) raise RuntimeError(msg) return res diff --git a/improver_tests/acceptance/SHA256SUMS b/improver_tests/acceptance/SHA256SUMS index 75d891727f..b14ce64e47 100644 --- a/improver_tests/acceptance/SHA256SUMS +++ b/improver_tests/acceptance/SHA256SUMS @@ -737,5 +737,4 @@ f20a9eabe867ea00e9570eb1177c01841b7606e2969287432d4a50ae888062a3 ./wxcode/nativ ec818f319f2c137ab356f97a3980788357254f17c0761908b4e656170c0e5d37 ./wxcode/native_units/probability_of_shower_condition_above_threshold.nc 1b8de3a02ac7dcf18ec9208a5e0b583b37c420093781b861b67eb5c8eaa37cd9 ./wxcode/native_units/probability_of_thickness_of_rainfall_amount_above_threshold.nc 554f9ed54f1092f31013b0e85de20e13932ca9533d50ac6316f170641901a209 ./wxcode/native_units/probability_of_visibility_in_air_below_threshold.nc -3de7bd3e25d9d5b9eafbef91ead03f1ebfa6202b9f90826dbe81ad99fdc74277 ./wxcode/wx_decision_tree_1h.json -80a0dc027f33a9f209d8304a92eb60fd204620492bea5bb9c0fe5c4d8a285edb ./wxcode/wx_decision_tree_3h.json +be969533132f0d1651959ef9db777b2c71b109673b75eb59edb0e87e78698a9f ./wxcode/wx_decision_tree.json diff --git a/improver_tests/acceptance/test_wxcode.py b/improver_tests/acceptance/test_wxcode.py index 5c1319fb7d..c423547c7a 100644 --- a/improver_tests/acceptance/test_wxcode.py +++ b/improver_tests/acceptance/test_wxcode.py @@ -62,7 +62,7 @@ def test_basic(tmp_path): param_paths = [ kgo_dir / "basic" / f"probability_of_{p}_threshold.nc" for p in ALL_PARAMS ] - wxtree = kgo_dir / "wx_decision_tree_1h.json" + wxtree = kgo_dir / "wx_decision_tree.json" output_path = tmp_path / "output.nc" args = [ *param_paths, @@ -70,6 +70,8 @@ def test_basic(tmp_path): wxtree, "--model-id-attr", "mosg__model_configuration", + "--target-period", + "3600", "--output", output_path, ] @@ -89,7 +91,7 @@ def test_native_units(tmp_path): kgo_dir / "native_units" / f"probability_of_{p}_threshold.nc" for p in ALL_PARAMS ] - wxtree = kgo_dir / "wx_decision_tree_1h.json" + wxtree = kgo_dir / "wx_decision_tree.json" output_path = tmp_path / "output.nc" args = [ *param_paths, @@ -97,6 +99,8 @@ def test_native_units(tmp_path): wxtree, "--model-id-attr", "mosg__model_configuration", + "--target-period", + "3600", "--output", output_path, ] @@ -112,7 +116,7 @@ def test_global(tmp_path): param_paths = [ kgo_dir / "global" / f"probability_of_{p}_threshold.nc" for p in params ] - wxtree = kgo_dir / "wx_decision_tree_3h.json" + wxtree = kgo_dir / "wx_decision_tree.json" output_path = tmp_path / "output.nc" args = [ *param_paths, @@ -120,6 +124,8 @@ def test_global(tmp_path): wxtree, "--model-id-attr", "mosg__model_configuration", + "--target-period", + "10800", "--output", output_path, ] @@ -139,7 +145,7 @@ def test_insufficient_files(tmp_path): param_paths = [ kgo_dir / "global" / f"probability_of_{p}_threshold.nc" for p in params ] - wxtree = kgo_dir / "wx_decision_tree_3h.json" + wxtree = kgo_dir / "wx_decision_tree.json" output_path = tmp_path / "output.nc" args = [ *param_paths, @@ -147,6 +153,8 @@ def test_insufficient_files(tmp_path): wxtree, "--model-id-attr", "mosg__model_configuration", + "--target-period", + "10800", "--output", output_path, ] @@ -164,7 +172,7 @@ def test_no_lightning(tmp_path): for p in ALL_PARAMS if "lightning" not in p ] - wxtree = kgo_dir / "wx_decision_tree_1h.json" + wxtree = kgo_dir / "wx_decision_tree.json" output_path = tmp_path / "output.nc" args = [ *param_paths, @@ -172,6 +180,8 @@ def test_no_lightning(tmp_path): wxtree, "--model-id-attr", "mosg__model_configuration", + "--target-period", + "3600", "--output", output_path, ] diff --git a/improver_tests/wxcode/wxcode/__init__.py b/improver_tests/wxcode/wxcode/__init__.py index 004ae166f7..dc415300d5 100644 --- a/improver_tests/wxcode/wxcode/__init__.py +++ b/improver_tests/wxcode/wxcode/__init__.py @@ -88,10 +88,15 @@ def set_up_wxcube( return cube -def wxcode_decision_tree() -> Dict[str, Dict[str, Any]]: +def wxcode_decision_tree(accumulation: bool = False) -> Dict[str, Dict[str, Any]]: """ Define an example decision tree to test the weather symbols code. + Args: + accumulation: + A boolean that if true means the "heavy_precipitation" node + should be modified to use a precipitation accumulation rather than + a rate. Returns: A dictionary containing the queries that comprise the decision tree. @@ -494,4 +499,18 @@ def wxcode_decision_tree() -> Dict[str, Dict[str, Any]]: }, } + if accumulation: + queries["heavy_precipitation"] = { + "if_true": "heavy_precipitation_cloud", + "if_false": "precipitation_in_vicinity", + "probability_thresholds": [0.5], + "threshold_condition": ">=", + "condition_combination": "", + "diagnostic_fields": [ + "probability_of_lwe_thickness_of_precipitation_amount_above_threshold" + ], + "diagnostic_thresholds": [[1.0, "mm", 3600]], + "diagnostic_conditions": ["above"], + } + return queries diff --git a/improver_tests/wxcode/wxcode/test_WeatherSymbols.py b/improver_tests/wxcode/wxcode/test_WeatherSymbols.py index 24d535dec1..262377f37b 100644 --- a/improver_tests/wxcode/wxcode/test_WeatherSymbols.py +++ b/improver_tests/wxcode/wxcode/test_WeatherSymbols.py @@ -862,10 +862,10 @@ def test_error(self): "", ] msg = ( - "Invalid condition chain found. First element has length > 1 ", - "but second element is not 'AND' or 'OR'.", + "Invalid condition chain found. First element has length > 1 " + "but second element is not 'AND' or 'OR'." ) - with self.assertRaises(RuntimeError, msg=msg): + with self.assertRaisesRegex(RuntimeError, msg): self.plugin.evaluate_condition_chain(self.cubes, chain) def test_with_operators(self): @@ -1203,8 +1203,8 @@ def test_unmatched_validity_times(self): cubes[-1].coord("time").points = cubes[-1].coord("time").points + 3600 msg = ( - "Weather symbol input cubes are valid at different times" - "['probability_of_lwe_snowfall_rate_above_threshold: 1507636800', " + "Weather symbol input cubes are valid at different times; \n" + "\\['probability_of_lwe_snowfall_rate_above_threshold: 1507636800', " "'probability_of_lwe_sleetfall_rate_above_threshold: 1507636800', " "'probability_of_rainfall_rate_above_threshold: 1507636800', " "'probability_of_lwe_precipitation_rate_in_vicinity_above_threshold: 1507636800', " @@ -1212,9 +1212,11 @@ def test_unmatched_validity_times(self): "'probability_of_low_type_cloud_area_fraction_above_threshold: 1507636800', " "'probability_of_visibility_in_air_below_threshold: 1507636800', " "'probability_of_lwe_precipitation_rate_above_threshold: 1507636800', " - "'probability_of_shower_condition_above_threshold: 1507640400']" + "'probability_of_shower_condition_above_threshold: 1507636800', " + "'probability_of_lwe_graupel_and_hail_fall_rate_in_vicinity_above_threshold: 1507636800', " # noqa: E501 + "'probability_of_lwe_precipitation_rate_max_above_threshold: 1507640400'\\]" ) - with self.assertRaises(ValueError, msg=msg): + with self.assertRaisesRegex(ValueError, msg): self.plugin.check_coincidence(cubes) def test_unmatched_periods(self): @@ -1233,15 +1235,29 @@ def test_unmatched_periods(self): msg = ( "Period diagnostics with different periods have been provided as " "input to the weather symbols code. Period diagnostics must all " - "describe the same period to be used together." - "['probability_of_number_of_lightning_flashes_per_unit_area_in_" - "vicinity_above_threshold: 3600', 'probability_of_shower_condition_" - "above_threshold: 7200', 'probability_of_shower_condition_above_" - "threshold: 7200']" + "describe the same period to be used together.\n" + "\\['probability_of_shower_condition_above_threshold: 7200', " + "'probability_of_number_of_lightning_flashes_per_unit_area_in_" + "vicinity_above_threshold: 3600', 'probability_of_lwe_graupel_and_" + "hail_fall_rate_in_vicinity_above_threshold: 3600', " + "'probability_of_lwe_precipitation_rate_max_above_threshold: 3600', " + "'probability_of_shower_condition_above_threshold: 7200'\\]" ) - with self.assertRaises(ValueError, msg=msg): + with self.assertRaisesRegex(ValueError, msg): self.plugin.check_coincidence(self.cubes) + def test_target_period_mismatch(self): + """Test that an exception is raised if the diagnostic periods do not + match the user specified target_period.""" + + plugin = WeatherSymbols(wxtree=wxcode_decision_tree(), target_period=10800) + msg = ( + "Diagnostic periods \\(3600\\) do not match " + "the user specified target_period \\(10800\\)." + ) + with self.assertRaisesRegex(ValueError, msg): + plugin.check_coincidence(self.cubes) + def test_no_period_diagnostics(self): """Test that the first cube in the diagnostic cube list is set as the global template cube if there are no period diagnostics.""" @@ -1354,11 +1370,9 @@ def test_error_on_unexpected_comparison(self): one of the expected strings.""" arr = np.array([0, 1, 2], dtype=np.int32) plugin = WeatherSymbols(wxtree=wxcode_decision_tree()) - msg = ( - "Invalid comparator: !=. ", - "Comparator must be one of '<', '>', '<=', '>='.", - ) - with self.assertRaises(ValueError, msg=msg): + msg = "Invalid comparator: !=. Comparator must be one of '<', '>', '<=', '>='." + + with self.assertRaisesRegex(ValueError, msg): plugin.compare_array_to_threshold(arr, "!=", 1) diff --git a/improver_tests/wxcode/wxcode/test_utilities.py b/improver_tests/wxcode/wxcode/test_utilities.py index 88be956cad..d34314c467 100644 --- a/improver_tests/wxcode/wxcode/test_utilities.py +++ b/improver_tests/wxcode/wxcode/test_utilities.py @@ -55,7 +55,7 @@ get_parameter_names, interrogate_decision_tree, update_daynight, - update_tree_units, + update_tree_thresholds, weather_code_attributes, ) @@ -100,6 +100,42 @@ def test_wxcode_values(self): self.assertEqual(WX_DICT[30], "Thunder") +@pytest.mark.parametrize( + "accumulation, target_period, expected_value, expected_unit", + ( + (False, None, 1, "mm/hr"), + (False, 10800, 1, "mm/hr"), + (True, 3600, 1, "mm"), + (True, 10800, 3, "mm"), + ), +) +def test_update_tree_thresholds( + accumulation, target_period, expected_value, expected_unit +): + """Test that updating tree thresholds returns iris AuxCoords with the + expected value and units. Includes a test that the threshold value is scaled + if it is defined with an associated period that differs from a user supplied + target period.""" + + tree = wxcode_decision_tree(accumulation=accumulation) + tree = update_tree_thresholds(tree, target_period=target_period) + (result,) = tree["heavy_precipitation"]["diagnostic_thresholds"] + + assert isinstance(result, iris.coords.AuxCoord) + assert result.points[0] == expected_value + assert result.units == expected_unit + + +def test_update_tree_thresholds_exception(): + """Test that updating tree thresholds raises an error if the input thresholds + are defined with an associated period, but no target_period is provided.""" + + tree = wxcode_decision_tree(accumulation=True) + expected = "The decision tree contains thresholds defined" + with pytest.raises(ValueError, match=expected): + update_tree_thresholds(tree, target_period=None) + + class Test_weather_code_attributes(IrisTest): """ Test weather_code_attributes is working correctly """ @@ -348,7 +384,51 @@ def test_interrogate_decision_tree(): "\u26C5 probability_of_shower_condition_above_threshold (1): 1.0\n" "\u26C5 probability_of_visibility_in_air_below_threshold (m): 1000.0, 5000.0\n" ) - tree = update_tree_units(wxcode_decision_tree()) + tree = update_tree_thresholds(wxcode_decision_tree(), None) + result = interrogate_decision_tree(tree) + assert result == expected + + +def test_interrogate_decision_tree_accumulation_1h(): + """Test that the function returns the right strings.""" + expected = ( + "\u26C5 probability_of_low_and_medium_type_cloud_area_fraction_above_threshold (1): 0.1875, 0.8125\n" # noqa: E501 + "\u26C5 probability_of_low_type_cloud_area_fraction_above_threshold (1): 0.85\n" + "\u26C5 probability_of_lwe_graupel_and_hail_fall_rate_in_vicinity_above_threshold (mm hr-1): 0.0\n" # noqa: E501 + "\u26C5 probability_of_lwe_precipitation_rate_above_threshold (mm hr-1): 0.03\n" # noqa: E501 + "\u26C5 probability_of_lwe_precipitation_rate_in_vicinity_above_threshold (mm hr-1): 0.1, 1.0\n" # noqa: E501 + "\u26C5 probability_of_lwe_precipitation_rate_max_above_threshold (mm hr-1): 1.0\n" # noqa: E501 + "\u26C5 probability_of_lwe_sleetfall_rate_above_threshold (mm hr-1): 0.03, 1.0\n" + "\u26C5 probability_of_lwe_snowfall_rate_above_threshold (mm hr-1): 0.03, 1.0\n" + "\u26C5 probability_of_lwe_thickness_of_precipitation_amount_above_threshold (mm): 1.0\n" # noqa: E501 + "\u26C5 probability_of_number_of_lightning_flashes_per_unit_area_in_vicinity_above_threshold (m-2): 0.0\n" # noqa: E501 + "\u26C5 probability_of_rainfall_rate_above_threshold (mm hr-1): 0.03, 1.0\n" + "\u26C5 probability_of_shower_condition_above_threshold (1): 1.0\n" + "\u26C5 probability_of_visibility_in_air_below_threshold (m): 1000.0, 5000.0\n" + ) + tree = update_tree_thresholds(wxcode_decision_tree(accumulation=True), 3600) + result = interrogate_decision_tree(tree) + assert result == expected + + +def test_interrogate_decision_tree_accumulation_3h(): + """Test that the function returns the right strings.""" + expected = ( + "\u26C5 probability_of_low_and_medium_type_cloud_area_fraction_above_threshold (1): 0.1875, 0.8125\n" # noqa: E501 + "\u26C5 probability_of_low_type_cloud_area_fraction_above_threshold (1): 0.85\n" + "\u26C5 probability_of_lwe_graupel_and_hail_fall_rate_in_vicinity_above_threshold (mm hr-1): 0.0\n" # noqa: E501 + "\u26C5 probability_of_lwe_precipitation_rate_above_threshold (mm hr-1): 0.03\n" # noqa: E501 + "\u26C5 probability_of_lwe_precipitation_rate_in_vicinity_above_threshold (mm hr-1): 0.1, 1.0\n" # noqa: E501 + "\u26C5 probability_of_lwe_precipitation_rate_max_above_threshold (mm hr-1): 1.0\n" # noqa: E501 + "\u26C5 probability_of_lwe_sleetfall_rate_above_threshold (mm hr-1): 0.03, 1.0\n" + "\u26C5 probability_of_lwe_snowfall_rate_above_threshold (mm hr-1): 0.03, 1.0\n" + "\u26C5 probability_of_lwe_thickness_of_precipitation_amount_above_threshold (mm): 3.0\n" # noqa: E501 + "\u26C5 probability_of_number_of_lightning_flashes_per_unit_area_in_vicinity_above_threshold (m-2): 0.0\n" # noqa: E501 + "\u26C5 probability_of_rainfall_rate_above_threshold (mm hr-1): 0.03, 1.0\n" + "\u26C5 probability_of_shower_condition_above_threshold (1): 1.0\n" + "\u26C5 probability_of_visibility_in_air_below_threshold (m): 1000.0, 5000.0\n" + ) + tree = update_tree_thresholds(wxcode_decision_tree(accumulation=True), 10800) result = interrogate_decision_tree(tree) assert result == expected From 8eb0e7470be7459b870adf4aa410a01cb5603210 Mon Sep 17 00:00:00 2001 From: Tom Gale Date: Fri, 28 Jan 2022 16:18:55 +1100 Subject: [PATCH 3/3] Change pandas DataFrame.at to DataFrame.loc (#1655) --- .../dataframe_utilities/test_dataframe_utilities.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/improver_tests/calibration/dataframe_utilities/test_dataframe_utilities.py b/improver_tests/calibration/dataframe_utilities/test_dataframe_utilities.py index 0bebcfee88..2ebad25b37 100644 --- a/improver_tests/calibration/dataframe_utilities/test_dataframe_utilities.py +++ b/improver_tests/calibration/dataframe_utilities/test_dataframe_utilities.py @@ -474,9 +474,9 @@ def test_site_coord_mismatch(self): and forecasts. In this case, the position (lat/lon/alt) from the forecast will be used.""" df = self.truth_subset_df.copy() - df.at[::3, "altitude"] = 45 - df.at[::3, "latitude"] = 52 - df.at[::3, "longitude"] = -12 + df.loc[::3, "altitude"] = 45 + df.loc[::3, "latitude"] = 52 + df.loc[::3, "longitude"] = -12 result = forecast_and_truth_dataframes_to_cubes( self.forecast_df, df,