diff --git a/doc/source/extended_documentation/calibration/reliability_calibration/reliability_calibration_examples.rst b/doc/source/extended_documentation/calibration/reliability_calibration/reliability_calibration_examples.rst index 5afd0485c4..ce65e10e0f 100644 --- a/doc/source/extended_documentation/calibration/reliability_calibration/reliability_calibration_examples.rst +++ b/doc/source/extended_documentation/calibration/reliability_calibration/reliability_calibration_examples.rst @@ -12,7 +12,7 @@ The reliability calibration tables returned by this plugin are structured as sho Auxiliary coordinates: table_row_name - x - - - Scalar coordinates: - cycle_hour: 22 + forecast_reference_time: 2017-11-11 00:00:00, bound=(2017-11-10 00:00:00, 2017-11-11 00:00:00) forecast_period: 68400 seconds Attributes: institution: Met Office diff --git a/improver/calibration/dataframe_utilities.py b/improver/calibration/dataframe_utilities.py index 9585b97d1c..2c520dff5c 100644 --- a/improver/calibration/dataframe_utilities.py +++ b/improver/calibration/dataframe_utilities.py @@ -326,17 +326,15 @@ def _prepare_dataframes( keep="last", ) # Sort to ensure a consistent ordering after removing duplicates. - forecast_df.sort_values( - by=["blend_time", "percentile", "wmo_id"], inplace=True, ignore_index=True, + forecast_df = forecast_df.sort_values( + by=["blend_time", "percentile", "wmo_id"], ignore_index=True, ) # Remove truth duplicates. truth_cols = ["diagnostic", "time", "wmo_id"] truth_df = truth_df.drop_duplicates(subset=truth_cols, keep="last",) # Sort to ensure a consistent ordering after removing duplicates. - truth_df.sort_values( - by=truth_cols, inplace=True, ignore_index=True, - ) + truth_df = truth_df.sort_values(by=truth_cols, ignore_index=True) # Find the common set of WMO IDs. common_wmo_ids = sorted( diff --git a/improver/calibration/ensemble_calibration.py b/improver/calibration/ensemble_calibration.py index 994a3f1939..f950700651 100644 --- a/improver/calibration/ensemble_calibration.py +++ b/improver/calibration/ensemble_calibration.py @@ -1130,7 +1130,7 @@ def mask_cube(cube: Cube, landsea_mask: Cube) -> None: IndexError: if the cube and landsea_mask shapes are not compatible. """ try: - cube.data[..., ~landsea_mask.data.astype(np.bool)] = np.nan + cube.data[..., ~landsea_mask.data.astype(bool)] = np.nan except IndexError as err: msg = "Cube and landsea_mask shapes are not compatible. {}".format(err) raise IndexError(msg) @@ -1380,7 +1380,6 @@ def process( forecast_var, number_of_realizations, ) - return coefficients_cubelist diff --git a/improver/calibration/reliability_calibration.py b/improver/calibration/reliability_calibration.py index b25b23588b..a2dd9f389b 100644 --- a/improver/calibration/reliability_calibration.py +++ b/improver/calibration/reliability_calibration.py @@ -233,10 +233,11 @@ def _create_reliability_table_cube( ) -> Cube: """ Construct a reliability table cube and populate it with the provided - data. The returned cube will include a cycle hour coordinate, which - describes the model cycle hour at which the forecast data was produced. - It will further include the forecast period, threshold coordinate, - and spatial coordinates from the forecast cube. + data. The returned cube will include a forecast_reference_time + coordinate, which will be the maximum range of bounds of the input + forecast reference times, with the point value set to the latest + of those in the inputs. It will further include the forecast period, + threshold coordinate, and spatial coordinates from the forecast cube. Args: forecast: @@ -443,11 +444,11 @@ def process(self, historic_forecasts: Cube, truths: Cube) -> Cube: whether the data is thresholded below or above a given diagnostic threshold. + `historic_forecasts` and `truths` should have matching validity times. + Args: historic_forecasts: A cube containing the historical forecasts used in calibration. - These are expected to all have a consistent cycle hour, that is - the hour in the forecast reference time. truths: A cube containing the thresholded gridded truths used in calibration. diff --git a/improver/cli/wxcode.py b/improver/cli/wxcode.py index 22baa7beb5..0e57214e19 100755 --- a/improver/cli/wxcode.py +++ b/improver/cli/wxcode.py @@ -63,11 +63,11 @@ def process( the decision tree. It will only be used if the decision tree provided has threshold values defined with an associated period. check_tree (bool): - If set the decision tree will be checked to see if it conforms to - the expected format; the only other argument required is the path - to the decision tree. If the tree is found to be valid the required - inputs will be listed. Setting this flag will prevent the CLI - performing any other actions. + If set, the decision tree will be checked to see if it conforms to + the expected format and that all nodes can be reached; the only other + argument required is the path to the decision tree. If the tree is found + to be valid the required inputs will be listed. Setting this flag will + prevent the CLI performing any other actions. Returns: iris.cube.Cube: diff --git a/improver/cli/wxcode_modal.py b/improver/cli/wxcode_modal.py index 206263495c..85e67cc1ba 100644 --- a/improver/cli/wxcode_modal.py +++ b/improver/cli/wxcode_modal.py @@ -36,7 +36,7 @@ @cli.clizefy @cli.with_output -def process(*cubes: cli.inputcube): +def process(*cubes: cli.inputcube, model_id_attr: str = None): """Generates a modal weather symbol for the period covered by the input weather symbol cubes. Where there are different weather codes available for night and day, the modal code returned is always a day code, regardless @@ -46,6 +46,10 @@ def process(*cubes: cli.inputcube): cubes (iris.cube.CubeList): A cubelist containing weather symbols cubes that cover the period over which a modal symbol is desired. + model_id_attr (str): + Name of attribute recording source models that should be + inherited by the output cube. The source models are expected as + a space-separated string. Returns: iris.cube.Cube: @@ -56,4 +60,4 @@ def process(*cubes: cli.inputcube): if not cubes: raise RuntimeError("Not enough input arguments. See help for more information.") - return ModalWeatherCode()(cubes) + return ModalWeatherCode(model_id_attr=model_id_attr)(cubes) diff --git a/improver/synthetic_data/set_up_test_cubes.py b/improver/synthetic_data/set_up_test_cubes.py index c416bf02ce..dbb253f00a 100644 --- a/improver/synthetic_data/set_up_test_cubes.py +++ b/improver/synthetic_data/set_up_test_cubes.py @@ -260,7 +260,7 @@ def _create_dimension_coord( coord_array = np.array(coord_array) - if issubclass(coord_array.dtype.type, np.float): + if issubclass(coord_array.dtype.type, float): # option needed for realizations percentile & probability cube setup # and heights coordinate coord_array = coord_array.astype(np.float32) diff --git a/improver/wxcode/modal_code.py b/improver/wxcode/modal_code.py index 25dc25d16f..024352cf20 100644 --- a/improver/wxcode/modal_code.py +++ b/improver/wxcode/modal_code.py @@ -73,10 +73,20 @@ class ModalWeatherCode(BasePlugin): covered by the input files. """ - def __init__(self): - """Create an aggregator instance for reuse""" + def __init__(self, model_id_attr: str = None): + """ + Set up plugin and create an aggregator instance for reuse + + Args: + model_id_attr: + Name of attribute recording source models that should be + inherited by the output cube. The source models are expected as + a space-separated string. + """ self.aggregator_instance = Aggregator("mode", self.mode_aggregator) + self.model_id_attr = model_id_attr + # Create the expected cell method for use with single cube inputs # that do not pass through the aggregator. self.mode_cell_method = iris.coords.CellMethod("mode", coords="time") @@ -205,6 +215,16 @@ def process(self, cubes: CubeList) -> Cube: result = cube.collapsed("time", self.aggregator_instance) self._set_blended_times(result) + if self.model_id_attr: + # Update contributing models + contributing_models = set() + for source_cube in cubes: + for model in source_cube.attributes[self.model_id_attr].split(" "): + contributing_models.update([model]) + result.attributes[self.model_id_attr] = " ".join( + sorted(list(contributing_models)) + ) + # Handle any unset points where it was hard to determine a suitable mode if (result.data == UNSET_CODE_INDICATOR).any(): self._group_codes(result, cube) diff --git a/improver/wxcode/utilities.py b/improver/wxcode/utilities.py index 6968c72a5a..b98d7c4085 100644 --- a/improver/wxcode/utilities.py +++ b/improver/wxcode/utilities.py @@ -34,6 +34,7 @@ from typing import Any, Dict, List, Optional import iris +import numpy as np from iris.cube import Cube REQUIRED_KEY_WORDS = [ @@ -393,6 +394,10 @@ def check_tree( raise ValueError("Decision tree is not a dictionary") issues = [] + start_node = list(wxtree.keys())[0] + all_targets = np.array( + [(n["if_true"], n["if_false"]) for n in wxtree.values()] + ).flatten() wxtree = update_tree_thresholds(wxtree, target_period) valid_codes = list(WX_DICT.keys()) @@ -403,6 +408,10 @@ def check_tree( if entry not in all_key_words: issues.append(f"Node {node} contains unknown key '{entry}'") + # Check that this node is reachable, or is the start_node + if not ((node == start_node) or node in all_targets): + issues.append(f"Unreachable node '{node}'") + # Check that if_diagnostic_missing key points at a if_true or if_false # node if "if_diagnostic_missing" in items: diff --git a/improver_tests/acceptance/SHA256SUMS b/improver_tests/acceptance/SHA256SUMS index 8bf0f11b93..29fe4596a9 100644 --- a/improver_tests/acceptance/SHA256SUMS +++ b/improver_tests/acceptance/SHA256SUMS @@ -94,10 +94,6 @@ b5ca2030a23ba6440c712a659f6f4f3dcbe8566afeef1a68e2e5a6a9b6e7484e ./combine/broa 88b1b0d76a4d1d90a59766d8ad166a9e001556f951201022976ab3fa4e45c1c8 ./combine/cellmethods/kgo.nc 7e4e3da97b7725de34d1e56fbdcb837cc2fc50eaddb29e3e8452ab72aff8e56b ./combine/cellmethods/precipitation_accumulation-PT01H.nc 45893b7a7c10936f33cac4899a01c6f2c5ee2b5abe20fa400144558d3b9a15b3 ./combine/cellmethods/precipitation_is_snow.nc -a4b991d8e0fa174cec415efcf936b38e67fb936c57f7a9e2794beb6629b9b824 ./combine/minimum_realizations/20220128T1900Z-PT0010H00M-temperature_at_screen_level_max-PT01H.nc -4bca51d7208294112240b10e3a36692491472dd3f7ebb072ca030e76a495f14f ./combine/minimum_realizations/20220128T2000Z-PT0011H00M-temperature_at_screen_level_max-PT01H.nc -41daf1da910869fbd9338e31a530aaa68a221828d6e6b8cdbb4302a8e65fc223 ./combine/minimum_realizations/20220128T2100Z-PT0012H00M-temperature_at_screen_level_max-PT01H.nc -b60f6046c86319f8b7ca3b5d7902dbaf3a52f571f30ba56a1a4bc814c42dd341 ./combine/minimum_realizations/kgo.nc 0bd96af6cb5c6caa045e397589dd0ce3b498af837d989fe73326f5e9459c6054 ./construct-reliability-tables/basic/forecast_0.nc fbc14286b4ce41e2e60df0870ae4911c1b00a38ec96912f43c6187fcaf7d02f6 ./construct-reliability-tables/basic/forecast_1.nc 902e5cb9d3dc5d2b78bb99aff8370f9815adf5064b2caeb7abed73a56a897a43 ./construct-reliability-tables/basic/kgo_single_value_bins.nc @@ -637,73 +633,73 @@ e653ae271dd789dde8b03e6237cc7b48552dd9d5206d3430ee68d283683247a0 ./wind_downsca de5cee66ebf03a6dc6fe4350fb8fedd782adc0bb5da72726a4098c3b62422130 ./wind_downscaling/with_realization/kgo.nc e50cf0c7e23b12317412e34ea45f959532beb6ba4247e075be08c2a6732e8cd8 ./wind_downscaling/with_realization/sigma.nc 933e1b15c22d3bf302c57dff3fc92c33d45d056b9b6eaa6d677e870537a42e7c ./wind_downscaling/with_realization/standard_orog.nc -97de7945072ae9a2845424904e925aaa1e722fac0c959bb3e534392d4e4a7795 ./wxcode-modal/blend_mismatch_inputs/20201209T0700Z-weather_symbols-PT01H.nc -f76ea9210c563997354ee5e5d3d819192a9d7bb329374839cdebe02d464643d6 ./wxcode-modal/blend_mismatch_inputs/20201209T0800Z-weather_symbols-PT01H.nc -b5f4fc01b1f03811505d4a64cd4e3aa49d94f7a0cb6882b22a5cfd826f82be0d ./wxcode-modal/blend_mismatch_inputs/20201209T0900Z-weather_symbols-PT01H.nc -80bc644d22207485bad81b2ee6d3abab8327ccde982c0771d05e9887d6b9ff7d ./wxcode-modal/blend_mismatch_inputs/20201209T1000Z-weather_symbols-PT01H.nc -cc20c480460357559bbe9151ad069e97535df17a0687ae9e0fa1f5ef3cf9f9a6 ./wxcode-modal/blend_mismatch_inputs/20201209T1100Z-weather_symbols-PT01H.nc -a4576afe3c045c00718692164a89bc0acb70191f70f34bcd300dfa899b9c6f15 ./wxcode-modal/blend_mismatch_inputs/20201209T1200Z-weather_symbols-PT01H.nc -b479b8d6e9b002e716575a8a9051a70891f4cc8862af8ee80b53ecb05d37e8a3 ./wxcode-modal/blend_mismatch_inputs/20201209T1300Z-weather_symbols-PT01H.nc -2b1241b79c97e991ed06c34e6109e85dc1517726d2d375f104a7ac85d939d0a2 ./wxcode-modal/blend_mismatch_inputs/20201209T1400Z-weather_symbols-PT01H.nc -e42830916b5bf8b3f97ace0e54dec8e80be3cf93d9faf420274bab1cfb1772e4 ./wxcode-modal/blend_mismatch_inputs/20201209T1500Z-weather_symbols-PT01H.nc -b4bba902e61db128924d33f145e73aecb6cdf27dc71db6d66d17c9dcfcc498b5 ./wxcode-modal/blend_mismatch_inputs/20201209T1600Z-weather_symbols-PT01H.nc -09da6234ded5f1572183b8e297aa8f633fc20fe3571b5fc3d22b3a8c95c80a8a ./wxcode-modal/blend_mismatch_inputs/20201209T1700Z-weather_symbols-PT01H.nc -56a8d8f763072027c4a4819e0cab298ea0b820cf32903ae010132fa0a72a2eb9 ./wxcode-modal/blend_mismatch_inputs/20201209T1800Z-weather_symbols-PT01H.nc -d081b4d643e2e7734a6a2084efd69a2a0a01ce90f8c234190729a4ceeba53c2b ./wxcode-modal/blend_mismatch_inputs/kgo.nc -c2d9ae0d65cd3da79a7484e8db57b351159d68ab66c3024294b66a95f166f957 ./wxcode-modal/gridded_input/20201209T0700Z-weather_symbols-PT01H.nc -61a2e23ae036a1dc91ce3acc4a8eb105405507718e19a0c07696a47017cd7d75 ./wxcode-modal/gridded_input/20201209T0800Z-weather_symbols-PT01H.nc -181011db593ba39126b67a0166f28844658fc4f9ab9479dd7bdc507f8222f3a5 ./wxcode-modal/gridded_input/20201209T0900Z-weather_symbols-PT01H.nc -fd430134e0fefe8e20a23cee7e301ef99526336c4864ac89d26494400fbecfe8 ./wxcode-modal/gridded_input/20201209T1000Z-weather_symbols-PT01H.nc -7ab5b13f76af59c38b5b725b8f7eee3e3dcd66c07df03c7c9540f7ff8d8a1a1c ./wxcode-modal/gridded_input/20201209T1100Z-weather_symbols-PT01H.nc -f71ebe242cc3f677e2418f04b67c88ce6f152ad31315f066329735512b9fee3d ./wxcode-modal/gridded_input/20201209T1200Z-weather_symbols-PT01H.nc -3845f7fa94d0b595d03438a5f6e36a6f2bcd94b313a50135ae475924f6c2b5cd ./wxcode-modal/gridded_input/20201209T1300Z-weather_symbols-PT01H.nc -9370226d29ba38f069c9e6ac44a85fb6969db0e24b99e6a55675a509677d55e1 ./wxcode-modal/gridded_input/20201209T1400Z-weather_symbols-PT01H.nc -9e91a6231b1ded5bfd6781a4dae87927981657d65d103cff7406d43663928261 ./wxcode-modal/gridded_input/20201209T1500Z-weather_symbols-PT01H.nc -9e84306e6cb7f0bed95c7cc02a6905e8c3389d778acbd14c5d9a4f0ce28d0b44 ./wxcode-modal/gridded_input/20201209T1600Z-weather_symbols-PT01H.nc -11847c4063f937ddb9a6612521228ee14f2658c7ca38a56ce2f26c799caedf85 ./wxcode-modal/gridded_input/20201209T1700Z-weather_symbols-PT01H.nc -e053c2d1a27f708a0bc24c52d2a069e2cf57e499014a7a905e72652ccdebf26b ./wxcode-modal/gridded_input/20201209T1800Z-weather_symbols-PT01H.nc -af44bfd1d30dbe0cd673f15a4e0d0d51473ba320365d6cfe9502c23c9d81dfbb ./wxcode-modal/gridded_input/kgo.nc -6da2c57289d7a5577b4b68f5def5364b35f600f20e823d6bf8803c266e670914 ./wxcode-modal/gridded_ties/20201209T0700Z-weather_symbols-PT01H.nc -c42d8c691b3cf06f12a8bb007bb68a2bf4d2e2d0b195938e852a3e269a1298e1 ./wxcode-modal/gridded_ties/20201209T0800Z-weather_symbols-PT01H.nc -e471cd8a242598db8350e73dedd8f144966b38fd8eb2cd193048e26de5b9b973 ./wxcode-modal/gridded_ties/20201209T0900Z-weather_symbols-PT01H.nc -0dc5ebf2212b3ac28dddefcf5d95ab85a93cf91d93604ce40ed67156c03ebfda ./wxcode-modal/gridded_ties/20201209T1000Z-weather_symbols-PT01H.nc -f6a8a6897507309a1f350f73fb5ba72f2d926f477f7b31026db7e740f36ddbc1 ./wxcode-modal/gridded_ties/20201209T1100Z-weather_symbols-PT01H.nc -ec0f5e8f13facb0f107fa3112e14c46ff34d4ac7531b6c36f9b186fc1e38d3a7 ./wxcode-modal/gridded_ties/20201209T1200Z-weather_symbols-PT01H.nc -9f930cd3fcd02769bc686994e9728d2e47eb7229704377d924391ac45c2321d0 ./wxcode-modal/gridded_ties/20201209T1300Z-weather_symbols-PT01H.nc -8e0af1a80bd199c6c8bfdf011a435203474d7a3f7778b5373b02a6d73a8761ff ./wxcode-modal/gridded_ties/20201209T1400Z-weather_symbols-PT01H.nc -a47daaf93e3fb3ac81b196a1e549b5870c3f21f43ad42e077567aaf0031cde43 ./wxcode-modal/gridded_ties/20201209T1500Z-weather_symbols-PT01H.nc -fa3ec96946235d6655cd82c1f687013b1b02596bde0629aa5081223c26f3c324 ./wxcode-modal/gridded_ties/20201209T1600Z-weather_symbols-PT01H.nc -a316682d14c5d45344ad3863dbb29ca9f414aa009536c602759a2c45eb5576bf ./wxcode-modal/gridded_ties/20201209T1700Z-weather_symbols-PT01H.nc -f61521064c98081d182630bae2ecc36150f6c53bbb6279725b4ebade326c3af7 ./wxcode-modal/gridded_ties/20201209T1800Z-weather_symbols-PT01H.nc -c28ef3583fc70dc5fa7ccaa4edc832da8467ac8afe170d7340b50aab43087fc4 ./wxcode-modal/gridded_ties/kgo.nc -7ba7e060ed581804c956ac2e3e66a41c625ba100541dc63256208cc801f6ea63 ./wxcode-modal/single_input/20201210T0000Z-weather_symbols-PT01H.nc -bb4a38f1b0f93ae6920fd00c6683c8258fe5b2edebb23195771a4cb7358e742d ./wxcode-modal/single_input/kgo.nc -97de7945072ae9a2845424904e925aaa1e722fac0c959bb3e534392d4e4a7795 ./wxcode-modal/spot_input/20201209T0700Z-weather_symbols-PT01H.nc -f76ea9210c563997354ee5e5d3d819192a9d7bb329374839cdebe02d464643d6 ./wxcode-modal/spot_input/20201209T0800Z-weather_symbols-PT01H.nc -b5f4fc01b1f03811505d4a64cd4e3aa49d94f7a0cb6882b22a5cfd826f82be0d ./wxcode-modal/spot_input/20201209T0900Z-weather_symbols-PT01H.nc -80bc644d22207485bad81b2ee6d3abab8327ccde982c0771d05e9887d6b9ff7d ./wxcode-modal/spot_input/20201209T1000Z-weather_symbols-PT01H.nc -cc20c480460357559bbe9151ad069e97535df17a0687ae9e0fa1f5ef3cf9f9a6 ./wxcode-modal/spot_input/20201209T1100Z-weather_symbols-PT01H.nc -950bee63559cf13b6af9f4b56d8d7a628a6afc77353345d44ea6798015c2148f ./wxcode-modal/spot_input/20201209T1200Z-weather_symbols-PT01H.nc -b479b8d6e9b002e716575a8a9051a70891f4cc8862af8ee80b53ecb05d37e8a3 ./wxcode-modal/spot_input/20201209T1300Z-weather_symbols-PT01H.nc -2b1241b79c97e991ed06c34e6109e85dc1517726d2d375f104a7ac85d939d0a2 ./wxcode-modal/spot_input/20201209T1400Z-weather_symbols-PT01H.nc -e42830916b5bf8b3f97ace0e54dec8e80be3cf93d9faf420274bab1cfb1772e4 ./wxcode-modal/spot_input/20201209T1500Z-weather_symbols-PT01H.nc -b4bba902e61db128924d33f145e73aecb6cdf27dc71db6d66d17c9dcfcc498b5 ./wxcode-modal/spot_input/20201209T1600Z-weather_symbols-PT01H.nc -09da6234ded5f1572183b8e297aa8f633fc20fe3571b5fc3d22b3a8c95c80a8a ./wxcode-modal/spot_input/20201209T1700Z-weather_symbols-PT01H.nc -56a8d8f763072027c4a4819e0cab298ea0b820cf32903ae010132fa0a72a2eb9 ./wxcode-modal/spot_input/20201209T1800Z-weather_symbols-PT01H.nc -d081b4d643e2e7734a6a2084efd69a2a0a01ce90f8c234190729a4ceeba53c2b ./wxcode-modal/spot_input/kgo.nc -c68ee173a914f9dd5c47d7a528db0c6335aabe8b1506095f009f7f2b308f11b1 ./wxcode-modal/spot_ties/20201209T0700Z-weather_symbols-PT01H.nc -5443a2d3502838bb83cc030f0e55867465dc4587a8407c9279204d0fcde7a27b ./wxcode-modal/spot_ties/20201209T0800Z-weather_symbols-PT01H.nc -35ff246048347536289053839c02b64255415aabe6763ca2b2289777b083a6ee ./wxcode-modal/spot_ties/20201209T0900Z-weather_symbols-PT01H.nc -1025491e3a2f207a2c1fd68d8a494bf54b9d0a4838bf35d54039eaeb0ab11778 ./wxcode-modal/spot_ties/20201209T1000Z-weather_symbols-PT01H.nc -2e6d8a6ed2b7e4ae88c74b25ad04dc38d7cce58574a0c52339bf3076fe0b8929 ./wxcode-modal/spot_ties/20201209T1100Z-weather_symbols-PT01H.nc -5bfc0319d35a7b7a230fdd51f3e8b6bfddf5632b32dde8e0ff023a6b106268f7 ./wxcode-modal/spot_ties/20201209T1200Z-weather_symbols-PT01H.nc -8f94e92886c8df8070db44b273037b032615e4b8768a3e45c708686061f0c8bc ./wxcode-modal/spot_ties/20201209T1300Z-weather_symbols-PT01H.nc -4fb839e28d711fd0ebbf87090358644b62f7dfa0d8dcdcad9c5dc322ce28eaac ./wxcode-modal/spot_ties/20201209T1400Z-weather_symbols-PT01H.nc -c798e3530eac9a45305711d5e91daa56d5005b826ce85bb48e2889e328ea965c ./wxcode-modal/spot_ties/20201209T1500Z-weather_symbols-PT01H.nc -83ba8460875232fc555976872cdb5ac59d5092ccc0da88f88beb9168a3403ac3 ./wxcode-modal/spot_ties/20201209T1600Z-weather_symbols-PT01H.nc -a6698b2aae8bc223a2d362ccb3047dd9882e6c0df21962e55c577e1ad5850e29 ./wxcode-modal/spot_ties/20201209T1700Z-weather_symbols-PT01H.nc -9290b2899eedfca038226c6707e100025ce68b0798972a60360845ccf36858d2 ./wxcode-modal/spot_ties/20201209T1800Z-weather_symbols-PT01H.nc -ceb7106e5b8919590327a75d4a8b740fb1a2c7cb5a271a5bf980f0c1396c7d22 ./wxcode-modal/spot_ties/kgo.nc +6513b722e1fdca6ad68acd768f3506ec814f78e9ccd4799972ec00a2316ef7cf ./wxcode-modal/blend_mismatch_inputs/20201209T0700Z-weather_symbols-PT01H.nc +d4c5d626ad2e5a805f90f2902f6da5d33651eb6fdcb1e16116e2290d0c608bf6 ./wxcode-modal/blend_mismatch_inputs/20201209T0800Z-weather_symbols-PT01H.nc +4642fc43faf4bdde6a678578868fdebb22163064fe6ef0eef41f2f39a959ca89 ./wxcode-modal/blend_mismatch_inputs/20201209T0900Z-weather_symbols-PT01H.nc +ce6a854694aecd515063c91774eeb6160f39c9dbf5e970f72f960a3527e157e9 ./wxcode-modal/blend_mismatch_inputs/20201209T1000Z-weather_symbols-PT01H.nc +300c38e7f522ff1e9257708b183148f3441bb1a861c15eaff2a5f1713bbbc872 ./wxcode-modal/blend_mismatch_inputs/20201209T1100Z-weather_symbols-PT01H.nc +df077177d1ac4b4faccb945f5e64c0bcf2bf19356dee5acef8caffc5db602823 ./wxcode-modal/blend_mismatch_inputs/20201209T1200Z-weather_symbols-PT01H.nc +5ab6b249ec615ddab9995629aaddbd85bba352ad52b7bcb0be5b3037d75c22ca ./wxcode-modal/blend_mismatch_inputs/20201209T1300Z-weather_symbols-PT01H.nc +340205e12625ff9466c1afba8c007343aa40e21b837ab2f2143fa032e6c47bd3 ./wxcode-modal/blend_mismatch_inputs/20201209T1400Z-weather_symbols-PT01H.nc +776739d288b6e43b3d9341ba301feead160986d2540514f0663f243ca003e55a ./wxcode-modal/blend_mismatch_inputs/20201209T1500Z-weather_symbols-PT01H.nc +4655358c22abdd0acfd5f4e08f156a7f4f629a1764dd8314f28f121964725078 ./wxcode-modal/blend_mismatch_inputs/20201209T1600Z-weather_symbols-PT01H.nc +28ee3f9d419bb9766cc68e1c74fa9e8e5010dc63a600635a54356a33795f6644 ./wxcode-modal/blend_mismatch_inputs/20201209T1700Z-weather_symbols-PT01H.nc +073fd0175e35afcf7f9fe61fedfa63bbd9ea5299f68d8b0287bc30085339b942 ./wxcode-modal/blend_mismatch_inputs/20201209T1800Z-weather_symbols-PT01H.nc +a6b3f48f8c2003c2d84da0a4cb8634bed24f0d54594d6cc9235c470858f49c19 ./wxcode-modal/blend_mismatch_inputs/kgo.nc +70adb1084eb1999513abe77fe43e385c04a5dd967ee57516f5113b6f8aec5454 ./wxcode-modal/gridded_input/20201209T0700Z-weather_symbols-PT01H.nc +03ab503a17fe14c03e3bafc03b44bc994e5a77d7d7ee9649d36c01bba91be813 ./wxcode-modal/gridded_input/20201209T0800Z-weather_symbols-PT01H.nc +2f23984f9e31a79d98e80a3426c551e4ddf32f44ec03cf095d11bf5e2e281113 ./wxcode-modal/gridded_input/20201209T0900Z-weather_symbols-PT01H.nc +4c21a13a71682482c8b5aac5babbe442ade17f8485c3d5e8d573f510d4a94080 ./wxcode-modal/gridded_input/20201209T1000Z-weather_symbols-PT01H.nc +6361353d199cb0c3d7c5e1d601e1b82377bcd7deba37f8f00e5910cc87649ab1 ./wxcode-modal/gridded_input/20201209T1100Z-weather_symbols-PT01H.nc +c649787aef87184392252660d9efeb1fd3c9933c308d02041a03a882ce1fbb59 ./wxcode-modal/gridded_input/20201209T1200Z-weather_symbols-PT01H.nc +c7cf5b8141df473b9854008aa25a9bdbdb162f840d6a829a46fbfccafa57d749 ./wxcode-modal/gridded_input/20201209T1300Z-weather_symbols-PT01H.nc +bc1022649e1b0ace39d9da1f3fbca03be855a6ec785e4487b584139e92d7fb59 ./wxcode-modal/gridded_input/20201209T1400Z-weather_symbols-PT01H.nc +724a98e9d311546af88b68796b4ba40968183b06ffd4688a0a24e1b6299864b1 ./wxcode-modal/gridded_input/20201209T1500Z-weather_symbols-PT01H.nc +fd4ee760bee0077bc4abf0e4be0a8703c43e10097fa8de08b1abf9a3ea4a780b ./wxcode-modal/gridded_input/20201209T1600Z-weather_symbols-PT01H.nc +5d2fa6f476337c2c3783b77d52615652f269174d3b8e804e7c6f931c3e7df4dd ./wxcode-modal/gridded_input/20201209T1700Z-weather_symbols-PT01H.nc +8d150f86a12e4b914e7f2ee269e643022db664d26c2539927d83f4424706cb3e ./wxcode-modal/gridded_input/20201209T1800Z-weather_symbols-PT01H.nc +a37d508c6c356325b1c31c245726aa5459484c4fb34fb2fbf309b10265d24712 ./wxcode-modal/gridded_input/kgo.nc +94a8758ca265acf625a21f4f9c6a246181b7b51e35710f312675d22b394f4d54 ./wxcode-modal/gridded_ties/20201209T0700Z-weather_symbols-PT01H.nc +0dbf6702561e52bcd97a86ef3d954e353a5218b78a1376792144eeb0a673a0c8 ./wxcode-modal/gridded_ties/20201209T0800Z-weather_symbols-PT01H.nc +08881255e7b71679aa1eda6e7bec8b85ceba5adc3547ae169e192abfc3238c39 ./wxcode-modal/gridded_ties/20201209T0900Z-weather_symbols-PT01H.nc +fc4a6da438ac558080c11413de21cd9943f356795d6f145f8e43e9ca290d1412 ./wxcode-modal/gridded_ties/20201209T1000Z-weather_symbols-PT01H.nc +ed07ba61392f465d5890a06c6ad3cee9d9adf31ab03ed365e1d9771d005d646c ./wxcode-modal/gridded_ties/20201209T1100Z-weather_symbols-PT01H.nc +350f0754f7bb247a0a81bb81d05f3e69da4d8d768d8269c82f6fef73713b5b06 ./wxcode-modal/gridded_ties/20201209T1200Z-weather_symbols-PT01H.nc +e8dc643ec6446d495a96d4dda6104b958d08137c15c3e832bb397a50229062eb ./wxcode-modal/gridded_ties/20201209T1300Z-weather_symbols-PT01H.nc +77375be1e8258f0b641ea6475c7739beec64252dc66ae4216a5cc02a3f01cd6e ./wxcode-modal/gridded_ties/20201209T1400Z-weather_symbols-PT01H.nc +148cdbc9282a8fb44e4793144852757ee36c5cd63d6efebfcb63337186e96a2f ./wxcode-modal/gridded_ties/20201209T1500Z-weather_symbols-PT01H.nc +44c510792f566329597239006b43ea736b5faa1a884202fd523aba31ebc4f445 ./wxcode-modal/gridded_ties/20201209T1600Z-weather_symbols-PT01H.nc +182211dba9b8d18c2d48dce89f2871854ae9baa9457bc9cfad16c1cea80faf09 ./wxcode-modal/gridded_ties/20201209T1700Z-weather_symbols-PT01H.nc +2ed494728e73e8f740680ae090504770ee6b60131d543698c640db5ed4e08598 ./wxcode-modal/gridded_ties/20201209T1800Z-weather_symbols-PT01H.nc +fab89f125fe7b016a35fcbf52cd9b713cfa28c7009b90150f289f47d7bea6b12 ./wxcode-modal/gridded_ties/kgo.nc +9fbcfc0e0c3f6af0d07690c6be12e2f641123168af0c7e165b8e0ec6a2ca6f98 ./wxcode-modal/single_input/20201210T0000Z-weather_symbols-PT01H.nc +4e52a42de445064a987da6f93b7e888c0a8ca974063a4d9cf9818a57d9b60367 ./wxcode-modal/single_input/kgo.nc +6513b722e1fdca6ad68acd768f3506ec814f78e9ccd4799972ec00a2316ef7cf ./wxcode-modal/spot_input/20201209T0700Z-weather_symbols-PT01H.nc +d4c5d626ad2e5a805f90f2902f6da5d33651eb6fdcb1e16116e2290d0c608bf6 ./wxcode-modal/spot_input/20201209T0800Z-weather_symbols-PT01H.nc +4642fc43faf4bdde6a678578868fdebb22163064fe6ef0eef41f2f39a959ca89 ./wxcode-modal/spot_input/20201209T0900Z-weather_symbols-PT01H.nc +ce6a854694aecd515063c91774eeb6160f39c9dbf5e970f72f960a3527e157e9 ./wxcode-modal/spot_input/20201209T1000Z-weather_symbols-PT01H.nc +300c38e7f522ff1e9257708b183148f3441bb1a861c15eaff2a5f1713bbbc872 ./wxcode-modal/spot_input/20201209T1100Z-weather_symbols-PT01H.nc +48ef16ad1fd0b499f1d38f585ddb0a4299c39e7d7df625ad0fa6c3dc24e0f094 ./wxcode-modal/spot_input/20201209T1200Z-weather_symbols-PT01H.nc +5ab6b249ec615ddab9995629aaddbd85bba352ad52b7bcb0be5b3037d75c22ca ./wxcode-modal/spot_input/20201209T1300Z-weather_symbols-PT01H.nc +340205e12625ff9466c1afba8c007343aa40e21b837ab2f2143fa032e6c47bd3 ./wxcode-modal/spot_input/20201209T1400Z-weather_symbols-PT01H.nc +776739d288b6e43b3d9341ba301feead160986d2540514f0663f243ca003e55a ./wxcode-modal/spot_input/20201209T1500Z-weather_symbols-PT01H.nc +4655358c22abdd0acfd5f4e08f156a7f4f629a1764dd8314f28f121964725078 ./wxcode-modal/spot_input/20201209T1600Z-weather_symbols-PT01H.nc +28ee3f9d419bb9766cc68e1c74fa9e8e5010dc63a600635a54356a33795f6644 ./wxcode-modal/spot_input/20201209T1700Z-weather_symbols-PT01H.nc +073fd0175e35afcf7f9fe61fedfa63bbd9ea5299f68d8b0287bc30085339b942 ./wxcode-modal/spot_input/20201209T1800Z-weather_symbols-PT01H.nc +a6b3f48f8c2003c2d84da0a4cb8634bed24f0d54594d6cc9235c470858f49c19 ./wxcode-modal/spot_input/kgo.nc +db1612786b08a929ad096f13ca603f37cfd78de681e1306d909654f345f9f409 ./wxcode-modal/spot_ties/20201209T0700Z-weather_symbols-PT01H.nc +b5beedd2fa6c58ecaaa51358b8b7abadddc1ad072af2d5141bbf0019b458e84e ./wxcode-modal/spot_ties/20201209T0800Z-weather_symbols-PT01H.nc +c92e6cb002b8ff8f185c097a55ff214793c7595667c8a7ba4afc1ae0f456c13e ./wxcode-modal/spot_ties/20201209T0900Z-weather_symbols-PT01H.nc +fd216b5dc3837202f3293f22e664d4fa623f7a349abda9ee8aef3fb19df3f660 ./wxcode-modal/spot_ties/20201209T1000Z-weather_symbols-PT01H.nc +fbff18f7ef5c2e3d462cbc0d78de30aca7534a37f427472011f6fa1835ec8020 ./wxcode-modal/spot_ties/20201209T1100Z-weather_symbols-PT01H.nc +129beabb8e059e46d3f00be1c5a5b8c1cdd44c4f1f08fc86a4249d4a0f854272 ./wxcode-modal/spot_ties/20201209T1200Z-weather_symbols-PT01H.nc +063e6f151af8a05f4c2276995384fc4058571144a851953752653eff7886130c ./wxcode-modal/spot_ties/20201209T1300Z-weather_symbols-PT01H.nc +4d23d6d1f8861feeb288ec7b68e57e893fa2ffd633de1b25ab4d4bc4aa3316cc ./wxcode-modal/spot_ties/20201209T1400Z-weather_symbols-PT01H.nc +71aac305fce32fb389ea63822cd88511137cf1260c1c0a55e28c1a5fcbc91cab ./wxcode-modal/spot_ties/20201209T1500Z-weather_symbols-PT01H.nc +6fe4b1563f0fbb4579dba46b12654f983cd6486f39986d52231aad5d18acf508 ./wxcode-modal/spot_ties/20201209T1600Z-weather_symbols-PT01H.nc +d506afe1572ded010805662f057926ba19367a8bf0c9f700380b6b63ab440399 ./wxcode-modal/spot_ties/20201209T1700Z-weather_symbols-PT01H.nc +1c0c3b4fcc95ea2d5c9658ebd42cbee8a2043afa607843e0e9013d273f7fc460 ./wxcode-modal/spot_ties/20201209T1800Z-weather_symbols-PT01H.nc +cb558fc0cb3d1f0535e2402fc293982a122a08380df7d5594cacb9cf032acc5a ./wxcode-modal/spot_ties/kgo.nc 803b2a645f8a33db4601a45dc16325f8a90f13fdea430b98ea94f032ee032180 ./wxcode/bad_wx_decision_tree.json 5a3a808f0cc7952e6cc4225d09136276f685c5fdd6798a795c0430aa65ca71fa ./wxcode/basic/kgo.nc 3b86d135373b44989c22bdfbc6cb0798e974ff1d77c4c6b6a83fce650863e59c ./wxcode/basic/kgo_no_lightning.nc @@ -742,5 +738,4 @@ f20a9eabe867ea00e9570eb1177c01841b7606e2969287432d4a50ae888062a3 ./wxcode/nativ ec818f319f2c137ab356f97a3980788357254f17c0761908b4e656170c0e5d37 ./wxcode/native_units/probability_of_shower_condition_above_threshold.nc 1b8de3a02ac7dcf18ec9208a5e0b583b37c420093781b861b67eb5c8eaa37cd9 ./wxcode/native_units/probability_of_thickness_of_rainfall_amount_above_threshold.nc 554f9ed54f1092f31013b0e85de20e13932ca9533d50ac6316f170641901a209 ./wxcode/native_units/probability_of_visibility_in_air_below_threshold.nc -3de7bd3e25d9d5b9eafbef91ead03f1ebfa6202b9f90826dbe81ad99fdc74277 ./wxcode/wx_decision_tree_1h.json -80a0dc027f33a9f209d8304a92eb60fd204620492bea5bb9c0fe5c4d8a285edb ./wxcode/wx_decision_tree_3h.json +be969533132f0d1651959ef9db777b2c71b109673b75eb59edb0e87e78698a9f ./wxcode/wx_decision_tree.json diff --git a/improver_tests/acceptance/acceptance.py b/improver_tests/acceptance/acceptance.py index b133f14a2d..df102691eb 100644 --- a/improver_tests/acceptance/acceptance.py +++ b/improver_tests/acceptance/acceptance.py @@ -69,7 +69,7 @@ def run_cli(cli_name, verbose=True): def run_function(args): if not checksum_ignore(): verify_checksums(args) - cli.main("improver", cli_name, *args, verbose=verbose) + return cli.main("improver", cli_name, *args, verbose=verbose) return run_function diff --git a/improver_tests/acceptance/test_wxcode.py b/improver_tests/acceptance/test_wxcode.py index c423547c7a..86236d879b 100644 --- a/improver_tests/acceptance/test_wxcode.py +++ b/improver_tests/acceptance/test_wxcode.py @@ -29,6 +29,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. """Tests for the wxcode CLI""" +import re import pytest @@ -187,3 +188,24 @@ def test_no_lightning(tmp_path): ] run_cli(args) acc.compare(output_path, kgo_path) + + +@pytest.mark.parametrize( + "wxtree,expected", + ( + ("wx_decision_tree.json", "Decision tree OK\nRequired inputs are:"), + ("bad_wx_decision_tree.json", "Unreachable node 'unreachable'"), + ), +) +def test_trees(wxtree, expected): + """Test the check-tree option""" + kgo_dir = acc.kgo_root() / "wxcode" + args = [ + "--wxtree", + kgo_dir / wxtree, + "--check-tree", + "--target-period", + "3600", + ] + result = run_cli(args) + assert re.match(expected, result) diff --git a/improver_tests/acceptance/test_wxcode_modal.py b/improver_tests/acceptance/test_wxcode_modal.py index 4c49d7668d..21320df4ac 100644 --- a/improver_tests/acceptance/test_wxcode_modal.py +++ b/improver_tests/acceptance/test_wxcode_modal.py @@ -68,6 +68,8 @@ def test_expected(tmp_path, test_path): output_path = tmp_path / "output.nc" args = [ *input_paths, + "--model-id-attr", + "mosg__model_configuration", "--output", output_path, ] diff --git a/improver_tests/calibration/dataframe_utilities/test_dataframe_utilities.py b/improver_tests/calibration/dataframe_utilities/test_dataframe_utilities.py index 0bebcfee88..2ebad25b37 100644 --- a/improver_tests/calibration/dataframe_utilities/test_dataframe_utilities.py +++ b/improver_tests/calibration/dataframe_utilities/test_dataframe_utilities.py @@ -474,9 +474,9 @@ def test_site_coord_mismatch(self): and forecasts. In this case, the position (lat/lon/alt) from the forecast will be used.""" df = self.truth_subset_df.copy() - df.at[::3, "altitude"] = 45 - df.at[::3, "latitude"] = 52 - df.at[::3, "longitude"] = -12 + df.loc[::3, "altitude"] = 45 + df.loc[::3, "latitude"] = 52 + df.loc[::3, "longitude"] = -12 result = forecast_and_truth_dataframes_to_cubes( self.forecast_df, df, diff --git a/improver_tests/wxcode/wxcode/test_ModalCode.py b/improver_tests/wxcode/wxcode/test_ModalCode.py index 7001a1898c..17b5230c73 100644 --- a/improver_tests/wxcode/wxcode/test_ModalCode.py +++ b/improver_tests/wxcode/wxcode/test_ModalCode.py @@ -33,10 +33,11 @@ from calendar import timegm from datetime import datetime as dt from datetime import timedelta +from typing import Tuple import numpy as np import pytest -from iris.cube import Cube +from iris.cube import CubeList from improver.spotdata.build_spotdata_cube import build_spotdata_cube from improver.synthetic_data.set_up_test_cubes import construct_scalar_time_coords @@ -44,11 +45,14 @@ from . import set_up_wxcube +MODEL_ID_ATTR = "mosg__model_configuration" TARGET_TIME = dt(2020, 6, 15, 18) @pytest.fixture(name="wxcode_series") -def wxcode_series_fixture(data, cube_type, offset_reference_times: bool) -> Cube: +def wxcode_series_fixture( + data, cube_type, offset_reference_times: bool, model_id_attr: bool, +) -> Tuple[bool, CubeList]: """Generate a time series of weather code cubes for combination to create a period representative code. When offset_reference_times is set, each successive cube will have a reference time one hour older.""" @@ -56,7 +60,7 @@ def wxcode_series_fixture(data, cube_type, offset_reference_times: bool) -> Cube time = TARGET_TIME ntimes = len(data) - wxcubes = [] + wxcubes = CubeList() for i in range(ntimes): wxtime = time - timedelta(hours=i) @@ -93,9 +97,13 @@ def wxcode_series_fixture(data, cube_type, offset_reference_times: bool) -> Cube scalar_coords=time_coords, ) ) - return wxcubes + if model_id_attr: + [c.attributes.update({MODEL_ID_ATTR: "uk_ens"}) for c in wxcubes] + wxcubes[0].attributes.update({MODEL_ID_ATTR: "uk_det uk_ens"}) + return model_id_attr, wxcubes +@pytest.mark.parametrize("model_id_attr", [False, True]) @pytest.mark.parametrize("offset_reference_times", [False, True]) @pytest.mark.parametrize("cube_type", ["gridded", "spot"]) @pytest.mark.parametrize( @@ -133,13 +141,15 @@ def wxcode_series_fixture(data, cube_type, offset_reference_times: bool) -> Cube ) def test_expected_values(wxcode_series, expected): """Test that the expected period representative symbol is returned.""" - result = ModalWeatherCode()(wxcode_series) + _, wxcode_cubes = wxcode_series + result = ModalWeatherCode()(wxcode_cubes) assert result.data.flatten()[0] == expected +@pytest.mark.parametrize("model_id_attr", [False, True]) @pytest.mark.parametrize("offset_reference_times", [False, True]) @pytest.mark.parametrize("cube_type", ["gridded", "spot"]) -@pytest.mark.parametrize("data", [np.ones((12)), np.ones((1))]) +@pytest.mark.parametrize("data", [np.ones(12), np.ones(1)]) def test_metadata(wxcode_series): """Check that the returned metadata is correct. In this case we expect a time coordinate with bounds that describe the full period over which the @@ -154,9 +164,16 @@ def test_metadata(wxcode_series): def as_utc_timestamp(time): return timegm(time.utctimetuple()) - result = ModalWeatherCode()(wxcode_series) + model_id_attr, wxcode_cubes = wxcode_series - n_times = len(wxcode_series) + if model_id_attr: + kwargs = {"model_id_attr": MODEL_ID_ATTR} + else: + kwargs = {} + + result = ModalWeatherCode(**kwargs)(wxcode_cubes) + + n_times = len(wxcode_cubes) expected_time = TARGET_TIME expected_bounds = [TARGET_TIME - timedelta(hours=n_times), TARGET_TIME] expected_reference_time = TARGET_TIME - timedelta(hours=18) @@ -180,3 +197,7 @@ def as_utc_timestamp(time): ) assert result.cell_methods[0].method == expected_cell_method[0] assert result.cell_methods[0].coord_names[0] == expected_cell_method[1] + if model_id_attr: + assert result.attributes[MODEL_ID_ATTR] == "uk_det uk_ens" + else: + assert MODEL_ID_ATTR not in result.attributes.keys() diff --git a/improver_tests/wxcode/wxcode/test_utilities.py b/improver_tests/wxcode/wxcode/test_utilities.py index d34314c467..0f7808bc2d 100644 --- a/improver_tests/wxcode/wxcode/test_utilities.py +++ b/improver_tests/wxcode/wxcode/test_utilities.py @@ -511,28 +511,29 @@ def modify_tree_fixture(node, key, value): ), ), ( - "lightning", + "lightning_shower", "if_true", 100, ( - "Node lightning results in an invalid weather code of 100 for the " + "Node lightning_shower results in an invalid weather code of 100 for the " "if_true condition" ), ), ( - "lightning", + "lightning_shower", "if_false", 100, ( - "Node lightning results in an invalid weather code of 100 for the " + "Node lightning_shower results in an invalid weather code of 100 for the " "if_false condition" ), ), ( - "lightning", + "lightning_shower", "if_false", "kittens", - "Node lightning has an invalid destination of kittens for the if_false condition", + "Node lightning_shower has an invalid destination of kittens for the if_false " + "condition", ), ( "snow_in_vicinity", @@ -590,6 +591,12 @@ def modify_tree_fixture(node, key, value): ["kittens"], "Node lightning has a non-numeric probability threshold ['kittens']", ), + ( + "mist_conditions", + "if_true", + "no_precipitation_cloud", + "Unreachable node 'fog_conditions'", + ), ), ) def test_check_tree(modify_tree, expected):