From f998ae3faf6ad8eef19aaa5b500438bc34df79ef Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Fri, 18 Aug 2023 13:56:28 +0200 Subject: [PATCH 01/64] First working prototype of ERA5 GRIB reader --- esmvalcore/_provenance.py | 12 +- esmvalcore/cmor/_fixes/native6/era5.py | 36 ++-- esmvalcore/config-developer.yml | 2 + .../config/extra_facets/native6-mappings.yml | 92 ++++++++++ esmvalcore/preprocessor/_io.py | 8 +- esmvalcore/preprocessor/_regrid.py | 169 ++++++++++++++++++ 6 files changed, 303 insertions(+), 16 deletions(-) create mode 100644 esmvalcore/config/extra_facets/native6-mappings.yml diff --git a/esmvalcore/_provenance.py b/esmvalcore/_provenance.py index dfe70e7220..75e661feaa 100644 --- a/esmvalcore/_provenance.py +++ b/esmvalcore/_provenance.py @@ -3,6 +3,7 @@ import logging import os from functools import total_ordering +from pathlib import Path from netCDF4 import Dataset from PIL import Image @@ -53,7 +54,7 @@ def attribute_to_authors(entity, authors): def attribute_to_projects(entity, projects): - """Attribute entity to projecs.""" + """Attribute entity to projects.""" namespace = 'project' create_namespace(entity.bundle, namespace) @@ -193,7 +194,7 @@ def initialize_provenance(self, activity): self._initialize_ancestors(activity) def _initialize_namespaces(self): - """Inialize the namespaces.""" + """Initialize the namespaces.""" for namespace in ('file', 'attribute', 'preprocessor', 'task'): create_namespace(self.provenance, namespace) @@ -206,9 +207,10 @@ def _initialize_entity(self): """Initialize the entity representing the file.""" if self.attributes is None: self.attributes = {} - with Dataset(self.filename, 'r') as dataset: - for attr in dataset.ncattrs(): - self.attributes[attr] = dataset.getncattr(attr) + if 'nc' in Path(self.filename).suffix: + with Dataset(self.filename, 'r') as dataset: + for attr in dataset.ncattrs(): + self.attributes[attr] = dataset.getncattr(attr) attributes = { 'attribute:' + str(k).replace(' ', '_'): str(v) diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py index d76fd21701..d29e217767 100644 --- a/esmvalcore/cmor/_fixes/native6/era5.py +++ b/esmvalcore/cmor/_fixes/native6/era5.py @@ -4,12 +4,14 @@ import iris import numpy as np +from iris.util import reverse from esmvalcore.iris_helpers import date2num +from ....preprocessor._regrid import _bilinear_unstructured_regrid +from ...table import CMOR_TABLES from ..fix import Fix from ..shared import add_scalar_height_coord -from ...table import CMOR_TABLES logger = logging.getLogger(__name__) @@ -349,21 +351,13 @@ class AllVars(Fix): def _fix_coordinates(self, cube): """Fix coordinates.""" - # Fix coordinate increasing direction - slices = [] - for coord in cube.coords(): - if coord.var_name in ('latitude', 'pressure_level'): - slices.append(slice(None, None, -1)) - else: - slices.append(slice(None)) - cube = cube[tuple(slices)] - # Add scalar height coordinates if 'height2m' in self.vardef.dimensions: add_scalar_height_coord(cube, 2.) if 'height10m' in self.vardef.dimensions: add_scalar_height_coord(cube, 10.) + # Fix coord metadata for coord_def in self.vardef.coordinates.values(): axis = coord_def.axis # ERA5 uses regular pressure level coordinate. In case the cmor @@ -388,6 +382,17 @@ def _fix_coordinates(self, cube): self._fix_monthly_time_coord(cube) + # Fix coordinate increasing direction + if cube.coords('latitude') and cube.coord('latitude').shape[0] > 1: + lat = cube.coord('latitude') + if lat.points[0] > lat.points[1]: + cube = reverse(cube, 'latitude') + if (cube.coords('air_pressure') and + cube.coord('air_pressure').shape[0] > 1): + plev = cube.coord('air_pressure') + if plev.points[0] < plev.points[1]: + cube = reverse(cube, 'air_pressure') + return cube @staticmethod @@ -421,6 +426,17 @@ def fix_metadata(self, cubes): cube.standard_name = self.vardef.standard_name cube.long_name = self.vardef.long_name + # If desired, regrid native ERA5 data in GRIB format (which is on a + # reduced Gaussian grid) + if ( + cube.coords('latitude') and + cube.coords('longitude') and + cube.coord('latitude').ndim == 1 and + cube.coord('longitude').ndim == 1 and + cube.coord_dims('latitude') == cube.coord_dims('longitude') + ): + cube = _bilinear_unstructured_regrid(cube, '0.25x0.25') + cube = self._fix_coordinates(cube) self._fix_units(cube) diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml index 69dfe9658d..32b5259deb 100644 --- a/esmvalcore/config-developer.yml +++ b/esmvalcore/config-developer.yml @@ -96,8 +96,10 @@ native6: cmor_strict: false input_dir: default: 'Tier{tier}/{dataset}/{version}/{frequency}/{short_name}' + DKRZ-ERA5-GRIB: '{family}/{level}/{type}/{tres}/{grib_id}' input_file: default: '*.nc' + DKRZ-ERA5-GRIB: '{family}{level}{typeid}_{tres}_*_{grib_id}.grb' output_file: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}' cmor_type: 'CMIP6' cmor_default_table_prefix: 'CMIP6_' diff --git a/esmvalcore/config/extra_facets/native6-mappings.yml b/esmvalcore/config/extra_facets/native6-mappings.yml new file mode 100644 index 0000000000..53152efaf6 --- /dev/null +++ b/esmvalcore/config/extra_facets/native6-mappings.yml @@ -0,0 +1,92 @@ +# Extra facets for native6 data + +# Notes: +# - All facets can also be specified in the recipes. The values given here are +# only defaults. + +# A complete list of supported keys is given in the documentation (see +# ESMValCore/doc/quickstart/find_data.rst). +--- + +ERA5: + + # Settings for all variables of all MIPs + '*': + '*': + family: E5 + type: an + typeid: '00' + version: '' # necessary to get a nice output file name + + # Variable-specific settings + ta: + level: pl + grib_id: '130' + tas: + level: sf + grib_id: '167' + + # MIP-specific settings + Amon: + '*': + tres: 1M + E1hr: + '*': + tres: 1H + + + # # Cell measures + # areacella: + # latitude: grid_latitude + # longitude: grid_longitude + # raw_name: cell_area + # areacello: + # latitude: grid_latitude + # longitude: grid_longitude + # raw_name: cell_area + + # # 2D dynamical/meteorological variables + # clivi: {var_type: atm_2d_ml} + # clt: {var_type: atm_2d_ml} + # clwvi: {var_type: atm_2d_ml} + # evspsbl: {var_type: atm_2d_ml} + # hfls: {var_type: atm_2d_ml} + # hfss: {var_type: atm_2d_ml} + # lwp: {raw_name: cllvi, var_type: atm_2d_ml} + # pr: {var_type: atm_2d_ml} + # prw: {var_type: atm_2d_ml} + # ps: {var_type: atm_2d_ml} + # psl: {var_type: atm_2d_ml} + # rlds: {var_type: atm_2d_ml} + # rldscs: {var_type: atm_2d_ml} + # rlus: {var_type: atm_2d_ml} + # rlut: {var_type: atm_2d_ml} + # rlutcs: {var_type: atm_2d_ml} + # rsds: {var_type: atm_2d_ml} + # rsdscs: {var_type: atm_2d_ml} + # rsdt: {var_type: atm_2d_ml} + # rsus: {var_type: atm_2d_ml} + # rsuscs: {var_type: atm_2d_ml} + # rsut: {var_type: atm_2d_ml} + # rsutcs: {var_type: atm_2d_ml} + # siconc: {raw_name: sic, raw_units: '1', var_type: atm_2d_ml} + # siconca: {raw_name: sic, raw_units: '1', var_type: atm_2d_ml} + # tas: {var_type: atm_2d_ml} + # tasmax: {var_type: atm_2d_ml} + # tasmin: {var_type: atm_2d_ml} + # tauu: {var_type: atm_2d_ml} + # tauv: {var_type: atm_2d_ml} + # ts: {var_type: atm_2d_ml} + # uas: {var_type: atm_2d_ml} + # vas: {var_type: atm_2d_ml} + + # # 3D dynamical/meteorological variables + # cl: {var_type: atm_3d_ml} + # cli: {var_type: atm_3d_ml} + # clw: {var_type: atm_3d_ml} + # hur: {raw_units: '1', var_type: atm_3d_ml} + # hus: {var_type: atm_3d_ml} + # ta: {var_type: atm_3d_ml} + # ua: {var_type: atm_3d_ml} + # va: {var_type: atm_3d_ml} + # wap: {var_type: atm_3d_ml} diff --git a/esmvalcore/preprocessor/_io.py b/esmvalcore/preprocessor/_io.py index 62eb52853e..c97f5f7d00 100644 --- a/esmvalcore/preprocessor/_io.py +++ b/esmvalcore/preprocessor/_io.py @@ -191,7 +191,13 @@ def load( # warnings.filterwarnings # (see https://github.com/SciTools/cf-units/issues/240) with suppress_errors(): - raw_cubes = iris.load_raw(file, callback=callback) + # GRIB files need to be loaded with iris.load, otherwise we will + # get separate (lat, lon) slices for each time step, pressure + # level, etc. + if file.suffix in ('.grib', '.grb', '.gb'): + raw_cubes = iris.load(file, callback=callback) + else: + raw_cubes = iris.load_raw(file, callback=callback) logger.debug("Done with loading %s", file) if not raw_cubes: diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py index 1008cf67ad..95f104538b 100644 --- a/esmvalcore/preprocessor/_regrid.py +++ b/esmvalcore/preprocessor/_regrid.py @@ -17,7 +17,9 @@ import stratify from geopy.geocoders import Nominatim from iris.analysis import AreaWeighted, Linear, Nearest, UnstructuredNearest +from iris.cube import Cube from iris.util import broadcast_to_shape +from scipy.spatial import Delaunay from ..cmor._fixes.shared import add_altitude_from_plev, add_plev_from_altitude from ..cmor.table import CMOR_TABLES @@ -1134,3 +1136,170 @@ def extract_coordinate_points(cube, definition, scheme): raise ValueError(msg) cube = cube.interpolate(definition.items(), scheme=scheme) return cube + + +def _bilinear_unstructured_regrid( + cube: Cube, + target_grid: str, + lat_offset: bool = True, + lon_offset: bool = True, +) -> Cube: + """Bilinear regridding for unstructured grids. + + Note + ---- + This private function has been introduced to regrid native ERA5 in GRIB + format similarly to how it is done if you download an interpolated versions + of ERA5 (see + https://confluence.ecmwf.int/display/CKB/ERA5%3A+What+is+the+spatial+reference). + + Currently, we do not support bilinear regridding for unstructured grids in + our `regrid` preprocessor (only nearest-neighbor). Since iris is currently + doing a massive overhaul of their in-built regridding + (https://github.com/SciTools/iris/issues/4754), it does not make sense to + include the following piece of code in their package just now. + + Thus, we provide this function here. Please be aware that it can be removed + at any point in time without prior warning (just like any other private + function). + + """ + # This function should only be called on unstructured grid cubes + msg = "No unstructured grid" + assert cube.coords('latitude'), msg + assert cube.coords('longitude'), msg + assert cube.coord('latitude').ndim == 1, msg + assert cube.coord('longitude').ndim == 1, msg + assert cube.coord_dims('latitude') == cube.coord_dims('longitude'), msg + udim = cube.coord_dims('latitude')[0] + + # Make sure the cube has lazy data and rechunk it properly (cube cannot be + # chunked along latitude and longitude dimension) + if not cube.has_lazy_data(): + cube.data = da.from_array(cube.data) + in_chunks = ['auto'] * cube.ndim + in_chunks[udim] = -1 # type: ignore + cube.data = cube.lazy_data().rechunk(in_chunks) + + # Generate a target grid from the provided cell-specification, and cache + # the resulting stock cube for later use + target_grid_cube = _CACHE.setdefault( + target_grid, + _global_stock_cube(target_grid, lat_offset, lon_offset), + ) + + # Put source and target grid in correct format and calculate vertices and + # interpolation weights + src_points = np.stack( + (cube.coord('latitude').points, cube.coord('longitude').points), + axis=-1, + ) + (tgt_lat, tgt_lon) = np.meshgrid( + target_grid_cube.coord('latitude').points, + target_grid_cube.coord('longitude').points, + indexing='ij', + ) + tgt_points = np.stack((tgt_lat.ravel(), tgt_lon.ravel()), axis=-1) + (indices, weights) = _get_linear_interpolation_weights( + src_points, tgt_points + ) + + # Perform actual regridding + regridded_data = da.apply_gufunc( + _interpolate, + '(i),(j,3),(j,3)->(j)', + cube.lazy_data(), + indices, + weights, + vectorize=True, + output_dtypes=cube.dtype, + ) + regridded_data = regridded_data.rechunk('auto') + + # Put regridded data in cube with correct metadata + regridded_cube = _get_dummy_regridded_cube(cube, target_grid_cube) + regridded_cube.data = regridded_data.reshape( + regridded_cube.shape, limit='128MiB' + ) + + return regridded_cube + + +def _get_dummy_regridded_cube(src_cube: Cube, tgt_cube: Cube) -> Cube: + """Get dummy regridded cube with correct shape and metadata.""" + src_cs = src_cube.coord_system() + xcoord = tgt_cube.coord(axis='x', dim_coords=True) + ycoord = tgt_cube.coord(axis='y', dim_coords=True) + xcoord.coord_system = src_cs + ycoord.coord_system = src_cs + return regrid(src_cube, tgt_cube, 'unstructured_nearest') + + +def _get_linear_interpolation_weights( + src_points: np.ndarray, + tgt_points: np.ndarray, + fill_value: float = np.nan, +) -> tuple[np.ndarray, np.ndarray]: + """Get vertices and weights for 2D linear regridding of unstructured grids. + + Taken from + https://stackoverflow.com/questions/20915502/speedup-scipy-griddata-for-multiple-interpolations-between-two-irregular-grids. + This is more than 80x faster than :func:`scipy.interpolate.griddata` and + gives identical results. + + Parameters + ---------- + src_points: np.ndarray + Points of the unstructured source grid. Must be an (N, 2) array, where + N is the number of source grid points. + tgt_points: np.ndarray + Points of the unstructured target grid. Must be an (M, 2) array, where + M is the number of target grid points. + fill_value: float + Fill value for extrapolated values. + + Returns + ------- + tuple[np.ndarray, np.ndarray] + Indices and interpolation weights. Both are (M, 3) arrays. + + """ + n_dims = 2 + tri = Delaunay(src_points) + simplex = tri.find_simplex(tgt_points) + extra_idx = (simplex == -1) + indices = np.take(tri.simplices, simplex, axis=0) + temp = np.take(tri.transform, simplex, axis=0) + delta = tgt_points - temp[:, n_dims] + bary = np.einsum('njk,nk->nj', temp[:, :n_dims, :], delta) + weights = np.hstack((bary, 1 - bary.sum(axis=1, keepdims=True))) + weights[extra_idx, :] = fill_value + return (indices, weights) + + +def _interpolate( + data: np.ndarray, + indices: np.ndarray, + weights: np.ndarray, +) -> np.ndarray: + """Interpolate data. + + Parameters + ---------- + data: np.ndarray + Data to interpolate. Must be an (N,) array, where N is the number of + source grid points. + indices: np.ndarray + Indices used to index the data. Must be an (M, 3) array, where M is the + number of target grid points. + weights: np.ndarray + Interpolation weights. Must be an (M, 3) array, where M is the number + of target grid points. + + Returns + ------- + np.ndarray + Interpolated data of shape (M,). + + """ + return np.einsum('nj,nj->n', np.take(data, indices), weights) From 8c48834dbdf3b22db6bc77a8c059c6c645535b69 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 21 Aug 2023 17:51:24 +0200 Subject: [PATCH 02/64] Extended list of supported variables for ERA5 GRIB support --- esmvalcore/cmor/_fixes/native6/era5.py | 98 +++++++- .../config/extra_facets/native6-mappings.yml | 223 +++++++++++++----- 2 files changed, 261 insertions(+), 60 deletions(-) diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py index d29e217767..054da2673e 100644 --- a/esmvalcore/cmor/_fixes/native6/era5.py +++ b/esmvalcore/cmor/_fixes/native6/era5.py @@ -25,15 +25,21 @@ def get_frequency(cube): time.convert_units('days since 1850-1-1 00:00:00.0') if len(time.points) == 1: - if cube.long_name != 'Geopotential': + acceptable_long_names = ( + 'Geopotential', + 'Percentage of the Grid Cell Occupied by Land (Including Lakes)', + ) + if cube.long_name not in acceptable_long_names: raise ValueError('Unable to infer frequency of cube ' f'with length 1 time dimension: {cube}') return 'fx' interval = time.points[1] - time.points[0] + if interval - 1 / 24 < 1e-4: return 'hourly' - + if interval - 1.0 < 1e-4: + return 'daily' return 'monthly' @@ -75,6 +81,27 @@ def divide_by_gravity(cube): return cube +class Albsn(Fix): + """Fixes for albsn.""" + + def fix_metadata(self, cubes): + """Fix metadata.""" + for cube in cubes: + # Invalid input cube units (ignored on load) were '0-1' + cube.units = '1' + return cubes + + +class Cli(Fix): + """Fixes for cli.""" + + def fix_metadata(self, cubes): + """Fix metadata.""" + for cube in cubes: + cube.units = 'kg kg-1' + return cubes + + class Clt(Fix): """Fixes for clt.""" @@ -88,6 +115,16 @@ def fix_metadata(self, cubes): return cubes +class Clw(Fix): + """Fixes for clw.""" + + def fix_metadata(self, cubes): + """Fix metadata.""" + for cube in cubes: + cube.units = 'kg kg-1' + return cubes + + class Cl(Fix): """Fixes for cl.""" @@ -148,6 +185,30 @@ def fix_metadata(self, cubes): return cubes +class Hus(Fix): + """Fixes for hus.""" + + def fix_metadata(self, cubes): + """Fix metadata.""" + for cube in cubes: + cube.units = 'kg kg-1' + return cubes + + +class O3(Fix): + """Fixes for o3.""" + + def fix_metadata(self, cubes): + """Convert mass mixing ratios to mole fractions.""" + for cube in cubes: + cube.units = 'kg kg-1' + # Convert to molar mixing ratios, which is almost identical to mole + # fraction for small amounts of substances (which we have here) + cube.data = cube.core_data() * 28.9644 / 47.9982 + cube.units = 'mol mol-1' + return cubes + + class Orog(Fix): """Fixes for orography.""" @@ -189,6 +250,26 @@ def fix_metadata(self, cubes): return cubes +class Prw(Fix): + """Fixes for prw.""" + + def fix_metadata(self, cubes): + """Fix metadata.""" + for cube in cubes: + cube.units = 'kg m-2' + return cubes + + +class Ps(Fix): + """Fixes for ps.""" + + def fix_metadata(self, cubes): + """Fix metadata.""" + for cube in cubes: + cube.units = 'Pa' + return cubes + + class Ptype(Fix): """Fixes for ptype.""" @@ -316,6 +397,17 @@ def fix_metadata(self, cubes): return cubes +class Sftlf(Fix): + """Fixes for sftlf.""" + + def fix_metadata(self, cubes): + """Fix metadata.""" + for cube in cubes: + # Invalid input cube units (ignored on load) were '0-1' + cube.units = '1' + return cubes + + class Tasmax(Fix): """Fixes for tasmax.""" @@ -427,7 +519,7 @@ def fix_metadata(self, cubes): cube.long_name = self.vardef.long_name # If desired, regrid native ERA5 data in GRIB format (which is on a - # reduced Gaussian grid) + # reduced Gaussian grid, i.e., unstructured grid) if ( cube.coords('latitude') and cube.coords('longitude') and diff --git a/esmvalcore/config/extra_facets/native6-mappings.yml b/esmvalcore/config/extra_facets/native6-mappings.yml index 53152efaf6..ad59266944 100644 --- a/esmvalcore/config/extra_facets/native6-mappings.yml +++ b/esmvalcore/config/extra_facets/native6-mappings.yml @@ -19,74 +19,183 @@ ERA5: version: '' # necessary to get a nice output file name # Variable-specific settings + albsn: + level: sf + grib_id: '032' + cl: + level: pl + grib_id: '248' + cli: + level: pl + grib_id: '247' + clt: + level: sf + grib_id: '164' + clw: + level: pl + grib_id: '246' + hur: + level: pl + grib_id: '157' + hus: + level: pl + grib_id: '133' + o3: + level: pl + grib_id: '203' + prw: + level: sf + grib_id: '137' + ps: + level: sf + grib_id: '134' + psl: + level: sf + grib_id: '151' + rainmxrat27: + level: pl + grib_id: '075' + sftlf: + level: sf + grib_id: '172' + siconc: + level: sf + grib_id: '031' + siconca: + level: sf + grib_id: '031' + snd: + level: sf + grib_id: '141' + snowmxrat27: + level: pl + grib_id: '076' ta: level: pl grib_id: '130' tas: level: sf grib_id: '167' + tdps: + level: sf + grib_id: '168' + tos: + level: sf + grib_id: '034' + toz: + level: sf + grib_id: '206' + ts: + level: sf + grib_id: '235' + ua: + level: pl + grib_id: '131' + uas: + level: sf + grib_id: '165' + va: + level: pl + grib_id: '132' + vas: + level: sf + grib_id: '166' + wap: + level: pl + grib_id: '135' + zg: + level: pl + grib_id: '129' + + # unclear: + # 075: specific rain water content = rainmxrat27 ?? + # 076: specific snow water content = snowmxrat27 ?? + # 136: total column water + # 186: low cloud cover p > 0.8 ps + # 187: medium cloud cover 0.45 ps < p < 0.8 ps + # 188: high cloud cover p < 0.45 ps + # 235: skin temperature = ts ?? + # 243: forecast albedo # MIP-specific settings + AERday: + '*': + tres: 1D + AERhr: + '*': + tres: 1H + AERmon: + '*': + tres: 1M + AERmonZ: + '*': + tres: 1M Amon: '*': tres: 1M + CFday: + '*': + tres: 1D + CFmon: + '*': + tres: 1M + day: + '*': + tres: 1D E1hr: '*': tres: 1H - - - # # Cell measures - # areacella: - # latitude: grid_latitude - # longitude: grid_longitude - # raw_name: cell_area - # areacello: - # latitude: grid_latitude - # longitude: grid_longitude - # raw_name: cell_area - - # # 2D dynamical/meteorological variables - # clivi: {var_type: atm_2d_ml} - # clt: {var_type: atm_2d_ml} - # clwvi: {var_type: atm_2d_ml} - # evspsbl: {var_type: atm_2d_ml} - # hfls: {var_type: atm_2d_ml} - # hfss: {var_type: atm_2d_ml} - # lwp: {raw_name: cllvi, var_type: atm_2d_ml} - # pr: {var_type: atm_2d_ml} - # prw: {var_type: atm_2d_ml} - # ps: {var_type: atm_2d_ml} - # psl: {var_type: atm_2d_ml} - # rlds: {var_type: atm_2d_ml} - # rldscs: {var_type: atm_2d_ml} - # rlus: {var_type: atm_2d_ml} - # rlut: {var_type: atm_2d_ml} - # rlutcs: {var_type: atm_2d_ml} - # rsds: {var_type: atm_2d_ml} - # rsdscs: {var_type: atm_2d_ml} - # rsdt: {var_type: atm_2d_ml} - # rsus: {var_type: atm_2d_ml} - # rsuscs: {var_type: atm_2d_ml} - # rsut: {var_type: atm_2d_ml} - # rsutcs: {var_type: atm_2d_ml} - # siconc: {raw_name: sic, raw_units: '1', var_type: atm_2d_ml} - # siconca: {raw_name: sic, raw_units: '1', var_type: atm_2d_ml} - # tas: {var_type: atm_2d_ml} - # tasmax: {var_type: atm_2d_ml} - # tasmin: {var_type: atm_2d_ml} - # tauu: {var_type: atm_2d_ml} - # tauv: {var_type: atm_2d_ml} - # ts: {var_type: atm_2d_ml} - # uas: {var_type: atm_2d_ml} - # vas: {var_type: atm_2d_ml} - - # # 3D dynamical/meteorological variables - # cl: {var_type: atm_3d_ml} - # cli: {var_type: atm_3d_ml} - # clw: {var_type: atm_3d_ml} - # hur: {raw_units: '1', var_type: atm_3d_ml} - # hus: {var_type: atm_3d_ml} - # ta: {var_type: atm_3d_ml} - # ua: {var_type: atm_3d_ml} - # va: {var_type: atm_3d_ml} - # wap: {var_type: atm_3d_ml} + E1hrClimMon: + '*': + tres: 1H + Eday: + '*': + tres: 1D + EdayZ: + '*': + tres: 1D + Efx: + '*': + tres: IV + Emon: + '*': + tres: 1M + EmonZ: + '*': + tres: 1M + fx: + '*': + tres: IV + IfxAnt: + '*': + tres: IV + IfxGre: + '*': + tres: IV + ImonAnt: + '*': + tres: 1M + ImonGre: + '*': + tres: 1M + LImon: + '*': + tres: 1M + Lmon: + '*': + tres: 1M + Oday: + '*': + tres: 1D + Ofx: + '*': + tres: IV + Omon: + '*': + tres: 1M + SIday: + '*': + tres: 1D + SImon: + '*': + tres: 1M From 60488dc4903f95e2b4a8f1b2f59ba558c86c9558 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Tue, 22 Aug 2023 16:27:54 +0200 Subject: [PATCH 03/64] Added public function to check for unstructured grids --- esmvalcore/iris_helpers.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/esmvalcore/iris_helpers.py b/esmvalcore/iris_helpers.py index e5bc3dbeea..b7d7b7bcab 100644 --- a/esmvalcore/iris_helpers.py +++ b/esmvalcore/iris_helpers.py @@ -157,3 +157,28 @@ def merge_cube_attributes( # Step 3: modify the cubes in-place for cube in cubes: cube.attributes = final_attributes + + +def has_unstructured_grid(cube: Cube) -> bool: + """Check if a cube has an unstructured grid. + + Parameters + ---------- + cube: + Cube to be checked. + + Returns + ------- + bool + ``True`` if input cube has an unstructured grid, else ``False``. + + """ + if not cube.coords('latitude') or not cube.coords('longitude'): + return False + lat = cube.coord('latitude') + lon = cube.coord('longitude') + if lat.ndim != 1 or lon.ndim != 1: + return False + if cube.coord_dims(lat) != cube.coord_dims(lon): + return False + return True From f9a4ab71236448d215d8c0a94ade0a50a36ca931 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Tue, 22 Aug 2023 16:28:42 +0200 Subject: [PATCH 04/64] Make regridding much faster --- esmvalcore/preprocessor/_regrid.py | 156 ++++++++++++++++++----------- 1 file changed, 100 insertions(+), 56 deletions(-) diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py index 95f104538b..d4497c59fd 100644 --- a/esmvalcore/preprocessor/_regrid.py +++ b/esmvalcore/preprocessor/_regrid.py @@ -21,6 +21,8 @@ from iris.util import broadcast_to_shape from scipy.spatial import Delaunay +from esmvalcore.iris_helpers import has_unstructured_grid + from ..cmor._fixes.shared import add_altitude_from_plev, add_plev_from_altitude from ..cmor.table import CMOR_TABLES from ._other import get_array_module @@ -1146,6 +1148,9 @@ def _bilinear_unstructured_regrid( ) -> Cube: """Bilinear regridding for unstructured grids. + The spatial dimension of the data (i.e., the one describing the + unstructured grid) needs to be the rightmost dimension. + Note ---- This private function has been introduced to regrid native ERA5 in GRIB @@ -1163,15 +1168,34 @@ def _bilinear_unstructured_regrid( at any point in time without prior warning (just like any other private function). + Warning + ------- + This function will drop all cell measures, ancillary variables and aux + factories, and any auxiliary coordinate that spans the spatial dimension. + """ # This function should only be called on unstructured grid cubes - msg = "No unstructured grid" - assert cube.coords('latitude'), msg - assert cube.coords('longitude'), msg - assert cube.coord('latitude').ndim == 1, msg - assert cube.coord('longitude').ndim == 1, msg - assert cube.coord_dims('latitude') == cube.coord_dims('longitude'), msg + if not has_unstructured_grid(cube): + raise ValueError( + f"Cube {cube.summary(shorten=True)} does not have unstructured " + f"grid" + ) + + # The unstructured grid dimension needs to be the rightmost dimension udim = cube.coord_dims('latitude')[0] + if udim != cube.ndim - 1: + raise ValueError( + f"The spatial dimension of cube {cube.summary(shorten=True)} " + f"(i.e, the one describing the unstructured grid) needs to be the " + f"rightmost dimension" + ) + + # Generate a target grid from the provided cell-specification, and cache + # the resulting stock cube for later use + tgt_cube = _CACHE.setdefault( + target_grid, + _global_stock_cube(target_grid, lat_offset, lon_offset), + ) # Make sure the cube has lazy data and rechunk it properly (cube cannot be # chunked along latitude and longitude dimension) @@ -1181,27 +1205,9 @@ def _bilinear_unstructured_regrid( in_chunks[udim] = -1 # type: ignore cube.data = cube.lazy_data().rechunk(in_chunks) - # Generate a target grid from the provided cell-specification, and cache - # the resulting stock cube for later use - target_grid_cube = _CACHE.setdefault( - target_grid, - _global_stock_cube(target_grid, lat_offset, lon_offset), - ) - - # Put source and target grid in correct format and calculate vertices and - # interpolation weights - src_points = np.stack( - (cube.coord('latitude').points, cube.coord('longitude').points), - axis=-1, - ) - (tgt_lat, tgt_lon) = np.meshgrid( - target_grid_cube.coord('latitude').points, - target_grid_cube.coord('longitude').points, - indexing='ij', - ) - tgt_points = np.stack((tgt_lat.ravel(), tgt_lon.ravel()), axis=-1) + # Calculate indices and interpolation weights (indices, weights) = _get_linear_interpolation_weights( - src_points, tgt_points + cube, target_grid, lat_offset, lon_offset ) # Perform actual regridding @@ -1216,54 +1222,88 @@ def _bilinear_unstructured_regrid( ) regridded_data = regridded_data.rechunk('auto') - # Put regridded data in cube with correct metadata - regridded_cube = _get_dummy_regridded_cube(cube, target_grid_cube) - regridded_cube.data = regridded_data.reshape( - regridded_cube.shape, limit='128MiB' + # Create new cube with correct metadata + dim_coords_and_dims = [ + (c, cube.coord_dims(c)) for c in cube.coords(dim_coords=True) if + udim not in cube.coord_dims(c) + ] + dim_coords_and_dims.extend([ + (tgt_cube.coord('latitude'), cube.ndim - 1), + (tgt_cube.coord('longitude'), cube.ndim), + ]) + aux_coords_and_dims = [ + (c, cube.coord_dims(c)) for c in cube.coords(dim_coords=False) if + udim not in cube.coord_dims(c) + ] + new_shape = cube.shape[:-1] + tgt_cube.shape + regridded_cube = Cube( + regridded_data.reshape(new_shape, limit='128MiB'), + standard_name=cube.standard_name, + long_name=cube.long_name, + var_name=cube.var_name, + units=cube.units, + attributes=cube.attributes, + cell_methods=cube.cell_methods, + dim_coords_and_dims=dim_coords_and_dims, + aux_coords_and_dims=aux_coords_and_dims, ) return regridded_cube -def _get_dummy_regridded_cube(src_cube: Cube, tgt_cube: Cube) -> Cube: - """Get dummy regridded cube with correct shape and metadata.""" - src_cs = src_cube.coord_system() - xcoord = tgt_cube.coord(axis='x', dim_coords=True) - ycoord = tgt_cube.coord(axis='y', dim_coords=True) - xcoord.coord_system = src_cs - ycoord.coord_system = src_cs - return regrid(src_cube, tgt_cube, 'unstructured_nearest') +_CACHE_WEIGHTS: Dict[str, tuple[np.ndarray, np.ndarray]] = {} def _get_linear_interpolation_weights( - src_points: np.ndarray, - tgt_points: np.ndarray, + src_cube: Cube, + target_grid: str, + lat_offset: bool = True, + lon_offset: bool = True, fill_value: float = np.nan, ) -> tuple[np.ndarray, np.ndarray]: """Get vertices and weights for 2D linear regridding of unstructured grids. - Taken from + Partly taken from https://stackoverflow.com/questions/20915502/speedup-scipy-griddata-for-multiple-interpolations-between-two-irregular-grids. This is more than 80x faster than :func:`scipy.interpolate.griddata` and gives identical results. - Parameters - ---------- - src_points: np.ndarray - Points of the unstructured source grid. Must be an (N, 2) array, where - N is the number of source grid points. - tgt_points: np.ndarray - Points of the unstructured target grid. Must be an (M, 2) array, where - M is the number of target grid points. - fill_value: float - Fill value for extrapolated values. + """ + # Cache result to avoid re-calculating this over and over + src_lat = src_cube.coord('latitude') + src_lon = src_cube.coord('longitude') + cache_key = ( + f"{src_lat.shape}_" + f"{target_grid}_" + f"{lat_offset}_" + f"{lon_offset}_" + f"{fill_value}_" + ) + if cache_key in _CACHE_WEIGHTS: + return _CACHE_WEIGHTS[cache_key] - Returns - ------- - tuple[np.ndarray, np.ndarray] - Indices and interpolation weights. Both are (M, 3) arrays. + # Generate a target grid from the provided cell-specification, and cache + # the resulting stock cube for later use + tgt_cube = _CACHE.setdefault( + target_grid, + _global_stock_cube(target_grid, lat_offset, lon_offset), + ) - """ + # Bring points into correct format + # src_points: (N, 2) where N is the number of source grid points + # tgt_points: (M, 2) where M is the number of target grid points + src_points = np.stack((src_lat.points, src_lon.points), axis=-1) + (tgt_lat, tgt_lon) = np.meshgrid( + tgt_cube.coord('latitude').points, + tgt_cube.coord('longitude').points, + indexing='ij', + ) + tgt_points = np.stack((tgt_lat.ravel(), tgt_lon.ravel()), axis=-1) + + # Actual indices and weights calculation using Delaunay triagulation + # Return shapes: + # indices: (M, 3) + # weights: (M, 3) n_dims = 2 tri = Delaunay(src_points) simplex = tri.find_simplex(tgt_points) @@ -1274,6 +1314,10 @@ def _get_linear_interpolation_weights( bary = np.einsum('njk,nk->nj', temp[:, :n_dims, :], delta) weights = np.hstack((bary, 1 - bary.sum(axis=1, keepdims=True))) weights[extra_idx, :] = fill_value + + # Cache result + _CACHE_WEIGHTS[cache_key] = (indices, weights) + return (indices, weights) From fc7384aaa5b086f4fcb47bae99ed28844a1da95e Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Tue, 22 Aug 2023 16:29:05 +0200 Subject: [PATCH 05/64] Add support for more variables and make regridding optional --- esmvalcore/cmor/_fixes/native6/era5.py | 72 ++++++++++++++----- .../config/extra_facets/native6-mappings.yml | 1 + 2 files changed, 54 insertions(+), 19 deletions(-) diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py index 054da2673e..e2e3ac3f78 100644 --- a/esmvalcore/cmor/_fixes/native6/era5.py +++ b/esmvalcore/cmor/_fixes/native6/era5.py @@ -6,12 +6,11 @@ import numpy as np from iris.util import reverse -from esmvalcore.iris_helpers import date2num - -from ....preprocessor._regrid import _bilinear_unstructured_regrid -from ...table import CMOR_TABLES -from ..fix import Fix -from ..shared import add_scalar_height_coord +from esmvalcore.cmor._fixes.fix import Fix +from esmvalcore.cmor._fixes.shared import add_scalar_height_coord +from esmvalcore.cmor.table import CMOR_TABLES +from esmvalcore.iris_helpers import date2num, has_unstructured_grid +from esmvalcore.preprocessor._regrid import _bilinear_unstructured_regrid logger = logging.getLogger(__name__) @@ -281,6 +280,16 @@ def fix_metadata(self, cubes): return cubes +class Rainmxrat27(Fix): + """Fixes for rainmxrat27.""" + + def fix_metadata(self, cubes): + """Fix metadata.""" + for cube in cubes: + cube.units = 'kg kg-1' + return cubes + + class Rlds(Fix): """Fixes for Rlds.""" @@ -408,6 +417,16 @@ def fix_metadata(self, cubes): return cubes +class Snowmxrat27(Fix): + """Fixes for snowmxrat27.""" + + def fix_metadata(self, cubes): + """Fix metadata.""" + for cube in cubes: + cube.units = 'kg kg-1' + return cubes + + class Tasmax(Fix): """Fixes for tasmax.""" @@ -428,6 +447,19 @@ def fix_metadata(self, cubes): return cubes +class Toz(Fix): + """Fixes for toz.""" + + def fix_metadata(self, cubes): + """Convert 'kg m-2' to 'm'.""" + for cube in cubes: + cube.units = 'kg m-2' + # 1 DU = 1e-5 m = 2.1415e-5 kg m-2 --> 1m = 2.1415 kg m-2 + cube.data = cube.core_data() / 2.1415 + cube.units = 'm' + return cubes + + class Zg(Fix): """Fixes for Geopotential.""" @@ -468,21 +500,24 @@ def _fix_coordinates(self, cube): coord.var_name = coord_def.out_name coord.long_name = coord_def.long_name coord.points = coord.core_points().astype('float64') - if (coord.bounds is None and len(coord.points) > 1 - and coord_def.must_have_bounds == "yes"): + if ( + coord.bounds is None and + len(coord.points) > 1 and + coord_def.must_have_bounds == "yes" and + not has_unstructured_grid(cube) + ): coord.guess_bounds() self._fix_monthly_time_coord(cube) # Fix coordinate increasing direction - if cube.coords('latitude') and cube.coord('latitude').shape[0] > 1: + if cube.coords('latitude') and not has_unstructured_grid(cube): lat = cube.coord('latitude') - if lat.points[0] > lat.points[1]: + if lat.points[0] > lat.points[-1]: cube = reverse(cube, 'latitude') - if (cube.coords('air_pressure') and - cube.coord('air_pressure').shape[0] > 1): + if cube.coords('air_pressure'): plev = cube.coord('air_pressure') - if plev.points[0] < plev.points[1]: + if plev.points[0] < plev.points[-1]: cube = reverse(cube, 'air_pressure') return cube @@ -521,13 +556,12 @@ def fix_metadata(self, cubes): # If desired, regrid native ERA5 data in GRIB format (which is on a # reduced Gaussian grid, i.e., unstructured grid) if ( - cube.coords('latitude') and - cube.coords('longitude') and - cube.coord('latitude').ndim == 1 and - cube.coord('longitude').ndim == 1 and - cube.coord_dims('latitude') == cube.coord_dims('longitude') + self.extra_facets.get('target_grid', False) and + has_unstructured_grid(cube) ): - cube = _bilinear_unstructured_regrid(cube, '0.25x0.25') + cube = _bilinear_unstructured_regrid( + cube, self.extra_facets['target_grid'] + ) cube = self._fix_coordinates(cube) self._fix_units(cube) diff --git a/esmvalcore/config/extra_facets/native6-mappings.yml b/esmvalcore/config/extra_facets/native6-mappings.yml index ad59266944..a7c7db95ff 100644 --- a/esmvalcore/config/extra_facets/native6-mappings.yml +++ b/esmvalcore/config/extra_facets/native6-mappings.yml @@ -14,6 +14,7 @@ ERA5: '*': '*': family: E5 + target_grid: 0.25x0.25 type: an typeid: '00' version: '' # necessary to get a nice output file name From e0c4da347c38011cf25f890ca46b17ed19342ec7 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Tue, 22 Aug 2023 16:29:12 +0200 Subject: [PATCH 06/64] Add doc --- doc/quickstart/find_data.rst | 103 +++++++++++++++++++++++++++++------ 1 file changed, 86 insertions(+), 17 deletions(-) diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index 5062d9fe15..5d4e2dfbb2 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -104,33 +104,102 @@ Supported native reanalysis/observational datasets The following native reanalysis/observational datasets are supported under the ``native6`` project. -To use these datasets, put the files containing the data in the directory that -you have configured for the ``native6`` project in your :ref:`user -configuration file`, in a subdirectory called -``Tier{tier}/{dataset}/{version}/{frequency}/{short_name}``. -Replace the items in curly braces by the values used in the variable/dataset -definition in the :ref:`recipe `. -Below is a list of native reanalysis/observational datasets currently -supported. -.. _read_native_era5: +.. _read_native_era5_nc: -ERA5 -^^^^ +ERA5 (in netCDF format downloaded from the CDS) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -- Supported variables: ``cl``, ``clt``, ``evspsbl``, ``evspsblpot``, ``mrro``, ``pr``, ``prsn``, ``ps``, ``psl``, ``ptype``, ``rls``, ``rlds``, ``rsds``, ``rsdt``, ``rss``, ``uas``, ``vas``, ``tas``, ``tasmax``, ``tasmin``, ``tdps``, ``ts``, ``tsn`` (``E1hr``/``Amon``), ``orog`` (``fx``) -- Tier: 3 +ERA5 data can be downloaded from the Copernicus Climate Data Store (CDS) using +the convenient tool `era5cli `__. +To read this data with ESMValCore, put the files containing the data in the +``rootpath`` that you have configured for the ``native6`` project in your +:ref:`user configuration file`, in a subdirectory called +``Tier3/ERA5/{version}/{frequency}/{short_name}`` (assuming your are using the +``default`` DRS for ``native6``). +Replace the items in curly braces by the values used in the variable/dataset +definition in the :ref:`recipe `. -.. note:: According to the description of Evapotranspiration and potential Evapotranspiration on the Copernicus page +Supported variables: ``cl``, ``clt``, ``evspsbl``, ``evspsblpot``, ``mrro``, ``pr``, ``prsn``, ``ps``, ``psl``, ``ptype``, ``rls``, ``rlds``, ``rsds``, ``rsdt``, ``rss``, ``uas``, ``vas``, ``tas``, ``tasmax``, ``tasmin``, ``tdps``, ``ts``, ``tsn`` (``E1hr``/``Amon``), ``orog`` (``fx``). + +.. note:: According to the description of Evapotranspiration and potential Evapotranspiration on the Copernicus page (https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-monthly-means?tab=overview): - "The ECMWF Integrated Forecasting System (IFS) convention is that downward fluxes are positive. + "The ECMWF Integrated Forecasting System (IFS) convention is that downward fluxes are positive. Therefore, negative values indicate evaporation and positive values indicate condensation." - + In the CMOR table, these fluxes are defined as positive, if they go from the surface into the atmosphere: - "Evaporation at surface (also known as evapotranspiration): flux of water into the atmosphere due to conversion + "Evaporation at surface (also known as evapotranspiration): flux of water into the atmosphere due to conversion of both liquid and solid phases to vapor (from underlying surface and vegetation)." Therefore, the ERA5 (and ERA5-Land) CMORizer switches the signs of ``evspsbl`` and ``evspsblpot`` to be compatible with the CMOR standard used e.g. by the CMIP models. +.. _read_native_era5_grib: + +ERA5 (in GRIB format available on DKRZ's Levante) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +ERA5 data in monthly, daily, and hourly resolution is available on `Levante +`__ +in its native GRIB format. +To read this data with ESMValCore, use the following settings in your +:ref:`user configuration file`: + +.. code-block:: yaml + + rootpath: + ... + native6: /pool/data/ERA5 + ... + + drs: + ... + native6: DKRZ-ERA5-GRIB + ... + +The `naming conventions +`__ +for input directories and files for native ERA5 data in GRIB format on Levante +are + +* input directories: ``{family}/{level}/{type}/{tres}/{grib_id}`` +* input files: ``{family}{level}{typeid}_{tres}_*_{grib_id}.grb`` + +All of these facets have reasonable defaults preconfigured in the corresponding +:ref:`extra facets` file, which is available here: +:download:`native6-mappings.yml +`. +If necessary, these facets can be overwritten in the recipe. + +Thus, example dataset entries could look like this: + +.. code-block:: yaml + + datasets: + - {project: native6, dataset: ERA5, timerange: '2000/2001', + short_name: tas, mip: Amon} + - {project: native6, dataset: ERA5, timerange: '2000/2001', + short_name: cl, mip: Amon, tres: 1H, frequency: 1hr} + - {project: native6, dataset: ERA5, timerange: '2000/2001', + short_name: ta, mip: Amon, type: fc, typeid: '12'} + +The native ERA5 output in GRIB format is stored on a `reduced Gaussian grid +`__. +By default, ESMValCore linearly interpolates the data to a regular 0.25° x +0.25° grid as `recommended by the ECMWF +`__. +If you want to use a different target resolution or completely disable this +feature, you can specify the optional facet ``target_grid`` in the recipe, +e.g., + +.. code-block:: yaml + + datasets: + - {project: native6, dataset: ERA5, timerange: '2000/2001', + short_name: tas, mip: Amon, target_grid: 1x1} + - {project: native6, dataset: ERA5, timerange: '2000/2001', + short_name: tas, mip: Amon, target_grid: false} # do NOT interpolate + +Supported variables: ``albsn``, ``cl``, ``cli``, ``clt``, ``clw``, ``hur``, ``hus``, ``o3``, ``prw``, ``ps``, ``psl``, ``rainmxrat27``, ``sftlf``, ``snd``, ``snowmxrat27``, ``ta``, ``tas``, ``tdps``, ``toz``, ``ts``, ``ua``, ``uas``, ``va``, ``vas``, ``wap``, ``zg``. + .. _read_native_mswep: MSWEP From 3db12bb262ed257b2cc2401a13a5108d1da55273 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 23 Aug 2023 09:49:52 +0200 Subject: [PATCH 07/64] Added first tests --- esmvalcore/cmor/_fixes/native6/era5.py | 13 ++-- tests/unit/provenance/test_trackedfile.py | 66 ++++++++++++----- tests/unit/test_iris_helpers.py | 90 +++++++++++++++++++++++ 3 files changed, 144 insertions(+), 25 deletions(-) diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py index e2e3ac3f78..1ba3d66862 100644 --- a/esmvalcore/cmor/_fixes/native6/era5.py +++ b/esmvalcore/cmor/_fixes/native6/era5.py @@ -500,13 +500,12 @@ def _fix_coordinates(self, cube): coord.var_name = coord_def.out_name coord.long_name = coord_def.long_name coord.points = coord.core_points().astype('float64') - if ( - coord.bounds is None and - len(coord.points) > 1 and - coord_def.must_have_bounds == "yes" and - not has_unstructured_grid(cube) - ): - coord.guess_bounds() + if (coord.bounds is None and len(coord.points) > 1 + and coord_def.must_have_bounds == "yes"): + # Do not guess bounds for lat and lon on unstructured grids + if (coord.name() not in ('latitude', 'longitude') + or not has_unstructured_grid(cube)): + coord.guess_bounds() self._fix_monthly_time_coord(cube) diff --git a/tests/unit/provenance/test_trackedfile.py b/tests/unit/provenance/test_trackedfile.py index 4e41f46cd2..09aa5be0e2 100644 --- a/tests/unit/provenance/test_trackedfile.py +++ b/tests/unit/provenance/test_trackedfile.py @@ -5,7 +5,7 @@ @pytest.fixture -def tracked_file(): +def tracked_file_nc(): file = TrackedFile( filename='/path/to/file.nc', attributes={'a': 'A'}, @@ -14,35 +14,65 @@ def tracked_file(): return file -def test_init(tracked_file): +@pytest.fixture +def tracked_file_grb(): + file = TrackedFile( + filename='/path/to/file.grb', + prov_filename='/original/path/to/file.grb', + ) + return file + + +def test_init_nc(tracked_file_nc): """Test `esmvalcore._provenance.TrackedFile.__init__`.""" - assert tracked_file.filename == '/path/to/file.nc' - assert tracked_file.attributes == {'a': 'A'} - assert tracked_file.prov_filename == '/original/path/to/file.nc' + assert tracked_file_nc.filename == '/path/to/file.nc' + assert tracked_file_nc.attributes == {'a': 'A'} + assert tracked_file_nc.prov_filename == '/original/path/to/file.nc' + + +def test_init_grb(tracked_file_grb): + """Test `esmvalcore._provenance.TrackedFile.__init__`.""" + assert tracked_file_grb.filename == '/path/to/file.grb' + assert tracked_file_grb.attributes is None + assert tracked_file_grb.prov_filename == '/original/path/to/file.grb' + + +def test_initialize_provenance_nc(tracked_file_nc): + """Test `esmvalcore._provenance.TrackedFile.initialize_provenance`.""" + provenance = ProvDocument() + provenance.add_namespace('task', uri=ESMVALTOOL_URI_PREFIX + 'task') + activity = provenance.activity('task:test-task-name') + + tracked_file_nc.initialize_provenance(activity) + assert isinstance(tracked_file_nc.provenance, ProvDocument) + assert tracked_file_nc.activity == activity + assert str(tracked_file_nc.entity.identifier) == 'file:/path/to/file.nc' + assert tracked_file_nc.attributes == {'a': 'A'} -def test_initialize_provenance(tracked_file): - """Test `esmvalcore._provenance.TrackedFile.initialize_provenancee`.""" +def test_initialize_provenance_grb(tracked_file_grb): + """Test `esmvalcore._provenance.TrackedFile.initialize_provenance`.""" provenance = ProvDocument() provenance.add_namespace('task', uri=ESMVALTOOL_URI_PREFIX + 'task') activity = provenance.activity('task:test-task-name') - tracked_file.initialize_provenance(activity) - assert isinstance(tracked_file.provenance, ProvDocument) - assert tracked_file.activity == activity - assert str(tracked_file.entity.identifier) == 'file:/path/to/file.nc' + tracked_file_grb.initialize_provenance(activity) + assert isinstance(tracked_file_grb.provenance, ProvDocument) + assert tracked_file_grb.activity == activity + assert str(tracked_file_grb.entity.identifier) == 'file:/path/to/file.grb' + assert tracked_file_grb.attributes == {} -def test_copy_provenance(tracked_file): +def test_copy_provenance(tracked_file_nc): """Test `esmvalcore._provenance.TrackedFile.copy_provenance`.""" provenance = ProvDocument() provenance.add_namespace('task', uri=ESMVALTOOL_URI_PREFIX + 'task') activity = provenance.activity('task:test-task-name') - tracked_file.initialize_provenance(activity) + tracked_file_nc.initialize_provenance(activity) - copied_file = tracked_file.copy_provenance() - assert copied_file.activity == tracked_file.activity - assert copied_file.entity == tracked_file.entity - assert copied_file.provenance == tracked_file.provenance - assert copied_file.provenance is not tracked_file.provenance + copied_file = tracked_file_nc.copy_provenance() + assert copied_file.activity == tracked_file_nc.activity + assert copied_file.entity == tracked_file_nc.entity + assert copied_file.provenance == tracked_file_nc.provenance + assert copied_file.provenance is not tracked_file_nc.provenance diff --git a/tests/unit/test_iris_helpers.py b/tests/unit/test_iris_helpers.py index 07e81608e8..2b9f149939 100644 --- a/tests/unit/test_iris_helpers.py +++ b/tests/unit/test_iris_helpers.py @@ -20,6 +20,7 @@ from esmvalcore.iris_helpers import ( add_leading_dim_to_cube, date2num, + has_unstructured_grid, merge_cube_attributes, ) @@ -220,3 +221,92 @@ def test_merge_cube_attributes_1_cube(): merge_cube_attributes(cubes) assert len(cubes) == 1 assert_attribues_equal(cubes[0].attributes, expected_attributes) + + +@pytest.fixture +def lat_coord_1d(): + """1D latitude coordinate.""" + return DimCoord([0, 1], standard_name='latitude') + + +@pytest.fixture +def lon_coord_1d(): + """1D longitude coordinate.""" + return DimCoord([0, 1], standard_name='longitude') + + +@pytest.fixture +def lat_coord_2d(): + """2D latitude coordinate.""" + return AuxCoord([[0, 1]], standard_name='latitude') + + +@pytest.fixture +def lon_coord_2d(): + """2D longitude coordinate.""" + return AuxCoord([[0, 1]], standard_name='longitude') + + +def test_has_unstructured_grid_no_lat_lon(): + """Test `has_unstructured_grid`.""" + cube = Cube(0) + assert has_unstructured_grid(cube) is False + + +def test_has_unstructured_grid_no_lat(lon_coord_1d): + """Test `has_unstructured_grid`.""" + cube = Cube([0, 1], dim_coords_and_dims=[(lon_coord_1d, 0)]) + assert has_unstructured_grid(cube) is False + + +def test_has_unstructured_grid_no_lon(lat_coord_1d): + """Test `has_unstructured_grid`.""" + cube = Cube([0, 1], dim_coords_and_dims=[(lat_coord_1d, 0)]) + assert has_unstructured_grid(cube) is False + + +def test_has_unstructured_grid_2d_lat(lat_coord_2d, lon_coord_1d): + """Test `has_unstructured_grid`.""" + cube = Cube( + [[0, 1]], + dim_coords_and_dims=[(lon_coord_1d, 1)], + aux_coords_and_dims=[(lat_coord_2d, (0, 1))], + ) + assert has_unstructured_grid(cube) is False + + +def test_has_unstructured_grid_2d_lon(lat_coord_1d, lon_coord_2d): + """Test `has_unstructured_grid`.""" + cube = Cube( + [[0, 1]], + dim_coords_and_dims=[(lat_coord_1d, 1)], + aux_coords_and_dims=[(lon_coord_2d, (0, 1))], + ) + assert has_unstructured_grid(cube) is False + + +def test_has_unstructured_grid_2d_lat_lon(lat_coord_2d, lon_coord_2d): + """Test `has_unstructured_grid`.""" + cube = Cube( + [[0, 1]], + aux_coords_and_dims=[(lat_coord_2d, (0, 1)), (lon_coord_2d, (0, 1))], + ) + assert has_unstructured_grid(cube) is False + + +def test_has_unstructured_grid_regular_grid(lat_coord_1d, lon_coord_1d): + """Test `has_unstructured_grid`.""" + cube = Cube( + [[0, 1], [2, 3]], + dim_coords_and_dims=[(lat_coord_1d, 0), (lon_coord_1d, 1)], + ) + assert has_unstructured_grid(cube) is False + + +def test_has_unstructured_grid_true(lat_coord_1d, lon_coord_1d): + """Test `has_unstructured_grid`.""" + cube = Cube( + [[0, 1], [2, 3]], + aux_coords_and_dims=[(lat_coord_1d, 0), (lon_coord_1d, 0)], + ) + assert has_unstructured_grid(cube) is True From 09aabcb24a354e67c0f68ac80544f336fed71bdd Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 23 Aug 2023 11:32:48 +0200 Subject: [PATCH 08/64] Added test for loading grib files --- esmvalcore/cmor/_fixes/native6/era5.py | 4 ++-- esmvalcore/preprocessor/_io.py | 3 ++- .../integration/preprocessor/_io/test_load.py | 20 ++++++++++++++++++ tests/sample_data/iris-sample-data/LICENSE | 10 +++++++++ .../iris-sample-data/polar_stereo.grib2 | Bin 0 -> 25934 bytes 5 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 tests/sample_data/iris-sample-data/LICENSE create mode 100644 tests/sample_data/iris-sample-data/polar_stereo.grib2 diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py index 1ba3d66862..d24b4169fd 100644 --- a/esmvalcore/cmor/_fixes/native6/era5.py +++ b/esmvalcore/cmor/_fixes/native6/era5.py @@ -503,8 +503,8 @@ def _fix_coordinates(self, cube): if (coord.bounds is None and len(coord.points) > 1 and coord_def.must_have_bounds == "yes"): # Do not guess bounds for lat and lon on unstructured grids - if (coord.name() not in ('latitude', 'longitude') - or not has_unstructured_grid(cube)): + if not (coord.name() in ('latitude', 'longitude') and + has_unstructured_grid(cube)): coord.guess_bounds() self._fix_monthly_time_coord(cube) diff --git a/esmvalcore/preprocessor/_io.py b/esmvalcore/preprocessor/_io.py index c97f5f7d00..416405503e 100644 --- a/esmvalcore/preprocessor/_io.py +++ b/esmvalcore/preprocessor/_io.py @@ -194,7 +194,8 @@ def load( # GRIB files need to be loaded with iris.load, otherwise we will # get separate (lat, lon) slices for each time step, pressure # level, etc. - if file.suffix in ('.grib', '.grb', '.gb'): + grib_formats = ('.grib2', '.grib', '.grb2', '.grb', '.gb2', '.gb') + if file.suffix in grib_formats: raw_cubes = iris.load(file, callback=callback) else: raw_cubes = iris.load_raw(file, callback=callback) diff --git a/tests/integration/preprocessor/_io/test_load.py b/tests/integration/preprocessor/_io/test_load.py index 99247bdc5a..ecb602d7cb 100644 --- a/tests/integration/preprocessor/_io/test_load.py +++ b/tests/integration/preprocessor/_io/test_load.py @@ -4,12 +4,14 @@ import tempfile import unittest import warnings +from pathlib import Path import iris import numpy as np from iris.coords import DimCoord from iris.cube import Cube, CubeList +import esmvalcore from esmvalcore.preprocessor._io import concatenate_callback, load @@ -51,6 +53,24 @@ def test_load(self): self.assertTrue((cube.coord('latitude').points == np.array([1, 2])).all()) + def test_load_grib(self): + """Test loading a grib file.""" + grib_path = Path( + Path(esmvalcore.__file__).parents[1], + 'tests', + 'sample_data', + 'iris-sample-data', + 'polar_stereo.grib2', + ) + cubes = load(grib_path) + + assert len(cubes) == 1 + cube = cubes[0] + assert cube.standard_name == 'air_temperature' + assert cube.units == 'K' + assert cube.shape == (200, 247) + assert 'source_file' in cube.attributes + def test_callback_remove_attributes(self): """Test callback remove unwanted attributes.""" attributes = ('history', 'creation_date', 'tracking_id', 'comment') diff --git a/tests/sample_data/iris-sample-data/LICENSE b/tests/sample_data/iris-sample-data/LICENSE new file mode 100644 index 0000000000..6ab33c6548 --- /dev/null +++ b/tests/sample_data/iris-sample-data/LICENSE @@ -0,0 +1,10 @@ +Data in this directory is taken from iris-sample-data (https://github.com/SciTools/iris-sample-data). + +It is licensed under the following UK's Open Government Licence (https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/): + + +(c) British Crown copyright, 2018. + +You may use and re-use the information featured in this repository (not including logos) free of charge in any format or medium, under the terms of the Open Government Licence. We encourage users to establish hypertext links to this website. + +Any email enquiries regarding the use and re-use of this information resource should be sent to: psi@nationalarchives.gsi.gov.uk. diff --git a/tests/sample_data/iris-sample-data/polar_stereo.grib2 b/tests/sample_data/iris-sample-data/polar_stereo.grib2 new file mode 100644 index 0000000000000000000000000000000000000000..ab02a2d13fe920e78498d11f07291890729c344b GIT binary patch literal 25934 zcmY(q1FSGSur0c6+qP}nwr$(CZQHhO+cv*#?|uJs?#s)2ldhF^X3}YsrcEWTBq;;{ z008tK{u5KV{|FTjfDHfu1OO0x7Zl}xwEtri1pbc(-v1*Mu)qJm{HJ^S4-z^2Z~wQl|H|l~{~Liq z6#sfH02lxe1_0pif84-j0Bgj+23bj`2?k;J+CEwFMO9zux}o03Zt(SXemx zf74$A0M!3fY(n{;L-PEo3@+a-tt6FYcp_+oCiNXvsws|XN+#((KTBcNz#r6uc_Gp$ zT^GG*;n%atNP`I31ET$(L_?|TU1DpOST4fn{fL)Y6gjk1*NS50OP_uLg@z{5@T*kt z6K~*0J^pl4$GQ#Fql!%|wYjkwS===V}CSU_4KDf7Kt^J*v5( z`FZ`G;1Or%zKEk3wo}+adlh0lU=W_eAyOqjyk_l+E=e@Y@z9TX!NbD~?y9b-T)jE( za2H1j0gxzDo#xPbD>WxA{g+lVuyY|qCKJ7?FBu6{0SRY9J_btR`66EeI*`1D=VYIn zEsP$1NtnrKW-8+pnPp%%jxc`Ziz`Tr}{~`YHYun+M zRsugaLEMy~fjy5qWUFgFC)H$FT#~}faV%}Oz$GcOX#lOFsQGvCIj5x3<2Vh<68oeV z*q1jDD5q66IjXPJxVwwaLX0`eehOfS$E4kaB0%{M5;0uG};`k#SOKNdpKEl zS+C@jN~dzQC&7{3aEm^I!-Q!RUPFvFK=nG{ZKrrbP|SOlD(u+^B4w8b z!n~KbO^17@vajk);@F#-IYfo>z--lB?KXQa&?fz15do4njyckhhqdom^j4z4RfNAuy1mHns++8U2?Y`weBkrjF0ey2Y=n{D;A^=(X`jDJ#!3$iiA)0Of@OeB62 zmhtzvt)YMW;cTXWNAa}DMf{^n(~za}J+*fiwXqIv?_KkK%dCb+`jCMvT8X1Co^Doy z{RhlH(2&kGf7NY(z~v*qj}N>Llp-==Ml|y&8I|*NtknbPB!U~*soqH%={1jpb4zp( zDiKnuoK{HUTzL=w!H<_&@bk4E`1zWJu?y1UL=k%yIm~S4T>(sDV-}I)a^{2&p6H2G zq@x=dMVo=BHTI-BTQzDOOTH74v6*J`s)D25khZw`&HY-l1j-*I2S4BastL~ze+Ng3 zgS5VHgx=Ej*D_CML=R@Wp9szEm}FlA;Jn}Ltc_)@qH1?5A?VHY0pTKln;kW6bd?HD z1NB^xWJtIj(*iil45-p?zNk8HRkS_UfmZ_>k8a`?Pn zqhLLjXxy;zuFuOUZ++*gn=r^$+wu(ovdw8d^zm{UiQVT|&v_;Z0_8Uj{j`z*NKqD2 z)KWx@_+H3DGC^D3faV;wFM_JvjH{JHR)D)CBT(PslRpxx@H#RV;ZJH}Eku3;-m^=( zo~|S;ldpK2&|h^A=Y|{({c0K&4KR9Qa7{Y0%|gPY4B5c;`>QOQd3wkt1f(J@<%Yt{ z#BFY;wkRY8!00@OXp37JrsK@-YTNEB2*lytARk{e`l(y9%8OZ=Kf+Vu2k#PdOt^Js z*wBF|@!D05s-XE*4aBeaX)U&qpc@vt+jS9`#Bw7*QZs? zozUM4r@>Yq=+^pDOT7oRa%mJyO);m@`I=bME170eF(r5PYVnP%PZ>YC>o>xq+7r?$ zT(=@O*lhT*&XLDz_B;NAqlK_^X(XvQz+Q1>0r>-?*WPKL}W12U~@>eS*u+E#!4ga|z z0f+?jA10}Q-hQ@*2e^Med{Ydo&*tBrHxNz;8Zd8aL)|bp1pGbmLLUmd;^74i8kD9~ zx?gYNX@md=E0aR0IiLG6qx>NaZ_h={x$KB9cl1`#H(F5l?hcMCLjz3_#f!S>dC=kRv?nuTU z7C~tOx^GkP#mtP&Y!tq__+TD8#knwNzOAmERY0(BSJ+}qHNuq4st!C_Y6{FAji{x= zJG6;Umfvle+e-)D9rSJ59w1-B{>@Rb;d8hy$4C;p*ZH3;iS9pKks#Mac7W-POE)HH z26BT^$}EL`<}qjzvrN?T#aJOBX4bXXo%7gIq|D-Qb51$ync=h@qm{OS1@iq9&`gQ9 z)vnv)&PvBAW)?3lVh#$7t4!>$+}tQLwb(*xeNF50nLk%E6~t#p`L{7G)=hNh-cc71 z>L-?ji>^kBg1XX+>5t$o@a#!8)CCl=TM+PWqoC$&2+2F?#O?C+@5b15a~bk0KwD66Qtu*ri7m@OgLerPlNI!H+xw|<6!UKEzKTObN z3tSEbw1p($zX+R$*|C-WA>o1~W;(kRgaRh9h{nHdw}UkyWbcqDUf2$ZiOQCiG^`_A z&?E~HORNC?CwbwS;^-rl-GPuiP1z_<*z6GX2YmGjKYPPweD<&nM6{ZIf_7j-;AY0} zkUUa|HTHBN$sSjVMIQ-}h^i*w)$CdZzFQ&N@f?|>ZZ9DEMKm1ClYOT;-b9kW!f?Td zkYQ$yOz|EAu{0vdLoQB6UFECdf(5bm4O*q5a(@g3aDMsdkTjhF4~t}=_9&ifO0xpp zAOgG6!sZ6M$<{d?YI( z22mWw!3(0R7{Sgii-H+e0XgYuV7Cf?xl=wDY*F5Y>EcZd{vDzF6H|(&Wh=lgL>a8!cqM4O~*wV zYJ$)L0c)<75pRkd=ng)A7@sfLunUSzT6V|+SQb3sW^ucX6|dfzH{~8>-SoXKB}0xD zI?o$Of;0ptdwe&h&vNd^tDDQTFgp~uoesaVSpXu;OF=2ID)>^sF!p;uWjXM zzn(VesR}o&38v>D=cHsz^=s-Rf0lw5+_Drli&Yqt-|6sAkou2CoBh38Yb>h?#*f(P zlsU*wNN{?W>%m~u`ot)vC}!|HK*?COYHS}7OF0vfZ7VrXP37^p7rCrulCzE}tPq5g z_$lbgiSS)&e%e3}Ppwr#{tW3NN!~Y?M``gO7n+mbFH`SuA@MAnhfH$e@1aN0r9I~h zAnu^{gUPVUGf5o04Gut^TN@2d6I`}F6~fi@?TXaQw+s|h8niMm!n^YOxbB(HRBwO^ z>qpcPe(RM9xNzFVB!- zDo(>7%SM4N6d{0o_hzt9d1$wAA4_uJnKvwC(^O7bCgiAMC9^@#0T4!g@vJuGYC%3m z3JZZjvS~19aCx=MssqI8|@k{1dx-DFpU!wghx-hZ_)9{d`;~ zeWdi7hPbI?X0r({HTb0x2F>i6mqkL3wv~E!h@rb&X#-l9a!wP>q_?iAKz0%v*HU&OH@m!;p zx`GKcFVd@zK3_JAq(=bw1`sdjUsbP)nc?3q=H4SM^?4_-8Q+(%=P{?I2By4E90O1Y z5CftX4HeWSQ;G@=Q{s?A6SLL_JXz&-1$rtK_$L+YV&MacHJ7B;yO1VI;`p1<<9W-;`(D=G`S$(V`K-iFDj-FVdvvQh_!@6m8*j5)VPyWpi zcvD(i8Tzu7nS_`4C%S&%y* zyA@J|{mT{G&nGjsaTmeWk~K^{-fziAl)b-A(H*k}a?rIXt!A={{MlT>X;m%C?az)j z`*~4B5NHx3%qAM(_&QC|!Y9fXpEZRO!TpU5Yv5cH;`*{SU7#CtnWsQO?^}~?|5)@o z3;W2=!KEZizflRNgK3e3kDXrv*5+n|#-H;qL_hW_2spm{UpH@cQ~v41&kF?n%#f$f zj-Yd8`avs0qS6gi8a(+DG-$&1NmK=gGyBz(F`Vt4ucQblAMv_4h}eYKSHR2@|3*lW zUAtFlrBg@OvE^hD-|tms25Nbp10^YoBD^i|Gl#$DUCX?Hwn5uCAf4q3yFlq%P{_*q zccxLC$k27W+oasfo|WzLwNCV@hYBRM=C&q=ODcjS|~X9iB~u>!=NA zH%keD%2snEt-aMFo(L&9XelIoo$9IZN8-xpz0Tcl1H}VEccegm=d-7BA+CZ^jV(eo zQCRtPmwVGup!i=EoRwhc<@yV*o~17($fK+?E4^$=aChLrnrx$-H1b!{8h9H9z-0V7 zdbI!S88s`;FNd+=fk+;5h@zcp2{mWF`clmElVQ*yG_egV$ea*2_~5BizHIn*+ zDqtNn+K)U9(oC_mFL`T-J*jw|Pt<<8Y$@BINxwCIDyS`FlA)Yq70 zO5VU;d049wU4g$Kbu9s@GTJhbb*p?lna{}VZIJVC`*j1P^m&kl9lH{>BkJ`0PNRFg zzmel`*^So+fH4^IfFzD&_yaLf5JkU#*`%w8mgcf@3BRG$8T0G_Q!)6N`i`!pFG*dU z)qC2JFc)PsRdXxXpveKTorUR3$=K0A_uw`2AvpArEKNn#EA?#;*>TZFscS&xSgf}g zo~-#`7^q0QB?gZ?K}bt9ree!&YVOAbht0 ze~AoT9lwek(xh{h56aTH4Hs{80BM2PoJ3qiFL0AkTq|Vd?yd9d>qA+Jy)0w^YccOv zv@CZEiOQfI_Ez$vD_{i)P)&DfJb&?(TOe;Hz*&sQ5w7;7%y(4%ay&_iw&L)|gC$gsz5UY)TttLv zndY{1GqnUlr8QY3Mh#g_N5n$cN_SI8GkAW5ov&yb^s#wgi8 zS=1KC5GES0Q8bZl7vO2CCF6M(b~kIj;E8$4lXqh86nSVf)yMCtb^<)2@H}n^%R$|^ z&K)W0PYSnY93HXeXoVpgc9TV>NZso+KJ5%2_YnFo&KSLsxEpn(fpbTG7V95iz!>B3 zbv^lF#0uYX6nN(Y$E$EHA|~>4!M?OD&FQaw;iB1kQIE_?24QACHa^c@eVF5(o{c z#+HS600O(6-ZpdF!E~8wuMtl+@)IP z5|%JefKX*jZ@UkYdJ1Bs4xpAwIKzwDB#MkCKSdRY=5=2~f%9c>Iyhb2?j_tC& zYuAe~*Bcr>`i+pB7lx3CakgyZxm0h^NO8GN+&KKySRw$dJy;VX4&}M_il+=_thj`$ zQJ*G0s>{1)oT&I;{`p_W&2l&|##L%>8tcagSr4@EkVpSpn_s6{tPVnmGEinuEvi|& zJ!Y$AEZ=&(M2fLuU@DzA=tcX(OQQ!8Ek3;YxZeUI6Ib)%Lp^F6T&!XnuEYce1=iSf zi8qEp%SLqy^%8o~6kg|{_|ay@lviSkGo^#jdojxl^)jDB#&--r(fYM;Z%Wd8kmyKX zuwA&Z@w;j1aqj2boHJA$5PJJ^#F-v_)`$o_D9R}$9=5iXaOODnJ(G26BfJ>ml3q&4 zj}|U)m{<08sA=AsAP5x&R1Xk{^vzYr@0JWR6{d(*g@;H|DSGt!GT+qT+RZ{@iF{xq z65X7{g7C$6TSkcJDgk~}y~4R8>NEL(l<-}Jaz9dovYWKh>O4-BCO#GkIV4X0J z>Vx2LKdd}JQ-quB%Kq;rWeZtM4WkGffR6&9bQ5YWSwjuKRSBd^ z;zJ0uXm|is$r(@U0RszXLd(+~DbXdB3^(SZ+nq#Th%MB#? z>$Nb+&lbi<_pa&PeAbfyH)hxXRU+tjT1En!3z^qN?ft_~SnU8f#y^MzZ-g0N``yEa_3w5B7O6A+ zK80=>(u#e7tOCCRSLZm!peGgZGpiUAluMq~o{ZGDxd0?3d_C(3^C0^>mxp!|vpIKN zApO(Y%NG|)(P13H50;?wu*M2XS8i+MsS@A(H3wQvbbQu(mJa3k6`uDReZlq?9ak`5 z#}gK8`->ITRz7UUXrQP|2^ZI=iZ~Mgc)M<4eQ(G&5jn&>$5zqPBw{x8M1Tj+WU~Vm z41Pyr%`J7peYeG@N}3G800lcQD_(q8*&&(Ru(w8hPDd|#)faNjOnQ4jDR#>#wGYZu zsBE{xIPka@MOD_YYyIA!QeF&XiXv%oU(z2{Jy*$fsVQ{&_#Z^nhR<)hn~algwk+%B zU|CD;#dKPs)V{nkF)iemn~|*A)ucwx^IND#litjjG+UB(OAfgVB<4laB4>QuyuCX} zO0$(ks>PRnvBCq~d1|b=ToBBCca>=O*ML74ocvJ}Gg9PN?ECvZKvl=QT`jS^x+s(& zeJuglBacTc>QzF@Aa1Rh7Q`!p+1s%X`_%!dH=G_hc*Di8$|2$V9fOpAp|%7D;eM4n z@zziPua9_t{t|84j#crep&CL+*Uumo=e}_VHQN#kIv;obd8`M4r`~wO?X#C4I+`y{ zyY9Eeh1hcnGmA9@yJcP=GNuSQ89*^qcrdT0DZ-enGQ6k1PqvP=_@`$vn0@ccf}_$EhsuF$Kz z&UdA-)oR@VW$93H3|7l$b|8R9y#()G@AVvbxO!@F1>QonC$GzLd5nHX-Gkm~OJ`O( z|Ay-mCV+eo^qir1C({UN;?9U4mS|wYaI`J$fl;6zBG2E?)mGgeP`e#?))HA-OJwQ3 z|J#!%u_3#zKA%ON&}>t*d$FkPE|D?N!Ew$^Pp>~D(p3RQD9~pca_xa=bzeVeu~7Lc z))M-~!dWFtf*LVju^}Mwa;cxB@ni>CpAB&Z1DH>J!Hs!g$yRe;WI`X$0TdG^nYz-as%-bpSr*a}uontL0}LPS_UOp$|L_hMlH zc)MVF6WVu5dbF}Ss80+N4&8Fw7!yyBlT@os)1#8Wo*04KifQbtDE0~mK}RCaQKxI*0n zqkVb`ZVhqyB72=g-qhR2;?Sak+2@Q&Oj_LQ8y654P~QfHC_M&_MMO7yzG@#n#7H~*dehHqlH~1EM}Jn!YJNS`g1sZx!j4}4pcIqBdTC$0zDMRULy6TKutX{_ksU- zXFlqu1H8F#x6TN7H%g30H6*}c@m`WQcPd>8iX~*SK+5z{lDa1BSB{oB1nW^W31DzY2*vJb%fE>XZv6_J%g zA=xRbDyY#>nqYB<7Xvgzb&3*%NEsbS?ZwojGW~TI*nqsQ|68w)Pjhv*8ba3l;_*Sv zb!0$ySaQ*mt5!(!%_pA3d(~Ofi~f|FL?KU-89NyY<7QbP$=KTXhc~Q(w6?;whvnX- zbWPqRCfZ0>?i5mKOVmH|US_3YXSgvZ%_9>@iTq{g^Rkm(s9CqGU%l%HJpY9F6Vnd4 zI_#hlphl77OU)%z>3v!H;}sF&6WH8ESO3f_?4c3CPgr*b95xh99pYbx(RctX)Ency z8+S~CEV(&Mu+sWYa2t~ZSFsLroyl;&pJLgPbs+RV^UTLmlY4{QH*D6MC5rh0ya9Rs92?f-5<^K+;-X(g{ zJm$Bm8XWEyv*u=j5mjZ#*>bdFx+P#MG)Xd`nw0SAuBf{!>o37*%hLC14r z0|4`>yhLG{4k(eel2m#t`#P0NA1e^@gw^m zguaR+_j_<_hkh#;NME7t%)g=Wrl~rH`~zz!u$H~8_`Or-HKCi5EV;%7pBx$_`Jto9 zB$&K`N{nMpxMzxn_fJyXWQRFYzGEx*suki?mh@>Phm3zG4|JjXuk*ir(R>EpM^>Lc z3ho8qTjXD=-Bc*@okJq_(9;>uE?YBvpjY*N^QHS(xY3*`J#|idtr%avD32|i&CUna zyv&&~&-J)+@UK^i?aLG`w| z2VV0tVnayn11ZH%XT5=d+YFOG^m46&1RLzHR2zz1Pw`V+W}(ER=62N@102ISh%UpL z2HVO+%DY!IFxbM~-mPm$Drr+h;XIYw5^ zbEu~kM0i%)!FJ!p&Fs@6+B;~ICX?7gz!t2FZX9(sK_nFgj>QVF$Hy9Qx{qSPH4Lvh z7OyOI>>MXM#=~}iJo7yr*ooYK8(JVzU>gb{H$(R6K~PaUmc0~Rp?SZ5wVlBHs+fm` z9b#4l^J662oIWA6S4#M+nPs1QD?*H{d3E{N?l9g z&6oHwRE321ktGovI$rGmLR*Sf$dZ(pZC5H%f6+~L-%%%gboXfd4MiZ%Y8n5x?Pd`g z!4!BrK8bJDRk7miXq(97OLyvev0yGmh%K5za{HxJURrT>padO{1A+oYM^;}2JejJk z4d8qUMiPkgrU6lDh6Y$jJbhVkF|<+l+Tsc%Sj>kekecTMC!*2lG;o_3ho*`8@C$Qh zhoiWL5K-G|K=`DUE6i2Cja2}JNCeeM`FTv$eo8FhCvPDoRD5- zpkU$}KQCv6CnS~&QgDe{FR;B5J>W*Mk#hXZ)$7L=XFgz?CEYbEew@~3TlXECwC(Bk zQR6tR)wIO7ccKNc>sP!+Y?U(?_*zls`Xp{<*=O@s{Ml7}Pm{%J|4hexK6`r-ILshe z5FWB-=FAVAfSG84TWOu3f-^aHsh{E+RXa;ro7FMy#-^Auvw_4F^qvKAPCs(fOYTuc zE?zwrzPrx8z6Ie)nWqj2<&Uozv|$8krl2d@Xjv;VXl=2p;#P|-EZ>2r5q5(J_hsI= zN6TEz>xMDv{K_3^m>=1niSBbRQcm`Ujh2Lh$*bXT#lz9?7Kzr}Q{~halkrOpN^`R^ zaD9YE?)$KUfkfS z55h%YaSipd>cQj@`goPqTru$N+NiGhC( z&RPv4EEI2gh~v$E2Q#6NUJpvD93>;}j1z*Ya0o4q8`(AEEEu}r{p(8MV6eoTS|F*F zpim2qbYsHo)?l<~Fyg@%>&EU;EhyPzp>Qk}<>CjDg~g*3$}f^F8v{9I1lSeLmlude z05#*i{@>Bqhk^A;0X!l+)BCTr`W1db#(XbU#b|N}?G{-N14{H|x8wmp2Fx^Tf~5XiKZGHbF7W__2oxo~5W3ZVE@k{2ZeZ?dz^>Lj0$Cnpq*Hb%Dc5m%m~&cvfPx zQsW>`=*|#r42Nu4QJPj82%?NPS~LT6u?*e=bY8j#i942G;NA6P%7_eb9)mkJEHbL_tVRK0?c@cPL0-5>eSgfF`^#+zFfUnbVJ0Csyt; z#pPB0l50=-So#(L+~wnmqz2;(!&}v~3`sqRVl+3@i6dCOxOy$V9R&zt83iG0nC>cQ z>`vRB222N+l1YJ3v|ZD??QSQ(B~ef(!PS2!955e!*hBg^5JHo6lmxrT@zaAz*@2_{Hq0?$GYIAt^f`JZv{{f)JmBWnO9H|505Bv0NoW+z;@8@C|Wc_ z38C21f^;R3lDl0$M9dpCAqL9LFoQ?neD9@#V0fhM-p+5uOtz;tyG5cKzDC2hTt|W? z(-P~i4Y?(i@pW&^GE1|!M9AAS31&tOo-Hj9-=F3c7YAlRxzlm*kuwK4Hcv;a`YWG* z^5c(Yn{6jwH&fsKKU!(ZPRO3uIpo?pG~&z`T~FsyPgDRX1HRcIUogN1q$|L>Qc5#Zh0_rU{3=A}k!yliSqYY9^i@E*Zo67~P}x7nfQ% z0ik;ea|oBZ8EplbcEf=RG*>Y`PlarI0G)-MRkBN)&e3Cr9?IO;s&t1LlDycM!pHq> z-=88oXc!$2gP6dD$)@a?y~3tkyEz6urDV(G<52C)IUph{uB#dg9B|S<*W@eJTaA`Y zfWjx4>=qwr(x>CF>mc_6QFexl@+|Y&!oF9M^~)rZCA0|jgrv^Uj^62ZJk6SO#=Cpl z8BtQBL}!kOoBJ6ode2XLcaEP(!PA@UZ#Hpk%@5IMm{QbYJvsqnLS4jW^)AiiJd zaN|f)B&PAgqwDZTws>Ha43t$N{nZP>Z{m1(X^HgsOpa+QeuKf;b4;-N2fRNnGFgvK zx}#~gI?r?^%$V)E(7HEt5N7UF^wF0m$ffay;B(%{DiL)p{&^>XXA(_CjvjjdMA;8B zxXwj^s|sx{A|$tVu6XH;9ot1_zA;OV;j!Bsgyr9C~gvAf&6<5um zNe*=9q^C114ev z7c-@sh(X;jB`S{Ah#yewT~%B(MEh8tbn_D*$0$c*

@*iEWsdr>GVDt}UbgL%&3=4@oRzS$%>W&B|W$JEY~$ z3r}A7Rrrf6MxG_BI4qzfb}FTyV%=}-0_ z|HEh$>h(|H>ZG;^#c0;>10Ij7 zp?MDKL?A3LX#hGDNM&DZKD_qSb`@h;Cs|1YWz*&XU+FvhGJ9Pcz5>f4etyN3nUv}e z(vuq|-KaDLaK*zs_wha6BULq5BK@RJGF(QiTzi?%J<$V7qURkHlBNgu{c~iN0vhwO z{zdNYv%bL`olC8Ex|0$0AmkQZY+Io(MLsiyS=tnU8nkAAcSd<|?|O(8;0*+3RQnyg zxredtaiF`N3Waj(yZrEFbM$k{5eO)FzhgQCTxx{p4f;T!+qna7hk&=6kt>X8n0Wf+ zK@0+JTra3Ol=61|_TgIpw#xDmV=i#OwuAnuI=9e_j9)A=O)(<&Y4)R5w!g-$Yn4T% z=G=sk*2#>R6u$9SDRpHKD;8&pyPM#y$Z|RZI(D-!4NaI$?LHO~0V*q(Dw4{hWm?G` z;$JIR6VZw8dx?<%QAF!8pIkv$h)8ggRfA0UB-?7kkPD0lvZ`hg1RPcL%=gQ%ZLEEg z_1o(V*ZgFC>zu2wD^Hm5r>mNHNGmXd2OU}+xUmZp>(Vki7n23!US~3jy>zpD^>77S zFS@nyIu^iI;vMiJnfPiqY_hO_tSP{j*g;SB11D!H5L3*m;&bf>sdH}dED0aIUhJ=L zpJ32C2P`LtHpi3zgglO9$*|(lU5NMNpWM~ifwJ0UYwL_Yh|@AAj4d`l#fStUKfN)nT^PT ze%`K7+L0J3AW5G}XQP_o<R~J~t-NBFU8$ z-LtvAOCZ1zV-gqLz;iQwiZQTEVSEoE@Hi87i*!*h024nnn5&QYj*2z{SF7A87tjDs z8j0Bj9N_qX8_;R<%KOgEd|EA>u(K<>nsqwF9dod8lu)9W{=+CfrO4dhI*sdL;gxUCiJP z{96bU7si9gWMyNobYOR=G~*2(BAGxiZf7L9RV58ge{Nw|Hvh@@e`HI$FjSs%Ql}k8 zq(iu?1wF~WePC<_jB0Aj@%eM-PTDYPqM}fqtSZ&s^O0=)CLx{jS)!?&F}VqnO$E+7 z&~)yY9V|VFkHcjGM4PVVE5=aylcpGH*4G$!WY`8c*J_>0r1SH$(&XYk~G| z6^rli9+~?7sFIC7NGzW-1zEDmye_Hwzrl(WCzIDL*ozM`;r-NJ#FJQw)?d4 zr>YL@8rnLiikl^Ke7F|F1=59gyh0z`A5oSwmI_k9Lz4Uc@u&9VY1ZiVvgcxZ&0YoX$nHwQbZsvpJ={UC}56|-{=1>qW6Kabf zM?)%<-E^_(C9raT<-lGoa)`iyxsL^xB$uzmWb+=J^O@XQ65<8dl?s3P=LH5XcB#0e zopAcN_V{2@(oXe=!ruj8B}wjUu~~}SzK5_ZZAK`315rd1o7^dgn!EmoJxn9SUhEBa zWsk|{1t&Rs7raR2q$xo{yI^nV(7481JTEjmJ)Xy%8%!=KT2Lj)j;6-jb1#;j=7q89)Oy*SgxZ&!OY&gRo?-{vUr* z~`WEDdmK5NPtP7cO zq^v(EDi`FItN;Ui*TdJ%Q7@8;(5O9+*ge^!?UKcE!?hMBjyQCHxhMak^BL#~fySKg zHHGZdyQR$NCFUKWmNM^l7W7oaAt0zpU?1OfLy6zF*TcZvK7Rk2qm;WC)}P}3PTdU~ zC=sQk7DVjnpa0Znu-99RI-OPz*_|_Y_MkE{5e)f~8d#4ECkmJu4@iKd+lVOzH$XNg z>te1%b=`ei_`SmI-bcL#-zj?cD1HIFPdMK93BQa|Hskt+glp*yRsj< zwPt-M`}@FW!s1Gr*HhUO^TfaWYyXk2;I$a{f5<*x)p)YdauNb^VMqqpd0;xv(}kt?8vEI zoflmQh<%w9QAaDPewP^#Q72nr>kitS!MTr_R&H-K%cseekn2?$fslE57s)&0wr8dC zE<6zfPVm!_xO2z^Y6rnNBTNi~m8Rp~33XV$P$(q-RFLs=?x3)fT#;$X732z#2tv^I zNxWQVsUaw4(^<~7*v<43O3FD`2WoG7Geap@_-f9R>Z1h3=$L0JTSRFpt~f!&)<6l? z`~9DZi#ly%)iH$A4(03j2wCstOvw4W>1Wv07djlSxHKYV7dq4`mYWm6yUIpRKlTp{ zlH(-0D+eufrbXsx*hS`Fic=d7#3o}x7!)+Qxw#=frC+SCUL8~auq3=k^W^HS1Q`4Y zOwxlriwF>C*r8~emCALUQH3jp#ZlHAQGM0m9Ri-fD;TL0*d&NF zh5Syx`=XatP@QcnX?U8SnTW6#4H_RBTltmMVVbTYwug`P;hLUqZ%OLh zZrb5K%Yvp%BYHIb^GLD_X`;@aAZ49eKezO5X^%wG((~wpB2tZB^BHxo@Ct^0#(aab zlzV&*ZF&bGhr(W{b@#FQRD zEg^GbH+%GeI7MY>vj%(7?*UpusP#!O{vBdW{JRuQ*L|7lN}{>W!^VUB5YW9G==(FG z1IpgIz8idrgwC*Iy$Po8re5d94@=$COrux@L&#GlrX3NJ{G*k8Ui2c6iY$X;7c zqzO7x2S3bF-*7KrrPVy$H|UP$LM@l;r@fegbYY>~3g&uoCjii5KHszF@$-^D0Ln$= z5hXX+Yk5>F!r?o|R{sSkcBvZJhSp{vb_mnxw+Qon^AE0~xFF3x^_Q`EwUTMww z@F#pEUzObf6GU@8n_$D4`1G-yjrjfVZ`XKPlxEgr9mj@ci;Nenn$4`A(+p8uIxn_g zCkOWsRW++c5X?i=`Lx0Z6xp& z;QCnzNAso(^*TB~!|bBAGARbwRzY>Xo)Acg#NcC}az?WCzd5vNW})U}72i;e7+Uae z;Bm|ld~`$#E9u?t=cV3K^kbnM*nq7hE$hZ%p4L-<`54)pe)OI+* zWhC+4$rSh`cC9>AszLNnH~gjR6!3;(!BqM~xob&eZ#O&zRf5MfHA$B{IPZ=aOs_wX z8NepYkx5`X0{$cM2Kne&MST` zuhf^5;Trh?2b{<}I^z={KadG`fVBGf?OIw{ZaiCPZ36B-y?0O+xx51=B-Vu%#TR1; zk`E>VtPQ{%LmrA)V9209;u|^b{Ok8pX9yNT7xRaF^@ZrFEJd4Fcrw|06x0ySN^7ub}G5BE1F5jm9I+hqU~AkGXQs*=@RJR zYRQu+rG%0O4^?iQjlk-JnazWhMs4474|6z)L@XO8nf$rv$5GERUs$=vO`l*=gK-tz zUKKhhYz}=`m}cKS9royAqa~Z35X?At>7GR;>FTHh4|50ZG5xM=FL=d(G5L z=VJUo6*aC>nvVg@*cRD`SW;C^;sQ71++`jUg+g!T4>V&PgW24`1wm@P(B-GSlAS!G z*Jpk1oj3;-sMHVBU=9m?BOSMrR{>gft-lW9S#cXUylehjzRIP6Ytf zlaHq@U#{EW64xap-ge#m85p>G#hBxjk~04X;qis&E!4Gqa7<3OID;rM_bLH*61T#B z>u}}WrA9`~Mq65u&U$_cyO2;XpF5dlZ0$OXWe z&{0YeZRpy(6h`qSsAfn5eS0;|MAlKydj$4Wsy5Or8tk?}B99I!T|_+g7d(^TdYe7Z zC(Kg`*Lg_w1IG`p6lE42yyn+}FnXx2kRwOR^V=*Y1mfn6P{Br?*8o(&T&J92N9WUv z&*6bX=eIq`LDYYeQ_>29%X=yCedc`G7h>-7GMrN3GX7Nz~y$BPnVChNn@vNN;C0>wv$Syp)iL zBW}j*M!WaJs?fMM<-i-O@=*}HA;`gLdjUuHR0gv}>HmYt8!Fr_-GIHscg3VlN1Z&H z#FrN0mfclqr4Aly1uSR{@a=-CE`MAgH|)+1U;2_pB~9_(&6Qz%N+ zK=vIVr@)C@W=p(ZdiFAmU^vh$I#M%l!pL0ol#83lWA*&HiOc%V@qT`W?s`p2DFd0! zgbNKx^6c}ZuW-f_3i#4362G;zs(sf-gn7ssgpck9fK#KGYb51ZA+@FuGV6c zr3%=-o3~hC6zSZ$cQG6VZh;zdI_bk7e?e!OU)nw)iWGp99RN2r@2VNWK6(tKRGjvy z)vy`|q}LVTkp>tA#S$OC2_9(27AvhbpNI2KYyp1bzh>{Vl)HKa&rXUt_;7(ayb3+5 z=Pz*K@a{=)?t>lOtJ8Gn4euyQcSlCmDN#)HLjs@&_s|7Q-cXDFxIX*9It)Rcp3d%} z=wt=7SlQv&Uw(AOD?-q(XcmqK+VJ?dURJ}0-*zA*8n*f4!%UDywdumqIc@0)D@=7F zL97*Y%!*gA1~9CpV|Nc0ALCanJ#edEbvUth51gTZODX?4iO9lZ2fA23&ig7}+pSRC zO0X|NpR!~_l<*{Hlsj$TROV$dtNAVbngEV52J?a;k2|eePbnC=U?*o*LS1q}JqoEY zwRg7vOx4&B`9|9p{85MEpP>t z_O$Q^N=7b~MH&>3WZ8HhJU>isiQ|DX!`32|XM^`KWZCrY7CMk&g#r#2I8G+WHclw_!_`x)oWMzEWowt0Pf%poJ7sfa1N9fqh%W-cN^F_O`t|Gu zu1r;L!tkSO`@mNdhi7xAlk~ap5XIIBJXFmoyCXFcHUB4IH0K2=N*5z}XZS{vnbT+! zn5c~xIZc2X%!EeL>ivYdQeoA%TSvCik;!wW zVzA!~f;-AadgwtX+=>#^;<5tDSDh8dspMT9z8>%5(R@yL?huSpmrBu_PJbIk>Hr&u z6C^Ter9S4db~~jH>}fk(iIhY7wQQ^bPJniOsn#Tq2X9kOc``0Oo`IV&WT6aSA~q#3 zjYESKpYFPmi!p*3_f|kPJ9~#ozsskdiAwvjb#OE*6JPin*Tw$^?ZQ)f5sw8onw@%~ z>SoJ+{eG;Nqy7QRiH--#+}`Dj7DWlXQM$hv?}!4&rHS*`NnqKg(5&Ake2vyt{OdFr zL{FC1F}^0G+u>4Cfsi50;8gEw@5k2)lY)4t4wq zNw?a>Q*sSaz>&#|9`GWMf(aC@{Wm`8a7 z?6+yYplic87AFH^BC%EoVYFql(vcVr zP{y;a{JOD&11oD$Af<1!KdEBA1ZlZzo;(Tc0xxM1*^^6wEuQ|-YLCxPDC%5p{JAFGB@I!{*x3L2_nHIOR66y8(gE6zvRC%VYZ`N03 zeM-SE;5Bjc zH!F>H2KI#XFy@@*{!Xt$MDPRQ%VSP|UKv7*#K{Sm++a?6(R>?NPk5J}<%KYdV$p2r zKMjiEbw2A(!x7D3`?D8|vc%pO-rG<79cW)eGN0Z+eE@wtaWVePXsjHr@ymcyXr60i zDJNtpc2nqk88SjAB#!Qb$U!Iqcon14^TGmKBJpLdIV}*(!TfM330cn}1_e61Eic;G z>xO|CU2hUaqF%&e_X4#Mx_^vmz$|;74~&ve^{nMGqWtNV+0}xjqmtZg##6S8c9vcuoKV$rlakrD!+G@{MPUP6kPSwPXg8zb<#6y=N# zCL?&<(E*s)|5S0VPzF0vrjsL=6rFQaD}1p8{WhYOP_p?T+WpOsJF8}OC@lw%NviEb z0$n|cio6}&Y$NjivfJPcR_bnmblq|`2b0$91%l_}P<23oIBc9n(au}zX>a=e4o zt*fq8uSM7MeirsWajB{j;j1ecRKoF;o0s?UPt0PYiPWzSwti+@0UbuG_KK$1v4DZ@ zcF@>h;$SSd6zq`%MAiR2WcL0`u^N_k$i%4of{_3vpu5I^M6iVi2N>NECDKL2W*ovV zJh(RnkISuT#+m1wk_xlc@rB8ce}fnX>iP6Vc{`xfJW#ia`?lI0Y{G?7@9AF*n&{9I zp^;y7@a3gu1{7mlKBADS4vFpn=H3H0jD>^XSLm##>h_70SzmHVeNtzWDfQ$jaqSch z$X%!4S>k9Jkex2#MQq|#HcOT*Q);7q9QYxSzvUwVb9>}=Z0{X%uTR>=dmRX~=p~LY zO30!o!1}qY`*4-dK;?Mj;R zJRjBV5ul9xG~Mg06U~=8KGH7fnrisr*zz_yaci{>oUNMjmQL(?g3R|^e~Bj=XOyg` zEmJ#$;MBh3tE|}2k#6iMnoy!gC_-Sjxz6XtUD*vK$KcJux|_0s(RJei}ts+DJwQvUA`0(uP{`@+S0yIL;CHm!A# z0>X2o2VFi|rCEV^vMK=Bh|KwR!ggU8U##uKhu4l0R-#9eu39t*={O@3sVDRuhdK__TQjN0rf>KSPBLx%;5s_^MGH zWLg|@Qlq=Xp)>uh-nLr+T{C0qA#m$c<&0*r?qW41<@8>K6QMHp7oTcmR7~*}@M|5G z7g-+FLOkhc4mq*e&%d7H+2hat!!Or8-^>?2Jj=exq%hloOMo_uk^ERM@YFm=699o) z1*C|AkCNuDJhRBwN&b}Ri0WM|U+P-%WMs{A%7jbOs>{%(!j8+C2NGJS2Ff$U?zJPi zy{H}NtN(D4jiRu6VpnDOZ(0bw+67nX>*nM9#ykCh_12B$;IkLuakI%%^qH*SNd>dw zMjBU@Sq~3D9E9LwM7r3A(ZqA1_~Y!h!Upr;*xvyY@Usn!(ncaEpVHWPNzI5tOr|7G zSuy_;WL$b03mDngJBcpdTi~wktXBFlVn^)IzpU#ZEpBV;`;^y8xz68AdTXOA_wwcj zvb?epr_du9v`r-}vQL)P^(Y~3jv0qq&4Yo+n^aU=GA3+YA1pJvvF3zxL$Ta&`4c^YhOhZqUv_q`nIKw866cOe-!ndLA|+3AO#*em>et~fhH z9w#1(`24QY);Cd=;6pFOm1!0gbw+%zMlvlw8I~Nn$z)rnd?SfM=_wFU2A81ksmy;E zf$$%y3GP>YO6RJ<-`3zxsphv?@4{7fIt2OsrAt&G3lvW0UQ0=uAq`^MCx`SxZT{(PhBZW@QW+EhuOL*zPocN=_Ar zE|Ne67pdo9g|{L-x##Zd2!x@wN%g4`dlmK;Qa)b~qZ)ddLT^*7?GOU6IH?=rrx)z1 zgIv#KWPa7%pYMjPVEXz)NLETup3Rra1;qKFUFcI(M~e-DDVL64E+e8L71l~gmBnKg!#_knU1}=Wt9g7EtzSs8+H*(8} z)ntw6`W}9f>v&RAa>}HS3oZJ<;?I~N(hX{lKukm4X8PSgN2oUuyArFd#eb7pr?k^& zxWy%ZD*ih{)Kw90upj>o?bH^a@^SbXEe1A09(V6Iu~W0bEBBlRonY_gLjn$Ioh^JU zTR*EzrI{XB0W(6b|Cl?0v!SjJmCRG)v|jc~iw4a4(wo z>@l{@$seeam!YCj^QY0mr11NNO0NMLjMP=kNv3hvXc?KZ95J~W#Pf^&bSyfM);6_J z2cN3#9J&JKf(tpm^P?n>?6nEV1^|eiNKcpf6+7i%xhXgmz)@tTbrSXR{T*xFZRQ-a zxj`96YX!JLN_DS}YkWc1M}oBB5;Lm*Qb2eYf_hoOhpHajC&=6{;5z(mP#PhHKL?>d zh(sK#?17&B?uiu|*9M{TSQ&*|FTo#4JfJ;Bk;2hZ$oKC~EUT$*!7fWPpg-=?&*mYy z#Ys=&A@G=&v-!9oECv${wR&8G(v>bU{T@CAL)kXOv;sm)MKpTO^*ht^?rbsX&$^Zp z4DnB^l+lon3K-;m;3)kb{~RQjMU~sdZqAWi|74urZO|Q(uxr|ANPo2qR@Oin7z5!lBkN&~ zNgv;^7@!6nLj^}lj_@bBQ?p7s8OD`RhSq84be|NKUzWwpp*Ty(|dNrxp@gq0j zF{!KNqiq}BGL@U)(^f`@)9&~3NoeZ3`JrR%IDxg!Xxlb=Y|9zci0!5^(PGvXSvR+u zuu2O3n+Mf=C5VA5^wXSX3|yT9_WfkBXLU+{-iyI0LFwn{@^dqiG+S!r{E8I&98+4R zTs=y9;^L54g~_NL4eF<_K9y5KOl~T)f9Mpja0VmKm_24wh`)Qnq2uQ@UZ&QvAzXBu zFn?QBTC2g~CRKoaZu7s92Q9YRV>PIr3jFn>>n2XvQM`a;w6qu`SRe3wuHTmykRKM7!%Sq2jFS-InR)KGL7Y^IViLdH(61;R zS5CYOAcHWi{bU)i_4oa^OU4@R)^xEb|07L3s@g|V*v(kt&f!KdkT#P3Gnpixa z6FM{;nEop;Thx3u>od)1%@INk`cfLz51O+lC80=_)NqP=vCeEShFSBvgXn7xAX0sQ(GyldF?^JHSRcn_q3kea6W8m2n?w zhdPCPC(nOTj9(%EQdI%oYmeQ$omdG53tFNx{~w{4=5yxfmzlNZf26B`obi9eu;W)+ zN?Wiu9{j{B&m~om5)=Jc1xf<5zfK)gY_UEVOPxX)qxU;Up|c#WNz2$y3}{X4Zmw|6 zp^6|a*$u0xF3U2ynh#-#35#$#dv;6bLdGqI$q|LF$pbBz6MWfVhQ4F~oyx5nQ)29d zb<kd9D}p4SU5vu5S8&(u+&_t-^hX4q@Ew)N`w;vaxzMr37> zd-tt6BtH8gv34;@t{mTy{;f86B%uYkVm?r-gDOCN3q{{ZSs_h=tf|cJb@EYw}y%SZ2Q^X-C4^@(Z4ci<+Th(w~#r z_FMLpLOxs{zHz!BJ?tJbODa&o_kpuoxizFuiE$b^Qud`T?z=Rqc=M< zeY>~t!zf95ENg=`%b5q|Z7$K858(FW{qOgbQa6u~49#P81LH!^cjK z^*3)}ki+kl9jAy?UJ52gwP$;(RT&dT278aj;j?niY;SLzij362xJbakl_VQ!pJ+pz zx33`x_Ztn!s^ewf%0i%*XwsT7_OO6Av2v^9ziM+OVRH2|~V98*3xKOBAhx`bi8HwKhy!Ra)^hQX*p=b@HZAmclcJw)9Eu6%Z~8I8o=y562n%2f}of#>!#*flB4hq zwvjr%du~F~L;|v)4FrsMK$>9Cc~kKI?Rp0|U*N~4oDJ_~8IH|f0RogFvvllc7v|9S z-A)txSoq{p&!{nxq%$!xQ4)C;?-5jCJ|bdchlS;<5Yi?G9`pF0oeX0YwA_@bZQ8z_iL!zy1(FrB^W)xrVqOKpbl4 zF4oIU;W&2)zOcocU-hjI22L_-?Uj&c&~|Bgh;~SRJ>pR8rS1e_+!$QD8KJ_O{*59OQC2 zj?-DDh4+wS+N?W_#|Q1P(0Sop^*du#B7rq=1qFkCh&6GU?xIfy_45rnp9o=O)o+4O z>ouPm7Bx^h+|eBFtIodu1s~w}jAGKumFZ8=kK#|EMaHiYBnLSM_Xv0sGZ1tHmaPK& zYV|H+RPvIP*AL0Z&UdXRrXu7Wk~{j}*KDYuLTm~n=vP}Lu5j<{>0*df7Xj}iG~WDGo48;Nal zZKPX^I11PeSh6g%lKiHX^jT3mjD6K#kfGHbMfS3eGm^7CHy~C<_4&&>3%|RCWeEEm z#O^HU@d-Jtc%FSw@;Y+E`hINXEQA}{MhI|iT)63lVvw%JqKd0_FZ9NJ*t%+%T$U4o zvW%1tYUs1*Xy{`=ZjA{@>Ms)8|u$pZ`Ak$8KVsHQKr#7WsP*T4!_Hj7;mX2$N0Q;2{k1bF8F6)+qr~ zM%SrQhQ*>2OTWNVxl%Ktl@BVz)0bFqw7rkQ&VqGMmnCFGNWcL)X-B7+$h|=RZASHb ziKahZ6HWdZg7wylGb!>pbUfMKRD&qo-Mqwe`WjE4rbQ3q3GUzsprhAbrALQzZS50p zyll0>G?;${&Eh1|Kv4YWGn=8~+H2x5i)??MH+SV{8p<`B=S!C>N zvO8Jh67{)7)ea;+>k-G4%U4^EN`7PmUGp-7#)DRY9eB#Dkzrwbe*)($02Gr&{~a{9 z_3ze0L?ybyjfk0r>AX!k6uOZpc6_IZm7n9%I&E->2)m-_wRCa)SDsolepAd2Mu1)S ztbW(nJ*~Yv z$rQlj_Hcp*Ab)_3xJ)qGOZ&WeF~f)SC8c224`F<(i#(DK-Zd|=>lYuJR+m|bH%yL& zS5j~De61(V_;?vj+ClJJj)F3$m2iqy1=o%%1O6Gf&&*)FtA!bcEA5n*1juq{F@~O1 z#AWFv@T*}0IF7SFm+I1xa&m_%b!~ozwvj6Es_e#ht=7v!1^5MD=Mw_d?$0DJ@E9_` zU|*MM#`=XmVPw5YF2CRS$v^*eb`D@O#ez*3a=Jiw?q4Dn*2>K_kvX$G69iHAd{I=E zhk#35LfZZ!4~m34VrN3+jk4+>;Hj`1nFgc@TYcA!F)(ybZN zkRi6+w8FX!9s8=y%>!p?;04Yuu<$>W|2cMDREKK0r}e17G>IdFRGsVwo;rAJE9Gxt zmohx_cA(4Y_Gl6`!r++ZpBbF&tl^vsaWhwTQ&D!DEvw!6JMcwHRyG`_XIL7If;_oi bV<`A7G(w&j?v4)$LIh3>fA9a Date: Wed, 23 Aug 2023 11:50:00 +0200 Subject: [PATCH 09/64] Added iris-grib to environment and setup.py --- environment.yml | 1 + setup.py | 1 + 2 files changed, 2 insertions(+) diff --git a/environment.yml b/environment.yml index cb066cba09..7cdbb920ef 100644 --- a/environment.yml +++ b/environment.yml @@ -21,6 +21,7 @@ dependencies: - humanfriendly - importlib_metadata # required for Python < 3.10 - iris >=3.6.0 + - iris-grib - iris-esmf-regrid >=0.7.0 - isodate - jinja2 diff --git a/setup.py b/setup.py index 092379b6ba..6d24a28626 100755 --- a/setup.py +++ b/setup.py @@ -40,6 +40,7 @@ 'geopy', 'humanfriendly', "importlib_metadata;python_version<'3.10'", + 'iris-grib', 'isodate', 'jinja2', 'nc-time-axis', # needed by iris.plot From 744c20b96b94b10e6f67477b1756189f6b90857b Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 23 Aug 2023 12:10:32 +0200 Subject: [PATCH 10/64] Fixed environment --- environment.yml | 3 ++- setup.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 7cdbb920ef..595e2e1a6f 100644 --- a/environment.yml +++ b/environment.yml @@ -21,8 +21,8 @@ dependencies: - humanfriendly - importlib_metadata # required for Python < 3.10 - iris >=3.6.0 - - iris-grib - iris-esmf-regrid >=0.7.0 + - iris-grib - isodate - jinja2 - libnetcdf !=4.9.1 # to avoid hdf5 warnings @@ -39,6 +39,7 @@ dependencies: - py-cordex - pybtex - python >=3.9 + - python-eccodes<1.6.0 - python-stratify >=0.3 - pyyaml - requests diff --git a/setup.py b/setup.py index 6d24a28626..202b4e53e3 100755 --- a/setup.py +++ b/setup.py @@ -54,6 +54,7 @@ 'psutil', 'py-cordex', 'pybtex', + 'python-eccodes<1.6.0', 'pyyaml', 'requests', 'scipy>=1.6', From 0c8ce648cb3965fe1c247dda9e20d15954cdd5bc Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 23 Aug 2023 12:42:34 +0200 Subject: [PATCH 11/64] Fixed eccodes dependency --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 202b4e53e3..1bfffbee02 100755 --- a/setup.py +++ b/setup.py @@ -31,6 +31,7 @@ 'cf-units', 'dask[array,distributed]', 'dask-jobqueue', + 'eccodes<1.6.0', 'esgf-pyclient>=0.3.1', 'esmf-regrid', 'esmpy!=8.1.0', @@ -54,7 +55,6 @@ 'psutil', 'py-cordex', 'pybtex', - 'python-eccodes<1.6.0', 'pyyaml', 'requests', 'scipy>=1.6', From 39c66779e1b9e050ae8d7badfa8c7b6d4c8328ef Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 23 Aug 2023 12:59:35 +0200 Subject: [PATCH 12/64] Next try to get environment working --- environment.yml | 1 - setup.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/environment.yml b/environment.yml index 595e2e1a6f..cf90f5a3af 100644 --- a/environment.yml +++ b/environment.yml @@ -39,7 +39,6 @@ dependencies: - py-cordex - pybtex - python >=3.9 - - python-eccodes<1.6.0 - python-stratify >=0.3 - pyyaml - requests diff --git a/setup.py b/setup.py index 1bfffbee02..092379b6ba 100755 --- a/setup.py +++ b/setup.py @@ -31,7 +31,6 @@ 'cf-units', 'dask[array,distributed]', 'dask-jobqueue', - 'eccodes<1.6.0', 'esgf-pyclient>=0.3.1', 'esmf-regrid', 'esmpy!=8.1.0', @@ -41,7 +40,6 @@ 'geopy', 'humanfriendly', "importlib_metadata;python_version<'3.10'", - 'iris-grib', 'isodate', 'jinja2', 'nc-time-axis', # needed by iris.plot From b7a0c6892eaadc7b7dee0c802958ceb92b0837a0 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 23 Aug 2023 13:34:04 +0200 Subject: [PATCH 13/64] Temporarily remove GRIB loading test --- .../integration/preprocessor/_io/test_load.py | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/tests/integration/preprocessor/_io/test_load.py b/tests/integration/preprocessor/_io/test_load.py index ecb602d7cb..f2e3763bfd 100644 --- a/tests/integration/preprocessor/_io/test_load.py +++ b/tests/integration/preprocessor/_io/test_load.py @@ -10,6 +10,7 @@ import numpy as np from iris.coords import DimCoord from iris.cube import Cube, CubeList +import pytest import esmvalcore from esmvalcore.preprocessor._io import concatenate_callback, load @@ -53,23 +54,23 @@ def test_load(self): self.assertTrue((cube.coord('latitude').points == np.array([1, 2])).all()) - def test_load_grib(self): - """Test loading a grib file.""" - grib_path = Path( - Path(esmvalcore.__file__).parents[1], - 'tests', - 'sample_data', - 'iris-sample-data', - 'polar_stereo.grib2', - ) - cubes = load(grib_path) - - assert len(cubes) == 1 - cube = cubes[0] - assert cube.standard_name == 'air_temperature' - assert cube.units == 'K' - assert cube.shape == (200, 247) - assert 'source_file' in cube.attributes + # def test_load_grib(self): + # """Test loading a grib file.""" + # grib_path = Path( + # Path(esmvalcore.__file__).parents[1], + # 'tests', + # 'sample_data', + # 'iris-sample-data', + # 'polar_stereo.grib2', + # ) + # cubes = load(grib_path) + + # assert len(cubes) == 1 + # cube = cubes[0] + # assert cube.standard_name == 'air_temperature' + # assert cube.units == 'K' + # assert cube.shape == (200, 247) + # assert 'source_file' in cube.attributes def test_callback_remove_attributes(self): """Test callback remove unwanted attributes.""" From e907ff180f3cdaf93b4cfee959ee3cca26bd88bd Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 23 Aug 2023 13:44:41 +0200 Subject: [PATCH 14/64] Fixed tests --- tests/integration/preprocessor/_io/test_load.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/integration/preprocessor/_io/test_load.py b/tests/integration/preprocessor/_io/test_load.py index f2e3763bfd..863f53ce1f 100644 --- a/tests/integration/preprocessor/_io/test_load.py +++ b/tests/integration/preprocessor/_io/test_load.py @@ -4,15 +4,14 @@ import tempfile import unittest import warnings -from pathlib import Path +# from pathlib import Path import iris import numpy as np from iris.coords import DimCoord from iris.cube import Cube, CubeList -import pytest -import esmvalcore +# import esmvalcore from esmvalcore.preprocessor._io import concatenate_callback, load From 0b8fbfa20173387b1158f55651a9f89f7f0aa44a Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 23 Aug 2023 17:31:36 +0200 Subject: [PATCH 15/64] Added missing tests --- esmvalcore/cmor/_fixes/native6/era5.py | 18 +- esmvalcore/preprocessor/_regrid.py | 2 + .../cmor/_fixes/native6/test_era5.py | 589 ++++++++++++------ .../integration/preprocessor/_io/test_load.py | 3 +- .../test_bilinear_unstructured_regrid.py | 141 +++++ 5 files changed, 560 insertions(+), 193 deletions(-) create mode 100644 tests/integration/preprocessor/_regrid/test_bilinear_unstructured_regrid.py diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py index d24b4169fd..4e27785913 100644 --- a/esmvalcore/cmor/_fixes/native6/era5.py +++ b/esmvalcore/cmor/_fixes/native6/era5.py @@ -171,26 +171,26 @@ def fix_metadata(self, cubes): return cubes -class Mrro(Fix): - """Fixes for mrro.""" +class Hus(Fix): + """Fixes for hus.""" def fix_metadata(self, cubes): """Fix metadata.""" for cube in cubes: - fix_hourly_time_coordinate(cube) - fix_accumulated_units(cube) - multiply_with_density(cube) - + cube.units = 'kg kg-1' return cubes -class Hus(Fix): - """Fixes for hus.""" +class Mrro(Fix): + """Fixes for mrro.""" def fix_metadata(self, cubes): """Fix metadata.""" for cube in cubes: - cube.units = 'kg kg-1' + fix_hourly_time_coordinate(cube) + fix_accumulated_units(cube) + multiply_with_density(cube) + return cubes diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py index d4497c59fd..54101e745d 100644 --- a/esmvalcore/preprocessor/_regrid.py +++ b/esmvalcore/preprocessor/_regrid.py @@ -1274,6 +1274,8 @@ def _get_linear_interpolation_weights( src_lon = src_cube.coord('longitude') cache_key = ( f"{src_lat.shape}_" + f"{src_lat.points[0]}-{src_lat.points[-1]}-{src_lat.units}_" + f"{src_lon.points[0]}-{src_lon.points[-1]}-{src_lon.units}_" f"{target_grid}_" f"{lat_offset}_" f"{lon_offset}_" diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py index cfe20e7fde..b05b68a1d4 100644 --- a/tests/integration/cmor/_fixes/native6/test_era5.py +++ b/tests/integration/cmor/_fixes/native6/test_era5.py @@ -1,10 +1,12 @@ """Tests for the fixes of ERA5.""" import datetime -import iris +import dask.array as da import numpy as np import pytest from cf_units import Unit +from iris.coords import AuxCoord, DimCoord +from iris.cube import Cube, CubeList from esmvalcore.cmor._fixes.native6.era5 import ( AllVars, @@ -33,12 +35,12 @@ def test_get_zg_fix(): def test_get_frequency_hourly(): """Test cubes with hourly frequency.""" - time = iris.coords.DimCoord( + time = DimCoord( [0, 1, 2], standard_name='time', units=Unit('hours since 1900-01-01'), ) - cube = iris.cube.Cube( + cube = Cube( [1, 6, 3], var_name='random_var', dim_coords_and_dims=[(time, 0)], @@ -48,14 +50,31 @@ def test_get_frequency_hourly(): assert get_frequency(cube) == 'hourly' +def test_get_frequency_daily(): + """Test cubes with daily frequency.""" + time = DimCoord( + [0, 1, 2], + standard_name='time', + units=Unit('days since 1900-01-01'), + ) + cube = Cube( + [1, 6, 3], + var_name='random_var', + dim_coords_and_dims=[(time, 0)], + ) + assert get_frequency(cube) == 'daily' + cube.coord('time').convert_units('hours since 1850-1-1 00:00:00.0') + assert get_frequency(cube) == 'daily' + + def test_get_frequency_monthly(): """Test cubes with monthly frequency.""" - time = iris.coords.DimCoord( + time = DimCoord( [0, 31, 59], standard_name='time', units=Unit('hours since 1900-01-01'), ) - cube = iris.cube.Cube( + cube = Cube( [1, 6, 3], var_name='random_var', dim_coords_and_dims=[(time, 0)], @@ -67,14 +86,14 @@ def test_get_frequency_monthly(): def test_get_frequency_fx(): """Test cubes with time invariant frequency.""" - cube = iris.cube.Cube(1., long_name='Cube without time coordinate') + cube = Cube(1., long_name='Cube without time coordinate') assert get_frequency(cube) == 'fx' - time = iris.coords.DimCoord( + time = DimCoord( 0, standard_name='time', units=Unit('hours since 1900-01-01'), ) - cube = iris.cube.Cube( + cube = Cube( [1], var_name='cube_with_length_1_time_coord', long_name='Geopotential', @@ -87,7 +106,7 @@ def test_get_frequency_fx(): def _era5_latitude(): - return iris.coords.DimCoord( + return DimCoord( np.array([90., 0., -90.]), standard_name='latitude', long_name='latitude', @@ -97,7 +116,7 @@ def _era5_latitude(): def _era5_longitude(): - return iris.coords.DimCoord( + return DimCoord( np.array([0, 180, 359.75]), standard_name='longitude', long_name='longitude', @@ -110,11 +129,15 @@ def _era5_longitude(): def _era5_time(frequency): if frequency == 'invariant': timestamps = [788928] # hours since 1900 at 1 january 1990 + elif frequency == 'daily': + timestamps = [788940, 788964, 788988] elif frequency == 'hourly': timestamps = [788928, 788929, 788930] elif frequency == 'monthly': timestamps = [788928, 789672, 790344] - return iris.coords.DimCoord( + else: + raise NotImplementedError(f"Invalid frequency {frequency}") + return DimCoord( np.array(timestamps, dtype='int32'), standard_name='time', long_name='time', @@ -129,7 +152,7 @@ def _era5_plev(): 1, 1000, ]) - return iris.coords.DimCoord( + return DimCoord( values, long_name="pressure", units=Unit("millibars"), @@ -145,7 +168,7 @@ def _era5_data(frequency): def _cmor_latitude(): - return iris.coords.DimCoord( + return DimCoord( np.array([-90., 0., 90.]), standard_name='latitude', long_name='Latitude', @@ -156,7 +179,7 @@ def _cmor_latitude(): def _cmor_longitude(): - return iris.coords.DimCoord( + return DimCoord( np.array([0, 180, 359.75]), standard_name='longitude', long_name='Longitude', @@ -176,28 +199,38 @@ def _cmor_time(mip, bounds=None, shifted=False): timestamps -= 1 / 48 if bounds is not None: bounds = [[t - 1 / 48, t + 1 / 48] for t in timestamps] - elif mip == 'Amon': + elif mip == 'Eday': + timestamps = np.array([51134.5, 51135.5, 51136.5]) + if bounds is not None: + bounds = np.array( + [[51134.0, 51135.0], + [51135.0, 51136.0], + [51136.0, 51137.0]] + ) + elif 'mon' in mip: timestamps = np.array([51149.5, 51179., 51208.5]) if bounds is not None: bounds = np.array([[51134., 51165.], [51165., 51193.], [51193., 51224.]]) + else: + raise NotImplementedError() - return iris.coords.DimCoord(np.array(timestamps, dtype=float), - standard_name='time', - long_name='time', - var_name='time', - units=Unit('days since 1850-1-1 00:00:00', - calendar='gregorian'), - bounds=bounds) + return DimCoord(np.array(timestamps, dtype=float), + standard_name='time', + long_name='time', + var_name='time', + units=Unit('days since 1850-1-1 00:00:00', + calendar='gregorian'), + bounds=bounds) def _cmor_aux_height(value): - return iris.coords.AuxCoord(value, - long_name="height", - standard_name="height", - units=Unit('m'), - var_name="height", - attributes={'positive': 'up'}) + return AuxCoord(value, + long_name="height", + standard_name="height", + units=Unit('m'), + var_name="height", + attributes={'positive': 'up'}) def _cmor_plev(): @@ -205,12 +238,12 @@ def _cmor_plev(): 100000.0, 100.0, ]) - return iris.coords.DimCoord(values, - long_name="pressure", - standard_name="air_pressure", - units=Unit("Pa"), - var_name="plev", - attributes={'positive': 'down'}) + return DimCoord(values, + long_name="pressure", + standard_name="air_pressure", + units=Unit("Pa"), + var_name="plev", + attributes={'positive': 'down'}) def _cmor_data(mip): @@ -219,10 +252,80 @@ def _cmor_data(mip): return np.arange(27).reshape(3, 3, 3)[:, ::-1, :] +def era5_2d(frequency): + cube = Cube( + _era5_data(frequency), + long_name=None, + var_name=None, + units='unknown', + dim_coords_and_dims=[ + (_era5_time(frequency), 0), + (_era5_latitude(), 1), + (_era5_longitude(), 2), + ], + ) + return CubeList([cube]) + + +def era5_3d(frequency): + cube = Cube( + np.ones((3, 2, 3, 3)), + long_name=None, + var_name=None, + units='unknown', + dim_coords_and_dims=[ + (_era5_time(frequency), 0), + (_era5_plev(), 1), + (_era5_latitude(), 2), + (_era5_longitude(), 3), + ], + ) + return CubeList([cube]) + + +def cmor_2d(mip, short_name): + cmor_table = CMOR_TABLES['native6'] + vardef = cmor_table.get_variable(mip, short_name) + cube = Cube( + _cmor_data(mip).astype('float32'), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[ + (_cmor_time(mip, bounds=True), 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2), + ], + attributes={'comment': COMMENT}, + ) + return CubeList([cube]) + + +def cmor_3d(mip, short_name): + cmor_table = CMOR_TABLES['native6'] + vardef = cmor_table.get_variable(mip, short_name) + cube = Cube( + np.ones((3, 2, 3, 3)), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[ + (_cmor_time(mip, bounds=True), 0), + (_cmor_plev(), 1), + (_cmor_latitude(), 2), + (_cmor_longitude(), 3), + ], + attributes={'comment': COMMENT}, + ) + return CubeList([cube]) + + def cl_era5_monthly(): time = _era5_time('monthly') data = np.ones((3, 2, 3, 3)) - cube = iris.cube.Cube( + cube = Cube( data, long_name='Percentage Cloud Cover', var_name='cl', @@ -234,7 +337,7 @@ def cl_era5_monthly(): (_era5_longitude(), 3), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def cl_cmor_amon(): @@ -243,7 +346,7 @@ def cl_cmor_amon(): time = _cmor_time('Amon', bounds=True) data = np.ones((3, 2, 3, 3)) data = data * 100.0 - cube = iris.cube.Cube( + cube = Cube( data.astype('float32'), long_name=vardef.long_name, var_name=vardef.short_name, @@ -257,12 +360,12 @@ def cl_cmor_amon(): ], attributes={'comment': COMMENT}, ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def clt_era5_hourly(): time = _era5_time('hourly') - cube = iris.cube.Cube( + cube = Cube( _era5_data('hourly'), long_name='cloud cover fraction', var_name='cloud_cover', @@ -273,7 +376,7 @@ def clt_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def clt_cmor_e1hr(): @@ -281,7 +384,7 @@ def clt_cmor_e1hr(): vardef = cmor_table.get_variable('E1hr', 'clt') time = _cmor_time('E1hr', bounds=True) data = _cmor_data('E1hr') * 100 - cube = iris.cube.Cube( + cube = Cube( data.astype('float32'), long_name=vardef.long_name, var_name=vardef.short_name, @@ -294,12 +397,12 @@ def clt_cmor_e1hr(): ], attributes={'comment': COMMENT}, ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def evspsbl_era5_hourly(): time = _era5_time('hourly') - cube = iris.cube.Cube( + cube = Cube( _era5_data('hourly') * -1., long_name='total evapotranspiration', var_name='e', @@ -310,7 +413,7 @@ def evspsbl_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def evspsbl_cmor_e1hr(): @@ -318,7 +421,7 @@ def evspsbl_cmor_e1hr(): vardef = cmor_table.get_variable('E1hr', 'evspsbl') time = _cmor_time('E1hr', shifted=True, bounds=True) data = _cmor_data('E1hr') * 1000 / 3600. - cube = iris.cube.Cube( + cube = Cube( data.astype('float32'), long_name=vardef.long_name, var_name=vardef.short_name, @@ -331,12 +434,12 @@ def evspsbl_cmor_e1hr(): ], attributes={'comment': COMMENT}, ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def evspsblpot_era5_hourly(): time = _era5_time('hourly') - cube = iris.cube.Cube( + cube = Cube( _era5_data('hourly') * -1., long_name='potential evapotranspiration', var_name='epot', @@ -347,7 +450,7 @@ def evspsblpot_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def evspsblpot_cmor_e1hr(): @@ -355,7 +458,7 @@ def evspsblpot_cmor_e1hr(): vardef = cmor_table.get_variable('E1hr', 'evspsblpot') time = _cmor_time('E1hr', shifted=True, bounds=True) data = _cmor_data('E1hr') * 1000 / 3600. - cube = iris.cube.Cube( + cube = Cube( data.astype('float32'), long_name=vardef.long_name, var_name=vardef.short_name, @@ -368,12 +471,12 @@ def evspsblpot_cmor_e1hr(): ], attributes={'comment': COMMENT}, ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def mrro_era5_hourly(): time = _era5_time('hourly') - cube = iris.cube.Cube( + cube = Cube( _era5_data('hourly'), long_name='runoff', var_name='runoff', @@ -384,7 +487,7 @@ def mrro_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def mrro_cmor_e1hr(): @@ -392,7 +495,7 @@ def mrro_cmor_e1hr(): vardef = cmor_table.get_variable('E1hr', 'mrro') time = _cmor_time('E1hr', shifted=True, bounds=True) data = _cmor_data('E1hr') * 1000 / 3600. - cube = iris.cube.Cube( + cube = Cube( data.astype('float32'), long_name=vardef.long_name, var_name=vardef.short_name, @@ -405,12 +508,20 @@ def mrro_cmor_e1hr(): ], attributes={'comment': COMMENT}, ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) + + +def o3_era5_monthly(): + cube = era5_3d('monthly')[0] + cube = cube[:, ::-1, ::-1, :] # test if correct order of plev and lat stay + cube.data = cube.data.astype('float32') + cube.data *= 47.9982 / 28.9644 + return CubeList([cube]) def orog_era5_hourly(): time = _era5_time('invariant') - cube = iris.cube.Cube( + cube = Cube( _era5_data('invariant'), long_name='geopotential height', var_name='zg', @@ -421,14 +532,14 @@ def orog_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def orog_cmor_fx(): cmor_table = CMOR_TABLES['native6'] vardef = cmor_table.get_variable('fx', 'orog') data = _cmor_data('fx') / 9.80665 - cube = iris.cube.Cube( + cube = Cube( data.astype('float32'), long_name=vardef.long_name, var_name=vardef.short_name, @@ -437,12 +548,12 @@ def orog_cmor_fx(): dim_coords_and_dims=[(_cmor_latitude(), 0), (_cmor_longitude(), 1)], attributes={'comment': COMMENT}, ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def pr_era5_monthly(): time = _era5_time('monthly') - cube = iris.cube.Cube( + cube = Cube( _era5_data('monthly'), long_name='total_precipitation', var_name='tp', @@ -453,7 +564,7 @@ def pr_era5_monthly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def pr_cmor_amon(): @@ -461,7 +572,7 @@ def pr_cmor_amon(): vardef = cmor_table.get_variable('Amon', 'pr') time = _cmor_time('Amon', bounds=True) data = _cmor_data('Amon') * 1000. / 3600. / 24. - cube = iris.cube.Cube( + cube = Cube( data.astype('float32'), long_name=vardef.long_name, var_name=vardef.short_name, @@ -474,12 +585,12 @@ def pr_cmor_amon(): ], attributes={'comment': COMMENT}, ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def pr_era5_hourly(): time = _era5_time('hourly') - cube = iris.cube.Cube( + cube = Cube( _era5_data('hourly'), long_name='total_precipitation', var_name='tp', @@ -490,7 +601,7 @@ def pr_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def pr_cmor_e1hr(): @@ -498,7 +609,7 @@ def pr_cmor_e1hr(): vardef = cmor_table.get_variable('E1hr', 'pr') time = _cmor_time('E1hr', bounds=True, shifted=True) data = _cmor_data('E1hr') * 1000. / 3600. - cube = iris.cube.Cube( + cube = Cube( data.astype('float32'), long_name=vardef.long_name, var_name=vardef.short_name, @@ -511,12 +622,12 @@ def pr_cmor_e1hr(): ], attributes={'comment': COMMENT}, ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def prsn_era5_hourly(): time = _era5_time('hourly') - cube = iris.cube.Cube( + cube = Cube( _era5_data('hourly'), long_name='snow', var_name='snow', @@ -527,7 +638,7 @@ def prsn_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def prsn_cmor_e1hr(): @@ -535,7 +646,7 @@ def prsn_cmor_e1hr(): vardef = cmor_table.get_variable('E1hr', 'prsn') time = _cmor_time('E1hr', shifted=True, bounds=True) data = _cmor_data('E1hr') * 1000 / 3600. - cube = iris.cube.Cube( + cube = Cube( data.astype('float32'), long_name=vardef.long_name, var_name=vardef.short_name, @@ -548,12 +659,12 @@ def prsn_cmor_e1hr(): ], attributes={'comment': COMMENT}, ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def ptype_era5_hourly(): time = _era5_time('hourly') - cube = iris.cube.Cube( + cube = Cube( _era5_data('hourly'), long_name='snow', var_name='snow', @@ -564,7 +675,7 @@ def ptype_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def ptype_cmor_e1hr(): @@ -572,7 +683,7 @@ def ptype_cmor_e1hr(): vardef = cmor_table.get_variable('E1hr', 'ptype') time = _cmor_time('E1hr', shifted=False, bounds=True) data = _cmor_data('E1hr') - cube = iris.cube.Cube( + cube = Cube( data.astype('float32'), long_name=vardef.long_name, var_name=vardef.short_name, @@ -586,12 +697,12 @@ def ptype_cmor_e1hr(): ) cube.coord('latitude').long_name = 'latitude' cube.coord('longitude').long_name = 'longitude' - return iris.cube.CubeList([cube]) + return CubeList([cube]) def rlds_era5_hourly(): time = _era5_time('hourly') - cube = iris.cube.Cube( + cube = Cube( _era5_data('hourly'), long_name='surface thermal radiation downwards', var_name='ssrd', @@ -602,7 +713,7 @@ def rlds_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def rlds_cmor_e1hr(): @@ -610,24 +721,21 @@ def rlds_cmor_e1hr(): vardef = cmor_table.get_variable('E1hr', 'rlds') time = _cmor_time('E1hr', shifted=True, bounds=True) data = _cmor_data('E1hr') / 3600 - cube = iris.cube.Cube(data.astype('float32'), - long_name=vardef.long_name, - var_name=vardef.short_name, - standard_name=vardef.standard_name, - units=Unit(vardef.units), - dim_coords_and_dims=[(time, 0), - (_cmor_latitude(), 1), - (_cmor_longitude(), 2)], - attributes={ - 'comment': COMMENT, - 'positive': 'down', - }) - return iris.cube.CubeList([cube]) + cube = Cube(data.astype('float32'), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[(time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2)], + attributes={'comment': COMMENT, 'positive': 'down'}) + return CubeList([cube]) def rls_era5_hourly(): time = _era5_time('hourly') - cube = iris.cube.Cube( + cube = Cube( _era5_data('hourly'), long_name='runoff', var_name='runoff', @@ -638,7 +746,7 @@ def rls_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def rls_cmor_e1hr(): @@ -646,26 +754,23 @@ def rls_cmor_e1hr(): vardef = cmor_table.get_variable('E1hr', 'rls') time = _cmor_time('E1hr', shifted=True, bounds=True) data = _cmor_data('E1hr') - cube = iris.cube.Cube(data.astype('float32'), - long_name=vardef.long_name, - var_name=vardef.short_name, - standard_name=vardef.standard_name, - units=Unit(vardef.units), - dim_coords_and_dims=[ - (time, 0), - (_cmor_latitude(), 1), - (_cmor_longitude(), 2), - ], - attributes={ - 'comment': COMMENT, - 'positive': 'down', - }) - return iris.cube.CubeList([cube]) + cube = Cube(data.astype('float32'), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[ + (time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2), + ], + attributes={'comment': COMMENT, 'positive': 'down'}) + return CubeList([cube]) def rsds_era5_hourly(): time = _era5_time('hourly') - cube = iris.cube.Cube( + cube = Cube( _era5_data('hourly'), long_name='solar_radiation_downwards', var_name='rlwd', @@ -676,7 +781,7 @@ def rsds_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def rsds_cmor_e1hr(): @@ -684,24 +789,21 @@ def rsds_cmor_e1hr(): vardef = cmor_table.get_variable('E1hr', 'rsds') time = _cmor_time('E1hr', shifted=True, bounds=True) data = _cmor_data('E1hr') / 3600 - cube = iris.cube.Cube(data.astype('float32'), - long_name=vardef.long_name, - var_name=vardef.short_name, - standard_name=vardef.standard_name, - units=Unit(vardef.units), - dim_coords_and_dims=[(time, 0), - (_cmor_latitude(), 1), - (_cmor_longitude(), 2)], - attributes={ - 'comment': COMMENT, - 'positive': 'down', - }) - return iris.cube.CubeList([cube]) + cube = Cube(data.astype('float32'), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[(time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2)], + attributes={'comment': COMMENT, 'positive': 'down'}) + return CubeList([cube]) def rsdt_era5_hourly(): time = _era5_time('hourly') - cube = iris.cube.Cube( + cube = Cube( _era5_data('hourly'), long_name='thermal_radiation_downwards', var_name='strd', @@ -709,7 +811,7 @@ def rsdt_era5_hourly(): dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1), (_era5_longitude(), 2)], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def rsdt_cmor_e1hr(): @@ -717,24 +819,21 @@ def rsdt_cmor_e1hr(): vardef = cmor_table.get_variable('E1hr', 'rsdt') time = _cmor_time('E1hr', shifted=True, bounds=True) data = _cmor_data('E1hr') / 3600 - cube = iris.cube.Cube(data.astype('float32'), - long_name=vardef.long_name, - var_name=vardef.short_name, - standard_name=vardef.standard_name, - units=Unit(vardef.units), - dim_coords_and_dims=[(time, 0), - (_cmor_latitude(), 1), - (_cmor_longitude(), 2)], - attributes={ - 'comment': COMMENT, - 'positive': 'down', - }) - return iris.cube.CubeList([cube]) + cube = Cube(data.astype('float32'), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[(time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2)], + attributes={'comment': COMMENT, 'positive': 'down'}) + return CubeList([cube]) def rss_era5_hourly(): time = _era5_time('hourly') - cube = iris.cube.Cube( + cube = Cube( _era5_data('hourly'), long_name='net_solar_radiation', var_name='ssr', @@ -745,7 +844,7 @@ def rss_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def rss_cmor_e1hr(): @@ -753,24 +852,50 @@ def rss_cmor_e1hr(): vardef = cmor_table.get_variable('E1hr', 'rss') time = _cmor_time('E1hr', shifted=True, bounds=True) data = _cmor_data('E1hr') / 3600 - cube = iris.cube.Cube(data.astype('float32'), - long_name=vardef.long_name, - var_name=vardef.short_name, - standard_name=vardef.standard_name, - units=Unit(vardef.units), - dim_coords_and_dims=[(time, 0), - (_cmor_latitude(), 1), - (_cmor_longitude(), 2)], - attributes={ - 'comment': COMMENT, - 'positive': 'down', - }) - return iris.cube.CubeList([cube]) + cube = Cube(data.astype('float32'), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[(time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2)], + attributes={'comment': COMMENT, 'positive': 'down'}) + return CubeList([cube]) + + +def sftlf_era5(): + cube = Cube( + np.ones((3, 3)), + long_name=None, + var_name=None, + units='unknown', + dim_coords_and_dims=[ + (_era5_latitude(), 0), + (_era5_longitude(), 1), + ], + ) + return CubeList([cube]) + + +def sftlf_cmor_fx(): + cmor_table = CMOR_TABLES['native6'] + vardef = cmor_table.get_variable('fx', 'sftlf') + cube = Cube( + np.ones((3, 3)).astype('float32') * 100.0, + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[(_cmor_latitude(), 0), (_cmor_longitude(), 1)], + attributes={'comment': COMMENT}, + ) + return CubeList([cube]) def tas_era5_hourly(): time = _era5_time('hourly') - cube = iris.cube.Cube( + cube = Cube( _era5_data('hourly'), long_name='2m_temperature', var_name='t2m', @@ -781,7 +906,7 @@ def tas_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def tas_cmor_e1hr(): @@ -789,22 +914,22 @@ def tas_cmor_e1hr(): vardef = cmor_table.get_variable('E1hr', 'tas') time = _cmor_time('E1hr') data = _cmor_data('E1hr') - cube = iris.cube.Cube(data.astype('float32'), - long_name=vardef.long_name, - var_name=vardef.short_name, - standard_name=vardef.standard_name, - units=Unit(vardef.units), - dim_coords_and_dims=[(time, 0), - (_cmor_latitude(), 1), - (_cmor_longitude(), 2)], - attributes={'comment': COMMENT}) + cube = Cube(data.astype('float32'), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[(time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2)], + attributes={'comment': COMMENT}) cube.add_aux_coord(_cmor_aux_height(2.)) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def tas_era5_monthly(): time = _era5_time('monthly') - cube = iris.cube.Cube( + cube = Cube( _era5_data('monthly'), long_name='2m_temperature', var_name='t2m', @@ -815,7 +940,7 @@ def tas_era5_monthly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def tas_cmor_amon(): @@ -823,7 +948,7 @@ def tas_cmor_amon(): vardef = cmor_table.get_variable('Amon', 'tas') time = _cmor_time('Amon', bounds=True) data = _cmor_data('Amon') - cube = iris.cube.Cube( + cube = Cube( data.astype('float32'), long_name=vardef.long_name, var_name=vardef.short_name, @@ -837,13 +962,20 @@ def tas_cmor_amon(): attributes={'comment': COMMENT}, ) cube.add_aux_coord(_cmor_aux_height(2.)) - return iris.cube.CubeList([cube]) + return CubeList([cube]) + + +def toz_era5_monthly(): + cube = era5_2d('monthly')[0] + cube.data = cube.data.astype('float32') + cube.data *= 2.1415 + return CubeList([cube]) def zg_era5_monthly(): time = _era5_time('monthly') data = np.ones((3, 2, 3, 3)) - cube = iris.cube.Cube( + cube = Cube( data, long_name='geopotential height', var_name='zg', @@ -855,7 +987,7 @@ def zg_era5_monthly(): (_era5_longitude(), 3), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def zg_cmor_amon(): @@ -864,7 +996,7 @@ def zg_cmor_amon(): time = _cmor_time('Amon', bounds=True) data = np.ones((3, 2, 3, 3)) data = data / 9.80665 - cube = iris.cube.Cube( + cube = Cube( data.astype('float32'), long_name=vardef.long_name, var_name=vardef.short_name, @@ -878,12 +1010,12 @@ def zg_cmor_amon(): ], attributes={'comment': COMMENT}, ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def tasmax_era5_hourly(): time = _era5_time('hourly') - cube = iris.cube.Cube( + cube = Cube( _era5_data('hourly'), long_name='maximum 2m temperature', var_name='mx2t', @@ -894,7 +1026,7 @@ def tasmax_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def tasmax_cmor_e1hr(): @@ -902,7 +1034,7 @@ def tasmax_cmor_e1hr(): vardef = cmor_table.get_variable('E1hr', 'tasmax') time = _cmor_time('E1hr', shifted=True, bounds=True) data = _cmor_data('E1hr') - cube = iris.cube.Cube( + cube = Cube( data.astype('float32'), long_name=vardef.long_name, var_name=vardef.short_name, @@ -916,12 +1048,12 @@ def tasmax_cmor_e1hr(): attributes={'comment': COMMENT}, ) cube.add_aux_coord(_cmor_aux_height(2.)) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def tasmin_era5_hourly(): time = _era5_time('hourly') - cube = iris.cube.Cube( + cube = Cube( _era5_data('hourly'), long_name='minimum 2m temperature', var_name='mn2t', @@ -932,7 +1064,7 @@ def tasmin_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def tasmin_cmor_e1hr(): @@ -940,7 +1072,7 @@ def tasmin_cmor_e1hr(): vardef = cmor_table.get_variable('E1hr', 'tasmin') time = _cmor_time('E1hr', shifted=True, bounds=True) data = _cmor_data('E1hr') - cube = iris.cube.Cube( + cube = Cube( data.astype('float32'), long_name=vardef.long_name, var_name=vardef.short_name, @@ -954,12 +1086,12 @@ def tasmin_cmor_e1hr(): attributes={'comment': COMMENT}, ) cube.add_aux_coord(_cmor_aux_height(2.)) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def uas_era5_hourly(): time = _era5_time('hourly') - cube = iris.cube.Cube( + cube = Cube( _era5_data('hourly'), long_name='10m_u_component_of_wind', var_name='u10', @@ -970,7 +1102,7 @@ def uas_era5_hourly(): (_era5_longitude(), 2), ], ) - return iris.cube.CubeList([cube]) + return CubeList([cube]) def uas_cmor_e1hr(): @@ -978,7 +1110,7 @@ def uas_cmor_e1hr(): vardef = cmor_table.get_variable('E1hr', 'uas') time = _cmor_time('E1hr') data = _cmor_data('E1hr') - cube = iris.cube.Cube( + cube = Cube( data.astype('float32'), long_name=vardef.long_name, var_name=vardef.short_name, @@ -992,31 +1124,44 @@ def uas_cmor_e1hr(): attributes={'comment': COMMENT}, ) cube.add_aux_coord(_cmor_aux_height(10.)) - return iris.cube.CubeList([cube]) + return CubeList([cube]) VARIABLES = [ pytest.param(a, b, c, d, id=c + '_' + d) for (a, b, c, d) in [ + (era5_2d('daily'), cmor_2d('Eday', 'albsn'), 'albsn', 'Eday'), (cl_era5_monthly(), cl_cmor_amon(), 'cl', 'Amon'), + (era5_3d('monthly'), cmor_3d('Amon', 'cli'), 'cli', 'Amon'), (clt_era5_hourly(), clt_cmor_e1hr(), 'clt', 'E1hr'), + (era5_3d('monthly'), cmor_3d('Amon', 'clw'), 'clw', 'Amon'), (evspsbl_era5_hourly(), evspsbl_cmor_e1hr(), 'evspsbl', 'E1hr'), (evspsblpot_era5_hourly(), evspsblpot_cmor_e1hr(), 'evspsblpot', 'E1hr'), + (era5_3d('monthly'), cmor_3d('Amon', 'hus'), 'hus', 'Amon'), (mrro_era5_hourly(), mrro_cmor_e1hr(), 'mrro', 'E1hr'), + (o3_era5_monthly(), cmor_3d('Amon', 'o3'), 'o3', 'Amon'), (orog_era5_hourly(), orog_cmor_fx(), 'orog', 'fx'), (pr_era5_monthly(), pr_cmor_amon(), 'pr', 'Amon'), (pr_era5_hourly(), pr_cmor_e1hr(), 'pr', 'E1hr'), (prsn_era5_hourly(), prsn_cmor_e1hr(), 'prsn', 'E1hr'), + (era5_2d('monthly'), cmor_2d('Amon', 'prw'), 'prw', 'Amon'), + (era5_2d('monthly'), cmor_2d('Amon', 'ps'), 'ps', 'Amon'), (ptype_era5_hourly(), ptype_cmor_e1hr(), 'ptype', 'E1hr'), + (era5_3d('monthly'), cmor_3d('Emon', 'rainmxrat27'), 'rainmxrat27', + 'Emon'), (rlds_era5_hourly(), rlds_cmor_e1hr(), 'rlds', 'E1hr'), (rls_era5_hourly(), rls_cmor_e1hr(), 'rls', 'E1hr'), (rsds_era5_hourly(), rsds_cmor_e1hr(), 'rsds', 'E1hr'), (rsdt_era5_hourly(), rsdt_cmor_e1hr(), 'rsdt', 'E1hr'), (rss_era5_hourly(), rss_cmor_e1hr(), 'rss', 'E1hr'), + (sftlf_era5(), sftlf_cmor_fx(), 'sftlf', 'fx'), + (era5_3d('monthly'), cmor_3d('Emon', 'snowmxrat27'), 'snowmxrat27', + 'Emon'), (tas_era5_hourly(), tas_cmor_e1hr(), 'tas', 'E1hr'), (tas_era5_monthly(), tas_cmor_amon(), 'tas', 'Amon'), (tasmax_era5_hourly(), tasmax_cmor_e1hr(), 'tasmax', 'E1hr'), (tasmin_era5_hourly(), tasmin_cmor_e1hr(), 'tasmin', 'E1hr'), + (toz_era5_monthly(), cmor_2d('AERmon', 'toz'), 'toz', 'AERmon'), (uas_era5_hourly(), uas_cmor_e1hr(), 'uas', 'E1hr'), (zg_era5_monthly(), zg_cmor_amon(), 'zg', 'Amon'), ] @@ -1039,3 +1184,81 @@ def test_cmorization(era5_cubes, cmor_cubes, var, mip): print('cmor_cube:', cmor_cube) print('fixed_cube:', fixed_cube) assert fixed_cube == cmor_cube + + +@pytest.fixture +def unstructured_grid_cubes(): + """Sample cubes with unstructured grid.""" + time = DimCoord( + [0.0, 31.0], standard_name='time', units='days since 1950-01-01' + ) + lat = AuxCoord( + [1.0, 1.0, -1.0, -1.0], standard_name='latitude', units='degrees_north' + ) + lon = AuxCoord( + [179.0, 180.0, 180.0, 179.0], + standard_name='longitude', + units='degrees_east', + ) + cube = Cube( + da.from_array([[0.0, 1.0, 2.0, 3.0], [0.0, 0.0, 0.0, 0.0]]), + standard_name='air_temperature', + units='K', + dim_coords_and_dims=[(time, 0)], + aux_coords_and_dims=[(lat, 1), (lon, 1)], + ) + return CubeList([cube]) + + +def test_automatic_regrid(unstructured_grid_cubes): + """Test automatic regridding of unstructured data.""" + fixed_cubes = fix_metadata( + unstructured_grid_cubes, + 'tas', + 'native6', + 'era5', + 'Amon', + target_grid='180x90', + ) + + assert len(fixed_cubes) == 1 + fixed_cube = fixed_cubes[0] + assert fixed_cube.shape == (2, 2, 2) + + assert fixed_cube.coords('time', dim_coords=True) + assert fixed_cube.coord_dims('time') == (0,) + assert fixed_cube.coords('latitude', dim_coords=True) + assert fixed_cube.coord_dims('latitude') == (1,) + assert fixed_cube.coords('longitude', dim_coords=True) + assert fixed_cube.coord_dims('longitude') == (2,) + + +def test_no_automatic_regrid(unstructured_grid_cubes): + """Test no automatic regridding of unstructured data.""" + fixed_cubes = fix_metadata( + unstructured_grid_cubes, + 'tas', + 'native6', + 'era5', + 'Amon', + ) + + assert len(fixed_cubes) == 1 + fixed_cube = fixed_cubes[0] + + assert fixed_cube.shape == (2, 4) + + assert fixed_cube.coords('time', dim_coords=True) + assert fixed_cube.coord_dims('time') == (0,) + + assert fixed_cube.coords('latitude', dim_coords=False) + assert fixed_cube.coord_dims('latitude') == (1,) + lat = fixed_cube.coord('latitude') + np.testing.assert_allclose(lat.points, [1, 1, -1, -1]) + assert lat.bounds is None + + assert fixed_cube.coords('longitude', dim_coords=False) + assert fixed_cube.coord_dims('longitude') == (1,) + lon = fixed_cube.coord('longitude') + np.testing.assert_allclose(lon.points, [179, 180, 180, 179]) + assert lon.bounds is None diff --git a/tests/integration/preprocessor/_io/test_load.py b/tests/integration/preprocessor/_io/test_load.py index 863f53ce1f..99d727b71f 100644 --- a/tests/integration/preprocessor/_io/test_load.py +++ b/tests/integration/preprocessor/_io/test_load.py @@ -4,7 +4,6 @@ import tempfile import unittest import warnings -# from pathlib import Path import iris import numpy as np @@ -14,6 +13,8 @@ # import esmvalcore from esmvalcore.preprocessor._io import concatenate_callback, load +# from pathlib import Path + def _create_sample_cube(): coord = DimCoord([1, 2], standard_name='latitude', units='degrees_north') diff --git a/tests/integration/preprocessor/_regrid/test_bilinear_unstructured_regrid.py b/tests/integration/preprocessor/_regrid/test_bilinear_unstructured_regrid.py new file mode 100644 index 0000000000..83f2ac5b69 --- /dev/null +++ b/tests/integration/preprocessor/_regrid/test_bilinear_unstructured_regrid.py @@ -0,0 +1,141 @@ +""" Integration tests for `bilinear_unstructured_regrid`.""" + +import numpy as np +import pytest +import dask.array as da + +from iris.coords import AuxCoord, DimCoord +from iris.cube import Cube + +import esmvalcore.preprocessor._regrid +from esmvalcore.preprocessor._regrid import ( + _bilinear_unstructured_regrid, + _get_linear_interpolation_weights, +) + + +@pytest.fixture(autouse=True) +def clear_cache(monkeypatch): + """Start each test with a clear cache.""" + monkeypatch.setattr(esmvalcore.preprocessor._regrid, '_CACHE_WEIGHTS', {}) + + +@pytest.fixture +def unstructured_grid_cube(): + """Sample cube with unstructured grid.""" + time = DimCoord( + [0.0, 1.0], standard_name='time', units='days since 1950-01-01' + ) + lat = AuxCoord( + [-1.0, -1.0, 1.0, 1.0], standard_name='latitude', units='degrees_north' + ) + lon = AuxCoord( + [179.0, 180.0, 180.0, 179.0], + standard_name='longitude', + units='degrees_east', + ) + cube = Cube( + da.from_array([[0.0, 1.0, 2.0, 3.0], [0.0, 0.0, 0.0, 0.0]]), + standard_name='air_temperature', + units='K', + dim_coords_and_dims=[(time, 0)], + aux_coords_and_dims=[(lat, 1), (lon, 1)], + ) + return cube + + +TARGET_GRID = '180x90' +LAT_OFFSET = False +LON_OFFSET = False + + +def test_use_cached_weights(unstructured_grid_cube, mocker): + """Test `_get_linear_interpolation_weights`.""" + cache = esmvalcore.preprocessor._regrid._CACHE_WEIGHTS + key = ( + '(4,)_-1.0-1.0-degrees_north_179.0-179.0-degrees_east_180x90_' + 'False_False_nan_' + ) + cache[key] = mocker.sentinel.cached_weights + + weights = _get_linear_interpolation_weights( + unstructured_grid_cube, TARGET_GRID, LAT_OFFSET, LON_OFFSET + ) + + assert weights == mocker.sentinel.cached_weights + + +def test_bilinear_unstructured_regrid(unstructured_grid_cube): + """Test `_bilinear_unstructured_regrid`.""" + new_cube = _bilinear_unstructured_regrid( + unstructured_grid_cube, TARGET_GRID, LAT_OFFSET, LON_OFFSET + ) + + assert new_cube.metadata == unstructured_grid_cube.metadata + assert new_cube.shape == (2, 3, 2) + + assert new_cube.coords('time') + assert new_cube.coord('time') == unstructured_grid_cube.coord('time') + + assert new_cube.coords('latitude') + lat = new_cube.coord('latitude') + np.testing.assert_allclose(lat.points, [-90, 0, 90]) + np.testing.assert_allclose(lat.bounds, [[-90, -45], [-45, 45], [45, 90]]) + + assert new_cube.coords('longitude') + lat = new_cube.coord('longitude') + np.testing.assert_allclose(lat.points, [0, 180]) + np.testing.assert_allclose(lat.bounds, [[-90, 90], [90, 270]]) + + np.testing.assert_allclose( + new_cube.data, + [ + [[np.nan, np.nan], [np.nan, 1.5], [np.nan, np.nan]], + [[np.nan, np.nan], [np.nan, 0.0], [np.nan, np.nan]], + ], + ) + + cache = esmvalcore.preprocessor._regrid._CACHE_WEIGHTS + assert len(cache) == 1 + key = ( + '(4,)_-1.0-1.0-degrees_north_179.0-179.0-degrees_east_180x90_' + 'False_False_nan_' + ) + assert key in cache + assert len(cache[key]) == 2 + np.testing.assert_equal( + cache[key][0], + [[3, 1, 0], + [3, 1, 0], + [3, 1, 0], + [1, 3, 2], + [3, 1, 0], + [3, 1, 0]], + ) + np.testing.assert_allclose( + cache[key][1], + [[np.nan, np.nan, np.nan], + [np.nan, np.nan, np.nan], + [np.nan, np.nan, np.nan], + [0.5, 0.0, 0.5], + [np.nan, np.nan, np.nan], + [np.nan, np.nan, np.nan]], + ) + + +def test_bilinear_unstructured_regrid_no_unstructured_grid(): + """Test `_bilinear_unstructured_regrid`.""" + with pytest.raises(ValueError): + _bilinear_unstructured_regrid( + Cube(0), TARGET_GRID, LAT_OFFSET, LON_OFFSET + ) + + +def test_bilinear_unstructured_regrid_invalid_dims(unstructured_grid_cube): + """Test `_bilinear_unstructured_regrid`.""" + cube = unstructured_grid_cube.copy() + cube.transpose() + with pytest.raises(ValueError): + _bilinear_unstructured_regrid( + cube, TARGET_GRID, LAT_OFFSET, LON_OFFSET + ) From bef5b5e501d55d1767f0b8ca2c54b7da7bc2e1b0 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 23 Aug 2023 17:53:35 +0200 Subject: [PATCH 16/64] Fixed test --- .../test_bilinear_unstructured_regrid.py | 46 ++++++++++++------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/tests/integration/preprocessor/_regrid/test_bilinear_unstructured_regrid.py b/tests/integration/preprocessor/_regrid/test_bilinear_unstructured_regrid.py index 83f2ac5b69..900bd3e525 100644 --- a/tests/integration/preprocessor/_regrid/test_bilinear_unstructured_regrid.py +++ b/tests/integration/preprocessor/_regrid/test_bilinear_unstructured_regrid.py @@ -2,8 +2,6 @@ import numpy as np import pytest -import dask.array as da - from iris.coords import AuxCoord, DimCoord from iris.cube import Cube @@ -35,7 +33,7 @@ def unstructured_grid_cube(): units='degrees_east', ) cube = Cube( - da.from_array([[0.0, 1.0, 2.0, 3.0], [0.0, 0.0, 0.0, 0.0]]), + np.array([[0.0, 1.0, 2.0, 3.0], [0.0, 0.0, 0.0, 0.0]]), standard_name='air_temperature', units='K', dim_coords_and_dims=[(time, 0)], @@ -44,17 +42,17 @@ def unstructured_grid_cube(): return cube -TARGET_GRID = '180x90' -LAT_OFFSET = False -LON_OFFSET = False +TARGET_GRID = '120x60' +LAT_OFFSET = True +LON_OFFSET = True def test_use_cached_weights(unstructured_grid_cube, mocker): """Test `_get_linear_interpolation_weights`.""" cache = esmvalcore.preprocessor._regrid._CACHE_WEIGHTS key = ( - '(4,)_-1.0-1.0-degrees_north_179.0-179.0-degrees_east_180x90_' - 'False_False_nan_' + '(4,)_-1.0-1.0-degrees_north_179.0-179.0-degrees_east_120x60_' + 'True_True_nan_' ) cache[key] = mocker.sentinel.cached_weights @@ -72,53 +70,67 @@ def test_bilinear_unstructured_regrid(unstructured_grid_cube): ) assert new_cube.metadata == unstructured_grid_cube.metadata - assert new_cube.shape == (2, 3, 2) + assert new_cube.shape == (2, 3, 3) assert new_cube.coords('time') assert new_cube.coord('time') == unstructured_grid_cube.coord('time') assert new_cube.coords('latitude') lat = new_cube.coord('latitude') - np.testing.assert_allclose(lat.points, [-90, 0, 90]) - np.testing.assert_allclose(lat.bounds, [[-90, -45], [-45, 45], [45, 90]]) + np.testing.assert_allclose(lat.points, [-60, 0, 60]) + np.testing.assert_allclose(lat.bounds, [[-90, -30], [-30, 30], [30, 90]]) assert new_cube.coords('longitude') lat = new_cube.coord('longitude') - np.testing.assert_allclose(lat.points, [0, 180]) - np.testing.assert_allclose(lat.bounds, [[-90, 90], [90, 270]]) + np.testing.assert_allclose(lat.points, [60, 180, 300]) + np.testing.assert_allclose(lat.bounds, [[0, 120], [120, 240], [240, 360]]) np.testing.assert_allclose( new_cube.data, [ - [[np.nan, np.nan], [np.nan, 1.5], [np.nan, np.nan]], - [[np.nan, np.nan], [np.nan, 0.0], [np.nan, np.nan]], + [ + [np.nan, np.nan, np.nan], + [np.nan, 1.5, np.nan], + [np.nan, np.nan, np.nan], + ], + [ + [np.nan, np.nan, np.nan], + [np.nan, 0.0, np.nan], + [np.nan, np.nan, np.nan], + ], ], ) cache = esmvalcore.preprocessor._regrid._CACHE_WEIGHTS assert len(cache) == 1 key = ( - '(4,)_-1.0-1.0-degrees_north_179.0-179.0-degrees_east_180x90_' - 'False_False_nan_' + '(4,)_-1.0-1.0-degrees_north_179.0-179.0-degrees_east_120x60_' + 'True_True_nan_' ) assert key in cache assert len(cache[key]) == 2 np.testing.assert_equal( cache[key][0], [[3, 1, 0], + [3, 1, 0], [3, 1, 0], [3, 1, 0], [1, 3, 2], [3, 1, 0], + [3, 1, 0], + [3, 1, 0], [3, 1, 0]], ) np.testing.assert_allclose( cache[key][1], [[np.nan, np.nan, np.nan], + [np.nan, np.nan, np.nan], [np.nan, np.nan, np.nan], [np.nan, np.nan, np.nan], [0.5, 0.0, 0.5], [np.nan, np.nan, np.nan], + [np.nan, np.nan, np.nan], + [np.nan, np.nan, np.nan], [np.nan, np.nan, np.nan]], ) From 2693066e3a966f745dbc468960a1a89dcab7bc97 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Thu, 24 Aug 2023 10:13:09 +0200 Subject: [PATCH 17/64] Improved test coverage of ERA5 CMORizer --- esmvalcore/cmor/_fixes/native6/era5.py | 5 + .../cmor/_fixes/native6/test_era5.py | 208 +++++++++++++++++- 2 files changed, 211 insertions(+), 2 deletions(-) diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py index 4e27785913..cf60a3ac9a 100644 --- a/esmvalcore/cmor/_fixes/native6/era5.py +++ b/esmvalcore/cmor/_fixes/native6/era5.py @@ -56,6 +56,11 @@ def fix_accumulated_units(cube): cube.units = cube.units * 'd-1' elif get_frequency(cube) == 'hourly': cube.units = cube.units * 'h-1' + elif get_frequency(cube) == 'daily': + raise NotImplementedError( + f"Fixing of accumulated units of cube " + f"{cube.summary(shorten=True)} is not implemented for daily data" + ) return cube diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py index b05b68a1d4..243dff09aa 100644 --- a/tests/integration/cmor/_fixes/native6/test_era5.py +++ b/tests/integration/cmor/_fixes/native6/test_era5.py @@ -12,6 +12,7 @@ AllVars, Evspsbl, Zg, + fix_accumulated_units, get_frequency, ) from esmvalcore.cmor.fix import Fix, fix_metadata @@ -88,6 +89,7 @@ def test_get_frequency_fx(): """Test cubes with time invariant frequency.""" cube = Cube(1., long_name='Cube without time coordinate') assert get_frequency(cube) == 'fx' + time = DimCoord( 0, standard_name='time', @@ -100,11 +102,33 @@ def test_get_frequency_fx(): dim_coords_and_dims=[(time, 0)], ) assert get_frequency(cube) == 'fx' + + cube.long_name = ( + 'Percentage of the Grid Cell Occupied by Land (Including Lakes)' + ) + assert get_frequency(cube) == 'fx' + cube.long_name = 'Not geopotential' with pytest.raises(ValueError): get_frequency(cube) +def test_fix_accumulated_units_fail(): + """Test `fix_accumulated_units`.""" + time = DimCoord( + [0, 1, 2], + standard_name='time', + units=Unit('days since 1900-01-01'), + ) + cube = Cube( + [1, 6, 3], + var_name='random_var', + dim_coords_and_dims=[(time, 0)], + ) + with pytest.raises(NotImplementedError): + fix_accumulated_units(cube) + + def _era5_latitude(): return DimCoord( np.array([90., 0., -90.]), @@ -253,13 +277,21 @@ def _cmor_data(mip): def era5_2d(frequency): + if frequency == 'monthly': + time = DimCoord( + [-31, 0, 31], + standard_name='time', + units='days since 1850-01-01' + ) + else: + time = _era5_time(frequency) cube = Cube( _era5_data(frequency), long_name=None, var_name=None, units='unknown', dim_coords_and_dims=[ - (_era5_time(frequency), 0), + (time, 0), (_era5_latitude(), 1), (_era5_longitude(), 2), ], @@ -286,6 +318,17 @@ def era5_3d(frequency): def cmor_2d(mip, short_name): cmor_table = CMOR_TABLES['native6'] vardef = cmor_table.get_variable(mip, short_name) + if 'mon' in mip: + time = DimCoord( + [-15.5, 15.5, 45.0], + bounds=[[-31.0, 0.0], [0.0, 31.0], [31.0, 59.0]], + standard_name='time', + long_name='time', + var_name='time', + units='days since 1850-01-01' + ) + else: + time = _cmor_time(mip, bounds=True) cube = Cube( _cmor_data(mip).astype('float32'), long_name=vardef.long_name, @@ -293,7 +336,7 @@ def cmor_2d(mip, short_name): standard_name=vardef.standard_name, units=Unit(vardef.units), dim_coords_and_dims=[ - (_cmor_time(mip, bounds=True), 0), + (time, 0), (_cmor_latitude(), 1), (_cmor_longitude(), 2), ], @@ -733,6 +776,78 @@ def rlds_cmor_e1hr(): return CubeList([cube]) +def rlns_era5_hourly(): + freq = 'hourly' + cube = Cube( + _era5_data(freq), + long_name=None, + var_name=None, + units='J m**-2', + dim_coords_and_dims=[ + (_era5_time(freq), 0), + (_era5_latitude(), 1), + (_era5_longitude(), 2), + ], + ) + return CubeList([cube]) + + +def rlns_cmor_e1hr(): + mip = 'E1hr' + short_name = 'rlns' + cmor_table = CMOR_TABLES['native6'] + vardef = cmor_table.get_variable(mip, short_name) + time = _cmor_time(mip, shifted=True, bounds=True) + data = _cmor_data(mip) / 3600 + cube = Cube(data.astype('float32'), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[(time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2)], + attributes={'comment': COMMENT, 'positive': 'down'}) + cube.coord('latitude').long_name = 'latitude' # from custom table + cube.coord('longitude').long_name = 'longitude' # from custom table + return CubeList([cube]) + + +def rlus_era5_hourly(): + freq = 'hourly' + cube = Cube( + _era5_data(freq), + long_name=None, + var_name=None, + units='J m**-2', + dim_coords_and_dims=[ + (_era5_time(freq), 0), + (_era5_latitude(), 1), + (_era5_longitude(), 2), + ], + ) + return CubeList([cube]) + + +def rlus_cmor_e1hr(): + mip = 'E1hr' + short_name = 'rlus' + cmor_table = CMOR_TABLES['native6'] + vardef = cmor_table.get_variable(mip, short_name) + time = _cmor_time(mip, shifted=True, bounds=True) + data = _cmor_data(mip) / 3600 + cube = Cube(data.astype('float32'), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[(time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2)], + attributes={'comment': COMMENT, 'positive': 'up'}) + return CubeList([cube]) + + def rls_era5_hourly(): time = _era5_time('hourly') cube = Cube( @@ -801,6 +916,78 @@ def rsds_cmor_e1hr(): return CubeList([cube]) +def rsns_era5_hourly(): + freq = 'hourly' + cube = Cube( + _era5_data(freq), + long_name=None, + var_name=None, + units='J m**-2', + dim_coords_and_dims=[ + (_era5_time(freq), 0), + (_era5_latitude(), 1), + (_era5_longitude(), 2), + ], + ) + return CubeList([cube]) + + +def rsns_cmor_e1hr(): + mip = 'E1hr' + short_name = 'rsns' + cmor_table = CMOR_TABLES['native6'] + vardef = cmor_table.get_variable(mip, short_name) + time = _cmor_time(mip, shifted=True, bounds=True) + data = _cmor_data(mip) / 3600 + cube = Cube(data.astype('float32'), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[(time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2)], + attributes={'comment': COMMENT, 'positive': 'down'}) + cube.coord('latitude').long_name = 'latitude' # from custom table + cube.coord('longitude').long_name = 'longitude' # from custom table + return CubeList([cube]) + + +def rsus_era5_hourly(): + freq = 'hourly' + cube = Cube( + _era5_data(freq), + long_name=None, + var_name=None, + units='J m**-2', + dim_coords_and_dims=[ + (_era5_time(freq), 0), + (_era5_latitude(), 1), + (_era5_longitude(), 2), + ], + ) + return CubeList([cube]) + + +def rsus_cmor_e1hr(): + mip = 'E1hr' + short_name = 'rsus' + cmor_table = CMOR_TABLES['native6'] + vardef = cmor_table.get_variable(mip, short_name) + time = _cmor_time(mip, shifted=True, bounds=True) + data = _cmor_data(mip) / 3600 + cube = Cube(data.astype('float32'), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[(time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2)], + attributes={'comment': COMMENT, 'positive': 'up'}) + return CubeList([cube]) + + def rsdt_era5_hourly(): time = _era5_time('hourly') cube = Cube( @@ -1150,8 +1337,12 @@ def uas_cmor_e1hr(): (era5_3d('monthly'), cmor_3d('Emon', 'rainmxrat27'), 'rainmxrat27', 'Emon'), (rlds_era5_hourly(), rlds_cmor_e1hr(), 'rlds', 'E1hr'), + (rlns_era5_hourly(), rlns_cmor_e1hr(), 'rlns', 'E1hr'), + (rlus_era5_hourly(), rlus_cmor_e1hr(), 'rlus', 'E1hr'), (rls_era5_hourly(), rls_cmor_e1hr(), 'rls', 'E1hr'), (rsds_era5_hourly(), rsds_cmor_e1hr(), 'rsds', 'E1hr'), + (rsns_era5_hourly(), rsns_cmor_e1hr(), 'rsns', 'E1hr'), + (rsus_era5_hourly(), rsus_cmor_e1hr(), 'rsus', 'E1hr'), (rsdt_era5_hourly(), rsdt_cmor_e1hr(), 'rsdt', 'E1hr'), (rss_era5_hourly(), rss_cmor_e1hr(), 'rss', 'E1hr'), (sftlf_era5(), sftlf_cmor_fx(), 'sftlf', 'fx'), @@ -1262,3 +1453,16 @@ def test_no_automatic_regrid(unstructured_grid_cubes): lon = fixed_cube.coord('longitude') np.testing.assert_allclose(lon.points, [179, 180, 180, 179]) assert lon.bounds is None + + +def test_no_automatic_regrid_regular_grid(): + """Test no automatic regridding of regular grid data.""" + cubes = era5_2d('monthly') + fixed_cubes = fix_metadata( + cubes, 'ps', 'native6', 'era5', 'Amon', target_grid='1x1', + ) + + assert len(fixed_cubes) == 1 + fixed_cube = fixed_cubes[0] + assert fixed_cube.shape == (3, 3, 3) + assert fixed_cube == cmor_2d('Amon', 'ps')[0] From e7e72856e94bc41e9423918b13d5e290433791d4 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Thu, 24 Aug 2023 10:33:33 +0200 Subject: [PATCH 18/64] Increased test coverage of regrid module --- tests/unit/preprocessor/_regrid/test_regrid.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/unit/preprocessor/_regrid/test_regrid.py b/tests/unit/preprocessor/_regrid/test_regrid.py index b6334d0340..d934ed55a1 100644 --- a/tests/unit/preprocessor/_regrid/test_regrid.py +++ b/tests/unit/preprocessor/_regrid/test_regrid.py @@ -15,6 +15,7 @@ from esmvalcore.preprocessor._regrid import ( _CACHE, HORIZONTAL_SCHEMES, + _attempt_irregular_regridding, _horizontal_grid_is_close, _rechunk, ) @@ -383,5 +384,11 @@ def test_no_rechunk_unsupported_grid(): assert result.core_data().chunks == expected_chunks +def test_attempt_irregular_regridding(): + """Test `_attempt_irregular_regridding`.""" + result = _attempt_irregular_regridding(iris.cube.Cube(0), 'linear') + assert result is False + + if __name__ == '__main__': unittest.main() From 911ed28199ea3c14991c344cdcd7d65ba5e4220b Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Thu, 24 Aug 2023 11:39:27 +0200 Subject: [PATCH 19/64] Optimized doc --- doc/quickstart/find_data.rst | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index 5d4e2dfbb2..6c543bec91 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -104,6 +104,13 @@ Supported native reanalysis/observational datasets The following native reanalysis/observational datasets are supported under the ``native6`` project. +To read these datasets with ESMValCore, put the files containing the data in +the ``rootpath`` that you have configured for the ``native6`` project in your +:ref:`user configuration file`, in a subdirectory called +``Tier{tier}/{dataset}/{version}/{frequency}/{short_name}`` (assuming your are +using the ``default`` DRS for ``native6``). +Replace the items in curly braces by the values used in the variable/dataset +definition in the :ref:`recipe `. .. _read_native_era5_nc: @@ -112,13 +119,10 @@ ERA5 (in netCDF format downloaded from the CDS) ERA5 data can be downloaded from the Copernicus Climate Data Store (CDS) using the convenient tool `era5cli `__. -To read this data with ESMValCore, put the files containing the data in the -``rootpath`` that you have configured for the ``native6`` project in your -:ref:`user configuration file`, in a subdirectory called -``Tier3/ERA5/{version}/{frequency}/{short_name}`` (assuming your are using the -``default`` DRS for ``native6``). -Replace the items in curly braces by the values used in the variable/dataset -definition in the :ref:`recipe `. +For example for monthly data, place the files in the +``/Tier3/ERA5/version/mon/pr`` subdirectory of your ``rootpath`` that you have +configured for the ``native6`` project (assuming your are using the ``default`` +DRS for ``native6``). Supported variables: ``cl``, ``clt``, ``evspsbl``, ``evspsblpot``, ``mrro``, ``pr``, ``prsn``, ``ps``, ``psl``, ``ptype``, ``rls``, ``rlds``, ``rsds``, ``rsdt``, ``rss``, ``uas``, ``vas``, ``tas``, ``tasmax``, ``tasmin``, ``tdps``, ``ts``, ``tsn`` (``E1hr``/``Amon``), ``orog`` (``fx``). @@ -137,7 +141,7 @@ Supported variables: ``cl``, ``clt``, ``evspsbl``, ``evspsblpot``, ``mrro``, ``p ERA5 (in GRIB format available on DKRZ's Levante) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -ERA5 data in monthly, daily, and hourly resolution is available on `Levante +ERA5 data in monthly, daily, and hourly resolution is `available on Levante `__ in its native GRIB format. To read this data with ESMValCore, use the following settings in your @@ -166,7 +170,7 @@ are All of these facets have reasonable defaults preconfigured in the corresponding :ref:`extra facets` file, which is available here: :download:`native6-mappings.yml -`. +`. If necessary, these facets can be overwritten in the recipe. Thus, example dataset entries could look like this: @@ -209,7 +213,10 @@ MSWEP - Supported frequencies: ``mon``, ``day``, ``3hr``. - Tier: 3 -For example for monthly data, place the files in the ``/Tier3/MSWEP/version/mon/pr`` subdirectory of your ``native6`` project location. +For example for monthly data, place the files in the +``/Tier3/MSWEP/version/mon/pr`` subdirectory of your ``rootpath`` that you have +configured for the ``native6`` project (assuming your are using the ``default`` +DRS for ``native6``). .. note:: For monthly data (``V220``), the data must be postfixed with the date, i.e. rename ``global_monthly_050deg.nc`` to ``global_monthly_050deg_197901-201710.nc`` From 7afa55114ffd5dca0b1eb3baf6629dcd36e45c00 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Fri, 25 Aug 2023 13:44:31 +0200 Subject: [PATCH 20/64] More customizable automatic regriddind for ERA5 GRIB --- doc/quickstart/find_data.rst | 25 ++++++++++++++----- esmvalcore/cmor/_fixes/native6/era5.py | 4 +-- .../config/extra_facets/native6-mappings.yml | 3 ++- .../cmor/_fixes/native6/test_era5.py | 4 +-- 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index 6c543bec91..d95ff40071 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -124,7 +124,11 @@ For example for monthly data, place the files in the configured for the ``native6`` project (assuming your are using the ``default`` DRS for ``native6``). -Supported variables: ``cl``, ``clt``, ``evspsbl``, ``evspsblpot``, ``mrro``, ``pr``, ``prsn``, ``ps``, ``psl``, ``ptype``, ``rls``, ``rlds``, ``rsds``, ``rsdt``, ``rss``, ``uas``, ``vas``, ``tas``, ``tasmax``, ``tasmin``, ``tdps``, ``ts``, ``tsn`` (``E1hr``/``Amon``), ``orog`` (``fx``). +- Supported variables: ``cl``, ``clt``, ``evspsbl``, ``evspsblpot``, ``mrro``, + ``pr``, ``prsn``, ``ps``, ``psl``, ``ptype``, ``rls``, ``rlds``, ``rsds``, + ``rsdt``, ``rss``, ``uas``, ``vas``, ``tas``, ``tasmax``, ``tasmin``, + ``tdps``, ``ts``, ``tsn`` (``E1hr``/``Amon``), ``orog`` (``fx``). +- Tier: 3 .. note:: According to the description of Evapotranspiration and potential Evapotranspiration on the Copernicus page (https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-single-levels-monthly-means?tab=overview): @@ -191,18 +195,27 @@ By default, ESMValCore linearly interpolates the data to a regular 0.25° x 0.25° grid as `recommended by the ECMWF `__. If you want to use a different target resolution or completely disable this -feature, you can specify the optional facet ``target_grid`` in the recipe, -e.g., +feature, you can specify the optional facet ``regrid`` in the recipe. +``regrid`` takes the following keyword arguments: ``target_grid``, +``lat_offset`` (optional; default: ``True``), and ``lon_offset`` (optional; +default: ``True``). +See :func:`esmvalcore.preprocessor.regrid` for details on these keywords. +Example: .. code-block:: yaml datasets: - {project: native6, dataset: ERA5, timerange: '2000/2001', - short_name: tas, mip: Amon, target_grid: 1x1} + short_name: tas, mip: Amon, regrid: {target_grid: 1x1}} + - {project: native6, dataset: ERA5, timerange: '2000/2001', + short_name: tas, mip: Amon, regrid: {target_grid: 1x1, lat_offset: false}} - {project: native6, dataset: ERA5, timerange: '2000/2001', - short_name: tas, mip: Amon, target_grid: false} # do NOT interpolate + short_name: tas, mip: Amon, regrid: false} # do NOT interpolate -Supported variables: ``albsn``, ``cl``, ``cli``, ``clt``, ``clw``, ``hur``, ``hus``, ``o3``, ``prw``, ``ps``, ``psl``, ``rainmxrat27``, ``sftlf``, ``snd``, ``snowmxrat27``, ``ta``, ``tas``, ``tdps``, ``toz``, ``ts``, ``ua``, ``uas``, ``va``, ``vas``, ``wap``, ``zg``. +- Supported variables: ``albsn``, ``cl``, ``cli``, ``clt``, ``clw``, ``hur``, + ``hus``, ``o3``, ``prw``, ``ps``, ``psl``, ``rainmxrat27``, ``sftlf``, + ``snd``, ``snowmxrat27``, ``ta``, ``tas``, ``tdps``, ``toz``, ``ts``, ``ua``, + ``uas``, ``va``, ``vas``, ``wap``, ``zg``. .. _read_native_mswep: diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py index cf60a3ac9a..112bf55d4e 100644 --- a/esmvalcore/cmor/_fixes/native6/era5.py +++ b/esmvalcore/cmor/_fixes/native6/era5.py @@ -560,11 +560,11 @@ def fix_metadata(self, cubes): # If desired, regrid native ERA5 data in GRIB format (which is on a # reduced Gaussian grid, i.e., unstructured grid) if ( - self.extra_facets.get('target_grid', False) and + self.extra_facets.get('regrid', False) is not False and has_unstructured_grid(cube) ): cube = _bilinear_unstructured_regrid( - cube, self.extra_facets['target_grid'] + cube, **self.extra_facets['regrid'] ) cube = self._fix_coordinates(cube) diff --git a/esmvalcore/config/extra_facets/native6-mappings.yml b/esmvalcore/config/extra_facets/native6-mappings.yml index a7c7db95ff..18b72908c6 100644 --- a/esmvalcore/config/extra_facets/native6-mappings.yml +++ b/esmvalcore/config/extra_facets/native6-mappings.yml @@ -14,7 +14,8 @@ ERA5: '*': '*': family: E5 - target_grid: 0.25x0.25 + regrid: + target_grid: 0.25x0.25 type: an typeid: '00' version: '' # necessary to get a nice output file name diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py index 243dff09aa..6f9b770b42 100644 --- a/tests/integration/cmor/_fixes/native6/test_era5.py +++ b/tests/integration/cmor/_fixes/native6/test_era5.py @@ -1409,7 +1409,7 @@ def test_automatic_regrid(unstructured_grid_cubes): 'native6', 'era5', 'Amon', - target_grid='180x90', + regrid={'target_grid': '180x90'}, ) assert len(fixed_cubes) == 1 @@ -1459,7 +1459,7 @@ def test_no_automatic_regrid_regular_grid(): """Test no automatic regridding of regular grid data.""" cubes = era5_2d('monthly') fixed_cubes = fix_metadata( - cubes, 'ps', 'native6', 'era5', 'Amon', target_grid='1x1', + cubes, 'ps', 'native6', 'era5', 'Amon', regrid={'target_grid': '1x1'}, ) assert len(fixed_cubes) == 1 From b4972c9b4cf064eb012ce54c0af0aeab881bf044 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Tue, 10 Oct 2023 14:26:33 +0200 Subject: [PATCH 21/64] Added iris-grib to setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 84379ea008..b3ddaa79e8 100755 --- a/setup.py +++ b/setup.py @@ -40,6 +40,7 @@ 'geopy', 'humanfriendly', "importlib_metadata;python_version<'3.10'", + 'iris-grib', 'isodate', 'jinja2', 'nc-time-axis', # needed by iris.plot From 6be70595670c8971bae0a63c72eb878e3c78ec3c Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 10 Oct 2023 17:30:06 +0100 Subject: [PATCH 22/64] turn on GA tests --- .github/workflows/run-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index aa80283eb9..15a0d3989e 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -21,6 +21,7 @@ on: push: branches: - main + - read_era5_grib # run the test only if the PR is to main # turn it on if required #pull_request: From 908afc499cad853121a0e17eccf34c512872762d Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Thu, 25 Apr 2024 14:24:28 +0200 Subject: [PATCH 23/64] Removed unneeded changes --- esmvalcore/iris_helpers.py | 146 ++- esmvalcore/preprocessor/_regrid.py | 911 ++++++++++-------- .../unit/preprocessor/_regrid/test_regrid.py | 314 +++--- tests/unit/test_iris_helpers.py | 183 ++++ 4 files changed, 979 insertions(+), 575 deletions(-) diff --git a/esmvalcore/iris_helpers.py b/esmvalcore/iris_helpers.py index b7d7b7bcab..9b5fddbfe1 100644 --- a/esmvalcore/iris_helpers.py +++ b/esmvalcore/iris_helpers.py @@ -1,13 +1,16 @@ """Auxiliary functions for :mod:`iris`.""" -from typing import Dict, List, Sequence +from __future__ import annotations + +from typing import Dict, Iterable, List, Literal, Sequence import dask.array as da import iris import iris.cube import iris.util import numpy as np +from iris.coords import Coord from iris.cube import Cube -from iris.exceptions import CoordinateMultiDimError +from iris.exceptions import CoordinateMultiDimError, CoordinateNotFoundError from esmvalcore.typing import NetCDFAttr @@ -159,6 +162,138 @@ def merge_cube_attributes( cube.attributes = final_attributes +def _rechunk( + array: da.core.Array, + complete_dims: list[int], + remaining_dims: int | Literal['auto'], +) -> da.core.Array: + """Rechunk a given array so that it is not chunked along given dims.""" + new_chunks: list[str | int] = [remaining_dims] * array.ndim + for dim in complete_dims: + new_chunks[dim] = -1 + return array.rechunk(new_chunks) + + +def _rechunk_dim_metadata( + cube: Cube, + complete_dims: Iterable[int], + remaining_dims: int | Literal['auto'] = 'auto', +) -> None: + """Rechunk dimensional metadata of a cube (in-place).""" + # Non-dimensional coords that span complete_dims + # Note: dimensional coords are always realized (i.e., numpy arrays), so no + # chunking is necessary + for coord in cube.coords(dim_coords=False): + dims = cube.coord_dims(coord) + complete_dims_ = [dims.index(d) for d in complete_dims if d in dims] + if complete_dims_: + if coord.has_lazy_points(): + coord.points = _rechunk( + coord.lazy_points(), complete_dims_, remaining_dims + ) + if coord.has_bounds() and coord.has_lazy_bounds(): + coord.bounds = _rechunk( + coord.lazy_bounds(), complete_dims_, remaining_dims + ) + + # Rechunk cell measures that span complete_dims + for measure in cube.cell_measures(): + dims = cube.cell_measure_dims(measure) + complete_dims_ = [dims.index(d) for d in complete_dims if d in dims] + if complete_dims_ and measure.has_lazy_data(): + measure.data = _rechunk( + measure.lazy_data(), complete_dims_, remaining_dims + ) + + # Rechunk ancillary variables that span complete_dims + for anc_var in cube.ancillary_variables(): + dims = cube.ancillary_variable_dims(anc_var) + complete_dims_ = [dims.index(d) for d in complete_dims if d in dims] + if complete_dims_ and anc_var.has_lazy_data(): + anc_var.data = _rechunk( + anc_var.lazy_data(), complete_dims_, remaining_dims + ) + + +def rechunk_cube( + cube: Cube, + complete_coords: Iterable[Coord | str], + remaining_dims: int | Literal['auto'] = 'auto', +) -> Cube: + """Rechunk cube so that it is not chunked along given dimensions. + + This will rechunk the cube's data, but also all non-dimensional + coordinates, cell measures, and ancillary variables that span at least one + of the given dimensions. + + Note + ---- + This will only rechunk `dask` arrays. `numpy` arrays are not changed. + + Parameters + ---------- + cube: + Input cube. + complete_coords: + (Names of) coordinates along which the output cubes should not be + chunked. The given coordinates must span exactly 1 dimension. + remaining_dims: + Chunksize of the remaining dimensions. + + Returns + ------- + Cube + Rechunked cube. This will always be a copy of the input cube. + + """ + cube = cube.copy() # do not modify input cube + + # Make sure that complete_coords span exactly 1 dimension + complete_dims = [] + for coord in complete_coords: + coord = cube.coord(coord) + dims = cube.coord_dims(coord) + if len(dims) != 1: + raise CoordinateMultiDimError( + f"Complete coordinates must be 1D coordinates, got " + f"{len(dims):d}D coordinate '{coord.name()}'" + ) + complete_dims.append(dims[0]) + + # Rechunk data + if cube.has_lazy_data(): + cube.data = _rechunk(cube.lazy_data(), complete_dims, remaining_dims) + + # Rechunk dimensional metadata + _rechunk_dim_metadata(cube, complete_dims, remaining_dims=remaining_dims) + + return cube + + +def has_irregular_grid(cube: Cube) -> bool: + """Check if a cube has an irregular grid. + + Parameters + ---------- + cube: + Cube to be checked. + + Returns + ------- + bool + ``True`` if input cube has an irregular grid, else ``False``. + + """ + try: + lat = cube.coord('latitude') + lon = cube.coord('longitude') + except CoordinateNotFoundError: + return False + if lat.ndim == 2 and lon.ndim == 2: + return True + return False + + def has_unstructured_grid(cube: Cube) -> bool: """Check if a cube has an unstructured grid. @@ -173,10 +308,11 @@ def has_unstructured_grid(cube: Cube) -> bool: ``True`` if input cube has an unstructured grid, else ``False``. """ - if not cube.coords('latitude') or not cube.coords('longitude'): + try: + lat = cube.coord('latitude') + lon = cube.coord('longitude') + except CoordinateNotFoundError: return False - lat = cube.coord('latitude') - lon = cube.coord('longitude') if lat.ndim != 1 or lon.ndim != 1: return False if cube.coord_dims(lat) != cube.coord_dims(lon): diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py index 54101e745d..de43db1427 100644 --- a/esmvalcore/preprocessor/_regrid.py +++ b/esmvalcore/preprocessor/_regrid.py @@ -1,34 +1,51 @@ """Horizontal and vertical regridding module.""" +from __future__ import annotations +import functools import importlib import inspect import logging import os import re import ssl +import warnings from copy import deepcopy from decimal import Decimal +from functools import partial from pathlib import Path -from typing import Dict +from typing import TYPE_CHECKING, Any, Optional import dask.array as da import iris import numpy as np import stratify from geopy.geocoders import Nominatim -from iris.analysis import AreaWeighted, Linear, Nearest, UnstructuredNearest +from iris.analysis import AreaWeighted, Linear, Nearest from iris.cube import Cube -from iris.util import broadcast_to_shape -from scipy.spatial import Delaunay -from esmvalcore.iris_helpers import has_unstructured_grid +from esmvalcore.cmor._fixes.shared import ( + add_altitude_from_plev, + add_plev_from_altitude, +) +from esmvalcore.cmor.table import CMOR_TABLES +from esmvalcore.exceptions import ESMValCoreDeprecationWarning +from esmvalcore.iris_helpers import has_irregular_grid, has_unstructured_grid +from esmvalcore.preprocessor._other import get_array_module +from esmvalcore.preprocessor._shared import preserve_float_dtype +from esmvalcore.preprocessor._supplementary_vars import ( + add_ancillary_variable, + add_cell_measure, +) +from esmvalcore.preprocessor.regrid_schemes import ( + ESMPyAreaWeighted, + ESMPyLinear, + ESMPyNearest, + GenericFuncScheme, + UnstructuredNearest, +) -from ..cmor._fixes.shared import add_altitude_from_plev, add_plev_from_altitude -from ..cmor.table import CMOR_TABLES -from ._other import get_array_module -from ._regrid_esmpy import ESMF_REGRID_METHODS -from ._regrid_esmpy import regrid as esmpy_regrid -from ._supplementary_vars import add_ancillary_variable, add_cell_measure +if TYPE_CHECKING: + from esmvalcore.dataset import Dataset logger = logging.getLogger(__name__) @@ -52,22 +69,29 @@ _LON_MAX = 360.0 _LON_RANGE = _LON_MAX - _LON_MIN -# A cached stock of standard horizontal target grids. -_CACHE: Dict[str, iris.cube.Cube] = {} - # Supported point interpolation schemes. POINT_INTERPOLATION_SCHEMES = { 'linear': Linear(extrapolation_mode='mask'), 'nearest': Nearest(extrapolation_mode='mask'), } -# Supported horizontal regridding schemes. -HORIZONTAL_SCHEMES = { +# Supported horizontal regridding schemes for regular grids +HORIZONTAL_SCHEMES_REGULAR = { + 'area_weighted': AreaWeighted(), 'linear': Linear(extrapolation_mode='mask'), - 'linear_extrapolate': Linear(extrapolation_mode='extrapolate'), 'nearest': Nearest(extrapolation_mode='mask'), - 'area_weighted': AreaWeighted(), - 'unstructured_nearest': UnstructuredNearest(), +} + +# Supported horizontal regridding schemes for irregular grids +HORIZONTAL_SCHEMES_IRREGULAR = { + 'area_weighted': ESMPyAreaWeighted(), + 'linear': ESMPyLinear(), + 'nearest': ESMPyNearest(), +} + +# Supported horizontal regridding schemes for unstructured grids +HORIZONTAL_SCHEMES_UNSTRUCTURED = { + 'nearest': UnstructuredNearest(), } # Supported vertical interpolation schemes. @@ -136,7 +160,7 @@ def _generate_cube_from_dimcoords(latdata, londata, circular: bool = False): Returns ------- - :class:`~iris.cube.Cube` + iris.cube.Cube """ lats = iris.coords.DimCoord(latdata, standard_name='latitude', @@ -159,11 +183,12 @@ def _generate_cube_from_dimcoords(latdata, londata, circular: bool = False): shape = (latdata.size, londata.size) dummy = np.empty(shape, dtype=np.dtype('int8')) coords_spec = [(lats, 0), (lons, 1)] - cube = iris.cube.Cube(dummy, dim_coords_and_dims=coords_spec) + cube = Cube(dummy, dim_coords_and_dims=coords_spec) return cube +@functools.lru_cache def _global_stock_cube(spec, lat_offset=True, lon_offset=True): """Create a stock cube. @@ -189,7 +214,7 @@ def _global_stock_cube(spec, lat_offset=True, lon_offset=True): Returns ------- - :class:`~iris.cube.Cube` + iris.cube.Cube """ dlon, dlat = parse_cell_spec(spec) mid_dlon, mid_dlat = dlon / 2, dlat / 2 @@ -284,7 +309,7 @@ def _regional_stock_cube(spec: dict): Returns ------- - :class:`~iris.cube.Cube`. + iris.cube.Cube """ latdata, londata = _spec_to_latlonvals(**spec) @@ -304,19 +329,6 @@ def add_bounds_from_step(coord, step): return cube -def _attempt_irregular_regridding(cube, scheme): - """Check if irregular regridding with ESMF should be used.""" - if isinstance(scheme, str) and scheme in ESMF_REGRID_METHODS: - try: - lat_dim = cube.coord('latitude').ndim - lon_dim = cube.coord('longitude').ndim - if lat_dim == lon_dim == 2: - return True - except iris.exceptions.CoordinateNotFoundError: - pass - return False - - def extract_location(cube, location, scheme): """Extract a point using a location name, with interpolation. @@ -419,7 +431,7 @@ def extract_point(cube, latitude, longitude, scheme): Returns ------- - :py:class:`~iris.cube.Cube` + iris.cube.Cube Returns a cube with the extracted point(s), and with adjusted latitude and longitude coordinates (see above). If desired point outside values for at least one coordinate, this cube will have fully @@ -474,19 +486,250 @@ def is_dataset(dataset): return hasattr(dataset, 'facets') -def regrid(cube, target_grid, scheme, lat_offset=True, lon_offset=True): +def _get_target_grid_cube( + cube: Cube, + target_grid: Cube | Dataset | Path | str | dict, + lat_offset: bool = True, + lon_offset: bool = True, +) -> Cube: + """Get target grid cube.""" + if is_dataset(target_grid): + target_grid = target_grid.copy() # type: ignore + target_grid.supplementaries.clear() # type: ignore + target_grid.files = [target_grid.files[0]] # type: ignore + target_grid_cube = target_grid.load() # type: ignore + elif isinstance(target_grid, (str, Path)) and os.path.isfile(target_grid): + target_grid_cube = iris.load_cube(target_grid) + elif isinstance(target_grid, str): + # Generate a target grid from the provided cell-specification + target_grid_cube = _global_stock_cube( + target_grid, lat_offset, lon_offset + ) + # Align the target grid coordinate system to the source + # coordinate system. + src_cs = cube.coord_system() + xcoord = target_grid_cube.coord(axis='x', dim_coords=True) + ycoord = target_grid_cube.coord(axis='y', dim_coords=True) + xcoord.coord_system = src_cs + ycoord.coord_system = src_cs + elif isinstance(target_grid, dict): + # Generate a target grid from the provided specification, + target_grid_cube = _regional_stock_cube(target_grid) + else: + target_grid_cube = target_grid + + if not isinstance(target_grid_cube, Cube): + raise ValueError(f'Expecting a cube, got {target_grid}.') + + return target_grid_cube + + +def _attempt_irregular_regridding(cube: Cube, scheme: str) -> bool: + """Check if irregular regridding with ESMF should be used.""" + if not has_irregular_grid(cube): + return False + if scheme not in HORIZONTAL_SCHEMES_IRREGULAR: + raise ValueError( + f"Regridding scheme '{scheme}' does not support irregular data, " + f"expected one of {list(HORIZONTAL_SCHEMES_IRREGULAR)}" + ) + return True + + +def _attempt_unstructured_regridding(cube: Cube, scheme: str) -> bool: + """Check if unstructured regridding should be used.""" + if not has_unstructured_grid(cube): + return False + if scheme not in HORIZONTAL_SCHEMES_UNSTRUCTURED: + raise ValueError( + f"Regridding scheme '{scheme}' does not support unstructured " + f"data, expected one of {list(HORIZONTAL_SCHEMES_UNSTRUCTURED)}" + ) + return True + + +def _load_scheme(src_cube: Cube, scheme: str | dict): + """Return scheme that can be used in :meth:`iris.cube.Cube.regrid`.""" + loaded_scheme: Any = None + + # Deprecations + if scheme == 'unstructured_nearest': + msg = ( + "The regridding scheme `unstructured_nearest` has been deprecated " + "in ESMValCore version 2.11.0 and is scheduled for removal in " + "version 2.13.0. Please use the scheme `nearest` instead. This is " + "an exact replacement for data on unstructured grids. Since " + "version 2.11.0, ESMValCore is able to determine the most " + "suitable regridding scheme based on the input data." + ) + warnings.warn(msg, ESMValCoreDeprecationWarning) + scheme = 'nearest' + + if scheme == 'linear_extrapolate': + msg = ( + "The regridding scheme `linear_extrapolate` has been deprecated " + "in ESMValCore version 2.11.0 and is scheduled for removal in " + "version 2.13.0. Please use a generic scheme with `reference: " + "iris.analysis:Linear` and `extrapolation_mode: extrapolate` " + "instead (see https://docs.esmvaltool.org/projects/ESMValCore/en/" + "latest/recipe/preprocessor.html#generic-regridding-schemes)." + "This is an exact replacement." + ) + warnings.warn(msg, ESMValCoreDeprecationWarning) + scheme = 'linear' + loaded_scheme = Linear(extrapolation_mode='extrapolate') + logger.debug("Loaded regridding scheme %s", loaded_scheme) + return loaded_scheme + + # Scheme is a dict -> assume this describes a generic regridding scheme + if isinstance(scheme, dict): + loaded_scheme = _load_generic_scheme(scheme) + + # Scheme is a str -> load appropriate regridding scheme depending on the + # type of input data + elif _attempt_irregular_regridding(src_cube, scheme): + loaded_scheme = HORIZONTAL_SCHEMES_IRREGULAR[scheme] + elif _attempt_unstructured_regridding(src_cube, scheme): + loaded_scheme = HORIZONTAL_SCHEMES_UNSTRUCTURED[scheme] + else: + loaded_scheme = HORIZONTAL_SCHEMES_REGULAR.get(scheme) + + if loaded_scheme is None: + raise ValueError( + f"Got invalid regridding scheme string '{scheme}', expected one " + f"of {list(HORIZONTAL_SCHEMES_REGULAR)}" + ) + + logger.debug("Loaded regridding scheme %s", loaded_scheme) + + return loaded_scheme + + +def _load_generic_scheme(scheme: dict): + """Load generic regridding scheme.""" + scheme = dict(scheme) # do not overwrite original scheme + + try: + object_ref = scheme.pop("reference") + except KeyError as key_err: + raise ValueError( + "No reference specified for generic regridding." + ) from key_err + module_name, separator, scheme_name = object_ref.partition(":") + try: + obj: Any = importlib.import_module(module_name) + except ImportError as import_err: + raise ValueError( + f"Could not import specified generic regridding module " + f"'{module_name}'. Please double check spelling and that the " + f"required module is installed." + ) from import_err + if separator: + for attr in scheme_name.split('.'): + obj = getattr(obj, attr) + + # If `obj` is a function that requires `src_cube` and `grid_cube`, use + # GenericFuncScheme + scheme_args = inspect.getfullargspec(obj).args + if 'src_cube' in scheme_args and 'grid_cube' in scheme_args: + loaded_scheme = GenericFuncScheme(obj, **scheme) + else: + loaded_scheme = obj(**scheme) + + return loaded_scheme + + +_CACHED_REGRIDDERS: dict[tuple, dict] = {} + + +def _get_regridder( + src_cube: Cube, + tgt_cube: Cube, + scheme: str | dict, + cache_weights: bool, +): + """Get regridder to actually perform regridding. + + Note + ---- + If possible, this uses an existing regridder to reduce runtime (see also + https://scitools-iris.readthedocs.io/en/latest/userguide/ + interpolation_and_regridding.html#caching-a-regridder.) + + """ + # (1) Weights caching enabled + if cache_weights: + # To search for a matching regridder in the cache, first check the + # regridding scheme name and shapes of source and target coordinates. + # Only if these match, check coordinates themselves (this is much more + # expensive). + coord_key = _get_coord_key(src_cube, tgt_cube) + name_shape_key = _get_name_and_shape_key(src_cube, tgt_cube, scheme) + if name_shape_key in _CACHED_REGRIDDERS: + # We cannot simply do a test for `coord_key in + # _CACHED_REGRIDDERS[shape_key]` below since the hash() of a + # coordinate is simply its id() (thus, coordinates loaded from two + # different files would never be considered equal) + for (key, regridder) in _CACHED_REGRIDDERS[name_shape_key].items(): + if key == coord_key: + return regridder + + # Regridder is not in cached -> return a new one and cache it + loaded_scheme = _load_scheme(src_cube, scheme) + regridder = loaded_scheme.regridder(src_cube, tgt_cube) + _CACHED_REGRIDDERS.setdefault(name_shape_key, {}) + _CACHED_REGRIDDERS[name_shape_key][coord_key] = regridder + + # (2) Weights caching disabled + else: + loaded_scheme = _load_scheme(src_cube, scheme) + regridder = loaded_scheme.regridder(src_cube, tgt_cube) + + return regridder + + +def _get_coord_key(src_cube: Cube, tgt_cube: Cube) -> tuple: + """Get dict key from coordinates.""" + src_lat = src_cube.coord('latitude') + src_lon = src_cube.coord('longitude') + tgt_lat = tgt_cube.coord('latitude') + tgt_lon = tgt_cube.coord('longitude') + return (src_lat, src_lon, tgt_lat, tgt_lon) + + +def _get_name_and_shape_key( + src_cube: Cube, + tgt_cube: Cube, + scheme: str | dict, +) -> tuple: + """Get dict key from scheme name and coordinate shapes.""" + name = str(scheme) + shapes = [c.shape for c in _get_coord_key(src_cube, tgt_cube)] + return (name, *shapes) + + +@preserve_float_dtype +def regrid( + cube: Cube, + target_grid: Cube | Dataset | Path | str | dict, + scheme: str | dict, + lat_offset: bool = True, + lon_offset: bool = True, + cache_weights: bool = False, +) -> Cube: """Perform horizontal regridding. - Note that the target grid can be a cube (:py:class:`~iris.cube.Cube`), - path to a cube (``str``), a grid spec (``str``) in the form - of `MxN`, or a ``dict`` specifying the target grid. + Note that the target grid can be a :class:`~iris.cube.Cube`, a + :class:`~esmvalcore.dataset.Dataset`, a path to a cube + (:class:`~pathlib.Path` or :obj:`str`), a grid spec (:obj:`str`) in the + form of `MxN`, or a :obj:`dict` specifying the target grid. - For the latter, the ``target_grid`` should be a ``dict`` with the + For the latter, the `target_grid` should be a :obj:`dict` with the following keys: - ``start_longitude``: longitude at the center of the first grid cell. - ``end_longitude``: longitude at the center of the last grid cell. - - ``step_longitude``: constant longitude distance between grid cell \ + - ``step_longitude``: constant longitude distance between grid cell centers. - ``start_latitude``: latitude at the center of the first grid cell. - ``end_latitude``: longitude at the center of the last grid cell. @@ -494,39 +737,48 @@ def regrid(cube, target_grid, scheme, lat_offset=True, lon_offset=True): Parameters ---------- - cube : :py:class:`~iris.cube.Cube` + cube: The source cube to be regridded. - target_grid : Cube or str or dict + target_grid: The (location of a) cube that specifies the target or reference grid for the regridding operation. + Alternatively, a :class:`~esmvalcore.dataset.Dataset` can be provided. Alternatively, a string cell specification may be provided, of the form ``MxN``, which specifies the extent of the cell, longitude by latitude (degrees) for a global, regular target grid. Alternatively, a dictionary with a regional target grid may be specified (see above). - scheme : str or dict - The regridding scheme to perform. If both source and target grid are - structured (regular or irregular), can be one of the built-in schemes - ``linear``, ``linear_extrapolate``, ``nearest``, ``area_weighted``, - ``unstructured_nearest``. - Alternatively, a `dict` that specifies generic regridding (see below). - lat_offset : bool - Offset the grid centers of the latitude coordinate w.r.t. the - pole by half a grid step. This argument is ignored if ``target_grid`` - is a cube or file. - lon_offset : bool + scheme: + The regridding scheme to perform. If the source grid is structured + (regular or irregular), can be one of the built-in schemes ``linear``, + ``nearest``, ``area_weighted``. If the source grid is unstructured, can + be one of the built-in schemes ``nearest``. Alternatively, a `dict` + that specifies generic regridding can be given (see below). + lat_offset: + Offset the grid centers of the latitude coordinate w.r.t. the pole by + half a grid step. This argument is ignored if `target_grid` is a cube + or file. + lon_offset: Offset the grid centers of the longitude coordinate w.r.t. Greenwich - meridian by half a grid step. - This argument is ignored if ``target_grid`` is a cube or file. + meridian by half a grid step. This argument is ignored if + `target_grid` is a cube or file. + cache_weights: + If ``True``, cache regridding weights for later usage. This can speed + up the regridding of different datasets with similar source and target + grids massively, but may take up a lot of memory for extremely + high-resolution data. This option is ignored for schemes that do not + support weights caching. More details on this are given in the section + on :ref:`caching_regridding_weights`. To clear the cache, use + :func:`esmvalcore.preprocessor.regrid.cache_clear`. Returns ------- - :py:class:`~iris.cube.Cube` + iris.cube.Cube Regridded cube. See Also -------- - extract_levels : Perform vertical regridding. + extract_levels: Perform vertical regridding. Notes ----- @@ -567,105 +819,44 @@ def regrid(cube, target_grid, scheme, lat_offset=True, lon_offset=True): reference: esmf_regrid.schemes:ESMFAreaWeighted """ - if is_dataset(target_grid): - target_grid = target_grid.copy() - target_grid.supplementaries.clear() - target_grid.files = [target_grid.files[0]] - target_grid = target_grid.load() - elif isinstance(target_grid, (str, Path)) and os.path.isfile(target_grid): - target_grid = iris.load_cube(target_grid) - elif isinstance(target_grid, str): - # Generate a target grid from the provided cell-specification, - # and cache the resulting stock cube for later use. - target_grid = _CACHE.setdefault( - target_grid, - _global_stock_cube(target_grid, lat_offset, lon_offset), - ) - # Align the target grid coordinate system to the source - # coordinate system. - src_cs = cube.coord_system() - xcoord = target_grid.coord(axis='x', dim_coords=True) - ycoord = target_grid.coord(axis='y', dim_coords=True) - xcoord.coord_system = src_cs - ycoord.coord_system = src_cs - elif isinstance(target_grid, dict): - # Generate a target grid from the provided specification, - target_grid = _regional_stock_cube(target_grid) - - if not isinstance(target_grid, iris.cube.Cube): - raise ValueError(f'Expecting a cube, got {target_grid}.') - - if isinstance(scheme, dict): - scheme = dict(scheme) # do not overwrite original scheme - try: - object_ref = scheme.pop("reference") - except KeyError as key_err: - raise ValueError( - "No reference specified for generic regridding.") from key_err - module_name, separator, scheme_name = object_ref.partition(":") - try: - obj = importlib.import_module(module_name) - except ImportError as import_err: - raise ValueError( - "Could not import specified generic regridding module. " - "Please double check spelling and that the required module is " - "installed.") from import_err - if separator: - for attr in scheme_name.split('.'): - obj = getattr(obj, attr) - - scheme_args = inspect.getfullargspec(obj).args - # Add source and target cubes as arguments if required - if 'src_cube' in scheme_args: - scheme['src_cube'] = cube - if 'grid_cube' in scheme_args: - scheme['grid_cube'] = target_grid - - loaded_scheme = obj(**scheme) - else: - loaded_scheme = HORIZONTAL_SCHEMES.get(scheme.lower()) - - if loaded_scheme is None: - emsg = 'Unknown regridding scheme, got {!r}.' - raise ValueError(emsg.format(scheme)) - - # Unstructured regridding requires x2 2d spatial coordinates, - # so ensure to purge any 1d native spatial dimension coordinates - # for the regridder. - if scheme == 'unstructured_nearest': - for axis in ['x', 'y']: - coords = cube.coords(axis=axis, dim_coords=True) - if coords: - [coord] = coords - cube.remove_coord(coord) + # Load target grid and select appropriate scheme + target_grid_cube = _get_target_grid_cube( + cube, target_grid, lat_offset=lat_offset, lon_offset=lon_offset, + ) # Horizontal grids from source and target (almost) match # -> Return source cube with target coordinates - if _horizontal_grid_is_close(cube, target_grid): + if _horizontal_grid_is_close(cube, target_grid_cube): for coord in ['latitude', 'longitude']: - cube.coord(coord).points = target_grid.coord(coord).points - cube.coord(coord).bounds = target_grid.coord(coord).bounds + cube.coord(coord).points = ( + target_grid_cube.coord(coord).core_points() + ) + cube.coord(coord).bounds = ( + target_grid_cube.coord(coord).core_bounds() + ) return cube - # Horizontal grids from source and target do not match - # -> Regrid - if _attempt_irregular_regridding(cube, scheme): - cube = esmpy_regrid(cube, target_grid, scheme) - elif isinstance(loaded_scheme, iris.cube.Cube): - # Return regridded cube in cases in which the - # scheme is a function f(src_cube, grid_cube) -> Cube - cube = loaded_scheme - else: - cube = _rechunk(cube, target_grid) - cube = cube.regrid(target_grid, loaded_scheme) + # Load scheme and reuse existing regridder if possible + if isinstance(scheme, str): + scheme = scheme.lower() + regridder = _get_regridder(cube, target_grid_cube, scheme, cache_weights) + + # Rechunk and actually perform the regridding + cube = _rechunk(cube, target_grid_cube) + cube = regridder(cube) return cube -def _rechunk( - cube: iris.cube.Cube, - target_grid: iris.cube.Cube, -) -> iris.cube.Cube: +def _cache_clear(): + """Clear regridding weights cache.""" + _CACHED_REGRIDDERS.clear() + + +regrid.cache_clear = _cache_clear # type: ignore + + +def _rechunk(cube: Cube, target_grid: Cube) -> Cube: """Re-chunk cube with optimal chunk sizes for target grid.""" if not cube.has_lazy_data() or cube.ndim < 3: # Only rechunk lazy multidimensional data @@ -702,29 +893,30 @@ def _rechunk( return cube -def _horizontal_grid_is_close(cube1, cube2): +def _horizontal_grid_is_close(cube1: Cube, cube2: Cube) -> bool: """Check if two cubes have the same horizontal grid definition. The result of the function is a boolean answer, if both cubes have the same horizontal grid definition. The function checks both longitude and latitude, based on extent and resolution. + Note + ---- + The current implementation checks if the bounds and the grid shapes are the + same. Exits on first difference. + Parameters ---------- - cube1 : cube + cube1: The first of the cubes to be checked. - cube2 : cube + cube2: The second of the cubes to be checked. Returns ------- bool + ``True`` if grids are close; ``False`` if not. - .. note:: - - The current implementation checks if the bounds and the - grid shapes are the same. - Exits on first difference. """ # Go through the 2 expected horizontal coordinates longitude and latitude. for coord in ['latitude', 'longitude']: @@ -783,7 +975,7 @@ def _create_cube(src_cube, data, src_levels, levels): # Construct the resultant cube with the interpolated data # and the source cube metadata. kwargs = deepcopy(src_cube.metadata)._asdict() - result = iris.cube.Cube(data, **kwargs) + result = Cube(data, **kwargs) # Add the appropriate coordinates to the cube, excluding # any coordinates that span the z-dimension of interpolation. @@ -832,28 +1024,90 @@ def _create_cube(src_cube, data, src_levels, levels): return result +def is_lazy_masked_data(array): + """Similar to `iris._lazy_data.is_lazy_masked_data`.""" + return isinstance(array, da.Array) and isinstance( + da.utils.meta_from_array(array), np.ma.MaskedArray) + + +def broadcast_to_shape(array, shape, dim_map, chunks=None): + """Copy of `iris.util.broadcast_to_shape` that allows specifying chunks.""" + if isinstance(array, da.Array): + if chunks is not None: + chunks = list(chunks) + for src_idx, tgt_idx in enumerate(dim_map): + # Only use the specified chunks along new dimensions or on + # dimensions that have size 1 in the source array. + if array.shape[src_idx] != 1: + chunks[tgt_idx] = array.chunks[src_idx] + broadcast = partial(da.broadcast_to, shape=shape, chunks=chunks) + else: + broadcast = partial(np.broadcast_to, shape=shape) + + n_orig_dims = len(array.shape) + n_new_dims = len(shape) - n_orig_dims + array = array.reshape(array.shape + (1,) * n_new_dims) + + # Get dims in required order. + array = np.moveaxis(array, range(n_orig_dims), dim_map) + new_array = broadcast(array) + + if np.ma.isMA(array): + # broadcast_to strips masks so we need to handle them explicitly. + mask = np.ma.getmask(array) + if mask is np.ma.nomask: + new_mask = np.ma.nomask + else: + new_mask = broadcast(mask) + new_array = np.ma.array(new_array, mask=new_mask) + + elif is_lazy_masked_data(array): + # broadcast_to strips masks so we need to handle them explicitly. + mask = da.ma.getmaskarray(array) + new_mask = broadcast(mask) + new_array = da.ma.masked_array(new_array, new_mask) + + return new_array + + def _vertical_interpolate(cube, src_levels, levels, interpolation, extrapolation): """Perform vertical interpolation.""" # Determine the source levels and axis for vertical interpolation. z_axis, = cube.coord_dims(cube.coord(axis='z', dim_coords=True)) - # Broadcast the 1d source cube vertical coordinate to fully - # describe the spatial extent that will be interpolated. - src_levels_broadcast = broadcast_to_shape(src_levels.points, cube.shape, - cube.coord_dims(src_levels)) + if cube.has_lazy_data(): + # Make source levels lazy if cube has lazy data. + src_points = src_levels.lazy_points() + else: + src_points = src_levels.core_points() + + # Broadcast the source cube vertical coordinate to fully describe the + # spatial extent that will be interpolated. + src_levels_broadcast = broadcast_to_shape( + src_points, + shape=cube.shape, + chunks=cube.lazy_data().chunks if cube.has_lazy_data() else None, + dim_map=cube.coord_dims(src_levels), + ) + + # Make the target levels lazy if the input data is lazy. + if cube.has_lazy_data() and isinstance(src_points, da.Array): + levels = da.asarray(levels) # force mask onto data as nan's npx = get_array_module(cube.core_data()) data = npx.ma.filled(cube.core_data(), np.nan) - # Now perform the actual vertical interpolation. - new_data = stratify.interpolate(levels, - src_levels_broadcast, - data, - axis=z_axis, - interpolation=interpolation, - extrapolation=extrapolation) + # Perform vertical interpolation. + new_data = stratify.interpolate( + levels, + src_levels_broadcast, + data, + axis=z_axis, + interpolation=interpolation, + extrapolation=extrapolation, + ) # Calculate the mask based on the any NaN values in the interpolated data. new_data = npx.ma.masked_where(npx.isnan(new_data), new_data) @@ -923,42 +1177,75 @@ def parse_vertical_scheme(scheme): return scheme, extrap_scheme -def extract_levels(cube, - levels, - scheme, - coordinate=None, - rtol=1e-7, - atol=None): +def _rechunk_aux_factory_dependencies( + cube: iris.cube.Cube, + coord_name: str, +) -> iris.cube.Cube: + """Rechunk coordinate aux factory dependencies. + + This ensures that the resulting coordinate has reasonably sized + chunks that are aligned with the cube data for optimal computational + performance. + """ + # Workaround for https://github.com/SciTools/iris/issues/5457 + try: + factory = cube.aux_factory(coord_name) + except iris.exceptions.CoordinateNotFoundError: + return cube + + cube = cube.copy() + cube_chunks = cube.lazy_data().chunks + for coord in factory.dependencies.values(): + coord_dims = cube.coord_dims(coord) + if coord_dims is not None: + coord = coord.copy() + chunks = tuple(cube_chunks[i] for i in coord_dims) + coord.points = coord.lazy_points().rechunk(chunks) + if coord.has_bounds(): + coord.bounds = coord.lazy_bounds().rechunk(chunks + (None, )) + cube.replace_coord(coord) + return cube + + +@preserve_float_dtype +def extract_levels( + cube: iris.cube.Cube, + levels: np.typing.ArrayLike | da.Array, + scheme: str, + coordinate: Optional[str] = None, + rtol: float = 1e-7, + atol: Optional[float] = None, +): """Perform vertical interpolation. Parameters ---------- - cube : iris.cube.Cube + cube: The source cube to be vertically interpolated. - levels : ArrayLike + levels: One or more target levels for the vertical interpolation. Assumed to be in the same S.I. units of the source cube vertical dimension coordinate. If the requested levels are sufficiently close to the levels of the cube, cube slicing will take place instead of interpolation. - scheme : str + scheme: The vertical interpolation scheme to use. Choose from 'linear', 'nearest', 'linear_extrapolate', 'nearest_extrapolate'. - coordinate : optional str + coordinate: The coordinate to interpolate. If specified, pressure levels (if present) can be converted to height levels and vice versa using the US standard atmosphere. E.g. 'coordinate = altitude' will convert existing pressure levels (air_pressure) to height levels (altitude); 'coordinate = air_pressure' will convert existing height levels (altitude) to pressure levels (air_pressure). - rtol : float + rtol: Relative tolerance for comparing the levels in `cube` to the requested levels. If the levels are sufficiently close, the requested levels will be assigned to the cube and no interpolation will take place. - atol : float + atol: Absolute tolerance for comparing the levels in `cube` to the requested levels. If the levels are sufficiently close, the requested levels will be assigned to the cube and no interpolation will take place. @@ -978,29 +1265,37 @@ def extract_levels(cube, interpolation, extrapolation = parse_vertical_scheme(scheme) # Ensure we have a non-scalar array of levels. - levels = np.array(levels, ndmin=1) - - # Get the source cube vertical coordinate, if available. - if coordinate: - coord_names = [coord.name() for coord in cube.coords()] - if coordinate not in coord_names: - # Try to calculate air_pressure from altitude coordinate or - # vice versa using US standard atmosphere for conversion. - if coordinate == 'air_pressure' and 'altitude' in coord_names: - # Calculate pressure level coordinate from altitude. - add_plev_from_altitude(cube) - if coordinate == 'altitude' and 'air_pressure' in coord_names: - # Calculate altitude coordinate from pressure levels. - add_altitude_from_plev(cube) - src_levels = cube.coord(coordinate) + if not isinstance(levels, da.Array): + levels = np.array(levels, ndmin=1) + + # Try to determine the name of the vertical coordinate automatically + if coordinate is None: + coordinate = cube.coord(axis='z', dim_coords=True).name() + + # Add extra coordinates + coord_names = [coord.name() for coord in cube.coords()] + if coordinate in coord_names: + cube = _rechunk_aux_factory_dependencies(cube, coordinate) else: - src_levels = cube.coord(axis='z', dim_coords=True) + # Try to calculate air_pressure from altitude coordinate or + # vice versa using US standard atmosphere for conversion. + if coordinate == 'air_pressure' and 'altitude' in coord_names: + # Calculate pressure level coordinate from altitude. + cube = _rechunk_aux_factory_dependencies(cube, 'altitude') + add_plev_from_altitude(cube) + if coordinate == 'altitude' and 'air_pressure' in coord_names: + # Calculate altitude coordinate from pressure levels. + cube = _rechunk_aux_factory_dependencies(cube, 'air_pressure') + add_altitude_from_plev(cube) + + src_levels = cube.coord(coordinate) if (src_levels.shape == levels.shape and np.allclose( - src_levels.points, + src_levels.core_points(), levels, rtol=rtol, - atol=1e-7 * np.mean(src_levels.points) if atol is None else atol, + atol=1e-7 * + np.mean(src_levels.core_points()) if atol is None else atol, )): # Only perform vertical extraction/interpolation if the source # and target levels are not "similar" enough. @@ -1011,7 +1306,9 @@ def extract_levels(cube, set(levels).issubset(set(src_levels.points)): # If all target levels exist in the source cube, simply extract them. name = src_levels.name() - coord_values = {name: lambda cell: cell.point in set(levels)} + coord_values = { + name: lambda cell: cell.point in set(levels) # type: ignore + } constraint = iris.Constraint(coord_values=coord_values) result = cube.extract(constraint) # Ensure the constraint did not fail. @@ -1101,6 +1398,7 @@ def get_reference_levels(dataset): return coord.points.tolist() +@preserve_float_dtype def extract_coordinate_points(cube, definition, scheme): """Extract points from any coordinate with interpolation. @@ -1121,7 +1419,7 @@ def extract_coordinate_points(cube, definition, scheme): Returns ------- - :py:class:`~iris.cube.Cube` + iris.cube.Cube Returns a cube with the extracted point(s), and with adjusted latitude and longitude coordinates (see above). If desired point outside values for at least one coordinate, this cube will have fully @@ -1138,214 +1436,3 @@ def extract_coordinate_points(cube, definition, scheme): raise ValueError(msg) cube = cube.interpolate(definition.items(), scheme=scheme) return cube - - -def _bilinear_unstructured_regrid( - cube: Cube, - target_grid: str, - lat_offset: bool = True, - lon_offset: bool = True, -) -> Cube: - """Bilinear regridding for unstructured grids. - - The spatial dimension of the data (i.e., the one describing the - unstructured grid) needs to be the rightmost dimension. - - Note - ---- - This private function has been introduced to regrid native ERA5 in GRIB - format similarly to how it is done if you download an interpolated versions - of ERA5 (see - https://confluence.ecmwf.int/display/CKB/ERA5%3A+What+is+the+spatial+reference). - - Currently, we do not support bilinear regridding for unstructured grids in - our `regrid` preprocessor (only nearest-neighbor). Since iris is currently - doing a massive overhaul of their in-built regridding - (https://github.com/SciTools/iris/issues/4754), it does not make sense to - include the following piece of code in their package just now. - - Thus, we provide this function here. Please be aware that it can be removed - at any point in time without prior warning (just like any other private - function). - - Warning - ------- - This function will drop all cell measures, ancillary variables and aux - factories, and any auxiliary coordinate that spans the spatial dimension. - - """ - # This function should only be called on unstructured grid cubes - if not has_unstructured_grid(cube): - raise ValueError( - f"Cube {cube.summary(shorten=True)} does not have unstructured " - f"grid" - ) - - # The unstructured grid dimension needs to be the rightmost dimension - udim = cube.coord_dims('latitude')[0] - if udim != cube.ndim - 1: - raise ValueError( - f"The spatial dimension of cube {cube.summary(shorten=True)} " - f"(i.e, the one describing the unstructured grid) needs to be the " - f"rightmost dimension" - ) - - # Generate a target grid from the provided cell-specification, and cache - # the resulting stock cube for later use - tgt_cube = _CACHE.setdefault( - target_grid, - _global_stock_cube(target_grid, lat_offset, lon_offset), - ) - - # Make sure the cube has lazy data and rechunk it properly (cube cannot be - # chunked along latitude and longitude dimension) - if not cube.has_lazy_data(): - cube.data = da.from_array(cube.data) - in_chunks = ['auto'] * cube.ndim - in_chunks[udim] = -1 # type: ignore - cube.data = cube.lazy_data().rechunk(in_chunks) - - # Calculate indices and interpolation weights - (indices, weights) = _get_linear_interpolation_weights( - cube, target_grid, lat_offset, lon_offset - ) - - # Perform actual regridding - regridded_data = da.apply_gufunc( - _interpolate, - '(i),(j,3),(j,3)->(j)', - cube.lazy_data(), - indices, - weights, - vectorize=True, - output_dtypes=cube.dtype, - ) - regridded_data = regridded_data.rechunk('auto') - - # Create new cube with correct metadata - dim_coords_and_dims = [ - (c, cube.coord_dims(c)) for c in cube.coords(dim_coords=True) if - udim not in cube.coord_dims(c) - ] - dim_coords_and_dims.extend([ - (tgt_cube.coord('latitude'), cube.ndim - 1), - (tgt_cube.coord('longitude'), cube.ndim), - ]) - aux_coords_and_dims = [ - (c, cube.coord_dims(c)) for c in cube.coords(dim_coords=False) if - udim not in cube.coord_dims(c) - ] - new_shape = cube.shape[:-1] + tgt_cube.shape - regridded_cube = Cube( - regridded_data.reshape(new_shape, limit='128MiB'), - standard_name=cube.standard_name, - long_name=cube.long_name, - var_name=cube.var_name, - units=cube.units, - attributes=cube.attributes, - cell_methods=cube.cell_methods, - dim_coords_and_dims=dim_coords_and_dims, - aux_coords_and_dims=aux_coords_and_dims, - ) - - return regridded_cube - - -_CACHE_WEIGHTS: Dict[str, tuple[np.ndarray, np.ndarray]] = {} - - -def _get_linear_interpolation_weights( - src_cube: Cube, - target_grid: str, - lat_offset: bool = True, - lon_offset: bool = True, - fill_value: float = np.nan, -) -> tuple[np.ndarray, np.ndarray]: - """Get vertices and weights for 2D linear regridding of unstructured grids. - - Partly taken from - https://stackoverflow.com/questions/20915502/speedup-scipy-griddata-for-multiple-interpolations-between-two-irregular-grids. - This is more than 80x faster than :func:`scipy.interpolate.griddata` and - gives identical results. - - """ - # Cache result to avoid re-calculating this over and over - src_lat = src_cube.coord('latitude') - src_lon = src_cube.coord('longitude') - cache_key = ( - f"{src_lat.shape}_" - f"{src_lat.points[0]}-{src_lat.points[-1]}-{src_lat.units}_" - f"{src_lon.points[0]}-{src_lon.points[-1]}-{src_lon.units}_" - f"{target_grid}_" - f"{lat_offset}_" - f"{lon_offset}_" - f"{fill_value}_" - ) - if cache_key in _CACHE_WEIGHTS: - return _CACHE_WEIGHTS[cache_key] - - # Generate a target grid from the provided cell-specification, and cache - # the resulting stock cube for later use - tgt_cube = _CACHE.setdefault( - target_grid, - _global_stock_cube(target_grid, lat_offset, lon_offset), - ) - - # Bring points into correct format - # src_points: (N, 2) where N is the number of source grid points - # tgt_points: (M, 2) where M is the number of target grid points - src_points = np.stack((src_lat.points, src_lon.points), axis=-1) - (tgt_lat, tgt_lon) = np.meshgrid( - tgt_cube.coord('latitude').points, - tgt_cube.coord('longitude').points, - indexing='ij', - ) - tgt_points = np.stack((tgt_lat.ravel(), tgt_lon.ravel()), axis=-1) - - # Actual indices and weights calculation using Delaunay triagulation - # Return shapes: - # indices: (M, 3) - # weights: (M, 3) - n_dims = 2 - tri = Delaunay(src_points) - simplex = tri.find_simplex(tgt_points) - extra_idx = (simplex == -1) - indices = np.take(tri.simplices, simplex, axis=0) - temp = np.take(tri.transform, simplex, axis=0) - delta = tgt_points - temp[:, n_dims] - bary = np.einsum('njk,nk->nj', temp[:, :n_dims, :], delta) - weights = np.hstack((bary, 1 - bary.sum(axis=1, keepdims=True))) - weights[extra_idx, :] = fill_value - - # Cache result - _CACHE_WEIGHTS[cache_key] = (indices, weights) - - return (indices, weights) - - -def _interpolate( - data: np.ndarray, - indices: np.ndarray, - weights: np.ndarray, -) -> np.ndarray: - """Interpolate data. - - Parameters - ---------- - data: np.ndarray - Data to interpolate. Must be an (N,) array, where N is the number of - source grid points. - indices: np.ndarray - Indices used to index the data. Must be an (M, 3) array, where M is the - number of target grid points. - weights: np.ndarray - Interpolation weights. Must be an (M, 3) array, where M is the number - of target grid points. - - Returns - ------- - np.ndarray - Interpolated data of shape (M,). - - """ - return np.einsum('nj,nj->n', np.take(data, indices), weights) diff --git a/tests/unit/preprocessor/_regrid/test_regrid.py b/tests/unit/preprocessor/_regrid/test_regrid.py index d934ed55a1..f7ffff1228 100644 --- a/tests/unit/preprocessor/_regrid/test_regrid.py +++ b/tests/unit/preprocessor/_regrid/test_regrid.py @@ -1,168 +1,26 @@ """Unit tests for the :func:`esmvalcore.preprocessor.regrid.regrid` function.""" - -import unittest -from unittest import mock - import dask import dask.array as da import iris import numpy as np import pytest -import tests +import esmvalcore.preprocessor._regrid from esmvalcore.preprocessor import regrid from esmvalcore.preprocessor._regrid import ( - _CACHE, - HORIZONTAL_SCHEMES, - _attempt_irregular_regridding, + _get_regridder, _horizontal_grid_is_close, _rechunk, ) -class Test(tests.Test): - - def _check(self, tgt_grid, scheme, spec=False): - expected_scheme = HORIZONTAL_SCHEMES[scheme] - - if spec: - spec = tgt_grid - self.assertIn(spec, _CACHE) - self.assertEqual(_CACHE[spec], self.tgt_grid) - self.coord_system.asset_called_once() - expected_calls = [ - mock.call(axis='x', dim_coords=True), - mock.call(axis='y', dim_coords=True) - ] - self.assertEqual(self.tgt_grid_coord.mock_calls, expected_calls) - self.regrid.assert_called_once_with(self.tgt_grid, expected_scheme) - else: - if scheme == 'unstructured_nearest': - expected_calls = [ - mock.call(axis='x', dim_coords=True), - mock.call(axis='y', dim_coords=True) - ] - self.assertEqual(self.coords.mock_calls, expected_calls) - expected_calls = [mock.call(self.coord), mock.call(self.coord)] - self.assertEqual(self.remove_coord.mock_calls, expected_calls) - self.regrid.assert_called_once_with(tgt_grid, expected_scheme) - - # Reset the mocks to enable multiple calls per test-case. - for mocker in self.mocks: - mocker.reset_mock() - - def setUp(self): - self.coord_system = mock.Mock(return_value=None) - self.coord = mock.sentinel.coord - self.coords = mock.Mock(return_value=[self.coord]) - self.remove_coord = mock.Mock() - self.regridded_cube = mock.Mock() - self.regridded_cube.data = mock.sentinel.data - self.regridded_cube_data = mock.Mock() - self.regridded_cube.core_data.return_value = self.regridded_cube_data - self.regrid = mock.Mock(return_value=self.regridded_cube) - self.src_cube = mock.Mock( - spec=iris.cube.Cube, - coord_system=self.coord_system, - coords=self.coords, - remove_coord=self.remove_coord, - regrid=self.regrid, - dtype=float, - ) - self.src_cube.ndim = 1 - self.tgt_grid_coord = mock.Mock() - self.tgt_grid = mock.Mock(spec=iris.cube.Cube, - coord=self.tgt_grid_coord) - self.regrid_schemes = [ - 'linear', 'linear_extrapolate', 'nearest', 'area_weighted', - 'unstructured_nearest' - ] - - def _mock_horizontal_grid_is_close(src, tgt): - return False - - self.patch('esmvalcore.preprocessor._regrid._horizontal_grid_is_close', - side_effect=_mock_horizontal_grid_is_close) - - def _return_mock_global_stock_cube( - spec, - lat_offset=True, - lon_offset=True, - ): - return self.tgt_grid - - self.mock_stock = self.patch( - 'esmvalcore.preprocessor._regrid._global_stock_cube', - side_effect=_return_mock_global_stock_cube) - self.mocks = [ - self.coord_system, self.coords, self.regrid, self.src_cube, - self.tgt_grid_coord, self.tgt_grid, self.mock_stock - ] - - def test_invalid_tgt_grid__unknown(self): - dummy = mock.sentinel.dummy - scheme = 'linear' - emsg = 'Expecting a cube' - with self.assertRaisesRegex(ValueError, emsg): - regrid(self.src_cube, dummy, scheme) - - def test_invalid_scheme__unknown(self): - emsg = 'Unknown regridding scheme' - with self.assertRaisesRegex(ValueError, emsg): - regrid(self.src_cube, self.src_cube, 'wibble') - - def test_horizontal_schemes(self): - self.assertEqual(set(HORIZONTAL_SCHEMES.keys()), - set(self.regrid_schemes)) - - def test_regrid__horizontal_schemes(self): - for scheme in self.regrid_schemes: - result = regrid(self.src_cube, self.tgt_grid, scheme) - self.assertEqual(result, self.regridded_cube) - self.assertEqual(result.data, mock.sentinel.data) - self._check(self.tgt_grid, scheme) - - def test_regrid__cell_specification(self): - # Clear cache before and after the test to avoid poisoning - # the cache with Mocked cubes - # https://github.com/ESMValGroup/ESMValCore/issues/953 - _CACHE.clear() - - specs = ['1x1', '2x2', '3x3', '4x4', '5x5'] - scheme = 'linear' - for spec in specs: - result = regrid(self.src_cube, spec, scheme) - self.assertEqual(result, self.regridded_cube) - self.assertEqual(result.data, mock.sentinel.data) - self._check(spec, scheme, spec=True) - self.assertEqual(set(_CACHE.keys()), set(specs)) - - _CACHE.clear() - - def test_regrid_generic_missing_reference(self): - emsg = "No reference specified for generic regridding." - with self.assertRaisesRegex(ValueError, emsg): - regrid(self.src_cube, '1x1', {}) - - def test_regrid_generic_invalid_reference(self): - emsg = "Could not import specified generic regridding module." - with self.assertRaisesRegex(ValueError, emsg): - regrid(self.src_cube, '1x1', {"reference": "this.does:not.exist"}) - - def test_regrid_generic_regridding(self): - regrid(self.src_cube, '1x1', {"reference": "iris.analysis:Linear"}) - - third_party_regridder = mock.Mock() - - def test_regrid_generic_third_party(self): - regrid( - self.src_cube, '1x1', { - "reference": "tests.unit.preprocessor._regrid.test_regrid:" - "Test.third_party_regridder", - "method": "good", - }) - self.third_party_regridder.assert_called_once_with(method="good") +@pytest.fixture(autouse=True) +def clear_regridder_cache(monkeypatch): + """Clear regridder cache before test runs.""" + monkeypatch.setattr( + esmvalcore.preprocessor._regrid, '_CACHED_REGRIDDERS', {} + ) def _make_coord(start: float, stop: float, step: int, *, name: str): @@ -182,8 +40,9 @@ def _make_cube(*, lat: tuple, lon: tuple): lon_coord = _make_coord(*lon, name='longitude') return iris.cube.Cube( - np.empty([len(lat_coord.points), - len(lon_coord.points)]), + np.zeros( + [len(lat_coord.points), len(lon_coord.points)], dtype=np.float32 + ), dim_coords_and_dims=[(lat_coord, 0), (lon_coord, 1)], ) @@ -200,6 +59,116 @@ def _make_cube(*, lat: tuple, lon: tuple): LAT_SPEC3 = (-90, 90, 18) LON_SPEC3 = (0, 360, 36) +# 30x30 +LAT_SPEC4 = (-75, 75, 30) +LON_SPEC4 = (15, 345, 30) + + +@pytest.fixture +def cube_10x10(): + """Test cube.""" + return _make_cube(lat=LAT_SPEC1, lon=LON_SPEC1) + + +@pytest.fixture +def cube_30x30(): + """Test cube.""" + return _make_cube(lat=LAT_SPEC4, lon=LON_SPEC4) + + +SCHEMES = ['area_weighted', 'linear', 'nearest'] + + +@pytest.mark.parametrize('cache_weights', [True, False]) +@pytest.mark.parametrize('scheme', SCHEMES) +def test_builtin_regridding(scheme, cache_weights, cube_10x10, cube_30x30): + """Test `regrid.`""" + _cached_regridders = esmvalcore.preprocessor._regrid._CACHED_REGRIDDERS + assert _cached_regridders == {} + + res = regrid(cube_10x10, cube_30x30, scheme, cache_weights=cache_weights) + + assert res.coord('latitude') == cube_30x30.coord('latitude') + assert res.coord('longitude') == cube_30x30.coord('longitude') + assert res.dtype == np.float32 + assert np.allclose(res.data, 0.0) + + if cache_weights: + assert len(_cached_regridders) == 1 + key = (scheme, (18,), (36,), (30,), (30,)) + assert key in _cached_regridders + else: + assert not _cached_regridders + + +@pytest.mark.parametrize('scheme', SCHEMES) +def test_invalid_target_grid(scheme, cube_10x10, mocker): + """Test `regrid.`""" + target_grid = mocker.sentinel.target_grid + msg = "Expecting a cube" + with pytest.raises(ValueError, match=msg): + regrid(cube_10x10, target_grid, scheme) + + +def test_invalid_scheme(cube_10x10, cube_30x30): + """Test `regrid.`""" + msg = "Got invalid regridding scheme string 'wibble'" + with pytest.raises(ValueError, match=msg): + regrid(cube_10x10, cube_30x30, 'wibble') + + +def test_regrid_generic_missing_reference(cube_10x10, cube_30x30): + """Test `regrid.`""" + msg = "No reference specified for generic regridding." + with pytest.raises(ValueError, match=msg): + regrid(cube_10x10, cube_30x30, {}) + + +def test_regrid_generic_invalid_reference(cube_10x10, cube_30x30): + """Test `regrid.`""" + msg = "Could not import specified generic regridding module." + with pytest.raises(ValueError, match=msg): + regrid(cube_10x10, cube_30x30, {'reference': 'this.does:not.exist'}) + + +@pytest.mark.parametrize('cache_weights', [True, False]) +def test_regrid_generic_regridding(cache_weights, cube_10x10, cube_30x30): + """Test `regrid.`""" + _cached_regridders = esmvalcore.preprocessor._regrid._CACHED_REGRIDDERS + assert _cached_regridders == {} + + cube_gen = regrid( + cube_10x10, + cube_30x30, + { + 'reference': 'iris.analysis:Linear', + 'extrapolation_mode': 'mask', + }, + cache_weights=cache_weights, + ) + cube_lin = regrid( + cube_10x10, cube_30x30, 'linear', cache_weights=cache_weights + ) + assert cube_gen.dtype == np.float32 + assert cube_lin.dtype == np.float32 + assert cube_gen == cube_lin + + if cache_weights: + assert len(_cached_regridders) == 2 + key_1 = ( + "{'reference': 'iris.analysis:Linear', 'extrapolation_mode': " + "'mask'}", + (18,), + (36,), + (30,), + (30,), + ) + key_2 = ('linear', (18,), (36,), (30,), (30,)) + assert key_1 in _cached_regridders + assert key_2 in _cached_regridders + else: + assert not _cached_regridders + @pytest.mark.parametrize( 'cube2_spec, expected', @@ -384,11 +353,40 @@ def test_no_rechunk_unsupported_grid(): assert result.core_data().chunks == expected_chunks -def test_attempt_irregular_regridding(): - """Test `_attempt_irregular_regridding`.""" - result = _attempt_irregular_regridding(iris.cube.Cube(0), 'linear') - assert result is False +@pytest.mark.parametrize('scheme', SCHEMES) +def test_regridding_weights_use_cache(scheme, cube_10x10, cube_30x30, mocker): + """Test `regrid.`""" + _cached_regridders = esmvalcore.preprocessor._regrid._CACHED_REGRIDDERS + assert _cached_regridders == {} + + src_lat = cube_10x10.coord('latitude') + src_lon = cube_10x10.coord('longitude') + tgt_lat = cube_30x30.coord('latitude') + tgt_lon = cube_30x30.coord('longitude') + key = (scheme, (18,), (36,), (30,), (30,)) + _cached_regridders[key] = {} + _cached_regridders[key][(src_lat, src_lon, tgt_lat, tgt_lon)] = ( + mocker.sentinel.regridder + ) + mock_load_scheme = mocker.patch.object( + esmvalcore.preprocessor._regrid, '_load_scheme', autospec=True + ) + + reg = _get_regridder(cube_10x10, cube_30x30, scheme, cache_weights=True) + + assert reg == mocker.sentinel.regridder + + assert len(_cached_regridders) == 1 + assert key in _cached_regridders + + mock_load_scheme.assert_not_called() + + +def test_clear_regridding_weights_cache(): + """Test `regrid.cache_clear().`""" + _cached_regridders = esmvalcore.preprocessor._regrid._CACHED_REGRIDDERS + _cached_regridders['test'] = 'test' + regrid.cache_clear() -if __name__ == '__main__': - unittest.main() + assert _cached_regridders == {} diff --git a/tests/unit/test_iris_helpers.py b/tests/unit/test_iris_helpers.py index 2b9f149939..0b742ffe30 100644 --- a/tests/unit/test_iris_helpers.py +++ b/tests/unit/test_iris_helpers.py @@ -4,6 +4,7 @@ from itertools import permutations from unittest import mock +import dask.array as da import numpy as np import pytest from cf_units import Unit @@ -20,8 +21,10 @@ from esmvalcore.iris_helpers import ( add_leading_dim_to_cube, date2num, + has_irregular_grid, has_unstructured_grid, merge_cube_attributes, + rechunk_cube, ) @@ -223,6 +226,122 @@ def test_merge_cube_attributes_1_cube(): assert_attribues_equal(cubes[0].attributes, expected_attributes) +@pytest.fixture +def cube_3d(): + """3D sample cube.""" + # DimCoords + x = DimCoord([0, 1, 2], var_name='x') + y = DimCoord([0, 1, 2], var_name='y') + z = DimCoord([0, 1, 2, 3], var_name='z') + + # AuxCoords + aux_x = AuxCoord( + da.ones(3, chunks=1), + bounds=da.ones((3, 3), chunks=(1, 1)), + var_name='aux_x', + ) + aux_z = AuxCoord(da.ones(4, chunks=1), var_name='aux_z') + aux_xy = AuxCoord(da.ones((3, 3), chunks=(1, 1)), var_name='xy') + aux_xz = AuxCoord(da.ones((3, 4), chunks=(1, 1)), var_name='xz') + aux_yz = AuxCoord(da.ones((3, 4), chunks=(1, 1)), var_name='yz') + aux_xyz = AuxCoord( + da.ones((3, 3, 4), chunks=(1, 1, 1)), + bounds=da.ones((3, 3, 4, 3), chunks=(1, 1, 1, 1)), + var_name='xyz', + ) + aux_coords_and_dims = [ + (aux_x, 0), + (aux_z, 2), + (aux_xy, (0, 1)), + (aux_xz, (0, 2)), + (aux_yz, (1, 2)), + (aux_xyz, (0, 1, 2)), + ] + + # CellMeasures and AncillaryVariables + cell_measure = CellMeasure( + da.ones((3, 4), chunks=(1, 1)), var_name='cell_measure' + ) + anc_var = AncillaryVariable( + da.ones((3, 4), chunks=(1, 1)), var_name='anc_var' + ) + + return Cube( + da.ones((3, 3, 4), chunks=(1, 1, 1)), + var_name='cube', + dim_coords_and_dims=[(x, 0), (y, 1), (z, 2)], + aux_coords_and_dims=aux_coords_and_dims, + cell_measures_and_dims=[(cell_measure, (1, 2))], + ancillary_variables_and_dims=[(anc_var, (0, 2))], + ) + + +def test_rechunk_cube_fully_lazy(cube_3d): + """Test ``rechunk_cube``.""" + input_cube = cube_3d.copy() + + x_coord = input_cube.coord('x') + result = rechunk_cube(input_cube, [x_coord, 'y'], remaining_dims=2) + + assert input_cube == cube_3d + assert result == cube_3d + assert result.core_data().chunksize == (3, 3, 2) + assert result.coord('aux_x').core_points().chunksize == (3,) + assert result.coord('aux_z').core_points().chunksize == (1,) + assert result.coord('xy').core_points().chunksize == (3, 3) + assert result.coord('xz').core_points().chunksize == (3, 2) + assert result.coord('yz').core_points().chunksize == (3, 2) + assert result.coord('xyz').core_points().chunksize == (3, 3, 2) + assert result.coord('aux_x').core_bounds().chunksize == (3, 2) + assert result.coord('aux_z').core_bounds() is None + assert result.coord('xy').core_bounds() is None + assert result.coord('xz').core_bounds() is None + assert result.coord('yz').core_bounds() is None + assert result.coord('xyz').core_bounds().chunksize == (3, 3, 2, 2) + assert result.cell_measure('cell_measure').core_data().chunksize == (3, 2) + assert result.ancillary_variable('anc_var').core_data().chunksize == (3, 2) + + +def test_rechunk_cube_partly_lazy(cube_3d): + """Test ``rechunk_cube``.""" + input_cube = cube_3d.copy() + + # Realize some arrays + input_cube.data + input_cube.coord('xyz').points + input_cube.coord('xyz').bounds + input_cube.cell_measure('cell_measure').data + + result = rechunk_cube(input_cube, ['x', 'y'], remaining_dims=2) + + assert input_cube == cube_3d + assert result == cube_3d + assert not result.has_lazy_data() + assert result.coord('aux_x').core_points().chunksize == (3,) + assert result.coord('aux_z').core_points().chunksize == (1,) + assert result.coord('xy').core_points().chunksize == (3, 3) + assert result.coord('xz').core_points().chunksize == (3, 2) + assert result.coord('yz').core_points().chunksize == (3, 2) + assert not result.coord('xyz').has_lazy_points() + assert result.coord('aux_x').core_bounds().chunksize == (3, 2) + assert result.coord('aux_z').core_bounds() is None + assert result.coord('xy').core_bounds() is None + assert result.coord('xz').core_bounds() is None + assert result.coord('yz').core_bounds() is None + assert not result.coord('xyz').has_lazy_bounds() + assert not result.cell_measure('cell_measure').has_lazy_data() + assert result.ancillary_variable('anc_var').core_data().chunksize == (3, 2) + + +def test_rechunk_cube_invalid_coord_fail(cube_3d): + """Test ``rechunk_cube``.""" + msg = ( + "Complete coordinates must be 1D coordinates, got 2D coordinate 'xy'" + ) + with pytest.raises(CoordinateMultiDimError, match=msg): + rechunk_cube(cube_3d, ['xy']) + + @pytest.fixture def lat_coord_1d(): """1D latitude coordinate.""" @@ -247,6 +366,70 @@ def lon_coord_2d(): return AuxCoord([[0, 1]], standard_name='longitude') +def test_has_irregular_grid_no_lat_lon(): + """Test `has_irregular_grid`.""" + cube = Cube(0) + assert has_irregular_grid(cube) is False + + +def test_has_irregular_grid_no_lat(lon_coord_2d): + """Test `has_irregular_grid`.""" + cube = Cube([[0, 1]], aux_coords_and_dims=[(lon_coord_2d, (0, 1))]) + assert has_irregular_grid(cube) is False + + +def test_has_irregular_grid_no_lon(lat_coord_2d): + """Test `has_irregular_grid`.""" + cube = Cube([[0, 1]], aux_coords_and_dims=[(lat_coord_2d, (0, 1))]) + assert has_irregular_grid(cube) is False + + +def test_has_irregular_grid_1d_lon(lat_coord_2d, lon_coord_1d): + """Test `has_irregular_grid`.""" + cube = Cube( + [[0, 1]], + dim_coords_and_dims=[(lon_coord_1d, 1)], + aux_coords_and_dims=[(lat_coord_2d, (0, 1))], + ) + assert has_irregular_grid(cube) is False + + +def test_has_irregular_grid_1d_lat(lat_coord_1d, lon_coord_2d): + """Test `has_irregular_grid`.""" + cube = Cube( + [[0, 1]], + dim_coords_and_dims=[(lat_coord_1d, 1)], + aux_coords_and_dims=[(lon_coord_2d, (0, 1))], + ) + assert has_irregular_grid(cube) is False + + +def test_has_irregular_grid_1d_lat_lon(lat_coord_1d, lon_coord_1d): + """Test `has_irregular_grid`.""" + cube = Cube( + [0, 1], aux_coords_and_dims=[(lat_coord_1d, 0), (lon_coord_1d, 0)] + ) + assert has_irregular_grid(cube) is False + + +def test_has_irregular_grid_regular_grid(lat_coord_1d, lon_coord_1d): + """Test `has_irregular_grid`.""" + cube = Cube( + [[0, 1], [2, 3]], + dim_coords_and_dims=[(lat_coord_1d, 0), (lon_coord_1d, 1)], + ) + assert has_irregular_grid(cube) is False + + +def test_has_irregular_grid_true(lat_coord_2d, lon_coord_2d): + """Test `has_irregular_grid`.""" + cube = Cube( + [[0, 1]], + aux_coords_and_dims=[(lat_coord_2d, (0, 1)), (lon_coord_2d, (0, 1))], + ) + assert has_irregular_grid(cube) is True + + def test_has_unstructured_grid_no_lat_lon(): """Test `has_unstructured_grid`.""" cube = Cube(0) From 660f870e432c62fbc27ac695bf83621cf7642396 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 10 Jun 2024 16:12:19 +0200 Subject: [PATCH 24/64] Remove unused test --- .../cmor/_fixes/native6/test_era5.py | 40 +------------------ 1 file changed, 2 insertions(+), 38 deletions(-) diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py index aa449f5eb2..df349fdf87 100644 --- a/tests/integration/cmor/_fixes/native6/test_era5.py +++ b/tests/integration/cmor/_fixes/native6/test_era5.py @@ -1459,31 +1459,8 @@ def unstructured_grid_cubes(): return CubeList([cube]) -def test_automatic_regrid(unstructured_grid_cubes): - """Test automatic regridding of unstructured data.""" - fixed_cubes = fix_metadata( - unstructured_grid_cubes, - 'tas', - 'native6', - 'era5', - 'Amon', - regrid={'target_grid': '180x90'}, - ) - - assert len(fixed_cubes) == 1 - fixed_cube = fixed_cubes[0] - assert fixed_cube.shape == (2, 2, 2) - - assert fixed_cube.coords('time', dim_coords=True) - assert fixed_cube.coord_dims('time') == (0,) - assert fixed_cube.coords('latitude', dim_coords=True) - assert fixed_cube.coord_dims('latitude') == (1,) - assert fixed_cube.coords('longitude', dim_coords=True) - assert fixed_cube.coord_dims('longitude') == (2,) - - -def test_no_automatic_regrid(unstructured_grid_cubes): - """Test no automatic regridding of unstructured data.""" +def test_unstructured_grid(unstructured_grid_cubes): + """Test processing unstructured data.""" fixed_cubes = fix_metadata( unstructured_grid_cubes, 'tas', @@ -1511,16 +1488,3 @@ def test_no_automatic_regrid(unstructured_grid_cubes): lon = fixed_cube.coord('longitude') np.testing.assert_allclose(lon.points, [179, 180, 180, 179]) assert lon.bounds is None - - -def test_no_automatic_regrid_regular_grid(): - """Test no automatic regridding of regular grid data.""" - cubes = era5_2d('monthly') - fixed_cubes = fix_metadata( - cubes, 'ps', 'native6', 'era5', 'Amon', regrid={'target_grid': '1x1'}, - ) - - assert len(fixed_cubes) == 1 - fixed_cube = fixed_cubes[0] - assert fixed_cube.shape == (3, 3, 3) - assert fixed_cube == cmor_2d('Amon', 'ps')[0] From 0488179902670d8aafa0e086fb4f03e2bc681a95 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Tue, 11 Jun 2024 09:52:47 +0200 Subject: [PATCH 25/64] Unrun GA tests --- .github/workflows/run-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 1d96b9ee6c..9cf1d6308b 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -21,7 +21,6 @@ on: push: branches: - main - - read_era5_grib # run the test only if the PR is to main # turn it on if required #pull_request: From cac1dff603f0733c93388cefd79d96d5097df15c Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Tue, 11 Jun 2024 10:00:23 +0200 Subject: [PATCH 26/64] Remove unused extra facets --- esmvalcore/config/extra_facets/native6-mappings.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/esmvalcore/config/extra_facets/native6-mappings.yml b/esmvalcore/config/extra_facets/native6-mappings.yml index 18b72908c6..e7140bdc20 100644 --- a/esmvalcore/config/extra_facets/native6-mappings.yml +++ b/esmvalcore/config/extra_facets/native6-mappings.yml @@ -14,8 +14,6 @@ ERA5: '*': '*': family: E5 - regrid: - target_grid: 0.25x0.25 type: an typeid: '00' version: '' # necessary to get a nice output file name @@ -109,16 +107,6 @@ ERA5: level: pl grib_id: '129' - # unclear: - # 075: specific rain water content = rainmxrat27 ?? - # 076: specific snow water content = snowmxrat27 ?? - # 136: total column water - # 186: low cloud cover p > 0.8 ps - # 187: medium cloud cover 0.45 ps < p < 0.8 ps - # 188: high cloud cover p < 0.45 ps - # 235: skin temperature = ts ?? - # 243: forecast albedo - # MIP-specific settings AERday: '*': From 846930b410dee2c6a7275d639e634d5cac24d567 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Tue, 11 Jun 2024 10:18:41 +0200 Subject: [PATCH 27/64] Update docs to latest changes --- doc/quickstart/find_data.rst | 39 +++++++++++++----------------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index 341ef5e161..58665b4b72 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -148,19 +148,15 @@ ERA5 (in GRIB format available on DKRZ's Levante) ERA5 data in monthly, daily, and hourly resolution is `available on Levante `__ in its native GRIB format. -To read this data with ESMValCore, use the following settings in your -:ref:`user configuration file`: +To read these data with ESMValCore, use the root path ``/pool/data/ERA5`` with +DRS ``DKRZ-ERA5-GRIB`` in your :ref:`user configuration file`, for example: .. code-block:: yaml rootpath: ... - native6: /pool/data/ERA5 - ... - - drs: - ... - native6: DKRZ-ERA5-GRIB + native6: + /pool/data/ERA5: DKRZ-ERA5-GRIB ... The `naming conventions @@ -191,26 +187,19 @@ Thus, example dataset entries could look like this: The native ERA5 output in GRIB format is stored on a `reduced Gaussian grid `__. -By default, ESMValCore linearly interpolates the data to a regular 0.25° x -0.25° grid as `recommended by the ECMWF -`__. -If you want to use a different target resolution or completely disable this -feature, you can specify the optional facet ``regrid`` in the recipe. -``regrid`` takes the following keyword arguments: ``target_grid``, -``lat_offset`` (optional; default: ``True``), and ``lon_offset`` (optional; -default: ``True``). -See :func:`esmvalcore.preprocessor.regrid` for details on these keywords. -Example: +To regrid the data to a regular 0.25°x0.25° grid as `recommended by the ECMWF +`__, +you can use the following preprocessor: .. code-block:: yaml - datasets: - - {project: native6, dataset: ERA5, timerange: '2000/2001', - short_name: tas, mip: Amon, regrid: {target_grid: 1x1}} - - {project: native6, dataset: ERA5, timerange: '2000/2001', - short_name: tas, mip: Amon, regrid: {target_grid: 1x1, lat_offset: false}} - - {project: native6, dataset: ERA5, timerange: '2000/2001', - short_name: tas, mip: Amon, regrid: false} # do NOT interpolate + preprocessors: + regrid_era5: # this is an arbitrary name + regrid: + target_grid: 0.25x0.25 + scheme: linear + +See :ref:`Horizontal regridding` for details. - Supported variables: ``albsn``, ``cl``, ``cli``, ``clt``, ``clw``, ``hur``, ``hus``, ``o3``, ``prw``, ``ps``, ``psl``, ``rainmxrat27``, ``sftlf``, From e6125ab15d2ffb9bc9c113b11bfca0739361434b Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Tue, 11 Jun 2024 11:01:14 +0200 Subject: [PATCH 28/64] Do not fix time bounds for variables with no time dim coord --- esmvalcore/cmor/_fixes/fix.py | 2 ++ tests/integration/cmor/test_fix.py | 30 +++++++++++++++++++++++++++--- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/esmvalcore/cmor/_fixes/fix.py b/esmvalcore/cmor/_fixes/fix.py index cf2aed42ec..fd279147c8 100644 --- a/esmvalcore/cmor/_fixes/fix.py +++ b/esmvalcore/cmor/_fixes/fix.py @@ -826,6 +826,8 @@ def _fix_time_bounds(self, cube: Cube, cube_coord: Coord) -> None: """Fix time bounds.""" times = {'time', 'time1', 'time2', 'time3'} key = times.intersection(self.vardef.coordinates) + if not key: + return cmor = self.vardef.coordinates[' '.join(key)] if cmor.must_have_bounds == 'yes' and not cube_coord.has_bounds(): cube_coord.bounds = get_time_bounds(cube_coord, self.frequency) diff --git a/tests/integration/cmor/test_fix.py b/tests/integration/cmor/test_fix.py index e52dc0fd42..972ab54c28 100644 --- a/tests/integration/cmor/test_fix.py +++ b/tests/integration/cmor/test_fix.py @@ -437,9 +437,6 @@ def test_fix_metadata_amon_ta_wrong_lat_units(self): with pytest.raises(CMORCheckError): cmor_check_metadata(fixed_cube, project, mip, short_name) - print(self.mock_debug.mock_calls) - print(self.mock_warning.mock_calls) - assert self.mock_debug.call_count == 3 assert self.mock_warning.call_count == 9 @@ -907,3 +904,30 @@ def test_deprecate_check_level_fix_data(self): 'Amon', check_level=CheckLevels.RELAXED, ) + + def test_fix_metadata_no_time_in_table(self): + """Test ``fix_data``.""" + short_name = 'sftlf' + project = 'CMIP6' + dataset = '__MODEL_WITH_NO_EXPLICIT_FIX__' + mip = 'fx' + cube = self.cubes_2d_latlon[0][0] + cube.units = '%' + cube.data = da.full(cube.shape, 1.0, dtype=cube.dtype) + + fixed_cubes = fix_metadata( + [cube], + short_name, + project, + dataset, + mip, + ) + + assert len(fixed_cubes) == 1 + fixed_cube = fixed_cubes[0] + assert fixed_cube.has_lazy_data() + + cmor_check_metadata(fixed_cube, project, mip, short_name) + + assert self.mock_debug.call_count == 3 + assert self.mock_warning.call_count == 6 From d6bbf215aa37beeba0c3f5bfde8203a0e5c8bd9c Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Tue, 11 Jun 2024 11:01:50 +0200 Subject: [PATCH 29/64] Add ERA5-GRIB to Levante-specific options --- esmvalcore/config-user.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/esmvalcore/config-user.yml b/esmvalcore/config-user.yml index ecdee818fc..56310fbbff 100644 --- a/esmvalcore/config-user.yml +++ b/esmvalcore/config-user.yml @@ -204,7 +204,9 @@ drs: # /work/bd0854/DATA/ESMValTool2/OBS: default # /work/bd0854/DATA/ESMValTool2/download: ESGF # ana4mips: /work/bd0854/DATA/ESMValTool2/OBS -# native6: /work/bd0854/DATA/ESMValTool2/RAWOBS +# native6: +# /work/bd0854/DATA/ESMValTool2/RAWOBS: default +# /pool/data/ERA5: DKRZ-ERA5-GRIB # RAWOBS: /work/bd0854/DATA/ESMValTool2/RAWOBS #drs: # ana4mips: default From cc632dc4aed304d8751f200e729bf08c59da414b Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Thu, 4 Jul 2024 16:47:57 +0200 Subject: [PATCH 30/64] Re-enable automatic regridding --- doc/quickstart/find_data.rst | 20 +++++----- esmvalcore/cmor/_fixes/fix.py | 2 +- esmvalcore/cmor/_fixes/native6/era5.py | 16 ++++++++ .../config/extra_facets/native6-mappings.yml | 1 + .../cmor/_fixes/native6/test_era5.py | 38 ++++++++++++++++++- tests/integration/cmor/test_table.py | 2 +- 6 files changed, 67 insertions(+), 12 deletions(-) diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index 58665b4b72..5ca044dadc 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -187,19 +187,21 @@ Thus, example dataset entries could look like this: The native ERA5 output in GRIB format is stored on a `reduced Gaussian grid `__. -To regrid the data to a regular 0.25°x0.25° grid as `recommended by the ECMWF -`__, -you can use the following preprocessor: +By default, these data is regridded to a regular 0.25°x0.25° grid as +`recommended by the ECMWF +`__ +using bilinear interpolation. + +To disable this, you can use the facet ``regrid: false`` in the recipe: .. code-block:: yaml - preprocessors: - regrid_era5: # this is an arbitrary name - regrid: - target_grid: 0.25x0.25 - scheme: linear + datasets: + - {project: native6, dataset: ERA5, timerange: '2000/2001', + short_name: tas, mip: Amon, regrid: false} -See :ref:`Horizontal regridding` for details. +It is recommended to disable the default regridding if regridding is setup in +the :ref:`preprocessor `. - Supported variables: ``albsn``, ``cl``, ``cli``, ``clt``, ``clw``, ``hur``, ``hus``, ``o3``, ``prw``, ``ps``, ``psl``, ``rainmxrat27``, ``sftlf``, diff --git a/esmvalcore/cmor/_fixes/fix.py b/esmvalcore/cmor/_fixes/fix.py index fd279147c8..771a362a63 100644 --- a/esmvalcore/cmor/_fixes/fix.py +++ b/esmvalcore/cmor/_fixes/fix.py @@ -826,7 +826,7 @@ def _fix_time_bounds(self, cube: Cube, cube_coord: Coord) -> None: """Fix time bounds.""" times = {'time', 'time1', 'time2', 'time3'} key = times.intersection(self.vardef.coordinates) - if not key: + if not key: # cube has time, but CMOR variable does not return cmor = self.vardef.coordinates[' '.join(key)] if cmor.must_have_bounds == 'yes' and not cube_coord.has_bounds(): diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py index 71152e8719..51d470639e 100644 --- a/esmvalcore/cmor/_fixes/native6/era5.py +++ b/esmvalcore/cmor/_fixes/native6/era5.py @@ -10,9 +10,12 @@ from esmvalcore.cmor._fixes.shared import add_scalar_height_coord from esmvalcore.cmor.table import CMOR_TABLES from esmvalcore.iris_helpers import date2num, has_unstructured_grid +from esmvalcore.preprocessor import regrid logger = logging.getLogger(__name__) +DEFAULT_ERA5_GRID = '0.25x0.25' + def get_frequency(cube): """Determine time frequency of input cube.""" @@ -566,3 +569,16 @@ def fix_metadata(self, cubes): fixed_cubes.append(cube) return fixed_cubes + + def fix_data(self, cube): + """Fix data.""" + regridding_enabled = self.extra_facets.get('regrid', True) + if regridding_enabled and has_unstructured_grid(cube): + logger.debug( + "Automatically regrid ERA5 data (variable %s) to %s° " + "grid using bilinear regridding", + self.vardef.short_name, + DEFAULT_ERA5_GRID, + ) + cube = regrid(cube, DEFAULT_ERA5_GRID, 'linear') + return cube diff --git a/esmvalcore/config/extra_facets/native6-mappings.yml b/esmvalcore/config/extra_facets/native6-mappings.yml index e7140bdc20..0ebfe5939e 100644 --- a/esmvalcore/config/extra_facets/native6-mappings.yml +++ b/esmvalcore/config/extra_facets/native6-mappings.yml @@ -14,6 +14,7 @@ ERA5: '*': '*': family: E5 + regrid: true type: an typeid: '00' version: '' # necessary to get a nice output file name diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py index df349fdf87..8a5aa78118 100644 --- a/tests/integration/cmor/_fixes/native6/test_era5.py +++ b/tests/integration/cmor/_fixes/native6/test_era5.py @@ -8,6 +8,7 @@ from iris.coords import AuxCoord, DimCoord from iris.cube import Cube, CubeList +import esmvalcore.cmor._fixes.native6.era5 from esmvalcore.cmor._fixes.fix import Fix, GenericFix from esmvalcore.cmor._fixes.native6.era5 import ( AllVars, @@ -16,7 +17,7 @@ fix_accumulated_units, get_frequency, ) -from esmvalcore.cmor.fix import fix_metadata +from esmvalcore.cmor.fix import fix_data, fix_metadata from esmvalcore.cmor.table import CMOR_TABLES, get_var_info from esmvalcore.preprocessor import cmor_check_metadata @@ -1488,3 +1489,38 @@ def test_unstructured_grid(unstructured_grid_cubes): lon = fixed_cube.coord('longitude') np.testing.assert_allclose(lon.points, [179, 180, 180, 179]) assert lon.bounds is None + + +@pytest.mark.parametrize('regrid', [None, False, True]) +def test_automatic_regridding_unstructured_cube( + regrid, unstructured_grid_cubes, monkeypatch +): + """Test automatic regridding.""" + monkeypatch.setattr( + esmvalcore.cmor._fixes.native6.era5, 'DEFAULT_ERA5_GRID', '60x60' + ) + cube = unstructured_grid_cubes[0] + + fix_kwargs = {} + if regrid is not None: + fix_kwargs['regrid'] = regrid + fixed_cube = fix_data(cube, 'tas', 'native6', 'era5', 'Amon', **fix_kwargs) + + if regrid is None or regrid is True: + assert fixed_cube.shape == (2, 3, 6) + else: + assert fixed_cube.shape == (2, 4) + + +@pytest.mark.parametrize('regrid', [None, False, True]) +def test_automatic_regridding_regular_cube(regrid): + """Test automatic regridding.""" + cube = era5_2d('monthly')[0] + + fix_kwargs = {} + if regrid is not None: + fix_kwargs['regrid'] = regrid + fixed_cube = fix_data(cube, 'tas', 'native6', 'era5', 'Amon', **fix_kwargs) + + assert fixed_cube.shape == (3, 3, 3) + assert fixed_cube is cube diff --git a/tests/integration/cmor/test_table.py b/tests/integration/cmor/test_table.py index 680e5b4c01..76fc3374a9 100644 --- a/tests/integration/cmor/test_table.py +++ b/tests/integration/cmor/test_table.py @@ -485,7 +485,7 @@ def test_get_variable_ch4s(self): 'Atmosphere CH4 surface') self.assertEqual(var.units, '1e-09') - def test_get_variable_tosStderr(self): + def test_get_variable_tosstderr(self): """Get tosStderr variable.""" CustomInfo() var = self.variables_info.get_variable('Omon', 'tosStderr') From 4c3f1beb0e55fa70d0e7c9bdfcc1096231ecce12 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 8 Jul 2024 14:01:50 +0200 Subject: [PATCH 31/64] Rename extra facets file and add link to Levante doc --- .../extra_facets/{native6-mappings.yml => native6-era5.yml} | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) rename esmvalcore/config/extra_facets/{native6-mappings.yml => native6-era5.yml} (93%) diff --git a/esmvalcore/config/extra_facets/native6-mappings.yml b/esmvalcore/config/extra_facets/native6-era5.yml similarity index 93% rename from esmvalcore/config/extra_facets/native6-mappings.yml rename to esmvalcore/config/extra_facets/native6-era5.yml index 0ebfe5939e..d15f4aab79 100644 --- a/esmvalcore/config/extra_facets/native6-mappings.yml +++ b/esmvalcore/config/extra_facets/native6-era5.yml @@ -1,4 +1,8 @@ -# Extra facets for native6 data +# Extra facets for native6 ERA5 data in GRIB format +# +# See +# https://docs.dkrz.de/doc/dataservices/finding_and_accessing_data/era_data/index.html#file-and-directory-names +# for details on these facets. # Notes: # - All facets can also be specified in the recipes. The values given here are From 317cf2252597859a5287ba25cf74b0342fd3637e Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 8 Jul 2024 14:57:35 +0200 Subject: [PATCH 32/64] Fix doc build --- doc/quickstart/configure.rst | 2 +- doc/quickstart/find_data.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/quickstart/configure.rst b/doc/quickstart/configure.rst index cb5c733404..4e31fb82b7 100644 --- a/doc/quickstart/configure.rst +++ b/doc/quickstart/configure.rst @@ -959,7 +959,7 @@ infrastructure. The following example illustrates the concept. .. _extra-facets-example-1: .. code-block:: yaml - :caption: Extra facet example file `native6-era5.yml` + :caption: Extra facet example file `native6-era5-example.yml` ERA5: Amon: diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index 5ca044dadc..08d1370657 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -169,8 +169,8 @@ are All of these facets have reasonable defaults preconfigured in the corresponding :ref:`extra facets` file, which is available here: -:download:`native6-mappings.yml -`. +:download:`native6-era5.yml +`. If necessary, these facets can be overwritten in the recipe. Thus, example dataset entries could look like this: From 0fea4abd583b8f5fb5c1a6d1a2d300d6090a23ff Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 24 Jul 2024 11:08:29 +0200 Subject: [PATCH 33/64] Update version of ERA5 GRIB data in extra facets --- esmvalcore/config/extra_facets/native6-era5.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/config/extra_facets/native6-era5.yml b/esmvalcore/config/extra_facets/native6-era5.yml index d15f4aab79..fc514acb11 100644 --- a/esmvalcore/config/extra_facets/native6-era5.yml +++ b/esmvalcore/config/extra_facets/native6-era5.yml @@ -21,7 +21,7 @@ ERA5: regrid: true type: an typeid: '00' - version: '' # necessary to get a nice output file name + version: v1 # Variable-specific settings albsn: From bf4e2e4b9d8ed1d06aa5acc0b0e35fec43f3cca3 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 12 Sep 2024 22:49:22 +0200 Subject: [PATCH 34/64] Replace yapf and isort by ruff, drop docformatter (cherry picked from commit c742deb15ef0a0d040d8ec2b1096fdcb6c4c31c9) --- .pre-commit-config.yaml | 18 +++++------------- environment.yml | 3 --- pyproject.toml | 8 ++++++++ setup.cfg | 9 --------- setup.py | 3 --- 5 files changed, 13 insertions(+), 28 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index be9ead4e6d..c2bc1189ee 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,20 +29,12 @@ repos: rev: 'v2.2.4' hooks: - id: codespell - - repo: https://github.com/PyCQA/isort - rev: '5.12.0' + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: "v0.4.10" hooks: - - id: isort - - repo: https://github.com/pre-commit/mirrors-yapf - rev: 'v0.32.0' - hooks: - - id: yapf - additional_dependencies: - - 'toml' - - repo: https://github.com/myint/docformatter - rev: 'v1.6.5' - hooks: - - id: docformatter + - id: ruff + args: [ --fix ] + - id: ruff-format - repo: https://github.com/pycqa/flake8 rev: '6.0.0' hooks: diff --git a/environment.yml b/environment.yml index 9cfe3bfe56..672577b664 100644 --- a/environment.yml +++ b/environment.yml @@ -65,14 +65,11 @@ dependencies: - types-PyYAML # Python packages needed for installing in development mode - codespell - - docformatter - - isort - pre-commit - pylint - flake8 >= 7 - pydocstyle # Not on conda forge - vprof - yamllint - - yapf - pip: - ESMValTool_sample_data diff --git a/pyproject.toml b/pyproject.toml index 7a7d8388dc..04a1344318 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,3 +24,11 @@ disable = [ "file-ignored", # Disable messages about disabling checks "locally-disabled", # Disable messages about disabling checks ] +[tool.ruff] +line-length = 79 +[tool.ruff.lint] +select = [ + "I" # isort +] +[tool.ruff.lint.isort] +known-first-party = ["esmvalcore"] diff --git a/setup.cfg b/setup.cfg index e558e6860e..67f99da824 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,20 +30,11 @@ exclude_lines = [pydocstyle] convention = numpy -[isort] -multi_line_output = 3 -include_trailing_comma = true - [mypy] # see mypy.readthedocs.io/en/stable/command_line.html python_version = 3.12 ignore_missing_imports = True files = esmvalcore, tests -[yapf] -based_on_style = pep8 -# see https://github.com/google/yapf/issues/744 -blank_line_before_nested_class_or_def = true - [codespell] ignore-words-list = vas,hist diff --git a/setup.py b/setup.py index 4f8081b382..1b546c7fe8 100755 --- a/setup.py +++ b/setup.py @@ -90,15 +90,12 @@ # Use pip install -e .[develop] to install in development mode 'develop': [ 'codespell', - 'docformatter', - 'isort', 'flake8>=7', 'pre-commit', 'pylint', 'pydocstyle', 'vprof', 'yamllint', - 'yapf', ], } From 272bd49e0ea182696a4784542358c57fd4631345 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Sat, 14 Sep 2024 00:28:07 +0200 Subject: [PATCH 35/64] Replace flake8 by ruff and move to pre-commit.ci (cherry picked from commit dc556907232305b60a95d87e44f79f49aba74760) --- .circleci/config.yml | 1 - .editorconfig | 1 - .github/CODEOWNERS | 2 - .github/workflows/create-condalock-file.yml | 2 - .../workflows/install-from-condalock-file.yml | 1 - .github/workflows/run-tests.yml | 2 - .pre-commit-config.yaml | 12 +- .prospector.yml | 12 +- CITATION.cff | 2 +- NOTICE | 2 - doc/contributing.rst | 22 +- doc/quickstart/install.rst | 2 +- doc/quickstart/run.rst | 2 +- doc/recipe/index.rst | 1 - environment.yml | 11 +- esmvalcore/_main.py | 3 +- esmvalcore/_recipe/to_datasets.py | 5 +- .../config/extra_facets/access-mappings.yml | 33 +- pyproject.toml | 12 +- setup.cfg | 9 - setup.py | 37 +- tests/integration/data_finder.yml | 6 +- .../esgf/search_results/expected.yml | 586 +++++++++--------- tests/integration/recipe/test_recipe.py | 2 +- 24 files changed, 369 insertions(+), 399 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c4c9208cd2..4be031b9a3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -28,7 +28,6 @@ commands: mkdir -p test-reports . /opt/conda/etc/profile.d/conda.sh conda activate esmvaltool - flake8 -j 4 pytest -n 4 --junitxml=test-reports/report.xml esmvaltool version - store_test_results: diff --git a/.editorconfig b/.editorconfig index 97c8ef6e5a..ddab414f89 100644 --- a/.editorconfig +++ b/.editorconfig @@ -27,4 +27,3 @@ indent_size = 2 [*.{md,Rmd}] trim_trailing_whitespace = false - diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1c8122dcbe..1e5f05f0eb 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,3 +1 @@ -esmvalcore/cmor @jvegasbsc .github/workflows @valeriupredoi - diff --git a/.github/workflows/create-condalock-file.yml b/.github/workflows/create-condalock-file.yml index 1d425591e5..5e1eaec889 100644 --- a/.github/workflows/create-condalock-file.yml +++ b/.github/workflows/create-condalock-file.yml @@ -71,8 +71,6 @@ jobs: run: | esmvaltool --help esmvaltool version - - name: Run flake8 - run: flake8 - name: Run pytests run: pytest -n 2 -m "not installation" # Automated PR diff --git a/.github/workflows/install-from-condalock-file.yml b/.github/workflows/install-from-condalock-file.yml index fbc4a71903..44a7839b55 100644 --- a/.github/workflows/install-from-condalock-file.yml +++ b/.github/workflows/install-from-condalock-file.yml @@ -51,7 +51,6 @@ jobs: - run: pip install -e .[develop] - run: esmvaltool --help - run: esmvaltool version 2>&1 | tee source_install_linux_artifacts_python_${{ matrix.python-version }}/version.txt - - run: flake8 - run: pytest -n 2 -m "not installation" - name: Upload artifacts if: ${{ always() }} # upload artifacts even if fail diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index ac294e9f44..2843334067 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -60,7 +60,6 @@ jobs: - run: conda list - run: pip install -e .[develop] 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/install.txt - run: conda list - - run: flake8 - run: pytest -n 2 -m "not installation" 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/test_report.txt - name: Upload artifacts if: ${{ always() }} # upload artifacts even if fail @@ -97,7 +96,6 @@ jobs: - run: conda list - run: pip install -e .[develop] 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/install.txt - run: conda list - - run: flake8 - run: pytest -n 2 -m "not installation" 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/test_report.txt - name: Upload artifacts if: ${{ always() }} # upload artifacts even if fail diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c2bc1189ee..727a99da5a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,9 @@ exclude: | (?x) ^doc/conf.py| ^esmvalcore/cmor/tables/| - ^esmvalcore/preprocessor/ne_masks/ + ^esmvalcore/preprocessor/ne_masks/| + ^esmvalcore/preprocessor/shapefiles/ + repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 @@ -33,14 +35,10 @@ repos: rev: "v0.4.10" hooks: - id: ruff - args: [ --fix ] + args: [--fix] - id: ruff-format - - repo: https://github.com/pycqa/flake8 - rev: '6.0.0' - hooks: - - id: flake8 - repo: https://github.com/pre-commit/mirrors-mypy - rev: 'v1.2.0' + rev: 'v1.11.2' hooks: - id: mypy additional_dependencies: diff --git a/.prospector.yml b/.prospector.yml index f1272ec938..2f0936ca0f 100644 --- a/.prospector.yml +++ b/.prospector.yml @@ -10,16 +10,16 @@ test-warnings: true member-warnings: false pyroma: - run: true + run: true pep8: - full: true + full: true mypy: run: true pep257: - # disable rules that are allowed by the numpy convention - # see https://github.com/PyCQA/pydocstyle/blob/master/src/pydocstyle/violations.py - # and http://pydocstyle.readthedocs.io/en/latest/error_codes.html - disable: ['D107', 'D203', 'D212', 'D213', 'D402', 'D413', 'D416'] + # disable rules that are allowed by the numpy convention + # see https://github.com/PyCQA/pydocstyle/blob/master/src/pydocstyle/violations.py + # and http://pydocstyle.readthedocs.io/en/latest/error_codes.html + disable: ['D107', 'D203', 'D212', 'D213', 'D402', 'D413', 'D416'] diff --git a/CITATION.cff b/CITATION.cff index 51f53cfb36..4d3da6e4c7 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -208,7 +208,7 @@ authors: affiliation: "DLR, Germany" family-names: Cammarano given-names: Diego - - + - affiliation: "ACCESS-NRI, Australia" family-names: Yousong given-names: Zeng diff --git a/NOTICE b/NOTICE index 5bf9f5d0cd..5e413cd5ba 100644 --- a/NOTICE +++ b/NOTICE @@ -50,5 +50,3 @@ In addition to using the Software, we encourage the community to join the Softwa To join the ESMValTool Development Team, please contact Dr. Birgit Hassler (birgit.hassler@dlr.de) and Dr. Axel Lauer (axel.lauer@dlr.de). ========================================== - - diff --git a/doc/contributing.rst b/doc/contributing.rst index 814ab79263..658306a53b 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -743,15 +743,15 @@ Perform the steps listed below with two persons, to reduce the risk of error. `PyPI `__, and `readthedocs `__. -The release of ESMValCore is tied to the release of ESMValTool. +The release of ESMValCore is tied to the release of ESMValTool. The detailed steps can be found in the ESMValTool :ref:`documentation `. -To start the procedure, ESMValCore gets released as a +To start the procedure, ESMValCore gets released as a release candidate to test the recipes in ESMValTool. If bugs are found -during the testing phase of the release candidate, make as many release -candidates for ESMValCore as needed in order to fix them. +during the testing phase of the release candidate, make as many release +candidates for ESMValCore as needed in order to fix them. -To make a new release of the package, be it a release candidate or the final release, +To make a new release of the package, be it a release candidate or the final release, follow these steps: 1. Check that all tests and builds work @@ -795,13 +795,13 @@ Use the script to create create a draft of the release notes. This script uses the titles and labels of merged pull requests since the previous release. -Open a discussion to allow members of the development team to nominate pull -requests as highlights. Add the most voted pull requests as highlights at the -beginning of changelog. After the highlights section, list any backward -incompatible changes that the release may include. The +Open a discussion to allow members of the development team to nominate pull +requests as highlights. Add the most voted pull requests as highlights at the +beginning of changelog. After the highlights section, list any backward +incompatible changes that the release may include. The :ref:`backward compatibility policy`. -lists the information that should be provided by the developer of any backward -incompatible change. Make sure to also list any deprecations that the release +lists the information that should be provided by the developer of any backward +incompatible change. Make sure to also list any deprecations that the release may include, as well as a brief description on how to upgrade a deprecated feature. Review the results, and if anything needs changing, change it on GitHub and re-run the script until the changelog looks acceptable. diff --git a/doc/quickstart/install.rst b/doc/quickstart/install.rst index 7ef5015fe3..804c86ed20 100644 --- a/doc/quickstart/install.rst +++ b/doc/quickstart/install.rst @@ -201,7 +201,7 @@ Pre-installed versions on HPC clusters / other servers If you would like to use pre-installed versions on HPC clusters (currently CEDA-JASMIN and DKRZ-Levante), -and other servers (currently Met Office Linux estate), please have a look at +and other servers (currently Met Office Linux estate), please have a look at :ref:`these instructions `. diff --git a/doc/quickstart/run.rst b/doc/quickstart/run.rst index ebde6d4075..5eca15e714 100644 --- a/doc/quickstart/run.rst +++ b/doc/quickstart/run.rst @@ -79,7 +79,7 @@ This feature is available for projects that are hosted on the ESGF, i.e. CMIP3, CMIP5, CMIP6, CORDEX, and obs4MIPs. To control the strictness of the CMOR checker and the checks during concatenation -on auxiliary coordinates, supplementary variables, and derived coordinates, +on auxiliary coordinates, supplementary variables, and derived coordinates, use the flag ``--check_level``: .. code:: bash diff --git a/doc/recipe/index.rst b/doc/recipe/index.rst index 98c3f6c237..bdb57e2336 100644 --- a/doc/recipe/index.rst +++ b/doc/recipe/index.rst @@ -8,4 +8,3 @@ The recipe format Overview Preprocessor - \ No newline at end of file diff --git a/environment.yml b/environment.yml index 672577b664..eaf317965f 100644 --- a/environment.yml +++ b/environment.yml @@ -50,26 +50,17 @@ dependencies: - sphinx >=6.1.3 - pydata-sphinx-theme # Python packages needed for testing - - mypy >=0.990 - pytest >=3.9,!=6.0.0rc1,!=6.0.0 - pytest-cov >=2.10.1 - pytest-env - pytest-html !=2.1.0 - pytest-metadata >=1.5.1 - pytest-mock - - pytest-mypy - pytest-xdist # Not on conda-forge - ESMValTool_sample_data==0.0.3 - # Still for testing, MyPy library stubs - - types-requests - - types-PyYAML # Python packages needed for installing in development mode - - codespell - pre-commit - pylint - - flake8 >= 7 - - pydocstyle # Not on conda forge - vprof - - yamllint - pip: - - ESMValTool_sample_data + - ESMValTool_sample_data diff --git a/esmvalcore/_main.py b/esmvalcore/_main.py index 75c0b20c94..da77ec81d5 100755 --- a/esmvalcore/_main.py +++ b/esmvalcore/_main.py @@ -25,7 +25,8 @@ For further help, please read the documentation at http://docs.esmvaltool.org. Have fun! -""" # noqa: line-too-long pylint: disable=line-too-long +""" + # pylint: disable=import-outside-toplevel import logging import os diff --git a/esmvalcore/_recipe/to_datasets.py b/esmvalcore/_recipe/to_datasets.py index 962d732a9d..47520e9b9c 100644 --- a/esmvalcore/_recipe/to_datasets.py +++ b/esmvalcore/_recipe/to_datasets.py @@ -493,8 +493,9 @@ def _report_unexpanded_globs( timerange = expanded_ds.facets.get('timerange') patterns = expanded_ds._file_globs msg = ( - f"{msg}\nNo files found matching:\n" + - "\n".join(str(p) for p in patterns) + ( # type:ignore + f"{msg}\nNo files found matching:\n" + + "\n".join(str(p) for p in patterns) # type: ignore[union-attr] + + ( # type:ignore f"\nwithin the requested timerange {timerange}." if timerange else "" ) diff --git a/esmvalcore/config/extra_facets/access-mappings.yml b/esmvalcore/config/extra_facets/access-mappings.yml index 9d4eb0621b..b82899c261 100644 --- a/esmvalcore/config/extra_facets/access-mappings.yml +++ b/esmvalcore/config/extra_facets/access-mappings.yml @@ -7,23 +7,23 @@ ACCESS-ESM1-5: '*': - + tas: raw_name: fld_s03i236 modeling_realm: atm - + pr: raw_name: fld_s05i216 modeling_realm: atm - + ps: raw_name: fld_s00i409 modeling_realm: atm - + clt: raw_name: fld_s02i204 modeling_realm: atm - + psl: raw_name: fld_s16i222 modeling_realm: atm @@ -31,7 +31,7 @@ ACCESS-ESM1-5: hus: raw_name: fld_s30i205 modeling_realm: atm - + zg: raw_name: fld_s30i207 modeling_realm: atm @@ -39,30 +39,29 @@ ACCESS-ESM1-5: va: raw_name: fld_s30i202 modeling_realm: atm - + ua: raw_name: fld_s30i201 modeling_realm: atm - + ta: raw_name: fld_s30i204 modeling_realm: atm - + rlus: - raw_name: - - fld_s02i207 - - fld_s02i201 - - fld_s03i332 + raw_name: + - fld_s02i207 + - fld_s02i201 + - fld_s03i332 - fld_s02i205 modeling_realm: atm - + rlds: raw_name: fld_s02i207 modeling_realm: atm rsus: - raw_name: + raw_name: - fld_s01i235 - - fld_s01i201 + - fld_s01i201 modeling_realm: atm - diff --git a/pyproject.toml b/pyproject.toml index 04a1344318..fb723f24fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,6 +5,10 @@ build-backend = "setuptools.build_meta" [tool.setuptools_scm] version_scheme = "release-branch-semver" +[tool.codespell] +skip = "*.ipynb,esmvalcore/config/extra_facets/ipslcm-mappings.yml" +ignore-words-list = "vas,hist" + [tool.pylint.main] jobs = 1 # Running more than one job in parallel crashes prospector. ignore-paths = [ @@ -28,7 +32,13 @@ disable = [ line-length = 79 [tool.ruff.lint] select = [ - "I" # isort + "E", # pycodestyle + "F", # pyflakes + "I", # isort + "W", # pycodestyle +] +ignore = [ + "E501", # Disable line-too-long as this is taken care of by the formatter. ] [tool.ruff.lint.isort] known-first-party = ["esmvalcore"] diff --git a/setup.cfg b/setup.cfg index 67f99da824..8e2877b27e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,5 @@ [tool:pytest] addopts = - --mypy --doctest-modules --ignore=esmvalcore/cmor/tables/ --cov=esmvalcore @@ -14,11 +13,6 @@ markers = installation: Test requires installation of dependencies use_sample_data: Run functional tests using real data -[flake8] -exclude = - .eggs/ - doc/conf.py - [coverage:run] parallel = true [coverage:report] @@ -35,6 +29,3 @@ convention = numpy python_version = 3.12 ignore_missing_imports = True files = esmvalcore, tests - -[codespell] -ignore-words-list = vas,hist diff --git a/setup.py b/setup.py index 1b546c7fe8..2290dae157 100755 --- a/setup.py +++ b/setup.py @@ -62,21 +62,15 @@ 'yamale', ], # Test dependencies - 'test': [ - 'flake8>=7.0.0', # not to pick up E231 - 'pytest>=3.9,!=6.0.0rc1,!=6.0.0', - 'pytest-cov>=2.10.1', - 'pytest-env', - 'pytest-html!=2.1.0', - 'pytest-metadata>=1.5.1', - 'pytest-mypy>=0.10.3', # gh issue/2314 - 'pytest-mock', - 'pytest-xdist', - 'ESMValTool_sample_data==0.0.3', - # MyPy library stubs - 'mypy>=0.990', - 'types-requests', - 'types-PyYAML', + "test": [ + "pytest>=3.9,!=6.0.0rc1,!=6.0.0", + "pytest-cov>=2.10.1", + "pytest-env", + "pytest-html!=2.1.0", + "pytest-metadata>=1.5.1", + "pytest-mock", + "pytest-xdist", + "ESMValTool_sample_data==0.0.3", ], # Documentation dependencies 'doc': [ @@ -88,14 +82,11 @@ ], # Development dependencies # Use pip install -e .[develop] to install in development mode - 'develop': [ - 'codespell', - 'flake8>=7', - 'pre-commit', - 'pylint', - 'pydocstyle', - 'vprof', - 'yamllint', + "develop": [ + "pre-commit", + "pylint", + "pydocstyle", + "vprof", ], } diff --git a/tests/integration/data_finder.yml b/tests/integration/data_finder.yml index 40e0c3e821..9b90bc7da6 100644 --- a/tests/integration/data_finder.yml +++ b/tests/integration/data_finder.yml @@ -452,7 +452,7 @@ get_input_filelist: - ta_Amon_HadGEM2-ES_historical_r1i1p1*.nc found_files: - historical/Amon/ta/HadGEM2-ES/r1i1p1/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc - + - drs: NCI variable: <<: *variable @@ -472,7 +472,7 @@ get_input_filelist: found_files: - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_195912-198411.nc - MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ta/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc - + - drs: NCI variable: <<: *variable @@ -593,7 +593,7 @@ get_input_filelist: found_files: - historical/atmos/mon/ta/HADGEM1/r1i1p1/ta_HADGEM1_195001-199912.nc - historical/atmos/mon/ta/HADGEM1/r1i1p1/ta_HADGEM1_200001-200112.nc - + - drs: NCI variable: variable_group: test diff --git a/tests/integration/esgf/search_results/expected.yml b/tests/integration/esgf/search_results/expected.yml index 9e463c0cf7..24f02b9181 100644 --- a/tests/integration/esgf/search_results/expected.yml +++ b/tests/integration/esgf/search_results/expected.yml @@ -1,301 +1,301 @@ Amon_r1i1p1_historical,rcp85_INM-CM4_CMIP5_tas.json: -- checksums: - - - SHA256 - - 0c7cc5410d6f03b3a49b8de9e0ae8090249a66a8ebd27e6d17a78fba96eba3f9 - - - SHA256 - - 0c7cc5410d6f03b3a49b8de9e0ae8090249a66a8ebd27e6d17a78fba96eba3f9 - - - SHA256 - - 0c7cc5410d6f03b3a49b8de9e0ae8090249a66a8ebd27e6d17a78fba96eba3f9 - - - MD5 - - fc448373de679c8fcbfe031364049df1 - dataset: cmip5.output1.INM.inmcm4.historical.mon.atmos.Amon.r1i1p1.v20130207 - facets: - dataset: inmcm4 - ensemble: r1i1p1 - exp: historical - frequency: mon - institute: INM - mip: Amon - product: output1 - project: CMIP5 - modeling_realm: atmos - short_name: tas - version: v20130207 - local_file: cmip5/output1/INM/inmcm4/historical/mon/atmos/Amon/r1i1p1/v20130207/tas_Amon_inmcm4_historical_r1i1p1_185001-200512.nc - name: tas_Amon_inmcm4_historical_r1i1p1_185001-200512.nc - size: 161801172 - urls: - - http://aims3.llnl.gov/thredds/fileServer/cmip5_css02_data/cmip5/output1/INM/inmcm4/historical/mon/atmos/Amon/r1i1p1/tas/1/tas_Amon_inmcm4_historical_r1i1p1_185001-200512.nc - - http://esgf-data1.ceda.ac.uk/thredds/fileServer/esg_dataroot/cmip5/output1/INM/inmcm4/historical/mon/atmos/Amon/r1i1p1/v20130207/tas/tas_Amon_inmcm4_historical_r1i1p1_185001-200512.nc - - http://esgf.nci.org.au/thredds/fileServer/replica/CMIP5/output1/INM/inmcm4/historical/mon/atmos/Amon/r1i1p1/v20130207/tas/tas_Amon_inmcm4_historical_r1i1p1_185001-200512.nc - - http://esgf2.dkrz.de/thredds/fileServer/lta_dataroot/cmip5/output1/INM/inmcm4/historical/mon/atmos/Amon/r1i1p1/v20130207/tas/tas_Amon_inmcm4_historical_r1i1p1_185001-200512.nc -- checksums: - - - SHA256 - - 4a39b0fb2698ab4305df9070240a1b9ef88c80422f379e3086f67a0dfc2b3047 - - - SHA256 - - 4a39b0fb2698ab4305df9070240a1b9ef88c80422f379e3086f67a0dfc2b3047 - - - SHA256 - - 4a39b0fb2698ab4305df9070240a1b9ef88c80422f379e3086f67a0dfc2b3047 - - - MD5 - - b8885d3860e66b036db76ca6a49e7c51 - dataset: cmip5.output1.INM.inmcm4.rcp85.mon.atmos.Amon.r1i1p1.v20130207 - facets: - dataset: inmcm4 - ensemble: r1i1p1 - exp: rcp85 - frequency: mon - institute: INM - mip: Amon - product: output1 - project: CMIP5 - modeling_realm: atmos - short_name: tas - version: v20130207 - local_file: cmip5/output1/INM/inmcm4/rcp85/mon/atmos/Amon/r1i1p1/v20130207/tas_Amon_inmcm4_rcp85_r1i1p1_200601-210012.nc - name: tas_Amon_inmcm4_rcp85_r1i1p1_200601-210012.nc - size: 98538788 - urls: - - http://aims3.llnl.gov/thredds/fileServer/cmip5_css02_data/cmip5/output1/INM/inmcm4/rcp85/mon/atmos/Amon/r1i1p1/tas/1/tas_Amon_inmcm4_rcp85_r1i1p1_200601-210012.nc - - http://esgf-data1.ceda.ac.uk/thredds/fileServer/esg_dataroot/cmip5/output1/INM/inmcm4/rcp85/mon/atmos/Amon/r1i1p1/v20130207/tas/tas_Amon_inmcm4_rcp85_r1i1p1_200601-210012.nc - - http://esgf.nci.org.au/thredds/fileServer/replica/CMIP5/output1/INM/inmcm4/rcp85/mon/atmos/Amon/r1i1p1/v20130207/tas/tas_Amon_inmcm4_rcp85_r1i1p1_200601-210012.nc - - http://esgf2.dkrz.de/thredds/fileServer/lta_dataroot/cmip5/output1/INM/inmcm4/rcp85/mon/atmos/Amon/r1i1p1/v20130207/tas/tas_Amon_inmcm4_rcp85_r1i1p1_200601-210012.nc + - checksums: + - - SHA256 + - 0c7cc5410d6f03b3a49b8de9e0ae8090249a66a8ebd27e6d17a78fba96eba3f9 + - - SHA256 + - 0c7cc5410d6f03b3a49b8de9e0ae8090249a66a8ebd27e6d17a78fba96eba3f9 + - - SHA256 + - 0c7cc5410d6f03b3a49b8de9e0ae8090249a66a8ebd27e6d17a78fba96eba3f9 + - - MD5 + - fc448373de679c8fcbfe031364049df1 + dataset: cmip5.output1.INM.inmcm4.historical.mon.atmos.Amon.r1i1p1.v20130207 + facets: + dataset: inmcm4 + ensemble: r1i1p1 + exp: historical + frequency: mon + institute: INM + mip: Amon + product: output1 + project: CMIP5 + modeling_realm: atmos + short_name: tas + version: v20130207 + local_file: cmip5/output1/INM/inmcm4/historical/mon/atmos/Amon/r1i1p1/v20130207/tas_Amon_inmcm4_historical_r1i1p1_185001-200512.nc + name: tas_Amon_inmcm4_historical_r1i1p1_185001-200512.nc + size: 161801172 + urls: + - http://aims3.llnl.gov/thredds/fileServer/cmip5_css02_data/cmip5/output1/INM/inmcm4/historical/mon/atmos/Amon/r1i1p1/tas/1/tas_Amon_inmcm4_historical_r1i1p1_185001-200512.nc + - http://esgf-data1.ceda.ac.uk/thredds/fileServer/esg_dataroot/cmip5/output1/INM/inmcm4/historical/mon/atmos/Amon/r1i1p1/v20130207/tas/tas_Amon_inmcm4_historical_r1i1p1_185001-200512.nc + - http://esgf.nci.org.au/thredds/fileServer/replica/CMIP5/output1/INM/inmcm4/historical/mon/atmos/Amon/r1i1p1/v20130207/tas/tas_Amon_inmcm4_historical_r1i1p1_185001-200512.nc + - http://esgf2.dkrz.de/thredds/fileServer/lta_dataroot/cmip5/output1/INM/inmcm4/historical/mon/atmos/Amon/r1i1p1/v20130207/tas/tas_Amon_inmcm4_historical_r1i1p1_185001-200512.nc + - checksums: + - - SHA256 + - 4a39b0fb2698ab4305df9070240a1b9ef88c80422f379e3086f67a0dfc2b3047 + - - SHA256 + - 4a39b0fb2698ab4305df9070240a1b9ef88c80422f379e3086f67a0dfc2b3047 + - - SHA256 + - 4a39b0fb2698ab4305df9070240a1b9ef88c80422f379e3086f67a0dfc2b3047 + - - MD5 + - b8885d3860e66b036db76ca6a49e7c51 + dataset: cmip5.output1.INM.inmcm4.rcp85.mon.atmos.Amon.r1i1p1.v20130207 + facets: + dataset: inmcm4 + ensemble: r1i1p1 + exp: rcp85 + frequency: mon + institute: INM + mip: Amon + product: output1 + project: CMIP5 + modeling_realm: atmos + short_name: tas + version: v20130207 + local_file: cmip5/output1/INM/inmcm4/rcp85/mon/atmos/Amon/r1i1p1/v20130207/tas_Amon_inmcm4_rcp85_r1i1p1_200601-210012.nc + name: tas_Amon_inmcm4_rcp85_r1i1p1_200601-210012.nc + size: 98538788 + urls: + - http://aims3.llnl.gov/thredds/fileServer/cmip5_css02_data/cmip5/output1/INM/inmcm4/rcp85/mon/atmos/Amon/r1i1p1/tas/1/tas_Amon_inmcm4_rcp85_r1i1p1_200601-210012.nc + - http://esgf-data1.ceda.ac.uk/thredds/fileServer/esg_dataroot/cmip5/output1/INM/inmcm4/rcp85/mon/atmos/Amon/r1i1p1/v20130207/tas/tas_Amon_inmcm4_rcp85_r1i1p1_200601-210012.nc + - http://esgf.nci.org.au/thredds/fileServer/replica/CMIP5/output1/INM/inmcm4/rcp85/mon/atmos/Amon/r1i1p1/v20130207/tas/tas_Amon_inmcm4_rcp85_r1i1p1_200601-210012.nc + - http://esgf2.dkrz.de/thredds/fileServer/lta_dataroot/cmip5/output1/INM/inmcm4/rcp85/mon/atmos/Amon/r1i1p1/v20130207/tas/tas_Amon_inmcm4_rcp85_r1i1p1_200601-210012.nc Amon_r1i1p1_historical_FIO-ESM_CMIP5_tas.json: -- checksums: - - - SHA256 - - b6d4b62ccba4cb8141e422d17a31d618a59161c72bd10476391811b693cbff6c - - - SHA256 - - b6d4b62ccba4cb8141e422d17a31d618a59161c72bd10476391811b693cbff6c - - - MD5 - - 970cc36b75466a30cf02b7ae0896a9b0 - - - SHA256 - - b6d4b62ccba4cb8141e422d17a31d618a59161c72bd10476391811b693cbff6c - dataset: cmip5.output1.FIO.FIO-ESM.historical.mon.atmos.Amon.r1i1p1.v20121010 - facets: - dataset: FIO-ESM - ensemble: r1i1p1 - exp: historical - frequency: mon - institute: FIO - mip: Amon - product: output1 - project: CMIP5 - modeling_realm: atmos - short_name: tas - version: v20121010 - local_file: cmip5/output1/FIO/FIO-ESM/historical/mon/atmos/Amon/r1i1p1/v20121010/tas_Amon_FIO-ESM_historical_r1i1p1_185001-200512.nc - name: tas_Amon_FIO-ESM_historical_r1i1p1_185001-200512.nc - size: 61398952 - urls: - - http://esgf-data1.ceda.ac.uk/thredds/fileServer/esg_dataroot/cmip5/output1/FIO/FIO-ESM/historical/mon/atmos/Amon/r1i1p1/v20121010/tas/tas_Amon_FIO-ESM_historical_r1i1p1_185001-200512.nc - - http://esgf.nci.org.au/thredds/fileServer/replica/CMIP5/output1/FIO/FIO-ESM/historical/mon/atmos/Amon/r1i1p1/v20121010/tas/tas_Amon_FIO-ESM_historical_r1i1p1_185001-200512.nc - - http://esgf2.dkrz.de/thredds/fileServer/lta_dataroot/cmip5/output1/FIO/FIO-ESM/historical/mon/atmos/Amon/r1i1p1/v20121010/tas/tas_Amon_FIO-ESM_historical_r1i1p1_185001-200512.nc - - http://aims3.llnl.gov/thredds/fileServer/cmip5_css02_data/cmip5/output1/FIO/fio-esm/historical/mon/atmos/Amon/r1i1p1/v20121010/tas/tas_Amon_FIO-ESM_historical_r1i1p1_185001-200512.nc + - checksums: + - - SHA256 + - b6d4b62ccba4cb8141e422d17a31d618a59161c72bd10476391811b693cbff6c + - - SHA256 + - b6d4b62ccba4cb8141e422d17a31d618a59161c72bd10476391811b693cbff6c + - - MD5 + - 970cc36b75466a30cf02b7ae0896a9b0 + - - SHA256 + - b6d4b62ccba4cb8141e422d17a31d618a59161c72bd10476391811b693cbff6c + dataset: cmip5.output1.FIO.FIO-ESM.historical.mon.atmos.Amon.r1i1p1.v20121010 + facets: + dataset: FIO-ESM + ensemble: r1i1p1 + exp: historical + frequency: mon + institute: FIO + mip: Amon + product: output1 + project: CMIP5 + modeling_realm: atmos + short_name: tas + version: v20121010 + local_file: cmip5/output1/FIO/FIO-ESM/historical/mon/atmos/Amon/r1i1p1/v20121010/tas_Amon_FIO-ESM_historical_r1i1p1_185001-200512.nc + name: tas_Amon_FIO-ESM_historical_r1i1p1_185001-200512.nc + size: 61398952 + urls: + - http://esgf-data1.ceda.ac.uk/thredds/fileServer/esg_dataroot/cmip5/output1/FIO/FIO-ESM/historical/mon/atmos/Amon/r1i1p1/v20121010/tas/tas_Amon_FIO-ESM_historical_r1i1p1_185001-200512.nc + - http://esgf.nci.org.au/thredds/fileServer/replica/CMIP5/output1/FIO/FIO-ESM/historical/mon/atmos/Amon/r1i1p1/v20121010/tas/tas_Amon_FIO-ESM_historical_r1i1p1_185001-200512.nc + - http://esgf2.dkrz.de/thredds/fileServer/lta_dataroot/cmip5/output1/FIO/FIO-ESM/historical/mon/atmos/Amon/r1i1p1/v20121010/tas/tas_Amon_FIO-ESM_historical_r1i1p1_185001-200512.nc + - http://aims3.llnl.gov/thredds/fileServer/cmip5_css02_data/cmip5/output1/FIO/fio-esm/historical/mon/atmos/Amon/r1i1p1/v20121010/tas/tas_Amon_FIO-ESM_historical_r1i1p1_185001-200512.nc Amon_r1i1p1_rcp85_HadGEM2-CC_CMIP5_tas.json: -- checksums: - - - SHA256 - - 10a94293f2ac844ab62496d5d5369ccc0e839c73882a323c21800d71d7780315 - - - SHA256 - - 10a94293f2ac844ab62496d5d5369ccc0e839c73882a323c21800d71d7780315 - dataset: cmip5.output1.MOHC.HadGEM2-CC.rcp85.mon.atmos.Amon.r1i1p1.v20120531 - facets: - dataset: HadGEM2-CC - ensemble: r1i1p1 - exp: rcp85 - frequency: mon - institute: MOHC - mip: Amon - product: output1 - project: CMIP5 - modeling_realm: atmos - short_name: tas - version: v20120531 - local_file: cmip5/output1/MOHC/HadGEM2-CC/rcp85/mon/atmos/Amon/r1i1p1/v20120531/tas_Amon_HadGEM2-CC_rcp85_r1i1p1_205512-208011.nc - name: tas_Amon_HadGEM2-CC_rcp85_r1i1p1_205512-208011.nc - size: 33432040 - urls: - - http://esgf-data1.ceda.ac.uk/thredds/fileServer/esg_dataroot/cmip5/output1/MOHC/HadGEM2-CC/rcp85/mon/atmos/Amon/r1i1p1/v20120531/tas/tas_Amon_HadGEM2-CC_rcp85_r1i1p1_205512-208011.nc - - http://esgf.nci.org.au/thredds/fileServer/replica/CMIP5/output1/MOHC/HadGEM2-CC/rcp85/mon/atmos/Amon/r1i1p1/v20120531/tas/tas_Amon_HadGEM2-CC_rcp85_r1i1p1_205512-208011.nc -- checksums: - - - SHA256 - - 568e590103ed3bec8692aad85686b576466bd76dea872a9b0411c4c1941f44ad - - - SHA256 - - 568e590103ed3bec8692aad85686b576466bd76dea872a9b0411c4c1941f44ad - dataset: cmip5.output1.MOHC.HadGEM2-CC.rcp85.mon.atmos.Amon.r1i1p1.v20120531 - facets: - dataset: HadGEM2-CC - ensemble: r1i1p1 - exp: rcp85 - frequency: mon - institute: MOHC - mip: Amon - product: output1 - project: CMIP5 - modeling_realm: atmos - short_name: tas - version: v20120531 - local_file: cmip5/output1/MOHC/HadGEM2-CC/rcp85/mon/atmos/Amon/r1i1p1/v20120531/tas_Amon_HadGEM2-CC_rcp85_r1i1p1_208012-209912.nc - name: tas_Amon_HadGEM2-CC_rcp85_r1i1p1_208012-209912.nc - size: 25523776 - urls: - - http://esgf-data1.ceda.ac.uk/thredds/fileServer/esg_dataroot/cmip5/output1/MOHC/HadGEM2-CC/rcp85/mon/atmos/Amon/r1i1p1/v20120531/tas/tas_Amon_HadGEM2-CC_rcp85_r1i1p1_208012-209912.nc - - http://esgf.nci.org.au/thredds/fileServer/replica/CMIP5/output1/MOHC/HadGEM2-CC/rcp85/mon/atmos/Amon/r1i1p1/v20120531/tas/tas_Amon_HadGEM2-CC_rcp85_r1i1p1_208012-209912.nc -- checksums: - - - SHA256 - - f718b1d91b25e5cc5acd3c0080dabb3762711676ad8efe0f54c0946f495f943a - - - SHA256 - - f718b1d91b25e5cc5acd3c0080dabb3762711676ad8efe0f54c0946f495f943a - dataset: cmip5.output1.MOHC.HadGEM2-CC.rcp85.mon.atmos.Amon.r1i1p1.v20120531 - facets: - dataset: HadGEM2-CC - ensemble: r1i1p1 - exp: rcp85 - frequency: mon - institute: MOHC - mip: Amon - product: output1 - project: CMIP5 - modeling_realm: atmos - short_name: tas - version: v20120531 - local_file: cmip5/output1/MOHC/HadGEM2-CC/rcp85/mon/atmos/Amon/r1i1p1/v20120531/tas_Amon_HadGEM2-CC_rcp85_r1i1p1_210001-210012.nc - name: tas_Amon_HadGEM2-CC_rcp85_r1i1p1_210001-210012.nc - size: 1353448 - urls: - - http://esgf-data1.ceda.ac.uk/thredds/fileServer/esg_dataroot/cmip5/output1/MOHC/HadGEM2-CC/rcp85/mon/atmos/Amon/r1i1p1/v20120531/tas/tas_Amon_HadGEM2-CC_rcp85_r1i1p1_210001-210012.nc - - http://esgf.nci.org.au/thredds/fileServer/replica/CMIP5/output1/MOHC/HadGEM2-CC/rcp85/mon/atmos/Amon/r1i1p1/v20120531/tas/tas_Amon_HadGEM2-CC_rcp85_r1i1p1_210001-210012.nc + - checksums: + - - SHA256 + - 10a94293f2ac844ab62496d5d5369ccc0e839c73882a323c21800d71d7780315 + - - SHA256 + - 10a94293f2ac844ab62496d5d5369ccc0e839c73882a323c21800d71d7780315 + dataset: cmip5.output1.MOHC.HadGEM2-CC.rcp85.mon.atmos.Amon.r1i1p1.v20120531 + facets: + dataset: HadGEM2-CC + ensemble: r1i1p1 + exp: rcp85 + frequency: mon + institute: MOHC + mip: Amon + product: output1 + project: CMIP5 + modeling_realm: atmos + short_name: tas + version: v20120531 + local_file: cmip5/output1/MOHC/HadGEM2-CC/rcp85/mon/atmos/Amon/r1i1p1/v20120531/tas_Amon_HadGEM2-CC_rcp85_r1i1p1_205512-208011.nc + name: tas_Amon_HadGEM2-CC_rcp85_r1i1p1_205512-208011.nc + size: 33432040 + urls: + - http://esgf-data1.ceda.ac.uk/thredds/fileServer/esg_dataroot/cmip5/output1/MOHC/HadGEM2-CC/rcp85/mon/atmos/Amon/r1i1p1/v20120531/tas/tas_Amon_HadGEM2-CC_rcp85_r1i1p1_205512-208011.nc + - http://esgf.nci.org.au/thredds/fileServer/replica/CMIP5/output1/MOHC/HadGEM2-CC/rcp85/mon/atmos/Amon/r1i1p1/v20120531/tas/tas_Amon_HadGEM2-CC_rcp85_r1i1p1_205512-208011.nc + - checksums: + - - SHA256 + - 568e590103ed3bec8692aad85686b576466bd76dea872a9b0411c4c1941f44ad + - - SHA256 + - 568e590103ed3bec8692aad85686b576466bd76dea872a9b0411c4c1941f44ad + dataset: cmip5.output1.MOHC.HadGEM2-CC.rcp85.mon.atmos.Amon.r1i1p1.v20120531 + facets: + dataset: HadGEM2-CC + ensemble: r1i1p1 + exp: rcp85 + frequency: mon + institute: MOHC + mip: Amon + product: output1 + project: CMIP5 + modeling_realm: atmos + short_name: tas + version: v20120531 + local_file: cmip5/output1/MOHC/HadGEM2-CC/rcp85/mon/atmos/Amon/r1i1p1/v20120531/tas_Amon_HadGEM2-CC_rcp85_r1i1p1_208012-209912.nc + name: tas_Amon_HadGEM2-CC_rcp85_r1i1p1_208012-209912.nc + size: 25523776 + urls: + - http://esgf-data1.ceda.ac.uk/thredds/fileServer/esg_dataroot/cmip5/output1/MOHC/HadGEM2-CC/rcp85/mon/atmos/Amon/r1i1p1/v20120531/tas/tas_Amon_HadGEM2-CC_rcp85_r1i1p1_208012-209912.nc + - http://esgf.nci.org.au/thredds/fileServer/replica/CMIP5/output1/MOHC/HadGEM2-CC/rcp85/mon/atmos/Amon/r1i1p1/v20120531/tas/tas_Amon_HadGEM2-CC_rcp85_r1i1p1_208012-209912.nc + - checksums: + - - SHA256 + - f718b1d91b25e5cc5acd3c0080dabb3762711676ad8efe0f54c0946f495f943a + - - SHA256 + - f718b1d91b25e5cc5acd3c0080dabb3762711676ad8efe0f54c0946f495f943a + dataset: cmip5.output1.MOHC.HadGEM2-CC.rcp85.mon.atmos.Amon.r1i1p1.v20120531 + facets: + dataset: HadGEM2-CC + ensemble: r1i1p1 + exp: rcp85 + frequency: mon + institute: MOHC + mip: Amon + product: output1 + project: CMIP5 + modeling_realm: atmos + short_name: tas + version: v20120531 + local_file: cmip5/output1/MOHC/HadGEM2-CC/rcp85/mon/atmos/Amon/r1i1p1/v20120531/tas_Amon_HadGEM2-CC_rcp85_r1i1p1_210001-210012.nc + name: tas_Amon_HadGEM2-CC_rcp85_r1i1p1_210001-210012.nc + size: 1353448 + urls: + - http://esgf-data1.ceda.ac.uk/thredds/fileServer/esg_dataroot/cmip5/output1/MOHC/HadGEM2-CC/rcp85/mon/atmos/Amon/r1i1p1/v20120531/tas/tas_Amon_HadGEM2-CC_rcp85_r1i1p1_210001-210012.nc + - http://esgf.nci.org.au/thredds/fileServer/replica/CMIP5/output1/MOHC/HadGEM2-CC/rcp85/mon/atmos/Amon/r1i1p1/v20120531/tas/tas_Amon_HadGEM2-CC_rcp85_r1i1p1_210001-210012.nc EUR-11_MOHC-HadGEM2-ES_r1i1p1_historical_CORDEX_RACMO22E_mon_tas.json: -- checksums: - - - SHA256 - - e27fb1414788529a714c27a7d11169136db9ece7247756ab26dcea70d1da53e3 - dataset: cordex.output.EUR-11.KNMI.MOHC-HadGEM2-ES.historical.r1i1p1.RACMO22E.v2.mon.tas.v20160620 - facets: - dataset: RACMO22E - domain: EUR-11 - driver: MOHC-HadGEM2-ES - ensemble: r1i1p1 - exp: historical - frequency: mon - institute: KNMI - product: output - project: CORDEX - rcm_version: v2 - short_name: tas - version: v20160620 - local_file: cordex/output/EUR-11/KNMI/MOHC-HadGEM2-ES/historical/r1i1p1/RACMO22E/v2/mon/tas/v20160620/tas_EUR-11_MOHC-HadGEM2-ES_historical_r1i1p1_KNMI-RACMO22E_v2_mon_195001-195012.nc - name: tas_EUR-11_MOHC-HadGEM2-ES_historical_r1i1p1_KNMI-RACMO22E_v2_mon_195001-195012.nc - size: 5982648 - urls: - - http://esgf1.dkrz.de/thredds/fileServer/cordex/cordex/output/EUR-11/KNMI/MOHC-HadGEM2-ES/historical/r1i1p1/KNMI-RACMO22E/v2/mon/tas/v20160620/tas_EUR-11_MOHC-HadGEM2-ES_historical_r1i1p1_KNMI-RACMO22E_v2_mon_195001-195012.nc -- checksums: - - - SHA256 - - f14160b5411dc0c7716f80709f309e14948736187087a4c50ec33e0aadcacf53 - dataset: cordex.output.EUR-11.KNMI.MOHC-HadGEM2-ES.historical.r1i1p1.RACMO22E.v2.mon.tas.v20160620 - facets: - dataset: RACMO22E - domain: EUR-11 - driver: MOHC-HadGEM2-ES - ensemble: r1i1p1 - exp: historical - frequency: mon - institute: KNMI - product: output - project: CORDEX - rcm_version: v2 - short_name: tas - version: v20160620 - local_file: cordex/output/EUR-11/KNMI/MOHC-HadGEM2-ES/historical/r1i1p1/RACMO22E/v2/mon/tas/v20160620/tas_EUR-11_MOHC-HadGEM2-ES_historical_r1i1p1_KNMI-RACMO22E_v2_mon_195101-196012.nc - name: tas_EUR-11_MOHC-HadGEM2-ES_historical_r1i1p1_KNMI-RACMO22E_v2_mon_195101-196012.nc - size: 41765410 - urls: - - http://esgf1.dkrz.de/thredds/fileServer/cordex/cordex/output/EUR-11/KNMI/MOHC-HadGEM2-ES/historical/r1i1p1/KNMI-RACMO22E/v2/mon/tas/v20160620/tas_EUR-11_MOHC-HadGEM2-ES_historical_r1i1p1_KNMI-RACMO22E_v2_mon_195101-196012.nc + - checksums: + - - SHA256 + - e27fb1414788529a714c27a7d11169136db9ece7247756ab26dcea70d1da53e3 + dataset: cordex.output.EUR-11.KNMI.MOHC-HadGEM2-ES.historical.r1i1p1.RACMO22E.v2.mon.tas.v20160620 + facets: + dataset: RACMO22E + domain: EUR-11 + driver: MOHC-HadGEM2-ES + ensemble: r1i1p1 + exp: historical + frequency: mon + institute: KNMI + product: output + project: CORDEX + rcm_version: v2 + short_name: tas + version: v20160620 + local_file: cordex/output/EUR-11/KNMI/MOHC-HadGEM2-ES/historical/r1i1p1/RACMO22E/v2/mon/tas/v20160620/tas_EUR-11_MOHC-HadGEM2-ES_historical_r1i1p1_KNMI-RACMO22E_v2_mon_195001-195012.nc + name: tas_EUR-11_MOHC-HadGEM2-ES_historical_r1i1p1_KNMI-RACMO22E_v2_mon_195001-195012.nc + size: 5982648 + urls: + - http://esgf1.dkrz.de/thredds/fileServer/cordex/cordex/output/EUR-11/KNMI/MOHC-HadGEM2-ES/historical/r1i1p1/KNMI-RACMO22E/v2/mon/tas/v20160620/tas_EUR-11_MOHC-HadGEM2-ES_historical_r1i1p1_KNMI-RACMO22E_v2_mon_195001-195012.nc + - checksums: + - - SHA256 + - f14160b5411dc0c7716f80709f309e14948736187087a4c50ec33e0aadcacf53 + dataset: cordex.output.EUR-11.KNMI.MOHC-HadGEM2-ES.historical.r1i1p1.RACMO22E.v2.mon.tas.v20160620 + facets: + dataset: RACMO22E + domain: EUR-11 + driver: MOHC-HadGEM2-ES + ensemble: r1i1p1 + exp: historical + frequency: mon + institute: KNMI + product: output + project: CORDEX + rcm_version: v2 + short_name: tas + version: v20160620 + local_file: cordex/output/EUR-11/KNMI/MOHC-HadGEM2-ES/historical/r1i1p1/RACMO22E/v2/mon/tas/v20160620/tas_EUR-11_MOHC-HadGEM2-ES_historical_r1i1p1_KNMI-RACMO22E_v2_mon_195101-196012.nc + name: tas_EUR-11_MOHC-HadGEM2-ES_historical_r1i1p1_KNMI-RACMO22E_v2_mon_195101-196012.nc + size: 41765410 + urls: + - http://esgf1.dkrz.de/thredds/fileServer/cordex/cordex/output/EUR-11/KNMI/MOHC-HadGEM2-ES/historical/r1i1p1/KNMI-RACMO22E/v2/mon/tas/v20160620/tas_EUR-11_MOHC-HadGEM2-ES_historical_r1i1p1_KNMI-RACMO22E_v2_mon_195101-196012.nc historical_gn_r4i1p1f1_CMIP6_CESM2_Amon_tas.json: -- checksums: - - - SHA256 - - 5f7cdf4fd94b995bfdbf0d9316555980f5b0d0d246c07e1cb6356cd7a4fbdce5 - - - SHA256 - - 5f7cdf4fd94b995bfdbf0d9316555980f5b0d0d246c07e1cb6356cd7a4fbdce5 - - - SHA256 - - 5f7cdf4fd94b995bfdbf0d9316555980f5b0d0d246c07e1cb6356cd7a4fbdce5 - - - SHA256 - - 5f7cdf4fd94b995bfdbf0d9316555980f5b0d0d246c07e1cb6356cd7a4fbdce5 - - - SHA256 - - 5f7cdf4fd94b995bfdbf0d9316555980f5b0d0d246c07e1cb6356cd7a4fbdce5 - - - SHA256 - - 5f7cdf4fd94b995bfdbf0d9316555980f5b0d0d246c07e1cb6356cd7a4fbdce5 - dataset: CMIP6.CMIP.NCAR.CESM2.historical.r4i1p1f1.Amon.tas.gn.v20190308 - facets: - activity: CMIP - dataset: CESM2 - ensemble: r4i1p1f1 - exp: historical - grid: gn - institute: NCAR - mip: Amon - project: CMIP6 - short_name: tas - version: v20190308 - local_file: CMIP6/CMIP/NCAR/CESM2/historical/r4i1p1f1/Amon/tas/gn/v20190308/tas_Amon_CESM2_historical_r4i1p1f1_gn_185001-201412.nc - name: tas_Amon_CESM2_historical_r4i1p1f1_gn_185001-201412.nc - size: 243060396 - urls: - - http://aims3.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/NCAR/CESM2/historical/r4i1p1f1/Amon/tas/gn/v20190308/tas_Amon_CESM2_historical_r4i1p1f1_gn_185001-201412.nc - - http://esgf-data.ucar.edu/thredds/fileServer/esg_dataroot/CMIP6/CMIP/NCAR/CESM2/historical/r4i1p1f1/Amon/tas/gn/v20190308/tas_Amon_CESM2_historical_r4i1p1f1_gn_185001-201412.nc - - http://esgf-data04.diasjp.net/thredds/fileServer/esg_dataroot/CMIP6/CMIP/NCAR/CESM2/historical/r4i1p1f1/Amon/tas/gn/v20190308/tas_Amon_CESM2_historical_r4i1p1f1_gn_185001-201412.nc - - https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/CMIP/NCAR/CESM2/historical/r4i1p1f1/Amon/tas/gn/v20190308/tas_Amon_CESM2_historical_r4i1p1f1_gn_185001-201412.nc - - http://esgf.nci.org.au/thredds/fileServer/replica/CMIP6/CMIP/NCAR/CESM2/historical/r4i1p1f1/Amon/tas/gn/v20190308/tas_Amon_CESM2_historical_r4i1p1f1_gn_185001-201412.nc - - http://esgf3.dkrz.de/thredds/fileServer/cmip6/CMIP/NCAR/CESM2/historical/r4i1p1f1/Amon/tas/gn/v20190308/tas_Amon_CESM2_historical_r4i1p1f1_gn_185001-201412.nc + - checksums: + - - SHA256 + - 5f7cdf4fd94b995bfdbf0d9316555980f5b0d0d246c07e1cb6356cd7a4fbdce5 + - - SHA256 + - 5f7cdf4fd94b995bfdbf0d9316555980f5b0d0d246c07e1cb6356cd7a4fbdce5 + - - SHA256 + - 5f7cdf4fd94b995bfdbf0d9316555980f5b0d0d246c07e1cb6356cd7a4fbdce5 + - - SHA256 + - 5f7cdf4fd94b995bfdbf0d9316555980f5b0d0d246c07e1cb6356cd7a4fbdce5 + - - SHA256 + - 5f7cdf4fd94b995bfdbf0d9316555980f5b0d0d246c07e1cb6356cd7a4fbdce5 + - - SHA256 + - 5f7cdf4fd94b995bfdbf0d9316555980f5b0d0d246c07e1cb6356cd7a4fbdce5 + dataset: CMIP6.CMIP.NCAR.CESM2.historical.r4i1p1f1.Amon.tas.gn.v20190308 + facets: + activity: CMIP + dataset: CESM2 + ensemble: r4i1p1f1 + exp: historical + grid: gn + institute: NCAR + mip: Amon + project: CMIP6 + short_name: tas + version: v20190308 + local_file: CMIP6/CMIP/NCAR/CESM2/historical/r4i1p1f1/Amon/tas/gn/v20190308/tas_Amon_CESM2_historical_r4i1p1f1_gn_185001-201412.nc + name: tas_Amon_CESM2_historical_r4i1p1f1_gn_185001-201412.nc + size: 243060396 + urls: + - http://aims3.llnl.gov/thredds/fileServer/css03_data/CMIP6/CMIP/NCAR/CESM2/historical/r4i1p1f1/Amon/tas/gn/v20190308/tas_Amon_CESM2_historical_r4i1p1f1_gn_185001-201412.nc + - http://esgf-data.ucar.edu/thredds/fileServer/esg_dataroot/CMIP6/CMIP/NCAR/CESM2/historical/r4i1p1f1/Amon/tas/gn/v20190308/tas_Amon_CESM2_historical_r4i1p1f1_gn_185001-201412.nc + - http://esgf-data04.diasjp.net/thredds/fileServer/esg_dataroot/CMIP6/CMIP/NCAR/CESM2/historical/r4i1p1f1/Amon/tas/gn/v20190308/tas_Amon_CESM2_historical_r4i1p1f1_gn_185001-201412.nc + - https://esgf.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/CMIP/NCAR/CESM2/historical/r4i1p1f1/Amon/tas/gn/v20190308/tas_Amon_CESM2_historical_r4i1p1f1_gn_185001-201412.nc + - http://esgf.nci.org.au/thredds/fileServer/replica/CMIP6/CMIP/NCAR/CESM2/historical/r4i1p1f1/Amon/tas/gn/v20190308/tas_Amon_CESM2_historical_r4i1p1f1_gn_185001-201412.nc + - http://esgf3.dkrz.de/thredds/fileServer/cmip6/CMIP/NCAR/CESM2/historical/r4i1p1f1/Amon/tas/gn/v20190308/tas_Amon_CESM2_historical_r4i1p1f1_gn_185001-201412.nc obs4MIPs_CERES-EBAF_mon_rsutcs.json: -- checksums: - - - SHA256 - - db1434a04f3c65eb43e85c0f5d5f344dec5c7813989a7e3bfb5aab6ac3a39414 - dataset: obs4MIPs.CERES-EBAF.v20160610 - facets: - dataset: CERES-EBAF - frequency: mon - institute: NASA-LaRC - project: obs4MIPs - modeling_realm: atmos - short_name: rsutcs - version: v20160610 - local_file: obs4MIPs/CERES-EBAF/v20160610/rsutcs_CERES-EBAF_L3B_Ed2-8_200003-201404.nc - name: rsutcs_CERES-EBAF_L3B_Ed2-8_200003-201404.nc - size: 44090540 - urls: - - https://dpesgf03.nccs.nasa.gov/thredds/fileServer/obs4MIPs/NASA-LaRC/observations/atmos/rsutcs/mon/grid/NASA-LaRC/CERES-EBAF/v20140728/rsutcs_CERES-EBAF_L3B_Ed2-8_200003-201404.nc + - checksums: + - - SHA256 + - db1434a04f3c65eb43e85c0f5d5f344dec5c7813989a7e3bfb5aab6ac3a39414 + dataset: obs4MIPs.CERES-EBAF.v20160610 + facets: + dataset: CERES-EBAF + frequency: mon + institute: NASA-LaRC + project: obs4MIPs + modeling_realm: atmos + short_name: rsutcs + version: v20160610 + local_file: obs4MIPs/CERES-EBAF/v20160610/rsutcs_CERES-EBAF_L3B_Ed2-8_200003-201404.nc + name: rsutcs_CERES-EBAF_L3B_Ed2-8_200003-201404.nc + size: 44090540 + urls: + - https://dpesgf03.nccs.nasa.gov/thredds/fileServer/obs4MIPs/NASA-LaRC/observations/atmos/rsutcs/mon/grid/NASA-LaRC/CERES-EBAF/v20140728/rsutcs_CERES-EBAF_L3B_Ed2-8_200003-201404.nc obs4MIPs_GPCP-V2.3_pr.json: -- checksums: - - - SHA256 - - 4dd4678b79ef139446c8406da5aae4fed210abb2f2160ef95f6988bf83e4525b - dataset: obs4MIPs.GPCP-V2.3.v20180519 - facets: - dataset: GPCP-V2.3 - frequency: mon - institute: NASA-GSFC - project: obs4MIPs - short_name: pr - version: v20180519 - local_file: obs4MIPs/GPCP-V2.3/v20180519/pr_GPCP-SG_L3_v2.3_197901-201710.nc - name: pr_GPCP-SG_L3_v2.3_197901-201710.nc - size: 19348352 - urls: - - https://dpesgf03.nccs.nasa.gov/thredds/fileServer/obs4MIPs/observations/NASA-GSFC/Obs-GPCP/GPCP/V2.3/atmos/pr/pr_GPCP-SG_L3_v2.3_197901-201710.nc + - checksums: + - - SHA256 + - 4dd4678b79ef139446c8406da5aae4fed210abb2f2160ef95f6988bf83e4525b + dataset: obs4MIPs.GPCP-V2.3.v20180519 + facets: + dataset: GPCP-V2.3 + frequency: mon + institute: NASA-GSFC + project: obs4MIPs + short_name: pr + version: v20180519 + local_file: obs4MIPs/GPCP-V2.3/v20180519/pr_GPCP-SG_L3_v2.3_197901-201710.nc + name: pr_GPCP-SG_L3_v2.3_197901-201710.nc + size: 19348352 + urls: + - https://dpesgf03.nccs.nasa.gov/thredds/fileServer/obs4MIPs/observations/NASA-GSFC/Obs-GPCP/GPCP/V2.3/atmos/pr/pr_GPCP-SG_L3_v2.3_197901-201710.nc run1_historical_cccma_cgcm3_1_CMIP3_mon_tas.json: -- checksums: - - - SHA256 - - ee398fdd869ff702c525ebac091e79e6ff69cf4487e3d042cf8dc1e2f105fcb4 - dataset: cmip3.CCCma.cccma_cgcm3_1.historical.mon.atmos.run1.tas.v1 - facets: - dataset: cccma_cgcm3_1 - ensemble: run1 - exp: historical - frequency: mon - institute: CCCma - project: CMIP3 - modeling_realm: atmos - short_name: tas - version: v1 - local_file: cmip3/CCCma/cccma_cgcm3_1/historical/mon/atmos/run1/tas/v1/tas_a1_20c3m_1_cgcm3.1_t47_1850_2000.nc - name: tas_a1_20c3m_1_cgcm3.1_t47_1850_2000.nc - size: 33448576 - urls: - - http://aims3.llnl.gov/thredds/fileServer/cmip3_data/data2/20c3m/atm/mo/tas/cccma_cgcm3_1/run1/tas_a1_20c3m_1_cgcm3.1_t47_1850_2000.nc + - checksums: + - - SHA256 + - ee398fdd869ff702c525ebac091e79e6ff69cf4487e3d042cf8dc1e2f105fcb4 + dataset: cmip3.CCCma.cccma_cgcm3_1.historical.mon.atmos.run1.tas.v1 + facets: + dataset: cccma_cgcm3_1 + ensemble: run1 + exp: historical + frequency: mon + institute: CCCma + project: CMIP3 + modeling_realm: atmos + short_name: tas + version: v1 + local_file: cmip3/CCCma/cccma_cgcm3_1/historical/mon/atmos/run1/tas/v1/tas_a1_20c3m_1_cgcm3.1_t47_1850_2000.nc + name: tas_a1_20c3m_1_cgcm3.1_t47_1850_2000.nc + size: 33448576 + urls: + - http://aims3.llnl.gov/thredds/fileServer/cmip3_data/data2/20c3m/atm/mo/tas/cccma_cgcm3_1/run1/tas_a1_20c3m_1_cgcm3.1_t47_1850_2000.nc diff --git a/tests/integration/recipe/test_recipe.py b/tests/integration/recipe/test_recipe.py index d8133fc2b7..32c381e5b1 100644 --- a/tests/integration/recipe/test_recipe.py +++ b/tests/integration/recipe/test_recipe.py @@ -1502,7 +1502,7 @@ def test_alias_generation(tmp_path, patched_datafinder, session): - {project: CORDEX, driver: ICHEC-EC-EARTH, dataset: RCA4, ensemble: r1, mip: mon, institute: SMHI} - {project: CORDEX, driver: MIROC-MIROC5, dataset: RCA4, ensemble: r1, mip: mon, institute: SMHI} scripts: null - """) # noqa: + """) recipe = get_recipe(tmp_path, content, session) assert len(recipe.datasets) == 14 From 15f80540d6c49f337af433863ca75905a3be981c Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Sat, 14 Sep 2024 00:29:24 +0200 Subject: [PATCH 36/64] Add yamllint configuration (cherry picked from commit fec33a16bafe08285eb18be6af6c147b581b68fd) --- .yamllint | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .yamllint diff --git a/.yamllint b/.yamllint new file mode 100644 index 0000000000..5192dee89a --- /dev/null +++ b/.yamllint @@ -0,0 +1,9 @@ +--- + +extends: default + +rules: + line-length: + level: warning + max: 120 + octal-values: enable From 3657745d156818effb62b770c771ee93012affbf Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Mon, 16 Sep 2024 11:23:55 +0200 Subject: [PATCH 37/64] Disable pycodestyle rules that conflict with ruff and fix some issues with wrong exceptions flagged by prospector (cherry picked from commit d39fc88bb8233dfa443b40f6e5f2570be67943e1) --- esmvalcore/_main.py | 28 ++-- esmvalcore/_recipe/to_datasets.py | 6 +- esmvalcore/esgf/_download.py | 7 +- esmvalcore/local.py | 2 +- esmvalcore/preprocessor/__init__.py | 10 +- esmvalcore/preprocessor/_derive/amoc.py | 5 +- esmvalcore/preprocessor/_mapping.py | 5 +- esmvalcore/preprocessor/_regrid.py | 5 +- esmvalcore/preprocessor/_regrid_esmpy.py | 5 +- esmvalcore/preprocessor/_units.py | 5 +- pyproject.toml | 10 +- setup.cfg | 3 + .../integration/cmor/_fixes/emac/test_emac.py | 136 ++++++++++-------- .../integration/cmor/_fixes/icon/test_icon.py | 81 ++++++----- .../cmor/_fixes/native6/test_era5.py | 11 +- tests/integration/test_task.py | 5 +- tests/unit/cmor/test_cmor_check.py | 11 +- tests/unit/esgf/test_download.py | 3 +- 18 files changed, 189 insertions(+), 149 deletions(-) diff --git a/esmvalcore/_main.py b/esmvalcore/_main.py index da77ec81d5..445b961b10 100755 --- a/esmvalcore/_main.py +++ b/esmvalcore/_main.py @@ -251,13 +251,16 @@ def get(recipe): from .config._diagnostics import DIAGNOSTICS from .config._logging import configure_logging - configure_logging(console_log_level='info') + from .exceptions import RecipeError + + configure_logging(console_log_level="info") installed_recipe = DIAGNOSTICS.recipes / recipe if not installed_recipe.exists(): - ValueError( - f'Recipe {recipe} not found. To list all available recipes, ' - 'execute "esmvaltool list"') - logger.info('Copying installed recipe to the current folder...') + raise RecipeError( + f"Recipe {recipe} not found. To list all available recipes, " + 'execute "esmvaltool list"' + ) + logger.info("Copying installed recipe to the current folder...") shutil.copy(installed_recipe, Path(recipe).name) logger.info('Recipe %s successfully copied', recipe) @@ -274,13 +277,16 @@ def show(recipe): """ from .config._diagnostics import DIAGNOSTICS from .config._logging import configure_logging - configure_logging(console_log_level='info') + from .exceptions import RecipeError + + configure_logging(console_log_level="info") installed_recipe = DIAGNOSTICS.recipes / recipe if not installed_recipe.exists(): - ValueError( - f'Recipe {recipe} not found. To list all available recipes, ' - 'execute "esmvaltool list"') - msg = f'Recipe {recipe}' + raise RecipeError( + f"Recipe {recipe} not found. To list all available recipes, " + 'execute "esmvaltool list"' + ) + msg = f"Recipe {recipe}" logger.info(msg) logger.info('=' * len(msg)) print(installed_recipe.read_text(encoding='utf-8')) @@ -514,8 +520,6 @@ def _log_header(self, config_file, log_files): def run(): """Run the `esmvaltool` program, logging any exceptions.""" - import sys - from .exceptions import RecipeError # Workaround to avoid using more for the output diff --git a/esmvalcore/_recipe/to_datasets.py b/esmvalcore/_recipe/to_datasets.py index 47520e9b9c..977ebc40b3 100644 --- a/esmvalcore/_recipe/to_datasets.py +++ b/esmvalcore/_recipe/to_datasets.py @@ -345,8 +345,10 @@ def _get_datasets_for_variable( ) -> list[Dataset]: """Read the datasets from a variable definition in the recipe.""" logger.debug( - "Populating list of datasets for variable %s in " - "diagnostic %s", variable_group, diagnostic_name) + "Populating list of datasets for variable %s in diagnostic %s", + variable_group, + diagnostic_name, + ) datasets = [] idx = 0 diff --git a/esmvalcore/esgf/_download.py b/esmvalcore/esgf/_download.py index 6b9bd1dfdd..16552e0a0a 100644 --- a/esmvalcore/esgf/_download.py +++ b/esmvalcore/esgf/_download.py @@ -362,8 +362,7 @@ def _get_relative_path(self) -> Path: def __repr__(self): """Represent the file as a string.""" hosts = [urlparse(u).hostname for u in self.urls] - return (f"ESGFFile:{self._get_relative_path()}" - f" on hosts {hosts}") + return f"ESGFFile:{self._get_relative_path()} on hosts {hosts}" def __eq__(self, other): """Compare `self` to `other`.""" @@ -499,9 +498,7 @@ def get_download_message(files): lines = [] for file in files: total_size += file.size - lines.append(f"{format_size(file.size)}" - "\t" - f"{file}") + lines.append(f"{format_size(file.size)}\t{file}") lines.insert(0, "Will download the following files:") lines.insert(0, f"Will download {format_size(total_size)}") diff --git a/esmvalcore/local.py b/esmvalcore/local.py index 539679f682..5843c3a8c6 100644 --- a/esmvalcore/local.py +++ b/esmvalcore/local.py @@ -636,7 +636,7 @@ def find_files( ------- list[LocalFile] The files that were found. - """ # pylint: disable=line-too-long + """ facets = dict(facets) if 'original_short_name' in facets: facets['short_name'] = facets['original_short_name'] diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 3800fb1413..8d6996ec9d 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -291,8 +291,9 @@ def check_preprocessor_settings(settings): signature.bind(None, **settings[step]) except TypeError: logger.error( - "Wrong preprocessor function arguments in " - "function '%s'", step) + "Wrong preprocessor function arguments in function '%s'", + step, + ) raise @@ -336,8 +337,9 @@ def _run_preproc_function(function, items, kwargs, input_files=None): if input_files is None: file_msg = "" else: - file_msg = (f"\nloaded from original input file(s)\n" - f"{pformat(input_files)}") + file_msg = ( + f"\nloaded from original input file(s)\n{pformat(input_files)}" + ) logger.debug( "Running preprocessor function '%s' on the data\n%s%s\nwith function " "argument(s)\n%s", function.__name__, pformat(items), file_msg, diff --git a/esmvalcore/preprocessor/_derive/amoc.py b/esmvalcore/preprocessor/_derive/amoc.py index 1e6e1261a5..fd1751332d 100644 --- a/esmvalcore/preprocessor/_derive/amoc.py +++ b/esmvalcore/preprocessor/_derive/amoc.py @@ -17,8 +17,9 @@ def required(project): required = [{'short_name': 'msftmz', 'optional': True}, {'short_name': 'msftyz', 'optional': True}] else: - raise ValueError(f"Project {project} can not be used " - f"for Amoc derivation.") + raise ValueError( + f"Project {project} can not be used for Amoc derivation." + ) return required diff --git a/esmvalcore/preprocessor/_mapping.py b/esmvalcore/preprocessor/_mapping.py index 28d1fefb83..0eb0e28e08 100644 --- a/esmvalcore/preprocessor/_mapping.py +++ b/esmvalcore/preprocessor/_mapping.py @@ -53,8 +53,9 @@ def ref_to_dims_index_as_index(cube, ref): try: dim = int(ref) except (ValueError, TypeError): - raise ValueError('{} Incompatible type {} for ' - 'slicing'.format(ref, type(ref))) + raise ValueError( + "{} Incompatible type {} for slicing".format(ref, type(ref)) + ) if dim < 0 or dim > cube.ndim: msg = ('Requested an iterator over a dimension ({}) ' 'which does not exist.'.format(dim)) diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py index 228bd7dc26..94471d6ad7 100644 --- a/esmvalcore/preprocessor/_regrid.py +++ b/esmvalcore/preprocessor/_regrid.py @@ -297,8 +297,9 @@ def _spec_to_latlonvals(*, start_latitude: float, end_latitude: float, List of latitudes """ if step_latitude == 0: - raise ValueError('Latitude step cannot be 0, ' - f'got step_latitude={step_latitude}.') + raise ValueError( + f"Latitude step cannot be 0, got step_latitude={step_latitude}." + ) if step_longitude == 0: raise ValueError('Longitude step cannot be 0, ' diff --git a/esmvalcore/preprocessor/_regrid_esmpy.py b/esmvalcore/preprocessor/_regrid_esmpy.py index cce6dae92f..cb423e452e 100755 --- a/esmvalcore/preprocessor/_regrid_esmpy.py +++ b/esmvalcore/preprocessor/_regrid_esmpy.py @@ -310,8 +310,9 @@ def is_lon_circular(lon): 'dimensional than 2d. Giving up.') circular = np.all(abs(seam) % 360. < 1.e-3) else: - raise ValueError('longitude is neither DimCoord nor AuxCoord. ' - 'Giving up.') + raise ValueError( + "longitude is neither DimCoord nor AuxCoord. Giving up." + ) return circular diff --git a/esmvalcore/preprocessor/_units.py b/esmvalcore/preprocessor/_units.py index 8c96f78ae1..1b597169dc 100644 --- a/esmvalcore/preprocessor/_units.py +++ b/esmvalcore/preprocessor/_units.py @@ -157,8 +157,9 @@ def accumulate_coordinate( coord = cube.coord(coordinate) except iris.exceptions.CoordinateNotFoundError as err: raise ValueError( - "Requested coordinate %s not found in cube %s", - coordinate, cube.summary(shorten=True)) from err + f"Requested coordinate {coordinate} not found in cube " + f"{cube.summary(shorten=True)}", + ) from err if coord.ndim > 1: raise NotImplementedError( diff --git a/pyproject.toml b/pyproject.toml index fb723f24fc..044d95fa64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,16 +26,18 @@ max-line-length = 79 disable = [ "import-error", # Needed because Codacy does not install dependencies "file-ignored", # Disable messages about disabling checks + "line-too-long", # Disable line-too-long as this is taken care of by the formatter. "locally-disabled", # Disable messages about disabling checks ] [tool.ruff] line-length = 79 [tool.ruff.lint] select = [ - "E", # pycodestyle - "F", # pyflakes - "I", # isort - "W", # pycodestyle + "E", # pycodestyle + "F", # pyflakes + "I", # isort + "ISC001", # pycodestyle + "W", # pycodestyle ] ignore = [ "E501", # Disable line-too-long as this is taken care of by the formatter. diff --git a/setup.cfg b/setup.cfg index 8e2877b27e..32cdf6d9b6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -21,6 +21,9 @@ exclude_lines = if __name__ == .__main__.: if TYPE_CHECKING: +[pycodestyle] +ignore = E203,E501,W503 # conflict with ruff formatter + [pydocstyle] convention = numpy diff --git a/tests/integration/cmor/_fixes/emac/test_emac.py b/tests/integration/cmor/_fixes/emac/test_emac.py index 6c6e006360..76ae272a31 100644 --- a/tests/integration/cmor/_fixes/emac/test_emac.py +++ b/tests/integration/cmor/_fixes/emac/test_emac.py @@ -850,10 +850,11 @@ def test_awhea_fix(cubes_2d): cube = fixed_cubes[0] assert cube.var_name == 'awhea' assert cube.standard_name is None - assert cube.long_name == ('Global Mean Net Surface Heat Flux Over Open ' - 'Water') - assert cube.units == 'W m-2' - assert 'positive' not in cube.attributes + assert cube.long_name == ( + "Global Mean Net Surface Heat Flux Over Open Water" + ) + assert cube.units == "W m-2" + assert "positive" not in cube.attributes np.testing.assert_allclose(cube.data, [[[1.0]]]) @@ -922,12 +923,13 @@ def test_clwvi_fix(cubes_2d): assert len(fixed_cubes) == 1 cube = fixed_cubes[0] - assert cube.var_name == 'clwvi' - assert cube.standard_name == ('atmosphere_mass_content_of_cloud_' - 'condensed_water') - assert cube.long_name == 'Condensed Water Path' - assert cube.units == 'kg m-2' - assert 'positive' not in cube.attributes + assert cube.var_name == "clwvi" + assert cube.standard_name == ( + "atmosphere_mass_content_of_cloud_condensed_water" + ) + assert cube.long_name == "Condensed Water Path" + assert cube.units == "kg m-2" + assert "positive" not in cube.attributes np.testing.assert_allclose(cube.data, [[[2.0]]]) @@ -1005,12 +1007,13 @@ def test_evspsbl_fix(cubes_2d): fix = get_fix('Amon', 'evspsbl') cube = fix.fix_data(cube) - assert cube.var_name == 'evspsbl' - assert cube.standard_name == 'water_evapotranspiration_flux' - assert cube.long_name == ('Evaporation Including Sublimation and ' - 'Transpiration') - assert cube.units == 'kg m-2 s-1' - assert 'positive' not in cube.attributes + assert cube.var_name == "evspsbl" + assert cube.standard_name == "water_evapotranspiration_flux" + assert cube.long_name == ( + "Evaporation Including Sublimation and Transpiration" + ) + assert cube.units == "kg m-2 s-1" + assert "positive" not in cube.attributes np.testing.assert_allclose(cube.data, [[[-1.0]]]) @@ -1107,12 +1110,13 @@ def test_od550aer_fix(cubes_3d): assert len(fixed_cubes) == 1 cube = fixed_cubes[0] - assert cube.var_name == 'od550aer' - assert cube.standard_name == ('atmosphere_optical_thickness_due_to_' - 'ambient_aerosol_particles') - assert cube.long_name == 'Ambient Aerosol Optical Thickness at 550nm' - assert cube.units == '1' - assert 'positive' not in cube.attributes + assert cube.var_name == "od550aer" + assert cube.standard_name == ( + "atmosphere_optical_thickness_due_to_ambient_aerosol_particles" + ) + assert cube.long_name == "Ambient Aerosol Optical Thickness at 550nm" + assert cube.units == "1" + assert "positive" not in cube.attributes check_lambda550nm(cube) @@ -1389,12 +1393,13 @@ def test_rlutcs_fix(cubes_2d): fix = get_fix('Amon', 'rlutcs') cube = fix.fix_data(cube) - assert cube.var_name == 'rlutcs' - assert cube.standard_name == ('toa_outgoing_longwave_flux_assuming_clear_' - 'sky') - assert cube.long_name == 'TOA Outgoing Clear-Sky Longwave Radiation' - assert cube.units == 'W m-2' - assert cube.attributes['positive'] == 'up' + assert cube.var_name == "rlutcs" + assert cube.standard_name == ( + "toa_outgoing_longwave_flux_assuming_clear_sky" + ) + assert cube.long_name == "TOA Outgoing Clear-Sky Longwave Radiation" + assert cube.units == "W m-2" + assert cube.attributes["positive"] == "up" np.testing.assert_allclose(cube.data, [[[-1.0]]]) @@ -1524,12 +1529,13 @@ def test_rsutcs_fix(cubes_2d): fix = get_fix('Amon', 'rsutcs') cube = fix.fix_data(cube) - assert cube.var_name == 'rsutcs' - assert cube.standard_name == ('toa_outgoing_shortwave_flux_assuming_clear_' - 'sky') - assert cube.long_name == 'TOA Outgoing Clear-Sky Shortwave Radiation' - assert cube.units == 'W m-2' - assert cube.attributes['positive'] == 'up' + assert cube.var_name == "rsutcs" + assert cube.standard_name == ( + "toa_outgoing_shortwave_flux_assuming_clear_sky" + ) + assert cube.long_name == "TOA Outgoing Clear-Sky Shortwave Radiation" + assert cube.units == "W m-2" + assert cube.attributes["positive"] == "up" np.testing.assert_allclose(cube.data, [[[-1.0]]]) @@ -1550,12 +1556,13 @@ def test_rtmt_fix(cubes_2d): assert len(fixed_cubes) == 1 cube = fixed_cubes[0] - assert cube.var_name == 'rtmt' - assert cube.standard_name == ('net_downward_radiative_flux_at_top_of_' - 'atmosphere_model') - assert cube.long_name == 'Net Downward Radiative Flux at Top of Model' - assert cube.units == 'W m-2' - assert cube.attributes['positive'] == 'down' + assert cube.var_name == "rtmt" + assert cube.standard_name == ( + "net_downward_radiative_flux_at_top_of_atmosphere_model" + ) + assert cube.long_name == "Net Downward Radiative Flux at Top of Model" + assert cube.units == "W m-2" + assert cube.attributes["positive"] == "down" np.testing.assert_allclose(cube.data, [[[2.0]]]) @@ -1835,12 +1842,13 @@ def test_toz_fix(cubes_2d): assert len(fixed_cubes) == 1 cube = fixed_cubes[0] - assert cube.var_name == 'toz' - assert cube.standard_name == ('equivalent_thickness_at_stp_of_atmosphere_' - 'ozone_content') - assert cube.long_name == 'Total Column Ozone' - assert cube.units == 'm' - assert 'positive' not in cube.attributes + assert cube.var_name == "toz" + assert cube.standard_name == ( + "equivalent_thickness_at_stp_of_atmosphere_ozone_content" + ) + assert cube.long_name == "Total Column Ozone" + assert cube.units == "m" + assert "positive" not in cube.attributes np.testing.assert_allclose(cube.data, [[[1e-5]]]) @@ -1946,10 +1954,11 @@ def test_MP_BC_tot_fix(cubes_1d): # noqa: N802 cube = fixed_cubes[0] assert cube.var_name == 'MP_BC_tot' assert cube.standard_name is None - assert cube.long_name == ('total mass of black carbon (sum of all aerosol ' - 'modes)') - assert cube.units == 'kg' - assert 'positive' not in cube.attributes + assert cube.long_name == ( + "total mass of black carbon (sum of all aerosol modes)" + ) + assert cube.units == "kg" + assert "positive" not in cube.attributes np.testing.assert_allclose(cube.data, [4.0]) @@ -2097,10 +2106,11 @@ def test_MP_DU_tot_fix(cubes_1d): # noqa: N802 cube = fixed_cubes[0] assert cube.var_name == 'MP_DU_tot' assert cube.standard_name is None - assert cube.long_name == ('total mass of mineral dust (sum of all aerosol ' - 'modes)') - assert cube.units == 'kg' - assert 'positive' not in cube.attributes + assert cube.long_name == ( + "total mass of mineral dust (sum of all aerosol modes)" + ) + assert cube.units == "kg" + assert "positive" not in cube.attributes np.testing.assert_allclose(cube.data, [4.0]) @@ -2344,10 +2354,11 @@ def test_MP_SO4mm_tot_fix(cubes_1d): # noqa: N802 cube = fixed_cubes[0] assert cube.var_name == 'MP_SO4mm_tot' assert cube.standard_name is None - assert cube.long_name == ('total mass of aerosol sulfate (sum of all ' - 'aerosol modes)') - assert cube.units == 'kg' - assert 'positive' not in cube.attributes + assert cube.long_name == ( + "total mass of aerosol sulfate (sum of all aerosol modes)" + ) + assert cube.units == "kg" + assert "positive" not in cube.attributes np.testing.assert_allclose(cube.data, [4.0]) @@ -2373,10 +2384,11 @@ def test_MP_SS_tot_fix(cubes_1d): # noqa: N802 cube = fixed_cubes[0] assert cube.var_name == 'MP_SS_tot' assert cube.standard_name is None - assert cube.long_name == ('total mass of sea salt (sum of all aerosol ' - 'modes)') - assert cube.units == 'kg' - assert 'positive' not in cube.attributes + assert cube.long_name == ( + "total mass of sea salt (sum of all aerosol modes)" + ) + assert cube.units == "kg" + assert "positive" not in cube.attributes np.testing.assert_allclose(cube.data, [3.0]) diff --git a/tests/integration/cmor/_fixes/icon/test_icon.py b/tests/integration/cmor/_fixes/icon/test_icon.py index 2ed06c57f6..d4dc3b7b7b 100644 --- a/tests/integration/cmor/_fixes/icon/test_icon.py +++ b/tests/integration/cmor/_fixes/icon/test_icon.py @@ -349,15 +349,20 @@ def check_lat_lon(cube): assert cube.coords('longitude', mesh_coords=True) # Check dimensional coordinate describing the mesh - assert cube.coords('first spatial index for variables stored on an ' - 'unstructured grid', dim_coords=True) - i_coord = cube.coord('first spatial index for variables stored on an ' - 'unstructured grid', dim_coords=True) - assert i_coord.var_name == 'i' + assert cube.coords( + "first spatial index for variables stored on an unstructured grid", + dim_coords=True, + ) + i_coord = cube.coord( + "first spatial index for variables stored on an unstructured grid", + dim_coords=True, + ) + assert i_coord.var_name == "i" assert i_coord.standard_name is None - assert i_coord.long_name == ('first spatial index for variables stored on ' - 'an unstructured grid') - assert i_coord.units == '1' + assert i_coord.long_name == ( + "first spatial index for variables stored on an unstructured grid" + ) + assert i_coord.units == "1" np.testing.assert_allclose(i_coord.points, [0, 1, 2, 3, 4, 5, 6, 7]) assert i_coord.bounds is None @@ -577,12 +582,13 @@ def test_clwvi_fix(cubes_regular_grid): assert len(fixed_cubes) == 1 cube = fixed_cubes[0] - assert cube.var_name == 'clwvi' - assert cube.standard_name == ('atmosphere_mass_content_of_cloud_' - 'condensed_water') - assert cube.long_name == 'Condensed Water Path' - assert cube.units == 'kg m-2' - assert 'positive' not in cube.attributes + assert cube.var_name == "clwvi" + assert cube.standard_name == ( + "atmosphere_mass_content_of_cloud_condensed_water" + ) + assert cube.long_name == "Condensed Water Path" + assert cube.units == "kg m-2" + assert "positive" not in cube.attributes np.testing.assert_allclose(cube.data, [[[0.0, 2000.0], [4000.0, 6000.0]]]) @@ -603,12 +609,13 @@ def test_lwp_fix(cubes_2d): assert len(fixed_cubes) == 1 cube = fixed_cubes[0] - assert cube.var_name == 'lwp' - assert cube.standard_name == ('atmosphere_mass_content_of_cloud_liquid_' - 'water') - assert cube.long_name == 'Liquid Water Path' - assert cube.units == 'kg m-2' - assert 'positive' not in cube.attributes + assert cube.var_name == "lwp" + assert cube.standard_name == ( + "atmosphere_mass_content_of_cloud_liquid_water" + ) + assert cube.long_name == "Liquid Water Path" + assert cube.units == "kg m-2" + assert "positive" not in cube.attributes check_time(cube) check_lat_lon(cube) @@ -807,15 +814,20 @@ def test_tas_dim_height2m_already_present(cubes_2d): assert cube.mesh is None - assert cube.coords('first spatial index for variables stored on an ' - 'unstructured grid', dim_coords=True) - i_coord = cube.coord('first spatial index for variables stored on an ' - 'unstructured grid', dim_coords=True) - assert i_coord.var_name == 'i' + assert cube.coords( + "first spatial index for variables stored on an unstructured grid", + dim_coords=True, + ) + i_coord = cube.coord( + "first spatial index for variables stored on an unstructured grid", + dim_coords=True, + ) + assert i_coord.var_name == "i" assert i_coord.standard_name is None - assert i_coord.long_name == ('first spatial index for variables stored on ' - 'an unstructured grid') - assert i_coord.units == '1' + assert i_coord.long_name == ( + "first spatial index for variables stored on an unstructured grid" + ) + assert i_coord.units == "1" np.testing.assert_allclose(i_coord.points, [0, 1, 2, 3, 4, 5, 6, 7]) assert i_coord.bounds is None @@ -2382,11 +2394,12 @@ def test_rtmt_fix(cubes_regular_grid): assert len(fixed_cubes) == 1 cube = fixed_cubes[0] - assert cube.var_name == 'rtmt' - assert cube.standard_name == ('net_downward_radiative_flux_at_top_of' - '_atmosphere_model') - assert cube.long_name == 'Net Downward Radiative Flux at Top of Model' - assert cube.units == 'W m-2' - assert cube.attributes['positive'] == 'down' + assert cube.var_name == "rtmt" + assert cube.standard_name == ( + "net_downward_radiative_flux_at_top_of_atmosphere_model" + ) + assert cube.long_name == "Net Downward Radiative Flux at Top of Model" + assert cube.units == "W m-2" + assert cube.attributes["positive"] == "down" np.testing.assert_allclose(cube.data, [[[0.0, -1.0], [-2.0, -3.0]]]) diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py index 70b432541d..2429d045bc 100644 --- a/tests/integration/cmor/_fixes/native6/test_era5.py +++ b/tests/integration/cmor/_fixes/native6/test_era5.py @@ -119,12 +119,11 @@ def _era5_time(frequency): elif frequency == 'monthly': timestamps = [788928, 789672, 790344] return iris.coords.DimCoord( - np.array(timestamps, dtype='int32'), - standard_name='time', - long_name='time', - var_name='time', - units=Unit('hours since 1900-01-01' - '00:00:00.0', calendar='gregorian'), + np.array(timestamps, dtype="int32"), + standard_name="time", + long_name="time", + var_name="time", + units=Unit("hours since 1900-01-0100:00:00.0", calendar="gregorian"), ) diff --git a/tests/integration/test_task.py b/tests/integration/test_task.py index f96cdb534d..b482449829 100644 --- a/tests/integration/test_task.py +++ b/tests/integration/test_task.py @@ -270,8 +270,9 @@ def test_diagnostic_diag_script_none(tmp_path): _get_single_diagnostic_task(tmp_path, diag_script, write_diag=False) diagnostics_root = DIAGNOSTICS.scripts script_file = os.path.abspath(os.path.join(diagnostics_root, diag_script)) - ept = ("Cannot execute script '{}' " - "({}): file does not exist.".format(script_file, script_file)) + ept = "Cannot execute script '{}' ({}): file does not exist.".format( + script_file, script_file + ) assert ept == str(err_msg.value) diff --git a/tests/unit/cmor/test_cmor_check.py b/tests/unit/cmor/test_cmor_check.py index 957136fd42..a533c1eedd 100644 --- a/tests/unit/cmor/test_cmor_check.py +++ b/tests/unit/cmor/test_cmor_check.py @@ -765,11 +765,12 @@ def test_bad_time(self): def test_wrong_parent_time_unit(self): """Test fail for wrong parent time units.""" - self.cube.coord('time').units = 'days since 1860-1-1 00:00:00' - self.cube.attributes['parent_time_units'] = 'days since ' \ - '1860-1-1-00-00-00' - self.cube.attributes['branch_time_in_parent'] = 0. - self.cube.attributes['branch_time_in_child'] = 0. + self.cube.coord("time").units = "days since 1860-1-1 00:00:00" + self.cube.attributes["parent_time_units"] = ( + "days since 1860-1-1-00-00-00" + ) + self.cube.attributes["branch_time_in_parent"] = 0.0 + self.cube.attributes["branch_time_in_child"] = 0.0 self._check_warnings_on_metadata() assert self.cube.attributes['branch_time_in_parent'] == 0. assert self.cube.attributes['branch_time_in_child'] == 0 diff --git a/tests/unit/esgf/test_download.py b/tests/unit/esgf/test_download.py index 66dd50a98f..0bcb17b76d 100644 --- a/tests/unit/esgf/test_download.py +++ b/tests/unit/esgf/test_download.py @@ -612,6 +612,5 @@ def test_download_noop(caplog): caplog.set_level('DEBUG') esmvalcore.esgf.download([], dest_folder='/does/not/exist') - msg = ("All required data is available locally," - " not downloading anything.") + msg = "All required data is available locally, not downloading anything." assert msg in caplog.text From 81a4347b7dff93dd6777917d6b2481598f687991 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Mon, 16 Sep 2024 15:09:07 +0200 Subject: [PATCH 38/64] Disable conflicting pycodestyle vs ruff settings also in prospector config (cherry picked from commit a4c9423bfd9d408fffb5936ccf8f169c62a4a543) --- .prospector.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.prospector.yml b/.prospector.yml index 2f0936ca0f..61dd478326 100644 --- a/.prospector.yml +++ b/.prospector.yml @@ -12,8 +12,9 @@ member-warnings: false pyroma: run: true -pep8: +pycodestyle: full: true + disable: ['E203', 'E501', 'W503'] # conflict with ruff formatter mypy: run: true From 8f49aa3b14f397d447e70be02289511525696b71 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Mon, 16 Sep 2024 15:18:46 +0200 Subject: [PATCH 39/64] Try renaming pycodestyle to pep8 (cherry picked from commit cc9e50db4be4bfe5366874fd3066fce851ade939) --- .prospector.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.prospector.yml b/.prospector.yml index 61dd478326..f7b0259898 100644 --- a/.prospector.yml +++ b/.prospector.yml @@ -12,7 +12,7 @@ member-warnings: false pyroma: run: true -pycodestyle: +pep8: full: true disable: ['E203', 'E501', 'W503'] # conflict with ruff formatter From 06a38cf6c2ebfd246ae7d8e4d4570fe90a44852b Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 18 Sep 2024 09:49:12 +0200 Subject: [PATCH 40/64] Another attempt (cherry picked from commit ebda3b9bc3869aca5cb70341d81e697eb014bc10) --- .prospector.yml | 1 - setup.cfg | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.prospector.yml b/.prospector.yml index f7b0259898..2f0936ca0f 100644 --- a/.prospector.yml +++ b/.prospector.yml @@ -14,7 +14,6 @@ pyroma: pep8: full: true - disable: ['E203', 'E501', 'W503'] # conflict with ruff formatter mypy: run: true diff --git a/setup.cfg b/setup.cfg index 32cdf6d9b6..10e8d8d09c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,7 +22,8 @@ exclude_lines = if TYPE_CHECKING: [pycodestyle] -ignore = E203,E501,W503 # conflict with ruff formatter +# ignore rules that conflict with ruff formatter +ignore = E203,E501,W503 [pydocstyle] convention = numpy From c73cff83a20c3f5107eddc2ff0e58d2be23bd44a Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 18 Sep 2024 10:03:01 +0200 Subject: [PATCH 41/64] And another one (cherry picked from commit 2edc268566c8fbaab8a3dc82dab86d38176a0164) --- .prospector.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.prospector.yml b/.prospector.yml index 2f0936ca0f..8c141202d3 100644 --- a/.prospector.yml +++ b/.prospector.yml @@ -14,6 +14,8 @@ pyroma: pep8: full: true + # ignore rules that conflict with ruff formatter + disable: ['E203', 'E501', 'W503'] mypy: run: true From 704f5430559557216e9041f915d08e9c069a6b38 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 18 Sep 2024 12:23:20 +0200 Subject: [PATCH 42/64] Update docs (cherry picked from commit 0c3f2ae9855c6a4c2220f9ff9cb4726d9bcc0d46) --- doc/contributing.rst | 58 +++++++++++++++++--------------------- doc/quickstart/install.rst | 2 ++ 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/doc/contributing.rst b/doc/contributing.rst index 658306a53b..39783b5114 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -210,6 +210,14 @@ This includes checks for invalid syntax and formatting errors. automatically just before you commit your code. It knows knows which tool to run for each filetype, and therefore provides a convenient way to check your code. +Install the pre-commit hooks by running + +.. code-block:: bash + + pre-commit install + +to make sure your code is formatted correctly and does not contain mistakes +whenever you commit some changes. Python ~~~~~~ @@ -229,20 +237,22 @@ the repository is cloned, e.g. ``cd ESMValCore``, and run `prospector `_ -to automatically check for bugs and formatting mistakes and +In addition to prospector, we use `ruff `_ +to automatically format the code and to check for certain bugs and `mypy `_ for checking that `type hints `_ are correct. Note that `type hints`_ are completely optional, but if you do choose to add them, they should be correct. +Both `ruff`_ and `mypy` are automatically run by pre-commit. When you make a pull request, adherence to the Python development best practices is checked in two ways: -#. As part of the unit tests, flake8_ and mypy_ are run by - `CircleCI `_, - see the section on Tests_ for more information. +#. A check that the code is formatted using the pre-commit hooks and does + not contain any mistakes that can be found by analyzing the code without + running it, is performed by + `pre-commit.ci `_. #. `Codacy `_ is a service that runs prospector (and other code quality tools) on changed files and reports the results. @@ -259,42 +269,25 @@ If you suspect prospector or Codacy may be wrong, please ask the Note that running prospector locally will give you quicker and sometimes more accurate results than waiting for Codacy. -Most formatting issues in Python code can be fixed automatically by -running the commands +Formatting issues in Python code can be fixed automatically by running the +command :: - isort some_file.py - -to sort the imports in `the standard way `__ -using `isort `__ and - -:: - - yapf -i some_file.py - -to add/remove whitespace as required by the standard using `yapf `__, - -:: - - docformatter -i some_file.py - -to run `docformatter `__ which helps -formatting the docstrings (such as line length, spaces). + pre-commit run --all YAML ~~~~ -Please use `yamllint `_ to check that your -YAML files do not contain mistakes. -``yamllint`` checks for valid syntax, common mistakes like key repetition and -cosmetic problems such as line length, trailing spaces, wrong indentation, etc. +We use `yamllint `_ to check that YAML files +do not contain mistakes. This is automatically run by pre-commit. Any text file ~~~~~~~~~~~~~ A generic tool to check for common spelling mistakes is `codespell `__. +This is automatically run by pre-commit. .. _documentation: @@ -379,13 +372,13 @@ the individual checks. To build the documentation on your own computer, go to the directory where the repository was cloned and run -:: +.. code-block:: bash sphinx-build doc doc/build or -:: +.. code-block:: bash sphinx-build -Ea doc doc/build @@ -393,7 +386,8 @@ to build it from scratch. Make sure that your newly added documentation builds without warnings or errors and looks correctly formatted. -CircleCI_ will build the documentation with the command: +`CircleCI `_ +will build the documentation with the command: .. code-block:: bash @@ -720,7 +714,7 @@ If the Codacy check keeps failing, please run prospector locally. If necessary, ask the pull request author to do the same and to address the reported issues. See the section on code_quality_ for more information. -Never merge a pull request with failing CircleCI or readthedocs checks. +Never merge a pull request with failing pre-commit, CircleCI, or readthedocs checks. .. _how-to-make-a-release: diff --git a/doc/quickstart/install.rst b/doc/quickstart/install.rst index 804c86ed20..0a821a0df9 100644 --- a/doc/quickstart/install.rst +++ b/doc/quickstart/install.rst @@ -195,6 +195,8 @@ To install from source for development, follow these instructions. e.g. ``pip install --trusted-host=pypi.python.org --trusted-host=pypi.org --trusted-host=files.pythonhosted.org -e .[develop]`` - Test that your installation was successful by running ``esmvaltool -h``. +- Install the :ref:`esmvaltool:pre-commit` hooks by running: + ``pre-commit install``. Pre-installed versions on HPC clusters / other servers ------------------------------------------------------ From cb4e7c6e46c689b9807141e288f9ff33356d1643 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 18 Sep 2024 13:41:15 +0200 Subject: [PATCH 43/64] Fix link (cherry picked from commit 2f940862fbc53b7c676e983c70f6ac799020386b) --- doc/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/contributing.rst b/doc/contributing.rst index 39783b5114..4188ebeb04 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -244,7 +244,7 @@ to automatically format the code and to check for certain bugs and correct. Note that `type hints`_ are completely optional, but if you do choose to add them, they should be correct. -Both `ruff`_ and `mypy` are automatically run by pre-commit. +Both `ruff`_ and `mypy`_ are automatically run by pre-commit. When you make a pull request, adherence to the Python development best practices is checked in two ways: From c0316adda7161d6229e6754b4d93f40f03eac55c Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 18 Sep 2024 13:58:16 +0200 Subject: [PATCH 44/64] Replace flake8 by pre-commit linting in GitHub Actions (cherry picked from commit f048a81aaf5e2d08a351279419bf57a8ec5f5186) --- .github/workflows/create-condalock-file.yml | 4 ++++ .github/workflows/install-from-condalock-file.yml | 3 +++ .github/workflows/run-tests.yml | 6 ++++++ 3 files changed, 13 insertions(+) diff --git a/.github/workflows/create-condalock-file.yml b/.github/workflows/create-condalock-file.yml index 5e1eaec889..184caf64d2 100644 --- a/.github/workflows/create-condalock-file.yml +++ b/.github/workflows/create-condalock-file.yml @@ -71,6 +71,10 @@ jobs: run: | esmvaltool --help esmvaltool version + - name: Run linters + run: | + pre-commit install + pre-commit run -a - name: Run pytests run: pytest -n 2 -m "not installation" # Automated PR diff --git a/.github/workflows/install-from-condalock-file.yml b/.github/workflows/install-from-condalock-file.yml index 44a7839b55..7e633c14e3 100644 --- a/.github/workflows/install-from-condalock-file.yml +++ b/.github/workflows/install-from-condalock-file.yml @@ -51,6 +51,9 @@ jobs: - run: pip install -e .[develop] - run: esmvaltool --help - run: esmvaltool version 2>&1 | tee source_install_linux_artifacts_python_${{ matrix.python-version }}/version.txt + - run: | + pre-commit install + pre-commit run -a - run: pytest -n 2 -m "not installation" - name: Upload artifacts if: ${{ always() }} # upload artifacts even if fail diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 2843334067..73e15c100c 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -60,6 +60,9 @@ jobs: - run: conda list - run: pip install -e .[develop] 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/install.txt - run: conda list + - run: | + pre-commit install + pre-commit run -a - run: pytest -n 2 -m "not installation" 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/test_report.txt - name: Upload artifacts if: ${{ always() }} # upload artifacts even if fail @@ -96,6 +99,9 @@ jobs: - run: conda list - run: pip install -e .[develop] 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/install.txt - run: conda list + - run: | + pre-commit install + pre-commit run -a - run: pytest -n 2 -m "not installation" 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/test_report.txt - name: Upload artifacts if: ${{ always() }} # upload artifacts even if fail From 10983ee18f70bc44fedfc43d14c7e46fe8a8a4a9 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 18 Sep 2024 14:45:25 +0200 Subject: [PATCH 45/64] Add Python 3.10 codespell pre-commit hook dependency (cherry picked from commit fe3ff7b098cd98376f0fe5324ffc0efefcb3062c) --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 727a99da5a..9fe68ac4c4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,6 +31,7 @@ repos: rev: 'v2.2.4' hooks: - id: codespell + additional_dependencies: [tomli] # required for Python 3.10 - repo: https://github.com/astral-sh/ruff-pre-commit rev: "v0.4.10" hooks: From e4c887bfb5cc55daaeb1928015543fb0254bd63a Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 18 Sep 2024 17:06:56 +0200 Subject: [PATCH 46/64] Remove pre-commit run from conda-lock files GitHub Actions Co-authored-by: Valeriu Predoi (cherry picked from commit 93cfe33535ca49e1f48870b2b8501c2750f6f350) --- .github/workflows/create-condalock-file.yml | 4 ---- .github/workflows/install-from-condalock-file.yml | 3 --- 2 files changed, 7 deletions(-) diff --git a/.github/workflows/create-condalock-file.yml b/.github/workflows/create-condalock-file.yml index 184caf64d2..5e1eaec889 100644 --- a/.github/workflows/create-condalock-file.yml +++ b/.github/workflows/create-condalock-file.yml @@ -71,10 +71,6 @@ jobs: run: | esmvaltool --help esmvaltool version - - name: Run linters - run: | - pre-commit install - pre-commit run -a - name: Run pytests run: pytest -n 2 -m "not installation" # Automated PR diff --git a/.github/workflows/install-from-condalock-file.yml b/.github/workflows/install-from-condalock-file.yml index 7e633c14e3..44a7839b55 100644 --- a/.github/workflows/install-from-condalock-file.yml +++ b/.github/workflows/install-from-condalock-file.yml @@ -51,9 +51,6 @@ jobs: - run: pip install -e .[develop] - run: esmvaltool --help - run: esmvaltool version 2>&1 | tee source_install_linux_artifacts_python_${{ matrix.python-version }}/version.txt - - run: | - pre-commit install - pre-commit run -a - run: pytest -n 2 -m "not installation" - name: Upload artifacts if: ${{ always() }} # upload artifacts even if fail From 63e1e42a49528044ffa0ad6558e30b0cb34f833a Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 25 Sep 2024 20:08:59 +0200 Subject: [PATCH 47/64] Add links with explanations why certain rules are ignored (cherry picked from commit 7b0f8cdf1d7373ded9cbcb4b40bd140fcf97446d) --- .prospector.yml | 3 +++ setup.cfg | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.prospector.yml b/.prospector.yml index 8c141202d3..51508847c2 100644 --- a/.prospector.yml +++ b/.prospector.yml @@ -15,6 +15,9 @@ pyroma: pep8: full: true # ignore rules that conflict with ruff formatter + # E203: https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#slices + # E501: https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules + # W503: https://pycodestyle.pycqa.org/en/latest/intro.html#error-codes disable: ['E203', 'E501', 'W503'] mypy: diff --git a/setup.cfg b/setup.cfg index 10e8d8d09c..15d02392d7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,6 +23,9 @@ exclude_lines = [pycodestyle] # ignore rules that conflict with ruff formatter +# E203: https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#slices +# E501: https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules +# W503: https://pycodestyle.pycqa.org/en/latest/intro.html#error-codes ignore = E203,E501,W503 [pydocstyle] From 167b8c031228668be5ebb4e8ca7adcc920407dcc Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Fri, 27 Sep 2024 10:09:04 +0200 Subject: [PATCH 48/64] Latest version of pre-commit --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9fe68ac4c4..6dac9fed16 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ exclude: | repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.6.0 hooks: - id: check-added-large-files - id: check-ast @@ -24,16 +24,16 @@ repos: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] - repo: https://github.com/adrienverge/yamllint - rev: 'v1.31.0' + rev: 'v1.35.1' hooks: - id: yamllint - repo: https://github.com/codespell-project/codespell - rev: 'v2.2.4' + rev: 'v2.3.0' hooks: - id: codespell additional_dependencies: [tomli] # required for Python 3.10 - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.4.10" + rev: "v0.6.8" hooks: - id: ruff args: [--fix] From 036b7c6e3605838c0ad648ee1f4b9986bb989533 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Fri, 27 Sep 2024 10:16:06 +0200 Subject: [PATCH 49/64] Ignore 'oce' for codespell --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 044d95fa64..5a45ca2ab9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ version_scheme = "release-branch-semver" [tool.codespell] skip = "*.ipynb,esmvalcore/config/extra_facets/ipslcm-mappings.yml" -ignore-words-list = "vas,hist" +ignore-words-list = "vas,hist,oce" [tool.pylint.main] jobs = 1 # Running more than one job in parallel crashes prospector. From f30e19f9761db8262f15667bc3a41fcdb420c925 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Fri, 27 Sep 2024 10:19:16 +0200 Subject: [PATCH 50/64] Update doc/conf.py --- doc/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/conf.py b/doc/conf.py index 3b0d93abb0..7e0b4b988d 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -90,7 +90,7 @@ templates_path = ['_templates'] # The suffix of source filenames. -source_suffix = '.rst' +source_suffix = {'.rst': 'restructuredtext'} # The encoding of source files. # source_encoding = 'utf-8-sig' From 4d449b741c74a7489446d9d4d8e6cb39ddab03cf Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Fri, 27 Sep 2024 15:23:36 +0200 Subject: [PATCH 51/64] Apply autoformatting --- doc/quickstart/find_data.rst | 2 +- esmvalcore/_provenance.py | 105 +- esmvalcore/cmor/_fixes/fix.py | 136 +- esmvalcore/cmor/_fixes/native6/era5.py | 165 +-- esmvalcore/preprocessor/_io.py | 255 ++-- .../cmor/_fixes/native6/test_era5.py | 1178 +++++++++-------- tests/integration/cmor/test_fix.py | 484 +++---- .../integration/preprocessor/_io/test_load.py | 69 +- tests/sample_data/iris-sample-data/LICENSE | 4 +- tests/unit/provenance/test_trackedfile.py | 38 +- 10 files changed, 1288 insertions(+), 1148 deletions(-) diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index 7c98db0569..aa91c1c6e0 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -682,7 +682,7 @@ Key Description Default value if not recipe if default DRS is used) ```special_attr`` A special attribute in the filename No default `ACCESS-ESM` raw data, it's related to - frquency of raw data + frequency of raw data ``sub_dataset`` Part of the ACCESS-ESM raw dataset No default root, need to specify if you want to use the cmoriser diff --git a/esmvalcore/_provenance.py b/esmvalcore/_provenance.py index 75e661feaa..89b5822c27 100644 --- a/esmvalcore/_provenance.py +++ b/esmvalcore/_provenance.py @@ -1,4 +1,5 @@ """Provenance module.""" + import copy import logging import os @@ -14,7 +15,7 @@ logger = logging.getLogger(__name__) -ESMVALTOOL_URI_PREFIX = 'https://www.esmvaltool.org/' +ESMVALTOOL_URI_PREFIX = "https://www.esmvaltool.org/" def create_namespace(provenance, namespace): @@ -25,11 +26,12 @@ def create_namespace(provenance, namespace): def get_esmvaltool_provenance(): """Create an esmvaltool run activity.""" provenance = ProvDocument() - namespace = 'software' + namespace = "software" create_namespace(provenance, namespace) attributes = {} # TODO: add dependencies with versions here - activity = provenance.activity(namespace + ':esmvaltool==' + __version__, - other_attributes=attributes) + activity = provenance.activity( + namespace + ":esmvaltool==" + __version__, other_attributes=attributes + ) return activity @@ -39,27 +41,27 @@ def get_esmvaltool_provenance(): def attribute_to_authors(entity, authors): """Attribute entity to authors.""" - namespace = 'author' + namespace = "author" create_namespace(entity.bundle, namespace) for author in authors: if isinstance(author, str): # This happens if the config-references.yml file is not available - author = {'name': author} + author = {"name": author} agent = entity.bundle.agent( - namespace + ':' + author['name'], - {'attribute:' + k: author[k] - for k in author if k != 'name'}) + namespace + ":" + author["name"], + {"attribute:" + k: author[k] for k in author if k != "name"}, + ) entity.wasAttributedTo(agent) def attribute_to_projects(entity, projects): """Attribute entity to projects.""" - namespace = 'project' + namespace = "project" create_namespace(entity.bundle, namespace) for project in projects: - agent = entity.bundle.agent(namespace + ':' + project) + agent = entity.bundle.agent(namespace + ":" + project) entity.wasAttributedTo(agent) @@ -67,17 +69,19 @@ def get_recipe_provenance(documentation, filename): """Create a provenance entity describing a recipe.""" provenance = ProvDocument() - for namespace in ('recipe', 'attribute'): + for namespace in ("recipe", "attribute"): create_namespace(provenance, namespace) entity = provenance.entity( - 'recipe:{}'.format(filename), { - 'attribute:description': documentation.get('description', ''), - 'attribute:references': str(documentation.get('references', [])), - }) + "recipe:{}".format(filename), + { + "attribute:description": documentation.get("description", ""), + "attribute:references": str(documentation.get("references", [])), + }, + ) - attribute_to_authors(entity, documentation.get('authors', [])) - attribute_to_projects(entity, documentation.get('projects', [])) + attribute_to_authors(entity, documentation.get("authors", [])) + attribute_to_projects(entity, documentation.get("projects", [])) return entity @@ -85,9 +89,9 @@ def get_recipe_provenance(documentation, filename): def get_task_provenance(task, recipe_entity): """Create a provenance activity describing a task.""" provenance = ProvDocument() - create_namespace(provenance, 'task') + create_namespace(provenance, "task") - activity = provenance.activity('task:' + task.name) + activity = provenance.activity("task:" + task.name) trigger = recipe_entity provenance.update(recipe_entity.bundle) @@ -104,11 +108,9 @@ def get_task_provenance(task, recipe_entity): class TrackedFile: """File with provenance tracking.""" - def __init__(self, - filename, - attributes=None, - ancestors=None, - prov_filename=None): + def __init__( + self, filename, attributes=None, ancestors=None, prov_filename=None + ): """Create an instance of a file with provenance tracking. Arguments @@ -147,11 +149,11 @@ def __repr__(self): def __eq__(self, other): """Check if `other` equals `self`.""" - return hasattr(other, 'filename') and self.filename == other.filename + return hasattr(other, "filename") and self.filename == other.filename def __lt__(self, other): """Check if `other` should be sorted before `self`.""" - return hasattr(other, 'filename') and self.filename < other.filename + return hasattr(other, "filename") and self.filename < other.filename def __hash__(self): """Return a unique hash for the file.""" @@ -175,7 +177,7 @@ def filename(self): @property def provenance_file(self): """Filename of provenance.""" - return os.path.splitext(self.filename)[0] + '_provenance.xml' + return os.path.splitext(self.filename)[0] + "_provenance.xml" def initialize_provenance(self, activity): """Initialize the provenance document. @@ -186,7 +188,8 @@ def initialize_provenance(self, activity): """ if self.provenance is not None: raise ValueError( - "Provenance of {} already initialized".format(self)) + "Provenance of {} already initialized".format(self) + ) self.provenance = ProvDocument() self._initialize_namespaces() self._initialize_activity(activity) @@ -195,7 +198,7 @@ def initialize_provenance(self, activity): def _initialize_namespaces(self): """Initialize the namespaces.""" - for namespace in ('file', 'attribute', 'preprocessor', 'task'): + for namespace in ("file", "attribute", "preprocessor", "task"): create_namespace(self.provenance, namespace) def _initialize_activity(self, activity): @@ -207,21 +210,22 @@ def _initialize_entity(self): """Initialize the entity representing the file.""" if self.attributes is None: self.attributes = {} - if 'nc' in Path(self.filename).suffix: - with Dataset(self.filename, 'r') as dataset: + if "nc" in Path(self.filename).suffix: + with Dataset(self.filename, "r") as dataset: for attr in dataset.ncattrs(): self.attributes[attr] = dataset.getncattr(attr) attributes = { - 'attribute:' + str(k).replace(' ', '_'): str(v) + "attribute:" + str(k).replace(" ", "_"): str(v) for k, v in self.attributes.items() - if k not in ('authors', 'projects') + if k not in ("authors", "projects") } - self.entity = self.provenance.entity(f'file:{self.filename}', - attributes) + self.entity = self.provenance.entity( + f"file:{self.filename}", attributes + ) - attribute_to_authors(self.entity, self.attributes.get('authors', [])) - attribute_to_projects(self.entity, self.attributes.get('projects', [])) + attribute_to_authors(self.entity, self.attributes.get("authors", [])) + attribute_to_projects(self.entity, self.attributes.get("projects", [])) def _initialize_ancestors(self, activity): """Register ancestor files for provenance tracking.""" @@ -247,15 +251,15 @@ def wasderivedfrom(self, other): def _select_for_include(self): attributes = { - 'software': "Created with ESMValTool v{}".format(__version__), + "software": "Created with ESMValTool v{}".format(__version__), } - if 'caption' in self.attributes: - attributes['caption'] = self.attributes['caption'] + if "caption" in self.attributes: + attributes["caption"] = self.attributes["caption"] return attributes @staticmethod def _include_provenance_nc(filename, attributes): - with Dataset(filename, 'a') as dataset: + with Dataset(filename, "a") as dataset: for key, value in attributes.items(): setattr(dataset, key, value) @@ -263,8 +267,8 @@ def _include_provenance_nc(filename, attributes): def _include_provenance_png(filename, attributes): pnginfo = PngInfo() exif_tags = { - 'caption': 'ImageDescription', - 'software': 'Software', + "caption": "ImageDescription", + "software": "Software", } for key, value in attributes.items(): pnginfo.add_text(exif_tags.get(key, key), value, zip=True) @@ -276,8 +280,8 @@ def _include_provenance(self): attributes = self._select_for_include() # Attach provenance to supported file types - ext = os.path.splitext(self.filename)[1].lstrip('.').lower() - write = getattr(self, '_include_provenance_' + ext, None) + ext = os.path.splitext(self.filename)[1].lstrip(".").lower() + write = getattr(self, "_include_provenance_" + ext, None) if write: write(self.filename, attributes) @@ -288,17 +292,18 @@ def save_provenance(self): namespaces=self.provenance.namespaces, ) self._include_provenance() - with open(self.provenance_file, 'wb') as file: + with open(self.provenance_file, "wb") as file: # Create file with correct permissions before saving. - self.provenance.serialize(file, format='xml') + self.provenance.serialize(file, format="xml") self.activity = None self.entity = None self.provenance = None def restore_provenance(self): """Import provenance information from a previously saved file.""" - self.provenance = ProvDocument.deserialize(self.provenance_file, - format='xml') + self.provenance = ProvDocument.deserialize( + self.provenance_file, format="xml" + ) entity_uri = f"{ESMVALTOOL_URI_PREFIX}file{self.prov_filename}" self.entity = self.provenance.get_record(entity_uri)[0] # Find the associated activity diff --git a/esmvalcore/cmor/_fixes/fix.py b/esmvalcore/cmor/_fixes/fix.py index 771a362a63..6d05be9da6 100644 --- a/esmvalcore/cmor/_fixes/fix.py +++ b/esmvalcore/cmor/_fixes/fix.py @@ -1,4 +1,5 @@ """Contains the base class for dataset fixes.""" + from __future__ import annotations import importlib @@ -33,7 +34,7 @@ from esmvalcore.config import Session logger = logging.getLogger(__name__) -generic_fix_logger = logging.getLogger(f'{__name__}.genericfix') +generic_fix_logger = logging.getLogger(f"{__name__}.genericfix") class Fix: @@ -235,9 +236,9 @@ def get_fixes( """ vardef = get_var_info(project, mip, short_name) - project = project.replace('-', '_').lower() - dataset = dataset.replace('-', '_').lower() - short_name = short_name.replace('-', '_').lower() + project = project.replace("-", "_").lower() + dataset = dataset.replace("-", "_").lower() + short_name = short_name.replace("-", "_").lower() if extra_facets is None: extra_facets = {} @@ -245,30 +246,40 @@ def get_fixes( fixes = [] fixes_modules = [] - if project == 'cordex': - driver = extra_facets['driver'].replace('-', '_').lower() - extra_facets['dataset'] = dataset + if project == "cordex": + driver = extra_facets["driver"].replace("-", "_").lower() + extra_facets["dataset"] = dataset try: - fixes_modules.append(importlib.import_module( - f'esmvalcore.cmor._fixes.{project}.{driver}.{dataset}' - )) + fixes_modules.append( + importlib.import_module( + f"esmvalcore.cmor._fixes.{project}.{driver}.{dataset}" + ) + ) except ImportError: pass - fixes_modules.append(importlib.import_module( - 'esmvalcore.cmor._fixes.cordex.cordex_fixes')) + fixes_modules.append( + importlib.import_module( + "esmvalcore.cmor._fixes.cordex.cordex_fixes" + ) + ) else: try: - fixes_modules.append(importlib.import_module( - f'esmvalcore.cmor._fixes.{project}.{dataset}')) + fixes_modules.append( + importlib.import_module( + f"esmvalcore.cmor._fixes.{project}.{dataset}" + ) + ) except ImportError: pass for fixes_module in fixes_modules: classes = dict( - (name.lower(), value) for (name, value) in - inspect.getmembers(fixes_module, inspect.isclass) + (name.lower(), value) + for (name, value) in inspect.getmembers( + fixes_module, inspect.isclass + ) ) - for fix_name in (short_name, mip.lower(), 'allvars'): + for fix_name in (short_name, mip.lower(), "allvars"): if fix_name in classes: fixes.append( classes[fix_name]( @@ -344,7 +355,7 @@ def fix_metadata(self, cubes: Sequence[Cube]) -> CubeList: """ # Make sure the this fix also works when no extra_facets are given - if 'project' in self.extra_facets and 'dataset' in self.extra_facets: + if "project" in self.extra_facets and "dataset" in self.extra_facets: dataset_str = ( f"{self.extra_facets['project']}:" f"{self.extra_facets['dataset']}" @@ -391,7 +402,7 @@ def fix_data(self, cube: Cube) -> Cube: @staticmethod def _msg_suffix(cube: Cube) -> str: """Get prefix for log messages.""" - if 'source_file' in cube.attributes: + if "source_file" in cube.attributes: return f"\n(for file {cube.attributes['source_file']})" return f"\n(for variable {cube.var_name})" @@ -430,8 +441,8 @@ def _reverse_coord(self, cube: Cube, coord: Coord) -> tuple[Cube, Coord]: def _get_effective_units(self) -> str: """Get effective units.""" - if self.vardef.units.lower() == 'psu': - return '1' + if self.vardef.units.lower() == "psu": + return "1" return self.vardef.units def _fix_units(self, cube: Cube) -> Cube: @@ -497,9 +508,9 @@ def _fix_long_name(self, cube: Cube) -> Cube: def _fix_psu_units(self, cube: Cube) -> Cube: """Fix psu units.""" - if cube.attributes.get('invalid_units', '').lower() == 'psu': - cube.units = '1' - cube.attributes.pop('invalid_units') + if cube.attributes.get("invalid_units", "").lower() == "psu": + cube.units = "1" + cube.attributes.pop("invalid_units") self._debug_msg(cube, "Units converted from 'psu' to '1'") return cube @@ -521,7 +532,7 @@ def _fix_alternative_generic_level_coords(self, cube: Cube) -> Cube: """Fix alternative generic level coordinates.""" # Avoid overriding existing variable information cmor_var_coordinates = self.vardef.coordinates.copy() - for (coord_name, cmor_coord) in cmor_var_coordinates.items(): + for coord_name, cmor_coord in cmor_var_coordinates.items(): if not cmor_coord.generic_level: continue # Ignore non-generic-level coordinates if not cmor_coord.generic_lev_coords: @@ -557,9 +568,10 @@ def _fix_alternative_generic_level_coords(self, cube: Cube) -> Cube: # Search for alternative coordinates (i.e., regular level # coordinates); if none found, do nothing try: - (alternative_coord, - cube_coord) = _get_alternative_generic_lev_coord( - cube, coord_name, self.vardef.table_type + (alternative_coord, cube_coord) = ( + _get_alternative_generic_lev_coord( + cube, coord_name, self.vardef.table_type + ) ) except ValueError: # no alternatives found continue @@ -578,11 +590,13 @@ def _fix_cmip6_multidim_lat_lon_coord( cube_coord: Coord, ) -> None: """Fix CMIP6 multidimensional latitude and longitude coordinates.""" - is_cmip6_multidim_lat_lon = all([ - 'CMIP6' in self.vardef.table_type, - cube_coord.ndim > 1, - cube_coord.standard_name in ('latitude', 'longitude'), - ]) + is_cmip6_multidim_lat_lon = all( + [ + "CMIP6" in self.vardef.table_type, + cube_coord.ndim > 1, + cube_coord.standard_name in ("latitude", "longitude"), + ] + ) if is_cmip6_multidim_lat_lon: self._debug_msg( cube, @@ -674,7 +688,7 @@ def _fix_longitude_0_360( cube_coord: Coord, ) -> tuple[Cube, Coord]: """Fix longitude coordinate to be in [0, 360].""" - if not cube_coord.standard_name == 'longitude': + if not cube_coord.standard_name == "longitude": return (cube, cube_coord) points = cube_coord.core_points() @@ -696,7 +710,7 @@ def _fix_longitude_0_360( # nbounds>2 implies an irregular grid with bounds given as vertices # of the cell polygon. if cube_coord.ndim == 1 and cube_coord.nbounds in (0, 2): - lon_extent = CoordExtent(cube_coord, 0.0, 360., True, False) + lon_extent = CoordExtent(cube_coord, 0.0, 360.0, True, False) cube = cube.intersection(lon_extent) else: new_lons = cube_coord.core_points().copy() @@ -724,12 +738,14 @@ def _fix_coord_bounds( cube_coord: Coord, ) -> None: """Fix coordinate bounds.""" - if cmor_coord.must_have_bounds != 'yes' or cube_coord.has_bounds(): + if cmor_coord.must_have_bounds != "yes" or cube_coord.has_bounds(): return # Skip guessing bounds for unstructured grids if has_unstructured_grid(cube) and cube_coord.standard_name in ( - 'latitude', 'longitude'): + "latitude", + "longitude", + ): self._debug_msg( cube, "Will not guess bounds for coordinate %s of unstructured grid", @@ -762,10 +778,11 @@ def _fix_coord_direction( # Skip fix for a variety of reasons if cube_coord.ndim > 1: return (cube, cube_coord) - if cube_coord.dtype.kind == 'U': + if cube_coord.dtype.kind == "U": return (cube, cube_coord) if has_unstructured_grid(cube) and cube_coord.standard_name in ( - 'latitude', 'longitude' + "latitude", + "longitude", ): return (cube, cube_coord) if len(cube_coord.core_points()) == 1: @@ -774,10 +791,10 @@ def _fix_coord_direction( return (cube, cube_coord) # Fix coordinates with wrong direction - if cmor_coord.stored_direction == 'increasing': + if cmor_coord.stored_direction == "increasing": if cube_coord.core_points()[0] > cube_coord.core_points()[1]: (cube, cube_coord) = self._reverse_coord(cube, cube_coord) - elif cmor_coord.stored_direction == 'decreasing': + elif cmor_coord.stored_direction == "decreasing": if cube_coord.core_points()[0] < cube_coord.core_points()[1]: (cube, cube_coord) = self._reverse_coord(cube, cube_coord) @@ -789,7 +806,7 @@ def _fix_time_units(self, cube: Cube, cube_coord: Coord) -> None: old_units = cube_coord.units cube_coord.convert_units( Unit( - 'days since 1850-1-1 00:00:00', + "days since 1850-1-1 00:00:00", calendar=cube_coord.units.calendar, ) ) @@ -800,9 +817,9 @@ def _fix_time_units(self, cube: Cube, cube_coord: Coord) -> None: # Fix units of time-related cube attributes attrs = cube.attributes - parent_time = 'parent_time_units' + parent_time = "parent_time_units" if parent_time in attrs: - if attrs[parent_time] in 'no parent': + if attrs[parent_time] in "no parent": pass else: try: @@ -810,26 +827,28 @@ def _fix_time_units(self, cube: Cube, cube_coord: Coord) -> None: except ValueError: pass else: - attrs[parent_time] = 'days since 1850-1-1 00:00:00' + attrs[parent_time] = "days since 1850-1-1 00:00:00" - branch_parent = 'branch_time_in_parent' + branch_parent = "branch_time_in_parent" if branch_parent in attrs: attrs[branch_parent] = parent_units.convert( - attrs[branch_parent], cube_coord.units) + attrs[branch_parent], cube_coord.units + ) - branch_child = 'branch_time_in_child' + branch_child = "branch_time_in_child" if branch_child in attrs: attrs[branch_child] = old_units.convert( - attrs[branch_child], cube_coord.units) + attrs[branch_child], cube_coord.units + ) def _fix_time_bounds(self, cube: Cube, cube_coord: Coord) -> None: """Fix time bounds.""" - times = {'time', 'time1', 'time2', 'time3'} + times = {"time", "time1", "time2", "time3"} key = times.intersection(self.vardef.coordinates) if not key: # cube has time, but CMOR variable does not return - cmor = self.vardef.coordinates[' '.join(key)] - if cmor.must_have_bounds == 'yes' and not cube_coord.has_bounds(): + cmor = self.vardef.coordinates[" ".join(key)] + if cmor.must_have_bounds == "yes" and not cube_coord.has_bounds(): cube_coord.bounds = get_time_bounds(cube_coord, self.frequency) self._warning_msg( cube, @@ -840,10 +859,10 @@ def _fix_time_bounds(self, cube: Cube, cube_coord: Coord) -> None: def _fix_time_coord(self, cube: Cube) -> Cube: """Fix time coordinate.""" # Make sure to get dimensional time coordinate if possible - if cube.coords('time', dim_coords=True): - cube_coord = cube.coord('time', dim_coords=True) - elif cube.coords('time'): - cube_coord = cube.coord('time') + if cube.coords("time", dim_coords=True): + cube_coord = cube.coord("time", dim_coords=True) + elif cube.coords("time"): + cube_coord = cube.coord("time") else: return cube @@ -855,7 +874,7 @@ def _fix_time_coord(self, cube: Cube) -> Cube: self._fix_time_units(cube, cube_coord) # Remove time_origin from coordinate attributes - cube_coord.attributes.pop('time_origin', None) + cube_coord.attributes.pop("time_origin", None) # Fix time bounds self._fix_time_bounds(cube, cube_coord) @@ -883,7 +902,6 @@ def _fix_coord( def _fix_coords(self, cube: Cube) -> Cube: """Fix non-time coordinates.""" for cmor_coord in self.vardef.coordinates.values(): - # Cannot fix generic level coords with no unique CMOR information if cmor_coord.generic_level and not cmor_coord.out_name: continue @@ -894,7 +912,7 @@ def _fix_coords(self, cube: Cube) -> Cube: cube_coord = cube.coord(var_name=cmor_coord.out_name) # Fixes for time coord are done separately - if cube_coord.var_name == 'time': + if cube_coord.var_name == "time": continue # Fixes diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py index 51d470639e..cf41c2d2e1 100644 --- a/esmvalcore/cmor/_fixes/native6/era5.py +++ b/esmvalcore/cmor/_fixes/native6/era5.py @@ -1,4 +1,5 @@ """Fixes for ERA5.""" + import datetime import logging @@ -14,51 +15,53 @@ logger = logging.getLogger(__name__) -DEFAULT_ERA5_GRID = '0.25x0.25' +DEFAULT_ERA5_GRID = "0.25x0.25" def get_frequency(cube): """Determine time frequency of input cube.""" try: - time = cube.coord(axis='T') + time = cube.coord(axis="T") except iris.exceptions.CoordinateNotFoundError: - return 'fx' + return "fx" - time.convert_units('days since 1850-1-1 00:00:00.0') + time.convert_units("days since 1850-1-1 00:00:00.0") if len(time.points) == 1: acceptable_long_names = ( - 'Geopotential', - 'Percentage of the Grid Cell Occupied by Land (Including Lakes)', + "Geopotential", + "Percentage of the Grid Cell Occupied by Land (Including Lakes)", ) if cube.long_name not in acceptable_long_names: - raise ValueError('Unable to infer frequency of cube ' - f'with length 1 time dimension: {cube}') - return 'fx' + raise ValueError( + "Unable to infer frequency of cube " + f"with length 1 time dimension: {cube}" + ) + return "fx" interval = time.points[1] - time.points[0] if interval - 1 / 24 < 1e-4: - return 'hourly' + return "hourly" if interval - 1.0 < 1e-4: - return 'daily' - return 'monthly' + return "daily" + return "monthly" def fix_hourly_time_coordinate(cube): """Shift aggregated variables 30 minutes back in time.""" - if get_frequency(cube) == 'hourly': - time = cube.coord(axis='T') + if get_frequency(cube) == "hourly": + time = cube.coord(axis="T") time.points = time.points - 1 / 48 return cube def fix_accumulated_units(cube): """Convert accumulations to fluxes.""" - if get_frequency(cube) == 'monthly': - cube.units = cube.units * 'd-1' - elif get_frequency(cube) == 'hourly': - cube.units = cube.units * 'h-1' - elif get_frequency(cube) == 'daily': + if get_frequency(cube) == "monthly": + cube.units = cube.units * "d-1" + elif get_frequency(cube) == "hourly": + cube.units = cube.units * "h-1" + elif get_frequency(cube) == "daily": raise NotImplementedError( f"Fixing of accumulated units of cube " f"{cube.summary(shorten=True)} is not implemented for daily data" @@ -69,20 +72,20 @@ def fix_accumulated_units(cube): def multiply_with_density(cube, density=1000): """Convert precipitatin from m to kg/m2.""" cube.data = cube.core_data() * density - cube.units *= 'kg m**-3' + cube.units *= "kg m**-3" return cube def remove_time_coordinate(cube): """Remove time coordinate for invariant parameters.""" cube = cube[0] - cube.remove_coord('time') + cube.remove_coord("time") return cube def divide_by_gravity(cube): """Convert geopotential to height.""" - cube.units = cube.units / 'm s-2' + cube.units = cube.units / "m s-2" cube.data = cube.core_data() / 9.80665 return cube @@ -94,7 +97,7 @@ def fix_metadata(self, cubes): """Fix metadata.""" for cube in cubes: # Invalid input cube units (ignored on load) were '0-1' - cube.units = '1' + cube.units = "1" return cubes @@ -104,7 +107,7 @@ class Cli(Fix): def fix_metadata(self, cubes): """Fix metadata.""" for cube in cubes: - cube.units = 'kg kg-1' + cube.units = "kg kg-1" return cubes @@ -115,8 +118,8 @@ def fix_metadata(self, cubes): """Fix metadata.""" for cube in cubes: # Invalid input cube units (ignored on load) were '0-1' - cube.units = '%' - cube.data = cube.core_data() * 100. + cube.units = "%" + cube.data = cube.core_data() * 100.0 return cubes @@ -127,7 +130,7 @@ class Clw(Fix): def fix_metadata(self, cubes): """Fix metadata.""" for cube in cubes: - cube.units = 'kg kg-1' + cube.units = "kg kg-1" return cubes @@ -138,8 +141,8 @@ def fix_metadata(self, cubes): """Fix metadata.""" for cube in cubes: # Invalid input cube units (ignored on load) were '0-1' - cube.units = '%' - cube.data = cube.core_data() * 100. + cube.units = "%" + cube.data = cube.core_data() * 100.0 return cubes @@ -151,7 +154,7 @@ def fix_metadata(self, cubes): """Fix metadata.""" for cube in cubes: # Set input cube units for invalid units were ignored on load - cube.units = 'm' + cube.units = "m" fix_hourly_time_coordinate(cube) fix_accumulated_units(cube) multiply_with_density(cube) @@ -168,7 +171,7 @@ def fix_metadata(self, cubes): """Fix metadata.""" for cube in cubes: # Set input cube units for invalid units were ignored on load - cube.units = 'm' + cube.units = "m" fix_hourly_time_coordinate(cube) fix_accumulated_units(cube) multiply_with_density(cube) @@ -184,7 +187,7 @@ class Hus(Fix): def fix_metadata(self, cubes): """Fix metadata.""" for cube in cubes: - cube.units = 'kg kg-1' + cube.units = "kg kg-1" return cubes @@ -207,11 +210,11 @@ class O3(Fix): def fix_metadata(self, cubes): """Convert mass mixing ratios to mole fractions.""" for cube in cubes: - cube.units = 'kg kg-1' + cube.units = "kg kg-1" # Convert to molar mixing ratios, which is almost identical to mole # fraction for small amounts of substances (which we have here) cube.data = cube.core_data() * 28.9644 / 47.9982 - cube.units = 'mol mol-1' + cube.units = "mol mol-1" return cubes @@ -248,7 +251,7 @@ def fix_metadata(self, cubes): """Fix metadata.""" for cube in cubes: # Set input cube units for invalid units were ignored on load - cube.units = 'm' + cube.units = "m" fix_hourly_time_coordinate(cube) fix_accumulated_units(cube) multiply_with_density(cube) @@ -262,7 +265,7 @@ class Prw(Fix): def fix_metadata(self, cubes): """Fix metadata.""" for cube in cubes: - cube.units = 'kg m-2' + cube.units = "kg m-2" return cubes @@ -272,7 +275,7 @@ class Ps(Fix): def fix_metadata(self, cubes): """Fix metadata.""" for cube in cubes: - cube.units = 'Pa' + cube.units = "Pa" return cubes @@ -293,7 +296,7 @@ class Rainmxrat27(Fix): def fix_metadata(self, cubes): """Fix metadata.""" for cube in cubes: - cube.units = 'kg kg-1' + cube.units = "kg kg-1" return cubes @@ -305,7 +308,7 @@ def fix_metadata(self, cubes): for cube in cubes: fix_hourly_time_coordinate(cube) fix_accumulated_units(cube) - cube.attributes['positive'] = 'down' + cube.attributes["positive"] = "down" return cubes @@ -318,7 +321,7 @@ def fix_metadata(self, cubes): for cube in cubes: fix_hourly_time_coordinate(cube) fix_accumulated_units(cube) - cube.attributes['positive'] = 'down' + cube.attributes["positive"] = "down" return cubes @@ -331,7 +334,7 @@ def fix_metadata(self, cubes): for cube in cubes: fix_hourly_time_coordinate(cube) fix_accumulated_units(cube) - cube.attributes['positive'] = 'up' + cube.attributes["positive"] = "up" return cubes @@ -343,7 +346,7 @@ def fix_metadata(self, cubes): """Fix metadata.""" for cube in cubes: fix_hourly_time_coordinate(cube) - cube.attributes['positive'] = 'down' + cube.attributes["positive"] = "down" return cubes @@ -356,7 +359,7 @@ def fix_metadata(self, cubes): for cube in cubes: fix_hourly_time_coordinate(cube) fix_accumulated_units(cube) - cube.attributes['positive'] = 'down' + cube.attributes["positive"] = "down" return cubes @@ -369,7 +372,7 @@ def fix_metadata(self, cubes): for cube in cubes: fix_hourly_time_coordinate(cube) fix_accumulated_units(cube) - cube.attributes['positive'] = 'down' + cube.attributes["positive"] = "down" return cubes @@ -382,7 +385,7 @@ def fix_metadata(self, cubes): for cube in cubes: fix_hourly_time_coordinate(cube) fix_accumulated_units(cube) - cube.attributes['positive'] = 'up' + cube.attributes["positive"] = "up" return cubes @@ -395,7 +398,7 @@ def fix_metadata(self, cubes): for cube in cubes: fix_hourly_time_coordinate(cube) fix_accumulated_units(cube) - cube.attributes['positive'] = 'down' + cube.attributes["positive"] = "down" return cubes @@ -408,7 +411,7 @@ def fix_metadata(self, cubes): for cube in cubes: fix_hourly_time_coordinate(cube) fix_accumulated_units(cube) - cube.attributes['positive'] = 'down' + cube.attributes["positive"] = "down" return cubes @@ -420,7 +423,7 @@ def fix_metadata(self, cubes): """Fix metadata.""" for cube in cubes: # Invalid input cube units (ignored on load) were '0-1' - cube.units = '1' + cube.units = "1" return cubes @@ -430,7 +433,7 @@ class Snowmxrat27(Fix): def fix_metadata(self, cubes): """Fix metadata.""" for cube in cubes: - cube.units = 'kg kg-1' + cube.units = "kg kg-1" return cubes @@ -460,10 +463,10 @@ class Toz(Fix): def fix_metadata(self, cubes): """Convert 'kg m-2' to 'm'.""" for cube in cubes: - cube.units = 'kg m-2' + cube.units = "kg m-2" # 1 DU = 1e-5 m = 2.1415e-5 kg m-2 --> 1m = 2.1415 kg m-2 cube.data = cube.core_data() / 2.1415 - cube.units = 'm' + cube.units = "m" return cubes @@ -483,10 +486,10 @@ class AllVars(Fix): def _fix_coordinates(self, cube): """Fix coordinates.""" # Add scalar height coordinates - if 'height2m' in self.vardef.dimensions: - add_scalar_height_coord(cube, 2.) - if 'height10m' in self.vardef.dimensions: - add_scalar_height_coord(cube, 10.) + if "height2m" in self.vardef.dimensions: + add_scalar_height_coord(cube, 2.0) + if "height10m" in self.vardef.dimensions: + add_scalar_height_coord(cube, 10.0) # Fix coord metadata for coord_def in self.vardef.coordinates.values(): @@ -497,42 +500,47 @@ def _fix_coordinates(self, cube): # (https://github.com/ESMValGroup/ESMValCore/issues/1029) if axis == "" and coord_def.name == "alevel": axis = "Z" - coord_def = CMOR_TABLES['CMIP6'].coords['plev19'] + coord_def = CMOR_TABLES["CMIP6"].coords["plev19"] coord = cube.coord(axis=axis) - if axis == 'T': - coord.convert_units('days since 1850-1-1 00:00:00.0') - if axis == 'Z': + if axis == "T": + coord.convert_units("days since 1850-1-1 00:00:00.0") + if axis == "Z": coord.convert_units(coord_def.units) coord.standard_name = coord_def.standard_name coord.var_name = coord_def.out_name coord.long_name = coord_def.long_name - coord.points = coord.core_points().astype('float64') - if (not coord.has_bounds() and len(coord.core_points()) > 1 - and coord_def.must_have_bounds == "yes"): + coord.points = coord.core_points().astype("float64") + if ( + not coord.has_bounds() + and len(coord.core_points()) > 1 + and coord_def.must_have_bounds == "yes" + ): # Do not guess bounds for lat and lon on unstructured grids - if not (coord.name() in ('latitude', 'longitude') and - has_unstructured_grid(cube)): + if not ( + coord.name() in ("latitude", "longitude") + and has_unstructured_grid(cube) + ): coord.guess_bounds() self._fix_monthly_time_coord(cube) # Fix coordinate increasing direction - if cube.coords('latitude') and not has_unstructured_grid(cube): - lat = cube.coord('latitude') + if cube.coords("latitude") and not has_unstructured_grid(cube): + lat = cube.coord("latitude") if lat.points[0] > lat.points[-1]: - cube = reverse(cube, 'latitude') - if cube.coords('air_pressure'): - plev = cube.coord('air_pressure') + cube = reverse(cube, "latitude") + if cube.coords("air_pressure"): + plev = cube.coord("air_pressure") if plev.points[0] < plev.points[-1]: - cube = reverse(cube, 'air_pressure') + cube = reverse(cube, "air_pressure") return cube @staticmethod def _fix_monthly_time_coord(cube): """Set the monthly time coordinates to the middle of the month.""" - if get_frequency(cube) == 'monthly': - coord = cube.coord(axis='T') + if get_frequency(cube) == "monthly": + coord = cube.coord(axis="T") end = [] for cell in coord.cells(): month = cell.point.month + 1 @@ -560,11 +568,12 @@ def fix_metadata(self, cubes): cube.long_name = self.vardef.long_name cube = self._fix_coordinates(cube) self._fix_units(cube) - cube.data = cube.core_data().astype('float32') + cube.data = cube.core_data().astype("float32") year = datetime.datetime.now().year - cube.attributes['comment'] = ( - 'Contains modified Copernicus Climate Change ' - f'Service Information {year}') + cube.attributes["comment"] = ( + "Contains modified Copernicus Climate Change " + f"Service Information {year}" + ) fixed_cubes.append(cube) @@ -572,7 +581,7 @@ def fix_metadata(self, cubes): def fix_data(self, cube): """Fix data.""" - regridding_enabled = self.extra_facets.get('regrid', True) + regridding_enabled = self.extra_facets.get("regrid", True) if regridding_enabled and has_unstructured_grid(cube): logger.debug( "Automatically regrid ERA5 data (variable %s) to %s° " @@ -580,5 +589,5 @@ def fix_data(self, cube): self.vardef.short_name, DEFAULT_ERA5_GRID, ) - cube = regrid(cube, DEFAULT_ERA5_GRID, 'linear') + cube = regrid(cube, DEFAULT_ERA5_GRID, "linear") return cube diff --git a/esmvalcore/preprocessor/_io.py b/esmvalcore/preprocessor/_io.py index 04ada11bd4..2847b4c88c 100644 --- a/esmvalcore/preprocessor/_io.py +++ b/esmvalcore/preprocessor/_io.py @@ -1,4 +1,5 @@ """Functions for loading and saving cubes.""" + from __future__ import annotations import copy @@ -19,21 +20,21 @@ from iris.cube import CubeList from esmvalcore.cmor.check import CheckLevels -from esmvalcore.iris_helpers import merge_cube_attributes from esmvalcore.esgf.facets import FACETS +from esmvalcore.iris_helpers import merge_cube_attributes from .._task import write_ncl_settings logger = logging.getLogger(__name__) -GLOBAL_FILL_VALUE = 1e+20 +GLOBAL_FILL_VALUE = 1e20 DATASET_KEYS = { - 'mip', + "mip", } VARIABLE_KEYS = { - 'reference_dataset', - 'alternative_dataset', + "reference_dataset", + "alternative_dataset", } iris.FUTURE.save_split_attrs = True @@ -51,17 +52,18 @@ def _get_attr_from_field_coord(ncfield, coord_name, attr): def _load_callback(raw_cube, field, _): """Use this callback to fix anything Iris tries to break.""" # Remove attributes that cause issues with merging and concatenation - _delete_attributes(raw_cube, - ('creation_date', 'tracking_id', 'history', 'comment')) + _delete_attributes( + raw_cube, ("creation_date", "tracking_id", "history", "comment") + ) for coord in raw_cube.coords(): # Iris chooses to change longitude and latitude units to degrees # regardless of value in file, so reinstating file value - if coord.standard_name in ['longitude', 'latitude']: - units = _get_attr_from_field_coord(field, coord.var_name, 'units') + if coord.standard_name in ["longitude", "latitude"]: + units = _get_attr_from_field_coord(field, coord.var_name, "units") if units is not None: coord.units = units # CMOR sometimes adds a history to the coordinates. - _delete_attributes(coord, ('history', )) + _delete_attributes(coord, ("history",)) def _delete_attributes(iris_object, atts): @@ -106,26 +108,32 @@ def load( ignore_warnings = list(ignore_warnings) # Default warnings ignored for every dataset - ignore_warnings.append({ - 'message': "Missing CF-netCDF measure variable .*", - 'category': UserWarning, - 'module': 'iris', - }) - ignore_warnings.append({ - 'message': "Ignoring netCDF variable '.*' invalid units '.*'", - 'category': UserWarning, - 'module': 'iris', - }) # iris < 3.8 - ignore_warnings.append({ - 'message': "Ignoring invalid units .* on netCDF variable .*", - 'category': UserWarning, - 'module': 'iris', - }) # iris >= 3.8 + ignore_warnings.append( + { + "message": "Missing CF-netCDF measure variable .*", + "category": UserWarning, + "module": "iris", + } + ) + ignore_warnings.append( + { + "message": "Ignoring netCDF variable '.*' invalid units '.*'", + "category": UserWarning, + "module": "iris", + } + ) # iris < 3.8 + ignore_warnings.append( + { + "message": "Ignoring invalid units .* on netCDF variable .*", + "category": UserWarning, + "module": "iris", + } + ) # iris >= 3.8 # Filter warnings with catch_warnings(): for warning_kwargs in ignore_warnings: - warning_kwargs.setdefault('action', 'ignore') + warning_kwargs.setdefault("action", "ignore") filterwarnings(**warning_kwargs) # Suppress UDUNITS-2 error messages that cannot be ignored with # warnings.filterwarnings @@ -134,7 +142,7 @@ def load( # GRIB files need to be loaded with iris.load, otherwise we will # get separate (lat, lon) slices for each time step, pressure # level, etc. - grib_formats = ('.grib2', '.grib', '.grb2', '.grb', '.gb2', '.gb') + grib_formats = (".grib2", ".grib", ".grb2", ".grb", ".gb2", ".gb") if file.suffix in grib_formats: raw_cubes = iris.load(file, callback=_load_callback) else: @@ -142,10 +150,10 @@ def load( logger.debug("Done with loading %s", file) if not raw_cubes: - raise ValueError(f'Can not load cubes from {file}') + raise ValueError(f"Can not load cubes from {file}") for cube in raw_cubes: - cube.attributes['source_file'] = str(file) + cube.attributes["source_file"] = str(file) return raw_cubes @@ -153,18 +161,19 @@ def load( def _concatenate_cubes(cubes, check_level): """Concatenate cubes according to the check_level.""" kwargs = { - 'check_aux_coords': True, - 'check_cell_measures': True, - 'check_ancils': True, - 'check_derived_coords': True + "check_aux_coords": True, + "check_cell_measures": True, + "check_ancils": True, + "check_derived_coords": True, } if check_level > CheckLevels.DEFAULT: kwargs = dict.fromkeys(kwargs, False) logger.debug( - 'Concatenation will be performed without checking ' - 'auxiliary coordinates, cell measures, ancillaries ' - 'and derived coordinates present in the cubes.', ) + "Concatenation will be performed without checking " + "auxiliary coordinates, cell measures, ancillaries " + "and derived coordinates present in the cubes.", + ) concatenated = iris.cube.CubeList(cubes).concatenate(**kwargs) @@ -172,7 +181,6 @@ def _concatenate_cubes(cubes, check_level): class _TimesHelper: - def __init__(self, time): self.times = time.core_points() self.units = str(time.units) @@ -232,7 +240,10 @@ def from_cube(cls, cube): # current cube ends after new one, just forget new cube logger.debug( "Discarding %s because the time range " - "is already covered by %s", new_cube.cube, current_cube.cube) + "is already covered by %s", + new_cube.cube, + current_cube.cube, + ) continue if new_cube.start == current_cube.start: # new cube completely covers current one @@ -240,20 +251,27 @@ def from_cube(cls, cube): current_cube = new_cube logger.debug( "Discarding %s because the time range is covered by %s", - current_cube.cube, new_cube.cube) + current_cube.cube, + new_cube.cube, + ) continue # new cube ends after current one, # use all of new cube, and shorten current cube to # eliminate overlap with new cube - cut_index = cftime.time2index( - new_cube.start, - _TimesHelper(current_cube.times), - current_cube.times.units.calendar, - select="before", - ) + 1 - logger.debug("Using %s shortened to %s due to overlap", - current_cube.cube, - current_cube.times.cell(cut_index).point) + cut_index = ( + cftime.time2index( + new_cube.start, + _TimesHelper(current_cube.times), + current_cube.times.units.calendar, + select="before", + ) + + 1 + ) + logger.debug( + "Using %s shortened to %s due to overlap", + current_cube.cube, + current_cube.times.cell(cut_index).point, + ) new_cubes.append(current_cube.cube[:cut_index]) current_cube = new_cube @@ -265,20 +283,23 @@ def from_cube(cls, cube): def _fix_calendars(cubes): """Check and homogenise calendars, if possible.""" - calendars = [cube.coord('time').units.calendar for cube in cubes] + calendars = [cube.coord("time").units.calendar for cube in cubes] unique_calendars = np.unique(calendars) calendar_ocurrences = np.array( - [calendars.count(calendar) for calendar in unique_calendars]) + [calendars.count(calendar) for calendar in unique_calendars] + ) calendar_index = int( - np.argwhere(calendar_ocurrences == calendar_ocurrences.max())) + np.argwhere(calendar_ocurrences == calendar_ocurrences.max()) + ) for cube in cubes: - time_coord = cube.coord('time') + time_coord = cube.coord("time") old_calendar = time_coord.units.calendar if old_calendar != unique_calendars[calendar_index]: new_unit = time_coord.units.change_calendar( - unique_calendars[calendar_index]) + unique_calendars[calendar_index] + ) time_coord.units = new_unit @@ -289,14 +310,14 @@ def _get_concatenation_error(cubes): iris.cube.CubeList(cubes).concatenate_cube() except iris.exceptions.ConcatenateError as exc: msg = str(exc) - logger.error('Can not concatenate cubes into a single one: %s', msg) - logger.error('Resulting cubes:') + logger.error("Can not concatenate cubes into a single one: %s", msg) + logger.error("Resulting cubes:") for cube in cubes: logger.error(cube) time = cube.coord("time") - logger.error('From %s to %s', time.cell(0), time.cell(-1)) + logger.error("From %s to %s", time.cell(0), time.cell(-1)) - raise ValueError(f'Can not concatenate cubes: {msg}') + raise ValueError(f"Can not concatenate cubes: {msg}") def _sort_cubes_by_time(cubes): @@ -304,12 +325,15 @@ def _sort_cubes_by_time(cubes): try: cubes = sorted(cubes, key=lambda c: c.coord("time").cell(0).point) except iris.exceptions.CoordinateNotFoundError as exc: - msg = "One or more cubes {} are missing".format(cubes) + \ - " time coordinate: {}".format(str(exc)) + msg = "One or more cubes {} are missing".format( + cubes + ) + " time coordinate: {}".format(str(exc)) raise ValueError(msg) except TypeError as error: - msg = ("Cubes cannot be sorted " - f"due to differing time units: {str(error)}") + msg = ( + "Cubes cannot be sorted " + f"due to differing time units: {str(error)}" + ) raise TypeError(msg) from error return cubes @@ -385,12 +409,9 @@ def concatenate(cubes, check_level=CheckLevels.DEFAULT): return result -def save(cubes, - filename, - optimize_access='', - compress=False, - alias='', - **kwargs): +def save( + cubes, filename, optimize_access="", compress=False, alias="", **kwargs +): """Save iris cubes to file. Parameters @@ -430,59 +451,71 @@ def save(cubes, raise ValueError(f"Cannot save empty cubes '{cubes}'") # Rename some arguments - kwargs['target'] = filename - kwargs['zlib'] = compress + kwargs["target"] = filename + kwargs["zlib"] = compress dirname = os.path.dirname(filename) if not os.path.exists(dirname): os.makedirs(dirname) - if (os.path.exists(filename) - and all(cube.has_lazy_data() for cube in cubes)): + if os.path.exists(filename) and all( + cube.has_lazy_data() for cube in cubes + ): logger.debug( "Not saving cubes %s to %s to avoid data loss. " - "The cube is probably unchanged.", cubes, filename) + "The cube is probably unchanged.", + cubes, + filename, + ) return filename for cube in cubes: - logger.debug("Saving cube:\n%s\nwith %s data to %s", cube, - "lazy" if cube.has_lazy_data() else "realized", filename) + logger.debug( + "Saving cube:\n%s\nwith %s data to %s", + cube, + "lazy" if cube.has_lazy_data() else "realized", + filename, + ) if optimize_access: cube = cubes[0] - if optimize_access == 'map': + if optimize_access == "map": dims = set( - cube.coord_dims('latitude') + cube.coord_dims('longitude')) - elif optimize_access == 'timeseries': - dims = set(cube.coord_dims('time')) + cube.coord_dims("latitude") + cube.coord_dims("longitude") + ) + elif optimize_access == "timeseries": + dims = set(cube.coord_dims("time")) else: dims = tuple() - for coord_dims in (cube.coord_dims(dimension) - for dimension in optimize_access.split(' ')): + for coord_dims in ( + cube.coord_dims(dimension) + for dimension in optimize_access.split(" ") + ): dims += coord_dims dims = set(dims) - kwargs['chunksizes'] = tuple( + kwargs["chunksizes"] = tuple( length if index in dims else 1 - for index, length in enumerate(cube.shape)) + for index, length in enumerate(cube.shape) + ) - kwargs['fill_value'] = GLOBAL_FILL_VALUE + kwargs["fill_value"] = GLOBAL_FILL_VALUE if alias: - for cube in cubes: - logger.debug('Changing var_name from %s to %s', cube.var_name, - alias) + logger.debug( + "Changing var_name from %s to %s", cube.var_name, alias + ) cube.var_name = alias # Ignore some warnings when saving with catch_warnings(): filterwarnings( - 'ignore', + "ignore", message=( ".* is being added as CF data variable attribute, but .* " "should only be a CF global attribute" ), category=UserWarning, - module='iris', + module="iris", ) iris.save(cubes, **kwargs) @@ -496,7 +529,7 @@ def _get_debug_filename(filename, step): num = int(sorted(os.listdir(dirname)).pop()[:2]) + 1 else: num = 0 - filename = os.path.join(dirname, '{:02}_{}.nc'.format(num, step)) + filename = os.path.join(dirname, "{:02}_{}.nc".format(num, step)) return filename @@ -505,8 +538,8 @@ def _sort_products(products): return sorted( products, key=lambda p: ( - p.attributes.get('recipe_dataset_index', 1e6), - p.attributes.get('dataset', ''), + p.attributes.get("recipe_dataset_index", 1e6), + p.attributes.get("dataset", ""), ), ) @@ -514,21 +547,22 @@ def _sort_products(products): def write_metadata(products, write_ncl=False): """Write product metadata to file.""" output_files = [] - for output_dir, prods in groupby(products, - lambda p: os.path.dirname(p.filename)): + for output_dir, prods in groupby( + products, lambda p: os.path.dirname(p.filename) + ): sorted_products = _sort_products(prods) metadata = {} for product in sorted_products: - if isinstance(product.attributes.get('exp'), (list, tuple)): + if isinstance(product.attributes.get("exp"), (list, tuple)): product.attributes = dict(product.attributes) - product.attributes['exp'] = '-'.join(product.attributes['exp']) - if 'original_short_name' in product.attributes: - del product.attributes['original_short_name'] + product.attributes["exp"] = "-".join(product.attributes["exp"]) + if "original_short_name" in product.attributes: + del product.attributes["original_short_name"] metadata[product.filename] = product.attributes - output_filename = os.path.join(output_dir, 'metadata.yml') + output_filename = os.path.join(output_dir, "metadata.yml") output_files.append(output_filename) - with open(output_filename, 'w', encoding='utf-8') as file: + with open(output_filename, "w", encoding="utf-8") as file: yaml.safe_dump(metadata, file) if write_ncl: output_files.append(_write_ncl_metadata(output_dir, metadata)) @@ -540,28 +574,31 @@ def _write_ncl_metadata(output_dir, metadata): """Write NCL metadata files to output_dir.""" variables = [copy.deepcopy(v) for v in metadata.values()] - info = {'input_file_info': variables} + info = {"input_file_info": variables} # Split input_file_info into dataset and variable properties # dataset keys and keys with non-identical values will be stored # in dataset_info, the rest in variable_info variable_info = {} - info['variable_info'] = [variable_info] - info['dataset_info'] = [] + info["variable_info"] = [variable_info] + info["dataset_info"] = [] for variable in variables: dataset_info = {} - info['dataset_info'].append(dataset_info) + info["dataset_info"].append(dataset_info) for key in variable: - dataset_specific = any(variable[key] != var.get(key, object()) - for var in variables) - if ((dataset_specific or key in DATASET_KEYS) - and key not in VARIABLE_KEYS): + dataset_specific = any( + variable[key] != var.get(key, object()) for var in variables + ) + if ( + dataset_specific or key in DATASET_KEYS + ) and key not in VARIABLE_KEYS: dataset_info[key] = variable[key] else: variable_info[key] = variable[key] - filename = os.path.join(output_dir, - variable_info['short_name'] + '_info.ncl') + filename = os.path.join( + output_dir, variable_info["short_name"] + "_info.ncl" + ) write_ncl_settings(info, filename) return filename diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py index 8a5aa78118..0139ae50a1 100644 --- a/tests/integration/cmor/_fixes/native6/test_era5.py +++ b/tests/integration/cmor/_fixes/native6/test_era5.py @@ -1,4 +1,5 @@ """Tests for the fixes of ERA5.""" + import datetime import dask.array as da @@ -21,21 +22,23 @@ from esmvalcore.cmor.table import CMOR_TABLES, get_var_info from esmvalcore.preprocessor import cmor_check_metadata -COMMENT = ('Contains modified Copernicus Climate Change Service Information ' - f'{datetime.datetime.now().year}') +COMMENT = ( + "Contains modified Copernicus Climate Change Service Information " + f"{datetime.datetime.now().year}" +) def test_get_evspsbl_fix(): """Test whether the right fixes are gathered for a single variable.""" - fix = Fix.get_fixes('native6', 'ERA5', 'E1hr', 'evspsbl') - vardef = get_var_info('native6', 'E1hr', 'evspsbl') + fix = Fix.get_fixes("native6", "ERA5", "E1hr", "evspsbl") + vardef = get_var_info("native6", "E1hr", "evspsbl") assert fix == [Evspsbl(vardef), AllVars(vardef), GenericFix(vardef)] def test_get_zg_fix(): """Test whether the right fix gets found again, for zg as well.""" - fix = Fix.get_fixes('native6', 'ERA5', 'Amon', 'zg') - vardef = get_var_info('native6', 'E1hr', 'evspsbl') + fix = Fix.get_fixes("native6", "ERA5", "Amon", "zg") + vardef = get_var_info("native6", "E1hr", "evspsbl") assert fix == [Zg(vardef), AllVars(vardef), GenericFix(vardef)] @@ -43,77 +46,77 @@ def test_get_frequency_hourly(): """Test cubes with hourly frequency.""" time = DimCoord( [0, 1, 2], - standard_name='time', - units=Unit('hours since 1900-01-01'), + standard_name="time", + units=Unit("hours since 1900-01-01"), ) cube = Cube( [1, 6, 3], - var_name='random_var', + var_name="random_var", dim_coords_and_dims=[(time, 0)], ) - assert get_frequency(cube) == 'hourly' - cube.coord('time').convert_units('days since 1850-1-1 00:00:00.0') - assert get_frequency(cube) == 'hourly' + assert get_frequency(cube) == "hourly" + cube.coord("time").convert_units("days since 1850-1-1 00:00:00.0") + assert get_frequency(cube) == "hourly" def test_get_frequency_daily(): """Test cubes with daily frequency.""" time = DimCoord( [0, 1, 2], - standard_name='time', - units=Unit('days since 1900-01-01'), + standard_name="time", + units=Unit("days since 1900-01-01"), ) cube = Cube( [1, 6, 3], - var_name='random_var', + var_name="random_var", dim_coords_and_dims=[(time, 0)], ) - assert get_frequency(cube) == 'daily' - cube.coord('time').convert_units('hours since 1850-1-1 00:00:00.0') - assert get_frequency(cube) == 'daily' + assert get_frequency(cube) == "daily" + cube.coord("time").convert_units("hours since 1850-1-1 00:00:00.0") + assert get_frequency(cube) == "daily" def test_get_frequency_monthly(): """Test cubes with monthly frequency.""" time = DimCoord( [0, 31, 59], - standard_name='time', - units=Unit('hours since 1900-01-01'), + standard_name="time", + units=Unit("hours since 1900-01-01"), ) cube = Cube( [1, 6, 3], - var_name='random_var', + var_name="random_var", dim_coords_and_dims=[(time, 0)], ) - assert get_frequency(cube) == 'monthly' - cube.coord('time').convert_units('days since 1850-1-1 00:00:00.0') - assert get_frequency(cube) == 'monthly' + assert get_frequency(cube) == "monthly" + cube.coord("time").convert_units("days since 1850-1-1 00:00:00.0") + assert get_frequency(cube) == "monthly" def test_get_frequency_fx(): """Test cubes with time invariant frequency.""" - cube = Cube(1., long_name='Cube without time coordinate') - assert get_frequency(cube) == 'fx' + cube = Cube(1.0, long_name="Cube without time coordinate") + assert get_frequency(cube) == "fx" time = DimCoord( 0, - standard_name='time', - units=Unit('hours since 1900-01-01'), + standard_name="time", + units=Unit("hours since 1900-01-01"), ) cube = Cube( [1], - var_name='cube_with_length_1_time_coord', - long_name='Geopotential', + var_name="cube_with_length_1_time_coord", + long_name="Geopotential", dim_coords_and_dims=[(time, 0)], ) - assert get_frequency(cube) == 'fx' + assert get_frequency(cube) == "fx" cube.long_name = ( - 'Percentage of the Grid Cell Occupied by Land (Including Lakes)' + "Percentage of the Grid Cell Occupied by Land (Including Lakes)" ) - assert get_frequency(cube) == 'fx' + assert get_frequency(cube) == "fx" - cube.long_name = 'Not geopotential' + cube.long_name = "Not geopotential" with pytest.raises(ValueError): get_frequency(cube) @@ -122,12 +125,12 @@ def test_fix_accumulated_units_fail(): """Test `fix_accumulated_units`.""" time = DimCoord( [0, 1, 2], - standard_name='time', - units=Unit('days since 1900-01-01'), + standard_name="time", + units=Unit("days since 1900-01-01"), ) cube = Cube( [1, 6, 3], - var_name='random_var', + var_name="random_var", dim_coords_and_dims=[(time, 0)], ) with pytest.raises(NotImplementedError): @@ -136,157 +139,162 @@ def test_fix_accumulated_units_fail(): def _era5_latitude(): return DimCoord( - np.array([90., 0., -90.]), - standard_name='latitude', - long_name='latitude', - var_name='latitude', - units=Unit('degrees'), + np.array([90.0, 0.0, -90.0]), + standard_name="latitude", + long_name="latitude", + var_name="latitude", + units=Unit("degrees"), ) def _era5_longitude(): return DimCoord( np.array([0, 180, 359.75]), - standard_name='longitude', - long_name='longitude', - var_name='longitude', - units=Unit('degrees'), + standard_name="longitude", + long_name="longitude", + var_name="longitude", + units=Unit("degrees"), circular=True, ) def _era5_time(frequency): - if frequency == 'invariant': + if frequency == "invariant": timestamps = [788928] # hours since 1900 at 1 january 1990 - elif frequency == 'daily': + elif frequency == "daily": timestamps = [788940, 788964, 788988] - elif frequency == 'hourly': + elif frequency == "hourly": timestamps = [788928, 788929, 788930] - elif frequency == 'monthly': + elif frequency == "monthly": timestamps = [788928, 789672, 790344] else: raise NotImplementedError(f"Invalid frequency {frequency}") return DimCoord( - np.array(timestamps, dtype='int32'), - standard_name='time', - long_name='time', - var_name='time', - units=Unit('hours since 1900-01-01' - '00:00:00.0', calendar='gregorian'), + np.array(timestamps, dtype="int32"), + standard_name="time", + long_name="time", + var_name="time", + units=Unit("hours since 1900-01-0100:00:00.0", calendar="gregorian"), ) def _era5_plev(): - values = np.array([ - 1, - 1000, - ]) + values = np.array( + [ + 1, + 1000, + ] + ) return DimCoord( values, long_name="pressure", units=Unit("millibars"), var_name="level", - attributes={'positive': 'down'}, + attributes={"positive": "down"}, ) def _era5_data(frequency): - if frequency == 'invariant': + if frequency == "invariant": return np.arange(9).reshape(1, 3, 3) return np.arange(27).reshape(3, 3, 3) def _cmor_latitude(): return DimCoord( - np.array([-90., 0., 90.]), - standard_name='latitude', - long_name='Latitude', - var_name='lat', - units=Unit('degrees_north'), - bounds=np.array([[-90., -45.], [-45., 45.], [45., 90.]]), + np.array([-90.0, 0.0, 90.0]), + standard_name="latitude", + long_name="Latitude", + var_name="lat", + units=Unit("degrees_north"), + bounds=np.array([[-90.0, -45.0], [-45.0, 45.0], [45.0, 90.0]]), ) def _cmor_longitude(): return DimCoord( np.array([0, 180, 359.75]), - standard_name='longitude', - long_name='Longitude', - var_name='lon', - units=Unit('degrees_east'), - bounds=np.array([[-0.125, 90.], [90., 269.875], [269.875, 359.875]]), + standard_name="longitude", + long_name="Longitude", + var_name="lon", + units=Unit("degrees_east"), + bounds=np.array([[-0.125, 90.0], [90.0, 269.875], [269.875, 359.875]]), circular=True, ) def _cmor_time(mip, bounds=None, shifted=False): """Provide expected time coordinate after fixes.""" - if mip == 'E1hr': + if mip == "E1hr": offset = 51134 # days since 1850 at 1 january 1990 timestamps = offset + np.arange(3) / 24 if shifted: timestamps -= 1 / 48 if bounds is not None: bounds = [[t - 1 / 48, t + 1 / 48] for t in timestamps] - elif mip == 'Eday': + elif mip == "Eday": timestamps = np.array([51134.5, 51135.5, 51136.5]) if bounds is not None: bounds = np.array( - [[51134.0, 51135.0], - [51135.0, 51136.0], - [51136.0, 51137.0]] + [[51134.0, 51135.0], [51135.0, 51136.0], [51136.0, 51137.0]] ) - elif 'mon' in mip: - timestamps = np.array([51149.5, 51179., 51208.5]) + elif "mon" in mip: + timestamps = np.array([51149.5, 51179.0, 51208.5]) if bounds is not None: - bounds = np.array([[51134., 51165.], [51165., 51193.], - [51193., 51224.]]) + bounds = np.array( + [[51134.0, 51165.0], [51165.0, 51193.0], [51193.0, 51224.0]] + ) else: raise NotImplementedError() - return DimCoord(np.array(timestamps, dtype=float), - standard_name='time', - long_name='time', - var_name='time', - units=Unit('days since 1850-1-1 00:00:00', - calendar='gregorian'), - bounds=bounds) + return DimCoord( + np.array(timestamps, dtype=float), + standard_name="time", + long_name="time", + var_name="time", + units=Unit("days since 1850-1-1 00:00:00", calendar="gregorian"), + bounds=bounds, + ) def _cmor_aux_height(value): - return AuxCoord(value, - long_name="height", - standard_name="height", - units=Unit('m'), - var_name="height", - attributes={'positive': 'up'}) + return AuxCoord( + value, + long_name="height", + standard_name="height", + units=Unit("m"), + var_name="height", + attributes={"positive": "up"}, + ) def _cmor_plev(): - values = np.array([ - 100000.0, - 100.0, - ]) - return DimCoord(values, - long_name="pressure", - standard_name="air_pressure", - units=Unit("Pa"), - var_name="plev", - attributes={'positive': 'down'}) + values = np.array( + [ + 100000.0, + 100.0, + ] + ) + return DimCoord( + values, + long_name="pressure", + standard_name="air_pressure", + units=Unit("Pa"), + var_name="plev", + attributes={"positive": "down"}, + ) def _cmor_data(mip): - if mip == 'fx': + if mip == "fx": return np.arange(9).reshape(3, 3)[::-1, :] return np.arange(27).reshape(3, 3, 3)[:, ::-1, :] def era5_2d(frequency): - if frequency == 'monthly': + if frequency == "monthly": time = DimCoord( - [-31, 0, 31], - standard_name='time', - units='days since 1850-01-01' + [-31, 0, 31], standard_name="time", units="days since 1850-01-01" ) else: time = _era5_time(frequency) @@ -294,7 +302,7 @@ def era5_2d(frequency): _era5_data(frequency), long_name=None, var_name=None, - units='unknown', + units="unknown", dim_coords_and_dims=[ (time, 0), (_era5_latitude(), 1), @@ -309,7 +317,7 @@ def era5_3d(frequency): np.ones((3, 2, 3, 3)), long_name=None, var_name=None, - units='unknown', + units="unknown", dim_coords_and_dims=[ (_era5_time(frequency), 0), (_era5_plev(), 1), @@ -321,21 +329,21 @@ def era5_3d(frequency): def cmor_2d(mip, short_name): - cmor_table = CMOR_TABLES['native6'] + cmor_table = CMOR_TABLES["native6"] vardef = cmor_table.get_variable(mip, short_name) - if 'mon' in mip: + if "mon" in mip: time = DimCoord( [-15.5, 15.5, 45.0], bounds=[[-31.0, 0.0], [0.0, 31.0], [31.0, 59.0]], - standard_name='time', - long_name='time', - var_name='time', - units='days since 1850-01-01' + standard_name="time", + long_name="time", + var_name="time", + units="days since 1850-01-01", ) else: time = _cmor_time(mip, bounds=True) cube = Cube( - _cmor_data(mip).astype('float32'), + _cmor_data(mip).astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, standard_name=vardef.standard_name, @@ -345,13 +353,13 @@ def cmor_2d(mip, short_name): (_cmor_latitude(), 1), (_cmor_longitude(), 2), ], - attributes={'comment': COMMENT}, + attributes={"comment": COMMENT}, ) return CubeList([cube]) def cmor_3d(mip, short_name): - cmor_table = CMOR_TABLES['native6'] + cmor_table = CMOR_TABLES["native6"] vardef = cmor_table.get_variable(mip, short_name) cube = Cube( np.ones((3, 2, 3, 3)), @@ -365,19 +373,19 @@ def cmor_3d(mip, short_name): (_cmor_latitude(), 2), (_cmor_longitude(), 3), ], - attributes={'comment': COMMENT}, + attributes={"comment": COMMENT}, ) return CubeList([cube]) def cl_era5_monthly(): - time = _era5_time('monthly') + time = _era5_time("monthly") data = np.ones((3, 2, 3, 3)) cube = Cube( data, - long_name='Percentage Cloud Cover', - var_name='cl', - units='%', + long_name="Percentage Cloud Cover", + var_name="cl", + units="%", dim_coords_and_dims=[ (time, 0), (_era5_plev(), 1), @@ -389,13 +397,13 @@ def cl_era5_monthly(): def cl_cmor_amon(): - cmor_table = CMOR_TABLES['native6'] - vardef = cmor_table.get_variable('Amon', 'cl') - time = _cmor_time('Amon', bounds=True) + cmor_table = CMOR_TABLES["native6"] + vardef = cmor_table.get_variable("Amon", "cl") + time = _cmor_time("Amon", bounds=True) data = np.ones((3, 2, 3, 3)) data = data * 100.0 cube = Cube( - data.astype('float32'), + data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, standard_name=vardef.standard_name, @@ -406,18 +414,18 @@ def cl_cmor_amon(): (_cmor_latitude(), 2), (_cmor_longitude(), 3), ], - attributes={'comment': COMMENT}, + attributes={"comment": COMMENT}, ) return CubeList([cube]) def clt_era5_hourly(): - time = _era5_time('hourly') + time = _era5_time("hourly") cube = Cube( - _era5_data('hourly'), - long_name='cloud cover fraction', - var_name='cloud_cover', - units='unknown', + _era5_data("hourly"), + long_name="cloud cover fraction", + var_name="cloud_cover", + units="unknown", dim_coords_and_dims=[ (time, 0), (_era5_latitude(), 1), @@ -428,12 +436,12 @@ def clt_era5_hourly(): def clt_cmor_e1hr(): - cmor_table = CMOR_TABLES['native6'] - vardef = cmor_table.get_variable('E1hr', 'clt') - time = _cmor_time('E1hr', bounds=True) - data = _cmor_data('E1hr') * 100 + cmor_table = CMOR_TABLES["native6"] + vardef = cmor_table.get_variable("E1hr", "clt") + time = _cmor_time("E1hr", bounds=True) + data = _cmor_data("E1hr") * 100 cube = Cube( - data.astype('float32'), + data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, standard_name=vardef.standard_name, @@ -443,18 +451,18 @@ def clt_cmor_e1hr(): (_cmor_latitude(), 1), (_cmor_longitude(), 2), ], - attributes={'comment': COMMENT}, + attributes={"comment": COMMENT}, ) return CubeList([cube]) def evspsbl_era5_hourly(): - time = _era5_time('hourly') + time = _era5_time("hourly") cube = Cube( - _era5_data('hourly') * -1., - long_name='total evapotranspiration', - var_name='e', - units='unknown', + _era5_data("hourly") * -1.0, + long_name="total evapotranspiration", + var_name="e", + units="unknown", dim_coords_and_dims=[ (time, 0), (_era5_latitude(), 1), @@ -465,12 +473,12 @@ def evspsbl_era5_hourly(): def evspsbl_cmor_e1hr(): - cmor_table = CMOR_TABLES['native6'] - vardef = cmor_table.get_variable('E1hr', 'evspsbl') - time = _cmor_time('E1hr', shifted=True, bounds=True) - data = _cmor_data('E1hr') * 1000 / 3600. + cmor_table = CMOR_TABLES["native6"] + vardef = cmor_table.get_variable("E1hr", "evspsbl") + time = _cmor_time("E1hr", shifted=True, bounds=True) + data = _cmor_data("E1hr") * 1000 / 3600.0 cube = Cube( - data.astype('float32'), + data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, standard_name=vardef.standard_name, @@ -480,18 +488,18 @@ def evspsbl_cmor_e1hr(): (_cmor_latitude(), 1), (_cmor_longitude(), 2), ], - attributes={'comment': COMMENT}, + attributes={"comment": COMMENT}, ) return CubeList([cube]) def evspsblpot_era5_hourly(): - time = _era5_time('hourly') + time = _era5_time("hourly") cube = Cube( - _era5_data('hourly') * -1., - long_name='potential evapotranspiration', - var_name='epot', - units='unknown', + _era5_data("hourly") * -1.0, + long_name="potential evapotranspiration", + var_name="epot", + units="unknown", dim_coords_and_dims=[ (time, 0), (_era5_latitude(), 1), @@ -502,12 +510,12 @@ def evspsblpot_era5_hourly(): def evspsblpot_cmor_e1hr(): - cmor_table = CMOR_TABLES['native6'] - vardef = cmor_table.get_variable('E1hr', 'evspsblpot') - time = _cmor_time('E1hr', shifted=True, bounds=True) - data = _cmor_data('E1hr') * 1000 / 3600. + cmor_table = CMOR_TABLES["native6"] + vardef = cmor_table.get_variable("E1hr", "evspsblpot") + time = _cmor_time("E1hr", shifted=True, bounds=True) + data = _cmor_data("E1hr") * 1000 / 3600.0 cube = Cube( - data.astype('float32'), + data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, standard_name=vardef.standard_name, @@ -517,18 +525,18 @@ def evspsblpot_cmor_e1hr(): (_cmor_latitude(), 1), (_cmor_longitude(), 2), ], - attributes={'comment': COMMENT}, + attributes={"comment": COMMENT}, ) return CubeList([cube]) def mrro_era5_hourly(): - time = _era5_time('hourly') + time = _era5_time("hourly") cube = Cube( - _era5_data('hourly'), - long_name='runoff', - var_name='runoff', - units='m', + _era5_data("hourly"), + long_name="runoff", + var_name="runoff", + units="m", dim_coords_and_dims=[ (time, 0), (_era5_latitude(), 1), @@ -539,12 +547,12 @@ def mrro_era5_hourly(): def mrro_cmor_e1hr(): - cmor_table = CMOR_TABLES['native6'] - vardef = cmor_table.get_variable('E1hr', 'mrro') - time = _cmor_time('E1hr', shifted=True, bounds=True) - data = _cmor_data('E1hr') * 1000 / 3600. + cmor_table = CMOR_TABLES["native6"] + vardef = cmor_table.get_variable("E1hr", "mrro") + time = _cmor_time("E1hr", shifted=True, bounds=True) + data = _cmor_data("E1hr") * 1000 / 3600.0 cube = Cube( - data.astype('float32'), + data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, standard_name=vardef.standard_name, @@ -554,26 +562,26 @@ def mrro_cmor_e1hr(): (_cmor_latitude(), 1), (_cmor_longitude(), 2), ], - attributes={'comment': COMMENT}, + attributes={"comment": COMMENT}, ) return CubeList([cube]) def o3_era5_monthly(): - cube = era5_3d('monthly')[0] + cube = era5_3d("monthly")[0] cube = cube[:, ::-1, ::-1, :] # test if correct order of plev and lat stay - cube.data = cube.data.astype('float32') + cube.data = cube.data.astype("float32") cube.data *= 47.9982 / 28.9644 return CubeList([cube]) def orog_era5_hourly(): - time = _era5_time('invariant') + time = _era5_time("invariant") cube = Cube( - _era5_data('invariant'), - long_name='geopotential height', - var_name='zg', - units='m**2 s**-2', + _era5_data("invariant"), + long_name="geopotential height", + var_name="zg", + units="m**2 s**-2", dim_coords_and_dims=[ (time, 0), (_era5_latitude(), 1), @@ -584,28 +592,28 @@ def orog_era5_hourly(): def orog_cmor_fx(): - cmor_table = CMOR_TABLES['native6'] - vardef = cmor_table.get_variable('fx', 'orog') - data = _cmor_data('fx') / 9.80665 + cmor_table = CMOR_TABLES["native6"] + vardef = cmor_table.get_variable("fx", "orog") + data = _cmor_data("fx") / 9.80665 cube = Cube( - data.astype('float32'), + data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, standard_name=vardef.standard_name, units=Unit(vardef.units), dim_coords_and_dims=[(_cmor_latitude(), 0), (_cmor_longitude(), 1)], - attributes={'comment': COMMENT}, + attributes={"comment": COMMENT}, ) return CubeList([cube]) def pr_era5_monthly(): - time = _era5_time('monthly') + time = _era5_time("monthly") cube = Cube( - _era5_data('monthly'), - long_name='total_precipitation', - var_name='tp', - units='m', + _era5_data("monthly"), + long_name="total_precipitation", + var_name="tp", + units="m", dim_coords_and_dims=[ (time, 0), (_era5_latitude(), 1), @@ -616,12 +624,12 @@ def pr_era5_monthly(): def pr_cmor_amon(): - cmor_table = CMOR_TABLES['native6'] - vardef = cmor_table.get_variable('Amon', 'pr') - time = _cmor_time('Amon', bounds=True) - data = _cmor_data('Amon') * 1000. / 3600. / 24. + cmor_table = CMOR_TABLES["native6"] + vardef = cmor_table.get_variable("Amon", "pr") + time = _cmor_time("Amon", bounds=True) + data = _cmor_data("Amon") * 1000.0 / 3600.0 / 24.0 cube = Cube( - data.astype('float32'), + data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, standard_name=vardef.standard_name, @@ -631,18 +639,18 @@ def pr_cmor_amon(): (_cmor_latitude(), 1), (_cmor_longitude(), 2), ], - attributes={'comment': COMMENT}, + attributes={"comment": COMMENT}, ) return CubeList([cube]) def pr_era5_hourly(): - time = _era5_time('hourly') + time = _era5_time("hourly") cube = Cube( - _era5_data('hourly'), - long_name='total_precipitation', - var_name='tp', - units='m', + _era5_data("hourly"), + long_name="total_precipitation", + var_name="tp", + units="m", dim_coords_and_dims=[ (time, 0), (_era5_latitude(), 1), @@ -653,12 +661,12 @@ def pr_era5_hourly(): def pr_cmor_e1hr(): - cmor_table = CMOR_TABLES['native6'] - vardef = cmor_table.get_variable('E1hr', 'pr') - time = _cmor_time('E1hr', bounds=True, shifted=True) - data = _cmor_data('E1hr') * 1000. / 3600. + cmor_table = CMOR_TABLES["native6"] + vardef = cmor_table.get_variable("E1hr", "pr") + time = _cmor_time("E1hr", bounds=True, shifted=True) + data = _cmor_data("E1hr") * 1000.0 / 3600.0 cube = Cube( - data.astype('float32'), + data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, standard_name=vardef.standard_name, @@ -668,18 +676,18 @@ def pr_cmor_e1hr(): (_cmor_latitude(), 1), (_cmor_longitude(), 2), ], - attributes={'comment': COMMENT}, + attributes={"comment": COMMENT}, ) return CubeList([cube]) def prsn_era5_hourly(): - time = _era5_time('hourly') + time = _era5_time("hourly") cube = Cube( - _era5_data('hourly'), - long_name='snow', - var_name='snow', - units='unknown', + _era5_data("hourly"), + long_name="snow", + var_name="snow", + units="unknown", dim_coords_and_dims=[ (time, 0), (_era5_latitude(), 1), @@ -690,12 +698,12 @@ def prsn_era5_hourly(): def prsn_cmor_e1hr(): - cmor_table = CMOR_TABLES['native6'] - vardef = cmor_table.get_variable('E1hr', 'prsn') - time = _cmor_time('E1hr', shifted=True, bounds=True) - data = _cmor_data('E1hr') * 1000 / 3600. + cmor_table = CMOR_TABLES["native6"] + vardef = cmor_table.get_variable("E1hr", "prsn") + time = _cmor_time("E1hr", shifted=True, bounds=True) + data = _cmor_data("E1hr") * 1000 / 3600.0 cube = Cube( - data.astype('float32'), + data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, standard_name=vardef.standard_name, @@ -705,18 +713,18 @@ def prsn_cmor_e1hr(): (_cmor_latitude(), 1), (_cmor_longitude(), 2), ], - attributes={'comment': COMMENT}, + attributes={"comment": COMMENT}, ) return CubeList([cube]) def ptype_era5_hourly(): - time = _era5_time('hourly') + time = _era5_time("hourly") cube = Cube( - _era5_data('hourly'), - long_name='snow', - var_name='snow', - units='unknown', + _era5_data("hourly"), + long_name="snow", + var_name="snow", + units="unknown", dim_coords_and_dims=[ (time, 0), (_era5_latitude(), 1), @@ -727,12 +735,12 @@ def ptype_era5_hourly(): def ptype_cmor_e1hr(): - cmor_table = CMOR_TABLES['native6'] - vardef = cmor_table.get_variable('E1hr', 'ptype') - time = _cmor_time('E1hr', shifted=False, bounds=True) - data = _cmor_data('E1hr') + cmor_table = CMOR_TABLES["native6"] + vardef = cmor_table.get_variable("E1hr", "ptype") + time = _cmor_time("E1hr", shifted=False, bounds=True) + data = _cmor_data("E1hr") cube = Cube( - data.astype('float32'), + data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, units=1, @@ -741,20 +749,20 @@ def ptype_cmor_e1hr(): (_cmor_latitude(), 1), (_cmor_longitude(), 2), ], - attributes={'comment': COMMENT}, + attributes={"comment": COMMENT}, ) - cube.coord('latitude').long_name = 'latitude' - cube.coord('longitude').long_name = 'longitude' + cube.coord("latitude").long_name = "latitude" + cube.coord("longitude").long_name = "longitude" return CubeList([cube]) def rlds_era5_hourly(): - time = _era5_time('hourly') + time = _era5_time("hourly") cube = Cube( - _era5_data('hourly'), - long_name='surface thermal radiation downwards', - var_name='ssrd', - units='J m**-2', + _era5_data("hourly"), + long_name="surface thermal radiation downwards", + var_name="ssrd", + units="J m**-2", dim_coords_and_dims=[ (time, 0), (_era5_latitude(), 1), @@ -765,29 +773,33 @@ def rlds_era5_hourly(): def rlds_cmor_e1hr(): - cmor_table = CMOR_TABLES['native6'] - vardef = cmor_table.get_variable('E1hr', 'rlds') - time = _cmor_time('E1hr', shifted=True, bounds=True) - data = _cmor_data('E1hr') / 3600 - cube = Cube(data.astype('float32'), - long_name=vardef.long_name, - var_name=vardef.short_name, - standard_name=vardef.standard_name, - units=Unit(vardef.units), - dim_coords_and_dims=[(time, 0), - (_cmor_latitude(), 1), - (_cmor_longitude(), 2)], - attributes={'comment': COMMENT, 'positive': 'down'}) + cmor_table = CMOR_TABLES["native6"] + vardef = cmor_table.get_variable("E1hr", "rlds") + time = _cmor_time("E1hr", shifted=True, bounds=True) + data = _cmor_data("E1hr") / 3600 + cube = Cube( + data.astype("float32"), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[ + (time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2), + ], + attributes={"comment": COMMENT, "positive": "down"}, + ) return CubeList([cube]) def rlns_era5_hourly(): - freq = 'hourly' + freq = "hourly" cube = Cube( _era5_data(freq), long_name=None, var_name=None, - units='J m**-2', + units="J m**-2", dim_coords_and_dims=[ (_era5_time(freq), 0), (_era5_latitude(), 1), @@ -798,33 +810,37 @@ def rlns_era5_hourly(): def rlns_cmor_e1hr(): - mip = 'E1hr' - short_name = 'rlns' - cmor_table = CMOR_TABLES['native6'] + mip = "E1hr" + short_name = "rlns" + cmor_table = CMOR_TABLES["native6"] vardef = cmor_table.get_variable(mip, short_name) time = _cmor_time(mip, shifted=True, bounds=True) data = _cmor_data(mip) / 3600 - cube = Cube(data.astype('float32'), - long_name=vardef.long_name, - var_name=vardef.short_name, - standard_name=vardef.standard_name, - units=Unit(vardef.units), - dim_coords_and_dims=[(time, 0), - (_cmor_latitude(), 1), - (_cmor_longitude(), 2)], - attributes={'comment': COMMENT, 'positive': 'down'}) - cube.coord('latitude').long_name = 'latitude' # from custom table - cube.coord('longitude').long_name = 'longitude' # from custom table + cube = Cube( + data.astype("float32"), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[ + (time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2), + ], + attributes={"comment": COMMENT, "positive": "down"}, + ) + cube.coord("latitude").long_name = "latitude" # from custom table + cube.coord("longitude").long_name = "longitude" # from custom table return CubeList([cube]) def rlus_era5_hourly(): - freq = 'hourly' + freq = "hourly" cube = Cube( _era5_data(freq), long_name=None, var_name=None, - units='J m**-2', + units="J m**-2", dim_coords_and_dims=[ (_era5_time(freq), 0), (_era5_latitude(), 1), @@ -835,31 +851,35 @@ def rlus_era5_hourly(): def rlus_cmor_e1hr(): - mip = 'E1hr' - short_name = 'rlus' - cmor_table = CMOR_TABLES['native6'] + mip = "E1hr" + short_name = "rlus" + cmor_table = CMOR_TABLES["native6"] vardef = cmor_table.get_variable(mip, short_name) time = _cmor_time(mip, shifted=True, bounds=True) data = _cmor_data(mip) / 3600 - cube = Cube(data.astype('float32'), - long_name=vardef.long_name, - var_name=vardef.short_name, - standard_name=vardef.standard_name, - units=Unit(vardef.units), - dim_coords_and_dims=[(time, 0), - (_cmor_latitude(), 1), - (_cmor_longitude(), 2)], - attributes={'comment': COMMENT, 'positive': 'up'}) + cube = Cube( + data.astype("float32"), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[ + (time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2), + ], + attributes={"comment": COMMENT, "positive": "up"}, + ) return CubeList([cube]) def rls_era5_hourly(): - time = _era5_time('hourly') + time = _era5_time("hourly") cube = Cube( - _era5_data('hourly'), - long_name='runoff', - var_name='runoff', - units='W m-2', + _era5_data("hourly"), + long_name="runoff", + var_name="runoff", + units="W m-2", dim_coords_and_dims=[ (time, 0), (_era5_latitude(), 1), @@ -870,31 +890,33 @@ def rls_era5_hourly(): def rls_cmor_e1hr(): - cmor_table = CMOR_TABLES['native6'] - vardef = cmor_table.get_variable('E1hr', 'rls') - time = _cmor_time('E1hr', shifted=True, bounds=True) - data = _cmor_data('E1hr') - cube = Cube(data.astype('float32'), - long_name=vardef.long_name, - var_name=vardef.short_name, - standard_name=vardef.standard_name, - units=Unit(vardef.units), - dim_coords_and_dims=[ - (time, 0), - (_cmor_latitude(), 1), - (_cmor_longitude(), 2), - ], - attributes={'comment': COMMENT, 'positive': 'down'}) + cmor_table = CMOR_TABLES["native6"] + vardef = cmor_table.get_variable("E1hr", "rls") + time = _cmor_time("E1hr", shifted=True, bounds=True) + data = _cmor_data("E1hr") + cube = Cube( + data.astype("float32"), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[ + (time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2), + ], + attributes={"comment": COMMENT, "positive": "down"}, + ) return CubeList([cube]) def rsds_era5_hourly(): - time = _era5_time('hourly') + time = _era5_time("hourly") cube = Cube( - _era5_data('hourly'), - long_name='solar_radiation_downwards', - var_name='rlwd', - units='J m**-2', + _era5_data("hourly"), + long_name="solar_radiation_downwards", + var_name="rlwd", + units="J m**-2", dim_coords_and_dims=[ (time, 0), (_era5_latitude(), 1), @@ -905,29 +927,33 @@ def rsds_era5_hourly(): def rsds_cmor_e1hr(): - cmor_table = CMOR_TABLES['native6'] - vardef = cmor_table.get_variable('E1hr', 'rsds') - time = _cmor_time('E1hr', shifted=True, bounds=True) - data = _cmor_data('E1hr') / 3600 - cube = Cube(data.astype('float32'), - long_name=vardef.long_name, - var_name=vardef.short_name, - standard_name=vardef.standard_name, - units=Unit(vardef.units), - dim_coords_and_dims=[(time, 0), - (_cmor_latitude(), 1), - (_cmor_longitude(), 2)], - attributes={'comment': COMMENT, 'positive': 'down'}) + cmor_table = CMOR_TABLES["native6"] + vardef = cmor_table.get_variable("E1hr", "rsds") + time = _cmor_time("E1hr", shifted=True, bounds=True) + data = _cmor_data("E1hr") / 3600 + cube = Cube( + data.astype("float32"), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[ + (time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2), + ], + attributes={"comment": COMMENT, "positive": "down"}, + ) return CubeList([cube]) def rsns_era5_hourly(): - freq = 'hourly' + freq = "hourly" cube = Cube( _era5_data(freq), long_name=None, var_name=None, - units='J m**-2', + units="J m**-2", dim_coords_and_dims=[ (_era5_time(freq), 0), (_era5_latitude(), 1), @@ -938,33 +964,37 @@ def rsns_era5_hourly(): def rsns_cmor_e1hr(): - mip = 'E1hr' - short_name = 'rsns' - cmor_table = CMOR_TABLES['native6'] + mip = "E1hr" + short_name = "rsns" + cmor_table = CMOR_TABLES["native6"] vardef = cmor_table.get_variable(mip, short_name) time = _cmor_time(mip, shifted=True, bounds=True) data = _cmor_data(mip) / 3600 - cube = Cube(data.astype('float32'), - long_name=vardef.long_name, - var_name=vardef.short_name, - standard_name=vardef.standard_name, - units=Unit(vardef.units), - dim_coords_and_dims=[(time, 0), - (_cmor_latitude(), 1), - (_cmor_longitude(), 2)], - attributes={'comment': COMMENT, 'positive': 'down'}) - cube.coord('latitude').long_name = 'latitude' # from custom table - cube.coord('longitude').long_name = 'longitude' # from custom table + cube = Cube( + data.astype("float32"), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[ + (time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2), + ], + attributes={"comment": COMMENT, "positive": "down"}, + ) + cube.coord("latitude").long_name = "latitude" # from custom table + cube.coord("longitude").long_name = "longitude" # from custom table return CubeList([cube]) def rsus_era5_hourly(): - freq = 'hourly' + freq = "hourly" cube = Cube( _era5_data(freq), long_name=None, var_name=None, - units='J m**-2', + units="J m**-2", dim_coords_and_dims=[ (_era5_time(freq), 0), (_era5_latitude(), 1), @@ -975,61 +1005,72 @@ def rsus_era5_hourly(): def rsus_cmor_e1hr(): - mip = 'E1hr' - short_name = 'rsus' - cmor_table = CMOR_TABLES['native6'] + mip = "E1hr" + short_name = "rsus" + cmor_table = CMOR_TABLES["native6"] vardef = cmor_table.get_variable(mip, short_name) time = _cmor_time(mip, shifted=True, bounds=True) data = _cmor_data(mip) / 3600 - cube = Cube(data.astype('float32'), - long_name=vardef.long_name, - var_name=vardef.short_name, - standard_name=vardef.standard_name, - units=Unit(vardef.units), - dim_coords_and_dims=[(time, 0), - (_cmor_latitude(), 1), - (_cmor_longitude(), 2)], - attributes={'comment': COMMENT, 'positive': 'up'}) + cube = Cube( + data.astype("float32"), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[ + (time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2), + ], + attributes={"comment": COMMENT, "positive": "up"}, + ) return CubeList([cube]) def rsdt_era5_hourly(): - time = _era5_time('hourly') + time = _era5_time("hourly") cube = Cube( - _era5_data('hourly'), - long_name='thermal_radiation_downwards', - var_name='strd', - units='J m**-2', - dim_coords_and_dims=[(time, 0), (_era5_latitude(), 1), - (_era5_longitude(), 2)], + _era5_data("hourly"), + long_name="thermal_radiation_downwards", + var_name="strd", + units="J m**-2", + dim_coords_and_dims=[ + (time, 0), + (_era5_latitude(), 1), + (_era5_longitude(), 2), + ], ) return CubeList([cube]) def rsdt_cmor_e1hr(): - cmor_table = CMOR_TABLES['native6'] - vardef = cmor_table.get_variable('E1hr', 'rsdt') - time = _cmor_time('E1hr', shifted=True, bounds=True) - data = _cmor_data('E1hr') / 3600 - cube = Cube(data.astype('float32'), - long_name=vardef.long_name, - var_name=vardef.short_name, - standard_name=vardef.standard_name, - units=Unit(vardef.units), - dim_coords_and_dims=[(time, 0), - (_cmor_latitude(), 1), - (_cmor_longitude(), 2)], - attributes={'comment': COMMENT, 'positive': 'down'}) + cmor_table = CMOR_TABLES["native6"] + vardef = cmor_table.get_variable("E1hr", "rsdt") + time = _cmor_time("E1hr", shifted=True, bounds=True) + data = _cmor_data("E1hr") / 3600 + cube = Cube( + data.astype("float32"), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[ + (time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2), + ], + attributes={"comment": COMMENT, "positive": "down"}, + ) return CubeList([cube]) def rss_era5_hourly(): - time = _era5_time('hourly') + time = _era5_time("hourly") cube = Cube( - _era5_data('hourly'), - long_name='net_solar_radiation', - var_name='ssr', - units='J m**-2', + _era5_data("hourly"), + long_name="net_solar_radiation", + var_name="ssr", + units="J m**-2", dim_coords_and_dims=[ (time, 0), (_era5_latitude(), 1), @@ -1040,19 +1081,23 @@ def rss_era5_hourly(): def rss_cmor_e1hr(): - cmor_table = CMOR_TABLES['native6'] - vardef = cmor_table.get_variable('E1hr', 'rss') - time = _cmor_time('E1hr', shifted=True, bounds=True) - data = _cmor_data('E1hr') / 3600 - cube = Cube(data.astype('float32'), - long_name=vardef.long_name, - var_name=vardef.short_name, - standard_name=vardef.standard_name, - units=Unit(vardef.units), - dim_coords_and_dims=[(time, 0), - (_cmor_latitude(), 1), - (_cmor_longitude(), 2)], - attributes={'comment': COMMENT, 'positive': 'down'}) + cmor_table = CMOR_TABLES["native6"] + vardef = cmor_table.get_variable("E1hr", "rss") + time = _cmor_time("E1hr", shifted=True, bounds=True) + data = _cmor_data("E1hr") / 3600 + cube = Cube( + data.astype("float32"), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[ + (time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2), + ], + attributes={"comment": COMMENT, "positive": "down"}, + ) return CubeList([cube]) @@ -1061,7 +1106,7 @@ def sftlf_era5(): np.ones((3, 3)), long_name=None, var_name=None, - units='unknown', + units="unknown", dim_coords_and_dims=[ (_era5_latitude(), 0), (_era5_longitude(), 1), @@ -1071,27 +1116,27 @@ def sftlf_era5(): def sftlf_cmor_fx(): - cmor_table = CMOR_TABLES['native6'] - vardef = cmor_table.get_variable('fx', 'sftlf') + cmor_table = CMOR_TABLES["native6"] + vardef = cmor_table.get_variable("fx", "sftlf") cube = Cube( - np.ones((3, 3)).astype('float32') * 100.0, + np.ones((3, 3)).astype("float32") * 100.0, long_name=vardef.long_name, var_name=vardef.short_name, standard_name=vardef.standard_name, units=Unit(vardef.units), dim_coords_and_dims=[(_cmor_latitude(), 0), (_cmor_longitude(), 1)], - attributes={'comment': COMMENT}, + attributes={"comment": COMMENT}, ) return CubeList([cube]) def tas_era5_hourly(): - time = _era5_time('hourly') + time = _era5_time("hourly") cube = Cube( - _era5_data('hourly'), - long_name='2m_temperature', - var_name='t2m', - units='K', + _era5_data("hourly"), + long_name="2m_temperature", + var_name="t2m", + units="K", dim_coords_and_dims=[ (time, 0), (_era5_latitude(), 1), @@ -1102,30 +1147,34 @@ def tas_era5_hourly(): def tas_cmor_e1hr(): - cmor_table = CMOR_TABLES['native6'] - vardef = cmor_table.get_variable('E1hr', 'tas') - time = _cmor_time('E1hr') - data = _cmor_data('E1hr') - cube = Cube(data.astype('float32'), - long_name=vardef.long_name, - var_name=vardef.short_name, - standard_name=vardef.standard_name, - units=Unit(vardef.units), - dim_coords_and_dims=[(time, 0), - (_cmor_latitude(), 1), - (_cmor_longitude(), 2)], - attributes={'comment': COMMENT}) - cube.add_aux_coord(_cmor_aux_height(2.)) + cmor_table = CMOR_TABLES["native6"] + vardef = cmor_table.get_variable("E1hr", "tas") + time = _cmor_time("E1hr") + data = _cmor_data("E1hr") + cube = Cube( + data.astype("float32"), + long_name=vardef.long_name, + var_name=vardef.short_name, + standard_name=vardef.standard_name, + units=Unit(vardef.units), + dim_coords_and_dims=[ + (time, 0), + (_cmor_latitude(), 1), + (_cmor_longitude(), 2), + ], + attributes={"comment": COMMENT}, + ) + cube.add_aux_coord(_cmor_aux_height(2.0)) return CubeList([cube]) def tas_era5_monthly(): - time = _era5_time('monthly') + time = _era5_time("monthly") cube = Cube( - _era5_data('monthly'), - long_name='2m_temperature', - var_name='t2m', - units='K', + _era5_data("monthly"), + long_name="2m_temperature", + var_name="t2m", + units="K", dim_coords_and_dims=[ (time, 0), (_era5_latitude(), 1), @@ -1136,12 +1185,12 @@ def tas_era5_monthly(): def tas_cmor_amon(): - cmor_table = CMOR_TABLES['native6'] - vardef = cmor_table.get_variable('Amon', 'tas') - time = _cmor_time('Amon', bounds=True) - data = _cmor_data('Amon') + cmor_table = CMOR_TABLES["native6"] + vardef = cmor_table.get_variable("Amon", "tas") + time = _cmor_time("Amon", bounds=True) + data = _cmor_data("Amon") cube = Cube( - data.astype('float32'), + data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, standard_name=vardef.standard_name, @@ -1151,27 +1200,27 @@ def tas_cmor_amon(): (_cmor_latitude(), 1), (_cmor_longitude(), 2), ], - attributes={'comment': COMMENT}, + attributes={"comment": COMMENT}, ) - cube.add_aux_coord(_cmor_aux_height(2.)) + cube.add_aux_coord(_cmor_aux_height(2.0)) return CubeList([cube]) def toz_era5_monthly(): - cube = era5_2d('monthly')[0] - cube.data = cube.data.astype('float32') + cube = era5_2d("monthly")[0] + cube.data = cube.data.astype("float32") cube.data *= 2.1415 return CubeList([cube]) def zg_era5_monthly(): - time = _era5_time('monthly') + time = _era5_time("monthly") data = np.ones((3, 2, 3, 3)) cube = Cube( data, - long_name='geopotential height', - var_name='zg', - units='m**2 s**-2', + long_name="geopotential height", + var_name="zg", + units="m**2 s**-2", dim_coords_and_dims=[ (time, 0), (_era5_plev(), 1), @@ -1183,13 +1232,13 @@ def zg_era5_monthly(): def zg_cmor_amon(): - cmor_table = CMOR_TABLES['native6'] - vardef = cmor_table.get_variable('Amon', 'zg') - time = _cmor_time('Amon', bounds=True) + cmor_table = CMOR_TABLES["native6"] + vardef = cmor_table.get_variable("Amon", "zg") + time = _cmor_time("Amon", bounds=True) data = np.ones((3, 2, 3, 3)) data = data / 9.80665 cube = Cube( - data.astype('float32'), + data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, standard_name=vardef.standard_name, @@ -1200,18 +1249,18 @@ def zg_cmor_amon(): (_cmor_latitude(), 2), (_cmor_longitude(), 3), ], - attributes={'comment': COMMENT}, + attributes={"comment": COMMENT}, ) return CubeList([cube]) def tasmax_era5_hourly(): - time = _era5_time('hourly') + time = _era5_time("hourly") cube = Cube( - _era5_data('hourly'), - long_name='maximum 2m temperature', - var_name='mx2t', - units='K', + _era5_data("hourly"), + long_name="maximum 2m temperature", + var_name="mx2t", + units="K", dim_coords_and_dims=[ (time, 0), (_era5_latitude(), 1), @@ -1222,12 +1271,12 @@ def tasmax_era5_hourly(): def tasmax_cmor_e1hr(): - cmor_table = CMOR_TABLES['native6'] - vardef = cmor_table.get_variable('E1hr', 'tasmax') - time = _cmor_time('E1hr', shifted=True, bounds=True) - data = _cmor_data('E1hr') + cmor_table = CMOR_TABLES["native6"] + vardef = cmor_table.get_variable("E1hr", "tasmax") + time = _cmor_time("E1hr", shifted=True, bounds=True) + data = _cmor_data("E1hr") cube = Cube( - data.astype('float32'), + data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, standard_name=vardef.standard_name, @@ -1237,19 +1286,19 @@ def tasmax_cmor_e1hr(): (_cmor_latitude(), 1), (_cmor_longitude(), 2), ], - attributes={'comment': COMMENT}, + attributes={"comment": COMMENT}, ) - cube.add_aux_coord(_cmor_aux_height(2.)) + cube.add_aux_coord(_cmor_aux_height(2.0)) return CubeList([cube]) def tasmin_era5_hourly(): - time = _era5_time('hourly') + time = _era5_time("hourly") cube = Cube( - _era5_data('hourly'), - long_name='minimum 2m temperature', - var_name='mn2t', - units='K', + _era5_data("hourly"), + long_name="minimum 2m temperature", + var_name="mn2t", + units="K", dim_coords_and_dims=[ (time, 0), (_era5_latitude(), 1), @@ -1260,12 +1309,12 @@ def tasmin_era5_hourly(): def tasmin_cmor_e1hr(): - cmor_table = CMOR_TABLES['native6'] - vardef = cmor_table.get_variable('E1hr', 'tasmin') - time = _cmor_time('E1hr', shifted=True, bounds=True) - data = _cmor_data('E1hr') + cmor_table = CMOR_TABLES["native6"] + vardef = cmor_table.get_variable("E1hr", "tasmin") + time = _cmor_time("E1hr", shifted=True, bounds=True) + data = _cmor_data("E1hr") cube = Cube( - data.astype('float32'), + data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, standard_name=vardef.standard_name, @@ -1275,19 +1324,19 @@ def tasmin_cmor_e1hr(): (_cmor_latitude(), 1), (_cmor_longitude(), 2), ], - attributes={'comment': COMMENT}, + attributes={"comment": COMMENT}, ) - cube.add_aux_coord(_cmor_aux_height(2.)) + cube.add_aux_coord(_cmor_aux_height(2.0)) return CubeList([cube]) def uas_era5_hourly(): - time = _era5_time('hourly') + time = _era5_time("hourly") cube = Cube( - _era5_data('hourly'), - long_name='10m_u_component_of_wind', - var_name='u10', - units='m s-1', + _era5_data("hourly"), + long_name="10m_u_component_of_wind", + var_name="u10", + units="m s-1", dim_coords_and_dims=[ (time, 0), (_era5_latitude(), 1), @@ -1298,12 +1347,12 @@ def uas_era5_hourly(): def uas_cmor_e1hr(): - cmor_table = CMOR_TABLES['native6'] - vardef = cmor_table.get_variable('E1hr', 'uas') - time = _cmor_time('E1hr') - data = _cmor_data('E1hr') + cmor_table = CMOR_TABLES["native6"] + vardef = cmor_table.get_variable("E1hr", "uas") + time = _cmor_time("E1hr") + data = _cmor_data("E1hr") cube = Cube( - data.astype('float32'), + data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, standard_name=vardef.standard_name, @@ -1313,19 +1362,19 @@ def uas_cmor_e1hr(): (_cmor_latitude(), 1), (_cmor_longitude(), 2), ], - attributes={'comment': COMMENT}, + attributes={"comment": COMMENT}, ) - cube.add_aux_coord(_cmor_aux_height(10.)) + cube.add_aux_coord(_cmor_aux_height(10.0)) return CubeList([cube]) def vas_era5_hourly(): - time = _era5_time('hourly') + time = _era5_time("hourly") cube = Cube( - _era5_data('hourly'), - long_name='10m_v_component_of_wind', - var_name='v10', - units='m s-1', + _era5_data("hourly"), + long_name="10m_v_component_of_wind", + var_name="v10", + units="m s-1", dim_coords_and_dims=[ (time, 0), (_era5_latitude(), 1), @@ -1336,12 +1385,12 @@ def vas_era5_hourly(): def vas_cmor_e1hr(): - cmor_table = CMOR_TABLES['native6'] - vardef = cmor_table.get_variable('E1hr', 'vas') - time = _cmor_time('E1hr') - data = _cmor_data('E1hr') + cmor_table = CMOR_TABLES["native6"] + vardef = cmor_table.get_variable("E1hr", "vas") + time = _cmor_time("E1hr") + data = _cmor_data("E1hr") cube = Cube( - data.astype('float32'), + data.astype("float32"), long_name=vardef.long_name, var_name=vardef.short_name, standard_name=vardef.standard_name, @@ -1351,81 +1400,94 @@ def vas_cmor_e1hr(): (_cmor_latitude(), 1), (_cmor_longitude(), 2), ], - attributes={'comment': COMMENT}, + attributes={"comment": COMMENT}, ) - cube.add_aux_coord(_cmor_aux_height(10.)) + cube.add_aux_coord(_cmor_aux_height(10.0)) return CubeList([cube]) VARIABLES = [ - pytest.param(a, b, c, d, id=c + '_' + d) for (a, b, c, d) in [ - (era5_2d('daily'), cmor_2d('Eday', 'albsn'), 'albsn', 'Eday'), - (cl_era5_monthly(), cl_cmor_amon(), 'cl', 'Amon'), - (era5_3d('monthly'), cmor_3d('Amon', 'cli'), 'cli', 'Amon'), - (clt_era5_hourly(), clt_cmor_e1hr(), 'clt', 'E1hr'), - (era5_3d('monthly'), cmor_3d('Amon', 'clw'), 'clw', 'Amon'), - (evspsbl_era5_hourly(), evspsbl_cmor_e1hr(), 'evspsbl', 'E1hr'), - (evspsblpot_era5_hourly(), evspsblpot_cmor_e1hr(), 'evspsblpot', - 'E1hr'), - (era5_3d('monthly'), cmor_3d('Amon', 'hus'), 'hus', 'Amon'), - (mrro_era5_hourly(), mrro_cmor_e1hr(), 'mrro', 'E1hr'), - (o3_era5_monthly(), cmor_3d('Amon', 'o3'), 'o3', 'Amon'), - (orog_era5_hourly(), orog_cmor_fx(), 'orog', 'fx'), - (pr_era5_monthly(), pr_cmor_amon(), 'pr', 'Amon'), - (pr_era5_hourly(), pr_cmor_e1hr(), 'pr', 'E1hr'), - (prsn_era5_hourly(), prsn_cmor_e1hr(), 'prsn', 'E1hr'), - (era5_2d('monthly'), cmor_2d('Amon', 'prw'), 'prw', 'Amon'), - (era5_2d('monthly'), cmor_2d('Amon', 'ps'), 'ps', 'Amon'), - (ptype_era5_hourly(), ptype_cmor_e1hr(), 'ptype', 'E1hr'), - (era5_3d('monthly'), cmor_3d('Emon', 'rainmxrat27'), 'rainmxrat27', - 'Emon'), - (rlds_era5_hourly(), rlds_cmor_e1hr(), 'rlds', 'E1hr'), - (rlns_era5_hourly(), rlns_cmor_e1hr(), 'rlns', 'E1hr'), - (rlus_era5_hourly(), rlus_cmor_e1hr(), 'rlus', 'E1hr'), - (rls_era5_hourly(), rls_cmor_e1hr(), 'rls', 'E1hr'), - (rsds_era5_hourly(), rsds_cmor_e1hr(), 'rsds', 'E1hr'), - (rsns_era5_hourly(), rsns_cmor_e1hr(), 'rsns', 'E1hr'), - (rsus_era5_hourly(), rsus_cmor_e1hr(), 'rsus', 'E1hr'), - (rsdt_era5_hourly(), rsdt_cmor_e1hr(), 'rsdt', 'E1hr'), - (rss_era5_hourly(), rss_cmor_e1hr(), 'rss', 'E1hr'), - (sftlf_era5(), sftlf_cmor_fx(), 'sftlf', 'fx'), - (era5_3d('monthly'), cmor_3d('Emon', 'snowmxrat27'), 'snowmxrat27', - 'Emon'), - (tas_era5_hourly(), tas_cmor_e1hr(), 'tas', 'E1hr'), - (tas_era5_monthly(), tas_cmor_amon(), 'tas', 'Amon'), - (tasmax_era5_hourly(), tasmax_cmor_e1hr(), 'tasmax', 'E1hr'), - (tasmin_era5_hourly(), tasmin_cmor_e1hr(), 'tasmin', 'E1hr'), - (toz_era5_monthly(), cmor_2d('AERmon', 'toz'), 'toz', 'AERmon'), - (uas_era5_hourly(), uas_cmor_e1hr(), 'uas', 'E1hr'), - (vas_era5_hourly(), vas_cmor_e1hr(), 'vas', 'E1hr'), - (zg_era5_monthly(), zg_cmor_amon(), 'zg', 'Amon'), + pytest.param(a, b, c, d, id=c + "_" + d) + for (a, b, c, d) in [ + (era5_2d("daily"), cmor_2d("Eday", "albsn"), "albsn", "Eday"), + (cl_era5_monthly(), cl_cmor_amon(), "cl", "Amon"), + (era5_3d("monthly"), cmor_3d("Amon", "cli"), "cli", "Amon"), + (clt_era5_hourly(), clt_cmor_e1hr(), "clt", "E1hr"), + (era5_3d("monthly"), cmor_3d("Amon", "clw"), "clw", "Amon"), + (evspsbl_era5_hourly(), evspsbl_cmor_e1hr(), "evspsbl", "E1hr"), + ( + evspsblpot_era5_hourly(), + evspsblpot_cmor_e1hr(), + "evspsblpot", + "E1hr", + ), + (era5_3d("monthly"), cmor_3d("Amon", "hus"), "hus", "Amon"), + (mrro_era5_hourly(), mrro_cmor_e1hr(), "mrro", "E1hr"), + (o3_era5_monthly(), cmor_3d("Amon", "o3"), "o3", "Amon"), + (orog_era5_hourly(), orog_cmor_fx(), "orog", "fx"), + (pr_era5_monthly(), pr_cmor_amon(), "pr", "Amon"), + (pr_era5_hourly(), pr_cmor_e1hr(), "pr", "E1hr"), + (prsn_era5_hourly(), prsn_cmor_e1hr(), "prsn", "E1hr"), + (era5_2d("monthly"), cmor_2d("Amon", "prw"), "prw", "Amon"), + (era5_2d("monthly"), cmor_2d("Amon", "ps"), "ps", "Amon"), + (ptype_era5_hourly(), ptype_cmor_e1hr(), "ptype", "E1hr"), + ( + era5_3d("monthly"), + cmor_3d("Emon", "rainmxrat27"), + "rainmxrat27", + "Emon", + ), + (rlds_era5_hourly(), rlds_cmor_e1hr(), "rlds", "E1hr"), + (rlns_era5_hourly(), rlns_cmor_e1hr(), "rlns", "E1hr"), + (rlus_era5_hourly(), rlus_cmor_e1hr(), "rlus", "E1hr"), + (rls_era5_hourly(), rls_cmor_e1hr(), "rls", "E1hr"), + (rsds_era5_hourly(), rsds_cmor_e1hr(), "rsds", "E1hr"), + (rsns_era5_hourly(), rsns_cmor_e1hr(), "rsns", "E1hr"), + (rsus_era5_hourly(), rsus_cmor_e1hr(), "rsus", "E1hr"), + (rsdt_era5_hourly(), rsdt_cmor_e1hr(), "rsdt", "E1hr"), + (rss_era5_hourly(), rss_cmor_e1hr(), "rss", "E1hr"), + (sftlf_era5(), sftlf_cmor_fx(), "sftlf", "fx"), + ( + era5_3d("monthly"), + cmor_3d("Emon", "snowmxrat27"), + "snowmxrat27", + "Emon", + ), + (tas_era5_hourly(), tas_cmor_e1hr(), "tas", "E1hr"), + (tas_era5_monthly(), tas_cmor_amon(), "tas", "Amon"), + (tasmax_era5_hourly(), tasmax_cmor_e1hr(), "tasmax", "E1hr"), + (tasmin_era5_hourly(), tasmin_cmor_e1hr(), "tasmin", "E1hr"), + (toz_era5_monthly(), cmor_2d("AERmon", "toz"), "toz", "AERmon"), + (uas_era5_hourly(), uas_cmor_e1hr(), "uas", "E1hr"), + (vas_era5_hourly(), vas_cmor_e1hr(), "vas", "E1hr"), + (zg_era5_monthly(), zg_cmor_amon(), "zg", "Amon"), ] ] -@pytest.mark.parametrize('era5_cubes, cmor_cubes, var, mip', VARIABLES) +@pytest.mark.parametrize("era5_cubes, cmor_cubes, var, mip", VARIABLES) def test_cmorization(era5_cubes, cmor_cubes, var, mip): """Verify that cmorization results in the expected target cube.""" - fixed_cubes = fix_metadata(era5_cubes, var, 'native6', 'era5', mip) + fixed_cubes = fix_metadata(era5_cubes, var, "native6", "era5", mip) assert len(fixed_cubes) == 1 fixed_cube = fixed_cubes[0] cmor_cube = cmor_cubes[0] # Test that CMOR checks are passing - fixed_cubes = cmor_check_metadata(fixed_cube, 'native6', mip, var) + fixed_cubes = cmor_check_metadata(fixed_cube, "native6", mip, var) - if fixed_cube.coords('time'): + if fixed_cube.coords("time"): for cube in [fixed_cube, cmor_cube]: - coord = cube.coord('time') + coord = cube.coord("time") coord.points = np.round(coord.points, decimals=7) if coord.bounds is not None: coord.bounds = np.round(coord.bounds, decimals=7) print("Test results for variable/MIP: ", var, mip) - print('cmor_cube:', cmor_cube) - print('fixed_cube:', fixed_cube) - print('cmor_cube data:', cmor_cube.data) - print('fixed_cube data:', fixed_cube.data) + print("cmor_cube:", cmor_cube) + print("fixed_cube:", fixed_cube) + print("cmor_cube data:", cmor_cube.data) + print("fixed_cube data:", fixed_cube.data) print("cmor_cube coords:") for coord in cmor_cube.coords(): print(coord) @@ -1440,20 +1502,20 @@ def test_cmorization(era5_cubes, cmor_cubes, var, mip): def unstructured_grid_cubes(): """Sample cubes with unstructured grid.""" time = DimCoord( - [0.0, 31.0], standard_name='time', units='days since 1950-01-01' + [0.0, 31.0], standard_name="time", units="days since 1950-01-01" ) lat = AuxCoord( - [1.0, 1.0, -1.0, -1.0], standard_name='latitude', units='degrees_north' + [1.0, 1.0, -1.0, -1.0], standard_name="latitude", units="degrees_north" ) lon = AuxCoord( [179.0, 180.0, 180.0, 179.0], - standard_name='longitude', - units='degrees_east', + standard_name="longitude", + units="degrees_east", ) cube = Cube( da.from_array([[0.0, 1.0, 2.0, 3.0], [0.0, 0.0, 0.0, 0.0]]), - standard_name='air_temperature', - units='K', + standard_name="air_temperature", + units="K", dim_coords_and_dims=[(time, 0)], aux_coords_and_dims=[(lat, 1), (lon, 1)], ) @@ -1464,10 +1526,10 @@ def test_unstructured_grid(unstructured_grid_cubes): """Test processing unstructured data.""" fixed_cubes = fix_metadata( unstructured_grid_cubes, - 'tas', - 'native6', - 'era5', - 'Amon', + "tas", + "native6", + "era5", + "Amon", ) assert len(fixed_cubes) == 1 @@ -1475,36 +1537,36 @@ def test_unstructured_grid(unstructured_grid_cubes): assert fixed_cube.shape == (2, 4) - assert fixed_cube.coords('time', dim_coords=True) - assert fixed_cube.coord_dims('time') == (0,) + assert fixed_cube.coords("time", dim_coords=True) + assert fixed_cube.coord_dims("time") == (0,) - assert fixed_cube.coords('latitude', dim_coords=False) - assert fixed_cube.coord_dims('latitude') == (1,) - lat = fixed_cube.coord('latitude') + assert fixed_cube.coords("latitude", dim_coords=False) + assert fixed_cube.coord_dims("latitude") == (1,) + lat = fixed_cube.coord("latitude") np.testing.assert_allclose(lat.points, [1, 1, -1, -1]) assert lat.bounds is None - assert fixed_cube.coords('longitude', dim_coords=False) - assert fixed_cube.coord_dims('longitude') == (1,) - lon = fixed_cube.coord('longitude') + assert fixed_cube.coords("longitude", dim_coords=False) + assert fixed_cube.coord_dims("longitude") == (1,) + lon = fixed_cube.coord("longitude") np.testing.assert_allclose(lon.points, [179, 180, 180, 179]) assert lon.bounds is None -@pytest.mark.parametrize('regrid', [None, False, True]) +@pytest.mark.parametrize("regrid", [None, False, True]) def test_automatic_regridding_unstructured_cube( regrid, unstructured_grid_cubes, monkeypatch ): """Test automatic regridding.""" monkeypatch.setattr( - esmvalcore.cmor._fixes.native6.era5, 'DEFAULT_ERA5_GRID', '60x60' + esmvalcore.cmor._fixes.native6.era5, "DEFAULT_ERA5_GRID", "60x60" ) cube = unstructured_grid_cubes[0] fix_kwargs = {} if regrid is not None: - fix_kwargs['regrid'] = regrid - fixed_cube = fix_data(cube, 'tas', 'native6', 'era5', 'Amon', **fix_kwargs) + fix_kwargs["regrid"] = regrid + fixed_cube = fix_data(cube, "tas", "native6", "era5", "Amon", **fix_kwargs) if regrid is None or regrid is True: assert fixed_cube.shape == (2, 3, 6) @@ -1512,15 +1574,15 @@ def test_automatic_regridding_unstructured_cube( assert fixed_cube.shape == (2, 4) -@pytest.mark.parametrize('regrid', [None, False, True]) +@pytest.mark.parametrize("regrid", [None, False, True]) def test_automatic_regridding_regular_cube(regrid): """Test automatic regridding.""" - cube = era5_2d('monthly')[0] + cube = era5_2d("monthly")[0] fix_kwargs = {} if regrid is not None: - fix_kwargs['regrid'] = regrid - fixed_cube = fix_data(cube, 'tas', 'native6', 'era5', 'Amon', **fix_kwargs) + fix_kwargs["regrid"] = regrid + fixed_cube = fix_data(cube, "tas", "native6", "era5", "Amon", **fix_kwargs) assert fixed_cube.shape == (3, 3, 3) assert fixed_cube is cube diff --git a/tests/integration/cmor/test_fix.py b/tests/integration/cmor/test_fix.py index 972ab54c28..350ecd8a82 100644 --- a/tests/integration/cmor/test_fix.py +++ b/tests/integration/cmor/test_fix.py @@ -22,8 +22,8 @@ @pytest.fixture(autouse=True) def disable_fix_cmor_checker(mocker): """Disable the CMOR checker in fixes (will be default in v2.12).""" - class MockChecker: + class MockChecker: def __init__(self, cube): self._cube = cube @@ -33,7 +33,7 @@ def check_metadata(self): def check_data(self): return self._cube - mock = mocker.patch('esmvalcore.cmor.fix._get_cmor_checker') + mock = mocker.patch("esmvalcore.cmor.fix._get_cmor_checker") mock.return_value = MockChecker @@ -44,58 +44,58 @@ class TestGenericFix: def setup(self, mocker): """Setup tests.""" self.mock_debug = mocker.patch( - 'esmvalcore.cmor._fixes.fix.GenericFix._debug_msg', autospec=True + "esmvalcore.cmor._fixes.fix.GenericFix._debug_msg", autospec=True ) self.mock_warning = mocker.patch( - 'esmvalcore.cmor._fixes.fix.GenericFix._warning_msg', + "esmvalcore.cmor._fixes.fix.GenericFix._warning_msg", autospec=True, ) # Create sample data with CMOR errors time_coord = DimCoord( [15, 45], - standard_name='time', - var_name='time', - units=Unit('days since 1851-01-01', calendar='noleap'), - attributes={'test': 1, 'time_origin': 'will_be_removed'}, + standard_name="time", + var_name="time", + units=Unit("days since 1851-01-01", calendar="noleap"), + attributes={"test": 1, "time_origin": "will_be_removed"}, ) plev_coord_rev = DimCoord( [250, 500, 850], - standard_name='air_pressure', - var_name='plev', - units='hPa', + standard_name="air_pressure", + var_name="plev", + units="hPa", ) lev_coord_hybrid_height = DimCoord( [1.0, 0.5, 0.0], - standard_name='atmosphere_hybrid_height_coordinate', - var_name='lev', - units='m', + standard_name="atmosphere_hybrid_height_coordinate", + var_name="lev", + units="m", ) lev_coord_hybrid_pressure = DimCoord( [0.0, 0.5, 1.0], - standard_name='atmosphere_hybrid_sigma_pressure_coordinate', - var_name='lev', - units='1', + standard_name="atmosphere_hybrid_sigma_pressure_coordinate", + var_name="lev", + units="1", ) ap_coord = AuxCoord( [0.0, 0.0, 0.0], - var_name='ap', - units='Pa', + var_name="ap", + units="Pa", ) b_coord = AuxCoord( [0.0, 0.5, 1.0], - var_name='b', - units='1', + var_name="b", + units="1", ) ps_coord = AuxCoord( np.full((2, 2, 2), 10), - var_name='ps', - units='Pa', + var_name="ps", + units="Pa", ) orog_coord = AuxCoord( np.full((2, 2), 10), - var_name='orog', - units='m', + var_name="orog", + units="m", ) hybrid_height_factory = HybridHeightFactory( delta=lev_coord_hybrid_height, @@ -109,46 +109,46 @@ def setup(self, mocker): ) lat_coord = DimCoord( [0, 10], - standard_name='latitude', - var_name='lat', - units='degrees', + standard_name="latitude", + var_name="lat", + units="degrees", ) lat_coord_rev = DimCoord( [10, -10], - standard_name='latitude', - var_name='lat', - units='degrees', + standard_name="latitude", + var_name="lat", + units="degrees", ) lat_coord_2d = AuxCoord( [[10, -10]], - standard_name='latitude', - var_name='wrong_name', - units='degrees', + standard_name="latitude", + var_name="wrong_name", + units="degrees", ) lon_coord = DimCoord( [-180, 0], - standard_name='longitude', - var_name='lon', - units='degrees', + standard_name="longitude", + var_name="lon", + units="degrees", ) lon_coord_unstructured = AuxCoord( [-180, 0], bounds=[[-200, -180, -160], [-20, 0, 20]], - standard_name='longitude', - var_name='lon', - units='degrees', + standard_name="longitude", + var_name="lon", + units="degrees", ) lon_coord_2d = AuxCoord( [[370, 380]], - standard_name='longitude', - var_name='wrong_name', - units='degrees', + standard_name="longitude", + var_name="wrong_name", + units="degrees", ) height2m_coord = AuxCoord( 2.0, - standard_name='height', - var_name='height', - units='m', + standard_name="height", + var_name="height", + units="m", ) coord_spec_3d = [ @@ -158,10 +158,10 @@ def setup(self, mocker): ] self.cube_3d = Cube( da.arange(2 * 2 * 2, dtype=np.float32).reshape(2, 2, 2), - standard_name='air_pressure', - long_name='Air Pressure', - var_name='tas', - units='celsius', + standard_name="air_pressure", + long_name="Air Pressure", + var_name="tas", + units="celsius", dim_coords_and_dims=coord_spec_3d, aux_coords_and_dims=[(height2m_coord, ())], attributes={}, @@ -175,10 +175,10 @@ def setup(self, mocker): ] cube_4d = Cube( da.arange(2 * 3 * 2 * 2, dtype=np.float32).reshape(2, 3, 2, 2), - standard_name='air_pressure', - long_name='Air Pressure', - var_name='ta', - units='celsius', + standard_name="air_pressure", + long_name="Air Pressure", + var_name="ta", + units="celsius", dim_coords_and_dims=coord_spec_4d, attributes={}, ) @@ -196,10 +196,10 @@ def setup(self, mocker): ] cube_hybrid_height_4d = Cube( da.arange(2 * 3 * 2 * 2, dtype=np.float32).reshape(2, 3, 2, 2), - standard_name='air_pressure', - long_name='Air Pressure', - var_name='ta', - units='celsius', + standard_name="air_pressure", + long_name="Air Pressure", + var_name="ta", + units="celsius", dim_coords_and_dims=coord_spec_hybrid_height_4d, aux_coords_and_dims=aux_coord_spec_hybrid_height_4d, aux_factories=[hybrid_height_factory], @@ -219,10 +219,10 @@ def setup(self, mocker): ] cube_hybrid_pressure_4d = Cube( da.arange(2 * 3 * 2 * 2, dtype=np.float32).reshape(2, 3, 2, 2), - standard_name='air_pressure', - long_name='Air Pressure', - var_name='ta', - units='celsius', + standard_name="air_pressure", + long_name="Air Pressure", + var_name="ta", + units="celsius", dim_coords_and_dims=coord_spec_hybrid_pressure_4d, aux_coords_and_dims=aux_coord_spec_hybrid_pressure_4d, aux_factories=[hybrid_pressure_factory], @@ -237,10 +237,10 @@ def setup(self, mocker): ] cube_unstructured = Cube( da.zeros((2, 2)), - standard_name='air_pressure', - long_name='Air Pressure', - var_name='tas', - units='celsius', + standard_name="air_pressure", + long_name="Air Pressure", + var_name="tas", + units="celsius", dim_coords_and_dims=[(time_coord, 0)], aux_coords_and_dims=coord_spec_unstrucutred, attributes={}, @@ -254,10 +254,10 @@ def setup(self, mocker): ] cube_2d_latlon = Cube( da.zeros((2, 1, 2)), - standard_name='air_pressure', - long_name='Air Pressure', - var_name='tas', - units='celsius', + standard_name="air_pressure", + long_name="Air Pressure", + var_name="tas", + units="celsius", dim_coords_and_dims=[(time_coord, 0)], aux_coords_and_dims=coord_spec_2d, attributes={}, @@ -266,51 +266,52 @@ def setup(self, mocker): def assert_time_metadata(self, cube): """Assert time metadata is correct.""" - assert cube.coord('time').standard_name == 'time' - assert cube.coord('time').var_name == 'time' - assert cube.coord('time').units == Unit( - 'days since 1850-01-01', calendar='365_day' + assert cube.coord("time").standard_name == "time" + assert cube.coord("time").var_name == "time" + assert cube.coord("time").units == Unit( + "days since 1850-01-01", calendar="365_day" ) - assert cube.coord('time').attributes == {'test': 1} + assert cube.coord("time").attributes == {"test": 1} def assert_time_data(self, cube, time_has_bounds=True): """Assert time data is correct.""" - np.testing.assert_allclose(cube.coord('time').points, [380, 410]) + np.testing.assert_allclose(cube.coord("time").points, [380, 410]) if time_has_bounds: np.testing.assert_allclose( - cube.coord('time').bounds, [[365, 396], [396, 424]], + cube.coord("time").bounds, + [[365, 396], [396, 424]], ) else: - assert cube.coord('time').bounds is None + assert cube.coord("time").bounds is None def assert_plev_metadata(self, cube): """Assert plev metadata is correct.""" - assert cube.coord('air_pressure').standard_name == 'air_pressure' - assert cube.coord('air_pressure').var_name == 'plev' - assert cube.coord('air_pressure').units == 'Pa' - assert cube.coord('air_pressure').attributes == {} + assert cube.coord("air_pressure").standard_name == "air_pressure" + assert cube.coord("air_pressure").var_name == "plev" + assert cube.coord("air_pressure").units == "Pa" + assert cube.coord("air_pressure").attributes == {} def assert_lat_metadata(self, cube): """Assert lat metadata is correct.""" - assert cube.coord('latitude').standard_name == 'latitude' - assert cube.coord('latitude').var_name == 'lat' - assert str(cube.coord('latitude').units) == 'degrees_north' - assert cube.coord('latitude').attributes == {} + assert cube.coord("latitude").standard_name == "latitude" + assert cube.coord("latitude").var_name == "lat" + assert str(cube.coord("latitude").units) == "degrees_north" + assert cube.coord("latitude").attributes == {} def assert_lon_metadata(self, cube): """Assert lon metadata is correct.""" - assert cube.coord('longitude').standard_name == 'longitude' - assert cube.coord('longitude').var_name == 'lon' - assert str(cube.coord('longitude').units) == 'degrees_east' - assert cube.coord('longitude').attributes == {} + assert cube.coord("longitude").standard_name == "longitude" + assert cube.coord("longitude").var_name == "lon" + assert str(cube.coord("longitude").units) == "degrees_east" + assert cube.coord("longitude").attributes == {} def assert_ta_metadata(self, cube): """Assert ta metadata is correct.""" # Variable metadata - assert cube.standard_name == 'air_temperature' - assert cube.long_name == 'Air Temperature' - assert cube.var_name == 'ta' - assert cube.units == 'K' + assert cube.standard_name == "air_temperature" + assert cube.long_name == "Air Temperature" + assert cube.var_name == "ta" + assert cube.units == "K" assert cube.attributes == {} def assert_ta_data(self, cube, time_has_bounds=True): @@ -318,18 +319,18 @@ def assert_ta_data(self, cube, time_has_bounds=True): assert cube.has_lazy_data() np.testing.assert_allclose( cube.data, - [[[[284.15, 283.15], - [282.15, 281.15]], - [[280.15, 279.15], - [278.15, 277.15]], - [[276.15, 275.15], - [274.15, 273.15]]], - [[[296.15, 295.15], - [294.15, 293.15]], - [[292.15, 291.15], - [290.15, 289.15]], - [[288.15, 287.15], - [286.15, 285.15]]]], + [ + [ + [[284.15, 283.15], [282.15, 281.15]], + [[280.15, 279.15], [278.15, 277.15]], + [[276.15, 275.15], [274.15, 273.15]], + ], + [ + [[296.15, 295.15], [294.15, 293.15]], + [[292.15, 291.15], [290.15, 289.15]], + [[288.15, 287.15], [286.15, 285.15]], + ], + ], ) # Time @@ -337,50 +338,50 @@ def assert_ta_data(self, cube, time_has_bounds=True): # Air pressure np.testing.assert_allclose( - cube.coord('air_pressure').points, + cube.coord("air_pressure").points, [85000.0, 50000.0, 25000.0], atol=1e-8, ) - assert cube.coord('air_pressure').bounds is None + assert cube.coord("air_pressure").bounds is None # Latitude np.testing.assert_allclose( - cube.coord('latitude').points, [-10.0, 10.0] + cube.coord("latitude").points, [-10.0, 10.0] ) np.testing.assert_allclose( - cube.coord('latitude').bounds, [[-20.0, 0.0], [0.0, 20.0]] + cube.coord("latitude").bounds, [[-20.0, 0.0], [0.0, 20.0]] ) # Longitude np.testing.assert_allclose( - cube.coord('longitude').points, [0.0, 180.0] + cube.coord("longitude").points, [0.0, 180.0] ) np.testing.assert_allclose( - cube.coord('longitude').bounds, [[-90.0, 90.0], [90.0, 270.0]] + cube.coord("longitude").bounds, [[-90.0, 90.0], [90.0, 270.0]] ) def assert_tas_metadata(self, cube): """Assert tas metadata is correct.""" - assert cube.standard_name == 'air_temperature' - assert cube.long_name == 'Near-Surface Air Temperature' - assert cube.var_name == 'tas' - assert cube.units == 'K' + assert cube.standard_name == "air_temperature" + assert cube.long_name == "Near-Surface Air Temperature" + assert cube.var_name == "tas" + assert cube.units == "K" assert cube.attributes == {} # Height 2m coordinate - assert cube.coord('height').standard_name == 'height' - assert cube.coord('height').var_name == 'height' - assert cube.coord('height').units == 'm' - assert cube.coord('height').attributes == {} - np.testing.assert_allclose(cube.coord('height').points, 2.0) - assert cube.coord('height').bounds is None + assert cube.coord("height").standard_name == "height" + assert cube.coord("height").var_name == "height" + assert cube.coord("height").units == "m" + assert cube.coord("height").attributes == {} + np.testing.assert_allclose(cube.coord("height").points, 2.0) + assert cube.coord("height").bounds is None def test_fix_metadata_amon_ta(self): """Test ``fix_metadata``.""" - short_name = 'ta' - project = 'CMIP6' - dataset = '__MODEL_WITH_NO_EXPLICIT_FIX__' - mip = 'Amon' + short_name = "ta" + project = "CMIP6" + dataset = "__MODEL_WITH_NO_EXPLICIT_FIX__" + mip = "Amon" fixed_cubes = fix_metadata( self.cubes_4d, @@ -407,13 +408,13 @@ def test_fix_metadata_amon_ta(self): def test_fix_metadata_amon_ta_wrong_lat_units(self): """Test ``fix_metadata``.""" - short_name = 'ta' - project = 'CMIP6' - dataset = '__MODEL_WITH_NO_EXPLICIT_FIX__' - mip = 'Amon' + short_name = "ta" + project = "CMIP6" + dataset = "__MODEL_WITH_NO_EXPLICIT_FIX__" + mip = "Amon" # Change units of latitude - self.cubes_4d[0].coord('latitude').units = 'K' + self.cubes_4d[0].coord("latitude").units = "K" fixed_cubes = fix_metadata( self.cubes_4d, @@ -433,7 +434,7 @@ def test_fix_metadata_amon_ta_wrong_lat_units(self): self.assert_ta_data(fixed_cube) # CMOR check will fail because of wrong latitude units - assert fixed_cube.coord('latitude').units == 'K' + assert fixed_cube.coord("latitude").units == "K" with pytest.raises(CMORCheckError): cmor_check_metadata(fixed_cube, project, mip, short_name) @@ -442,10 +443,10 @@ def test_fix_metadata_amon_ta_wrong_lat_units(self): def test_fix_metadata_cfmon_ta_hybrid_height(self): """Test ``fix_metadata`` with hybrid height coordinate.""" - short_name = 'ta' - project = 'CMIP6' - dataset = '__MODEL_WITH_NO_EXPLICIT_FIX__' - mip = 'CFmon' + short_name = "ta" + project = "CMIP6" + dataset = "__MODEL_WITH_NO_EXPLICIT_FIX__" + mip = "CFmon" fixed_cubes = fix_metadata( self.cubes_hybrid_height_4d, @@ -458,13 +459,13 @@ def test_fix_metadata_cfmon_ta_hybrid_height(self): assert len(fixed_cubes) == 1 fixed_cube = fixed_cubes[0] - hybrid_coord = fixed_cube.coord('atmosphere_hybrid_height_coordinate') - assert hybrid_coord.var_name == 'lev' + hybrid_coord = fixed_cube.coord("atmosphere_hybrid_height_coordinate") + assert hybrid_coord.var_name == "lev" assert hybrid_coord.long_name is None - assert hybrid_coord.units == 'm' + assert hybrid_coord.units == "m" np.testing.assert_allclose(hybrid_coord.points, [0.0, 0.5, 1.0]) - assert fixed_cube.coords('altitude') - assert fixed_cube.coord_dims('altitude') == (1, 2, 3) + assert fixed_cube.coords("altitude") + assert fixed_cube.coord_dims("altitude") == (1, 2, 3) self.assert_ta_metadata(fixed_cube) self.assert_time_metadata(fixed_cube) @@ -478,10 +479,10 @@ def test_fix_metadata_cfmon_ta_hybrid_height(self): def test_fix_metadata_cfmon_ta_hybrid_pressure(self): """Test ``fix_metadata`` with hybrid pressure coordinate.""" - short_name = 'ta' - project = 'CMIP6' - dataset = '__MODEL_WITH_NO_EXPLICIT_FIX__' - mip = 'CFmon' + short_name = "ta" + project = "CMIP6" + dataset = "__MODEL_WITH_NO_EXPLICIT_FIX__" + mip = "CFmon" fixed_cubes = fix_metadata( self.cubes_hybrid_pressure_4d, @@ -495,14 +496,14 @@ def test_fix_metadata_cfmon_ta_hybrid_pressure(self): fixed_cube = fixed_cubes[0] hybrid_coord = fixed_cube.coord( - 'atmosphere_hybrid_sigma_pressure_coordinate' + "atmosphere_hybrid_sigma_pressure_coordinate" ) - assert hybrid_coord.var_name == 'lev' + assert hybrid_coord.var_name == "lev" assert hybrid_coord.long_name is None - assert hybrid_coord.units == '1' + assert hybrid_coord.units == "1" np.testing.assert_allclose(hybrid_coord.points, [1.0, 0.5, 0.0]) - assert fixed_cube.coords('air_pressure') - assert fixed_cube.coord_dims('air_pressure') == (0, 1, 2, 3) + assert fixed_cube.coords("air_pressure") + assert fixed_cube.coord_dims("air_pressure") == (0, 1, 2, 3) self.assert_ta_metadata(fixed_cube) self.assert_time_metadata(fixed_cube) @@ -516,10 +517,10 @@ def test_fix_metadata_cfmon_ta_hybrid_pressure(self): def test_fix_metadata_cfmon_ta_alternative(self): """Test ``fix_metadata`` with alternative generic level coordinate.""" - short_name = 'ta' - project = 'CMIP6' - dataset = '__MODEL_WITH_NO_EXPLICIT_FIX__' - mip = 'CFmon' + short_name = "ta" + project = "CMIP6" + dataset = "__MODEL_WITH_NO_EXPLICIT_FIX__" + mip = "CFmon" fixed_cubes = fix_metadata( self.cubes_4d, @@ -546,13 +547,13 @@ def test_fix_metadata_cfmon_ta_alternative(self): def test_fix_metadata_cfmon_ta_no_alternative(self, mocker): """Test ``fix_metadata`` with no alternative coordinate.""" - short_name = 'ta' - project = 'CMIP6' - dataset = '__MODEL_WITH_NO_EXPLICIT_FIX__' - mip = 'CFmon' + short_name = "ta" + project = "CMIP6" + dataset = "__MODEL_WITH_NO_EXPLICIT_FIX__" + mip = "CFmon" # Remove alternative coordinate - self.cubes_4d[0].remove_coord('air_pressure') + self.cubes_4d[0].remove_coord("air_pressure") fixed_cubes = fix_metadata( self.cubes_4d, @@ -572,7 +573,7 @@ def test_fix_metadata_cfmon_ta_no_alternative(self, mocker): self.assert_lon_metadata(fixed_cube) # CMOR check will fail because of missing alevel coordinate - assert not fixed_cube.coords('air_pressure') + assert not fixed_cube.coords("air_pressure") with pytest.raises(CMORCheckError): cmor_check_metadata(fixed_cube, project, mip, short_name) @@ -581,14 +582,16 @@ def test_fix_metadata_cfmon_ta_no_alternative(self, mocker): def test_fix_metadata_e1hr_ta(self): """Test ``fix_metadata`` with plev3.""" - short_name = 'ta' - project = 'CMIP6' - dataset = '__MODEL_WITH_NO_EXPLICIT_FIX__' - mip = 'E1hr' + short_name = "ta" + project = "CMIP6" + dataset = "__MODEL_WITH_NO_EXPLICIT_FIX__" + mip = "E1hr" # Slightly adapt plev to test fixing of requested levels - self.cubes_4d[0].coord('air_pressure').points = [ - 250.0 + 9e-8, 500.0 + 9e-8, 850.0 + 9e-8 + self.cubes_4d[0].coord("air_pressure").points = [ + 250.0 + 9e-8, + 500.0 + 9e-8, + 850.0 + 9e-8, ] fixed_cubes = fix_metadata( @@ -597,7 +600,7 @@ def test_fix_metadata_e1hr_ta(self): project, dataset, mip, - frequency='mon', + frequency="mon", ) assert len(fixed_cubes) == 1 @@ -611,7 +614,7 @@ def test_fix_metadata_e1hr_ta(self): self.assert_ta_data(fixed_cube, time_has_bounds=False) cmor_check_metadata( - fixed_cube, project, mip, short_name, frequency='mon' + fixed_cube, project, mip, short_name, frequency="mon" ) assert self.mock_debug.call_count == 4 @@ -619,10 +622,10 @@ def test_fix_metadata_e1hr_ta(self): def test_fix_metadata_amon_tas_unstructured(self): """Test ``fix_metadata`` with unstructured grid.""" - short_name = 'tas' - project = 'CMIP6' - dataset = '__MODEL_WITH_NO_EXPLICIT_FIX__' - mip = 'Amon' + short_name = "tas" + project = "CMIP6" + dataset = "__MODEL_WITH_NO_EXPLICIT_FIX__" + mip = "Amon" fixed_cubes = fix_metadata( self.cubes_unstructured, @@ -642,16 +645,16 @@ def test_fix_metadata_amon_tas_unstructured(self): # Latitude np.testing.assert_allclose( - fixed_cube.coord('latitude').points, [10.0, -10.0] + fixed_cube.coord("latitude").points, [10.0, -10.0] ) - assert fixed_cube.coord('latitude').bounds is None + assert fixed_cube.coord("latitude").bounds is None # Longitude np.testing.assert_allclose( - fixed_cube.coord('longitude').points, [180.0, 0.0] + fixed_cube.coord("longitude").points, [180.0, 0.0] ) np.testing.assert_allclose( - fixed_cube.coord('longitude').bounds, + fixed_cube.coord("longitude").bounds, [[160.0, 180.0, 200.0], [340.0, 0.0, 20.0]], ) @@ -668,10 +671,10 @@ def test_fix_metadata_amon_tas_unstructured(self): def test_fix_metadata_amon_tas_2d_latlon(self): """Test ``fix_metadata`` with 2D latitude/longitude.""" - short_name = 'tas' - project = 'CMIP6' - dataset = '__MODEL_WITH_NO_EXPLICIT_FIX__' - mip = 'Amon' + short_name = "tas" + project = "CMIP6" + dataset = "__MODEL_WITH_NO_EXPLICIT_FIX__" + mip = "Amon" fixed_cubes = fix_metadata( self.cubes_2d_latlon, @@ -691,15 +694,15 @@ def test_fix_metadata_amon_tas_2d_latlon(self): # Latitude np.testing.assert_allclose( - fixed_cube.coord('latitude').points, [[10.0, -10.0]] + fixed_cube.coord("latitude").points, [[10.0, -10.0]] ) - assert fixed_cube.coord('latitude').bounds is None + assert fixed_cube.coord("latitude").bounds is None # Longitude np.testing.assert_allclose( - fixed_cube.coord('longitude').points, [[10.0, 20.0]] + fixed_cube.coord("longitude").points, [[10.0, 20.0]] ) - assert fixed_cube.coord('longitude').bounds is None + assert fixed_cube.coord("longitude").bounds is None # Variable data assert fixed_cube.has_lazy_data() @@ -714,17 +717,17 @@ def test_fix_metadata_amon_tas_2d_latlon(self): def test_fix_metadata_amon_tas_invalid_time_units(self): """Test ``fix_metadata`` with invalid time units.""" - short_name = 'tas' - project = 'CMIP6' - dataset = '__MODEL_WITH_NO_EXPLICIT_FIX__' - mip = 'Amon' + short_name = "tas" + project = "CMIP6" + dataset = "__MODEL_WITH_NO_EXPLICIT_FIX__" + mip = "Amon" - self.cubes_2d_latlon[0].remove_coord('time') + self.cubes_2d_latlon[0].remove_coord("time") aux_time_coord = AuxCoord( [1, 2], - standard_name='time', - var_name='time', - units='kg', + standard_name="time", + var_name="time", + units="kg", ) self.cubes_2d_latlon[0].add_aux_coord(aux_time_coord, 0) @@ -742,7 +745,7 @@ def test_fix_metadata_amon_tas_invalid_time_units(self): self.assert_lat_metadata(fixed_cube) self.assert_lon_metadata(fixed_cube) - assert fixed_cube.coord('time').units == 'kg' + assert fixed_cube.coord("time").units == "kg" # CMOR checks fail because calendar is not defined with pytest.raises(ValueError): @@ -753,15 +756,15 @@ def test_fix_metadata_amon_tas_invalid_time_units(self): def test_fix_metadata_amon_tas_invalid_time_attrs(self): """Test ``fix_metadata`` with invalid time attributes.""" - short_name = 'tas' - project = 'CMIP6' - dataset = '__MODEL_WITH_NO_EXPLICIT_FIX__' - mip = 'Amon' + short_name = "tas" + project = "CMIP6" + dataset = "__MODEL_WITH_NO_EXPLICIT_FIX__" + mip = "Amon" self.cubes_2d_latlon[0].attributes = { - 'parent_time_units': 'this is certainly not a unit', - 'branch_time_in_parent': 'BRANCH TIME IN PARENT', - 'branch_time_in_child': 'BRANCH TIME IN CHILD', + "parent_time_units": "this is certainly not a unit", + "branch_time_in_parent": "BRANCH TIME IN PARENT", + "branch_time_in_child": "BRANCH TIME IN CHILD", } fixed_cubes = fix_metadata( @@ -780,9 +783,9 @@ def test_fix_metadata_amon_tas_invalid_time_attrs(self): self.assert_lon_metadata(fixed_cube) assert fixed_cube.attributes == { - 'parent_time_units': 'this is certainly not a unit', - 'branch_time_in_parent': 'BRANCH TIME IN PARENT', - 'branch_time_in_child': 'BRANCH TIME IN CHILD', + "parent_time_units": "this is certainly not a unit", + "branch_time_in_parent": "BRANCH TIME IN PARENT", + "branch_time_in_child": "BRANCH TIME IN CHILD", } cmor_check_metadata(fixed_cube, project, mip, short_name) @@ -792,21 +795,21 @@ def test_fix_metadata_amon_tas_invalid_time_attrs(self): def test_fix_metadata_oimon_ssi(self): """Test ``fix_metadata`` with psu units.""" - short_name = 'ssi' - project = 'CMIP5' - dataset = '__MODEL_WITH_NO_EXPLICIT_FIX__' - mip = 'OImon' + short_name = "ssi" + project = "CMIP5" + dataset = "__MODEL_WITH_NO_EXPLICIT_FIX__" + mip = "OImon" - self.cubes_2d_latlon[0].var_name = 'ssi' + self.cubes_2d_latlon[0].var_name = "ssi" self.cubes_2d_latlon[0].attributes = { - 'invalid_units': 'psu', - 'parent_time_units': 'no parent', + "invalid_units": "psu", + "parent_time_units": "no parent", } # Also test 2D longitude that already has bounds - self.cubes_2d_latlon[0].coord('latitude').var_name = 'lat' - self.cubes_2d_latlon[0].coord('longitude').var_name = 'lon' - self.cubes_2d_latlon[0].coord('longitude').bounds = [ + self.cubes_2d_latlon[0].coord("latitude").var_name = "lat" + self.cubes_2d_latlon[0].coord("longitude").var_name = "lon" + self.cubes_2d_latlon[0].coord("longitude").bounds = [ [[365.0, 375.0], [375.0, 400.0]] ] @@ -822,11 +825,11 @@ def test_fix_metadata_oimon_ssi(self): fixed_cube = fixed_cubes[0] # Variable metadata - assert fixed_cube.standard_name == 'sea_ice_salinity' - assert fixed_cube.long_name == 'Sea Ice Salinity' - assert fixed_cube.var_name == 'ssi' - assert fixed_cube.units == '1' - assert fixed_cube.attributes == {'parent_time_units': 'no parent'} + assert fixed_cube.standard_name == "sea_ice_salinity" + assert fixed_cube.long_name == "Sea Ice Salinity" + assert fixed_cube.var_name == "ssi" + assert fixed_cube.units == "1" + assert fixed_cube.attributes == {"parent_time_units": "no parent"} # Coordinates self.assert_time_metadata(fixed_cube) @@ -835,23 +838,24 @@ def test_fix_metadata_oimon_ssi(self): # Latitude np.testing.assert_allclose( - fixed_cube.coord('latitude').points, [[10.0, -10.0]] + fixed_cube.coord("latitude").points, [[10.0, -10.0]] ) - assert fixed_cube.coord('latitude').bounds is None + assert fixed_cube.coord("latitude").bounds is None # Longitude np.testing.assert_allclose( - fixed_cube.coord('longitude').points, [[10.0, 20.0]] + fixed_cube.coord("longitude").points, [[10.0, 20.0]] ) np.testing.assert_allclose( - fixed_cube.coord('longitude').bounds, + fixed_cube.coord("longitude").bounds, [[[5.0, 15.0], [15.0, 40.0]]], ) # Variable data assert fixed_cube.has_lazy_data() np.testing.assert_allclose( - fixed_cube.data, [[[0.0, 0.0]], [[0.0, 0.0]]], + fixed_cube.data, + [[[0.0, 0.0]], [[0.0, 0.0]]], ) cmor_check_metadata(fixed_cube, project, mip, short_name) @@ -861,10 +865,10 @@ def test_fix_metadata_oimon_ssi(self): def test_fix_data_amon_tas(self): """Test ``fix_data``.""" - short_name = 'tas' - project = 'CMIP6' - dataset = '__MODEL_WITH_NO_EXPLICIT_FIX__' - mip = 'Amon' + short_name = "tas" + project = "CMIP6" + dataset = "__MODEL_WITH_NO_EXPLICIT_FIX__" + mip = "Amon" fixed_cube = fix_data( self.cube_3d, @@ -886,10 +890,10 @@ def test_deprecate_check_level_fix_metadata(self): with pytest.warns(ESMValCoreDeprecationWarning): fix_metadata( self.cubes_4d, - 'ta', - 'CMIP6', - 'MODEL', - 'Amon', + "ta", + "CMIP6", + "MODEL", + "Amon", check_level=CheckLevels.RELAXED, ) @@ -898,21 +902,21 @@ def test_deprecate_check_level_fix_data(self): with pytest.warns(ESMValCoreDeprecationWarning): fix_metadata( self.cubes_4d, - 'ta', - 'CMIP6', - 'MODEL', - 'Amon', + "ta", + "CMIP6", + "MODEL", + "Amon", check_level=CheckLevels.RELAXED, ) def test_fix_metadata_no_time_in_table(self): """Test ``fix_data``.""" - short_name = 'sftlf' - project = 'CMIP6' - dataset = '__MODEL_WITH_NO_EXPLICIT_FIX__' - mip = 'fx' + short_name = "sftlf" + project = "CMIP6" + dataset = "__MODEL_WITH_NO_EXPLICIT_FIX__" + mip = "fx" cube = self.cubes_2d_latlon[0][0] - cube.units = '%' + cube.units = "%" cube.data = da.full(cube.shape, 1.0, dtype=cube.dtype) fixed_cubes = fix_metadata( diff --git a/tests/integration/preprocessor/_io/test_load.py b/tests/integration/preprocessor/_io/test_load.py index fe8c5b3591..00244801c7 100644 --- a/tests/integration/preprocessor/_io/test_load.py +++ b/tests/integration/preprocessor/_io/test_load.py @@ -16,8 +16,8 @@ def _create_sample_cube(): - coord = DimCoord([1, 2], standard_name='latitude', units='degrees_north') - cube = Cube([1, 2], var_name='sample', dim_coords_and_dims=((coord, 0), )) + coord = DimCoord([1, 2], standard_name="latitude", units="degrees_north") + cube = Cube([1, 2], var_name="sample", dim_coords_and_dims=((coord, 0),)) return cube @@ -34,7 +34,7 @@ def tearDown(self): os.remove(temp_file) def _save_cube(self, cube): - descriptor, temp_file = tempfile.mkstemp('.nc') + descriptor, temp_file = tempfile.mkstemp(".nc") os.close(descriptor) iris.save(cube, temp_file) self.temp_files.append(temp_file) @@ -48,32 +48,33 @@ def test_load(self): cubes = load(temp_file) cube = cubes[0] self.assertEqual(1, len(cubes)) - self.assertEqual(temp_file, cube.attributes['source_file']) + self.assertEqual(temp_file, cube.attributes["source_file"]) self.assertTrue((cube.data == np.array([1, 2])).all()) - self.assertTrue((cube.coord('latitude').points == np.array([1, - 2])).all()) + self.assertTrue( + (cube.coord("latitude").points == np.array([1, 2])).all() + ) def test_load_grib(self): """Test loading a grib file.""" grib_path = Path( Path(esmvalcore.__file__).parents[1], - 'tests', - 'sample_data', - 'iris-sample-data', - 'polar_stereo.grib2', + "tests", + "sample_data", + "iris-sample-data", + "polar_stereo.grib2", ) cubes = load(grib_path) assert len(cubes) == 1 cube = cubes[0] - assert cube.standard_name == 'air_temperature' - assert cube.units == 'K' + assert cube.standard_name == "air_temperature" + assert cube.units == "K" assert cube.shape == (200, 247) - assert 'source_file' in cube.attributes + assert "source_file" in cube.attributes def test_callback_remove_attributes(self): """Test callback remove unwanted attributes.""" - attributes = ('history', 'creation_date', 'tracking_id', 'comment') + attributes = ("history", "creation_date", "tracking_id", "comment") for _ in range(2): cube = _create_sample_cube() for attr in attributes: @@ -85,13 +86,14 @@ def test_callback_remove_attributes(self): self.assertEqual(1, len(cubes)) self.assertTrue((cube.data == np.array([1, 2])).all()) self.assertTrue( - (cube.coord('latitude').points == np.array([1, 2])).all()) + (cube.coord("latitude").points == np.array([1, 2])).all() + ) for attr in attributes: self.assertTrue(attr not in cube.attributes) def test_callback_remove_attributes_from_coords(self): """Test callback remove unwanted attributes from coords.""" - attributes = ('history', ) + attributes = ("history",) for _ in range(2): cube = _create_sample_cube() for coord in cube.coords(): @@ -104,7 +106,8 @@ def test_callback_remove_attributes_from_coords(self): self.assertEqual(1, len(cubes)) self.assertTrue((cube.data == np.array([1, 2])).all()) self.assertTrue( - (cube.coord('latitude').points == np.array([1, 2])).all()) + (cube.coord("latitude").points == np.array([1, 2])).all() + ) for coord in cube.coords(): for attr in attributes: self.assertTrue(attr not in cube.attributes) @@ -118,50 +121,52 @@ def test_callback_fix_lat_units(self): cube = cubes[0] self.assertEqual(1, len(cubes)) self.assertTrue((cube.data == np.array([1, 2])).all()) - self.assertTrue((cube.coord('latitude').points == np.array([1, - 2])).all()) - self.assertEqual(cube.coord('latitude').units, 'degrees_north') + self.assertTrue( + (cube.coord("latitude").points == np.array([1, 2])).all() + ) + self.assertEqual(cube.coord("latitude").units, "degrees_north") - @unittest.mock.patch('iris.load_raw', autospec=True) + @unittest.mock.patch("iris.load_raw", autospec=True) def test_fail_empty_cubes(self, mock_load_raw): """Test that ValueError is raised when cubes are empty.""" mock_load_raw.return_value = CubeList([]) msg = "Can not load cubes from myfilename" with self.assertRaises(ValueError, msg=msg): - load('myfilename') + load("myfilename") @staticmethod def load_with_warning(*_, **__): """Mock load with a warning.""" - warnings.warn("This is a custom expected warning", - category=UserWarning) + warnings.warn( + "This is a custom expected warning", category=UserWarning + ) return CubeList([Cube(0)]) - @unittest.mock.patch('iris.load_raw', autospec=True) + @unittest.mock.patch("iris.load_raw", autospec=True) def test_do_not_ignore_warnings(self, mock_load_raw): """Test do not ignore specific warnings.""" mock_load_raw.side_effect = self.load_with_warning - ignore_warnings = [{'message': "non-relevant warning"}] + ignore_warnings = [{"message": "non-relevant warning"}] # Warning is not ignored -> assert warning has been issued with self.assertWarns(UserWarning): - cubes = load('myfilename', ignore_warnings=ignore_warnings) + cubes = load("myfilename", ignore_warnings=ignore_warnings) # Check output self.assertEqual(len(cubes), 1) - self.assertEqual(cubes[0].attributes, {'source_file': 'myfilename'}) + self.assertEqual(cubes[0].attributes, {"source_file": "myfilename"}) - @unittest.mock.patch('iris.load_raw', autospec=True) + @unittest.mock.patch("iris.load_raw", autospec=True) def test_ignore_warnings(self, mock_load_raw): """Test ignore specific warnings.""" mock_load_raw.side_effect = self.load_with_warning - ignore_warnings = [{'message': "This is a custom expected warning"}] + ignore_warnings = [{"message": "This is a custom expected warning"}] # Warning is ignored -> assert warning has not been issued with self.assertRaises(AssertionError): with self.assertWarns(UserWarning): - cubes = load('myfilename', ignore_warnings=ignore_warnings) + cubes = load("myfilename", ignore_warnings=ignore_warnings) # Check output self.assertEqual(len(cubes), 1) - self.assertEqual(cubes[0].attributes, {'source_file': 'myfilename'}) + self.assertEqual(cubes[0].attributes, {"source_file": "myfilename"}) diff --git a/tests/sample_data/iris-sample-data/LICENSE b/tests/sample_data/iris-sample-data/LICENSE index 6ab33c6548..2fc090dfca 100644 --- a/tests/sample_data/iris-sample-data/LICENSE +++ b/tests/sample_data/iris-sample-data/LICENSE @@ -5,6 +5,6 @@ It is licensed under the following UK's Open Government Licence (https://www.nat (c) British Crown copyright, 2018. -You may use and re-use the information featured in this repository (not including logos) free of charge in any format or medium, under the terms of the Open Government Licence. We encourage users to establish hypertext links to this website. +You may use and reuse the information featured in this repository (not including logos) free of charge in any format or medium, under the terms of the Open Government Licence. We encourage users to establish hypertext links to this website. -Any email enquiries regarding the use and re-use of this information resource should be sent to: psi@nationalarchives.gsi.gov.uk. +Any email enquiries regarding the use and reuse of this information resource should be sent to: psi@nationalarchives.gsi.gov.uk. diff --git a/tests/unit/provenance/test_trackedfile.py b/tests/unit/provenance/test_trackedfile.py index 09aa5be0e2..de75328ed3 100644 --- a/tests/unit/provenance/test_trackedfile.py +++ b/tests/unit/provenance/test_trackedfile.py @@ -7,9 +7,9 @@ @pytest.fixture def tracked_file_nc(): file = TrackedFile( - filename='/path/to/file.nc', - attributes={'a': 'A'}, - prov_filename='/original/path/to/file.nc', + filename="/path/to/file.nc", + attributes={"a": "A"}, + prov_filename="/original/path/to/file.nc", ) return file @@ -17,57 +17,57 @@ def tracked_file_nc(): @pytest.fixture def tracked_file_grb(): file = TrackedFile( - filename='/path/to/file.grb', - prov_filename='/original/path/to/file.grb', + filename="/path/to/file.grb", + prov_filename="/original/path/to/file.grb", ) return file def test_init_nc(tracked_file_nc): """Test `esmvalcore._provenance.TrackedFile.__init__`.""" - assert tracked_file_nc.filename == '/path/to/file.nc' - assert tracked_file_nc.attributes == {'a': 'A'} - assert tracked_file_nc.prov_filename == '/original/path/to/file.nc' + assert tracked_file_nc.filename == "/path/to/file.nc" + assert tracked_file_nc.attributes == {"a": "A"} + assert tracked_file_nc.prov_filename == "/original/path/to/file.nc" def test_init_grb(tracked_file_grb): """Test `esmvalcore._provenance.TrackedFile.__init__`.""" - assert tracked_file_grb.filename == '/path/to/file.grb' + assert tracked_file_grb.filename == "/path/to/file.grb" assert tracked_file_grb.attributes is None - assert tracked_file_grb.prov_filename == '/original/path/to/file.grb' + assert tracked_file_grb.prov_filename == "/original/path/to/file.grb" def test_initialize_provenance_nc(tracked_file_nc): """Test `esmvalcore._provenance.TrackedFile.initialize_provenance`.""" provenance = ProvDocument() - provenance.add_namespace('task', uri=ESMVALTOOL_URI_PREFIX + 'task') - activity = provenance.activity('task:test-task-name') + provenance.add_namespace("task", uri=ESMVALTOOL_URI_PREFIX + "task") + activity = provenance.activity("task:test-task-name") tracked_file_nc.initialize_provenance(activity) assert isinstance(tracked_file_nc.provenance, ProvDocument) assert tracked_file_nc.activity == activity - assert str(tracked_file_nc.entity.identifier) == 'file:/path/to/file.nc' - assert tracked_file_nc.attributes == {'a': 'A'} + assert str(tracked_file_nc.entity.identifier) == "file:/path/to/file.nc" + assert tracked_file_nc.attributes == {"a": "A"} def test_initialize_provenance_grb(tracked_file_grb): """Test `esmvalcore._provenance.TrackedFile.initialize_provenance`.""" provenance = ProvDocument() - provenance.add_namespace('task', uri=ESMVALTOOL_URI_PREFIX + 'task') - activity = provenance.activity('task:test-task-name') + provenance.add_namespace("task", uri=ESMVALTOOL_URI_PREFIX + "task") + activity = provenance.activity("task:test-task-name") tracked_file_grb.initialize_provenance(activity) assert isinstance(tracked_file_grb.provenance, ProvDocument) assert tracked_file_grb.activity == activity - assert str(tracked_file_grb.entity.identifier) == 'file:/path/to/file.grb' + assert str(tracked_file_grb.entity.identifier) == "file:/path/to/file.grb" assert tracked_file_grb.attributes == {} def test_copy_provenance(tracked_file_nc): """Test `esmvalcore._provenance.TrackedFile.copy_provenance`.""" provenance = ProvDocument() - provenance.add_namespace('task', uri=ESMVALTOOL_URI_PREFIX + 'task') - activity = provenance.activity('task:test-task-name') + provenance.add_namespace("task", uri=ESMVALTOOL_URI_PREFIX + "task") + activity = provenance.activity("task:test-task-name") tracked_file_nc.initialize_provenance(activity) From 7a59ea1dbce4b540ad6e997737623d984c6b0ebb Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Fri, 27 Sep 2024 15:24:43 +0200 Subject: [PATCH 52/64] Restored original LICENSE file --- pyproject.toml | 2 +- tests/sample_data/iris-sample-data/LICENSE | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5a45ca2ab9..848a3180b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" version_scheme = "release-branch-semver" [tool.codespell] -skip = "*.ipynb,esmvalcore/config/extra_facets/ipslcm-mappings.yml" +skip = "*.ipynb,esmvalcore/config/extra_facets/ipslcm-mappings.yml,tests/sample_data/iris-sample-data/LICENSE" ignore-words-list = "vas,hist,oce" [tool.pylint.main] diff --git a/tests/sample_data/iris-sample-data/LICENSE b/tests/sample_data/iris-sample-data/LICENSE index 2fc090dfca..6ab33c6548 100644 --- a/tests/sample_data/iris-sample-data/LICENSE +++ b/tests/sample_data/iris-sample-data/LICENSE @@ -5,6 +5,6 @@ It is licensed under the following UK's Open Government Licence (https://www.nat (c) British Crown copyright, 2018. -You may use and reuse the information featured in this repository (not including logos) free of charge in any format or medium, under the terms of the Open Government Licence. We encourage users to establish hypertext links to this website. +You may use and re-use the information featured in this repository (not including logos) free of charge in any format or medium, under the terms of the Open Government Licence. We encourage users to establish hypertext links to this website. -Any email enquiries regarding the use and reuse of this information resource should be sent to: psi@nationalarchives.gsi.gov.uk. +Any email enquiries regarding the use and re-use of this information resource should be sent to: psi@nationalarchives.gsi.gov.uk. From 872fc50daade108e9b54f9d178102869d61fa2f9 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Fri, 27 Sep 2024 15:39:56 +0200 Subject: [PATCH 53/64] Pin iris-grib --- environment.yml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index eaf317965f..92f72051d3 100644 --- a/environment.yml +++ b/environment.yml @@ -20,7 +20,7 @@ dependencies: - humanfriendly - iris >=3.10.0 - iris-esmf-regrid >=0.11.0 - - iris-grib + - iris-grib >=0.20.0 # github.com/SciTools/iris-grib/issues/520 - isodate - jinja2 - libnetcdf !=4.9.1 # to avoid hdf5 warnings diff --git a/setup.py b/setup.py index 32d2e15e27..c07ac8362b 100755 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ "fire", "geopy", "humanfriendly", - "iris-grib", + "iris-grib>=0.20.0", # iris-grib/issues/520 "isodate", "jinja2", "nc-time-axis", # needed by iris.plot From f7fcc699500e4fdbbfea6213e612f30843230486 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 7 Oct 2024 13:55:32 +0200 Subject: [PATCH 54/64] Remove superfluous spaces --- doc/quickstart/find_data.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index d27246b9a5..abc23bb9c8 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -104,8 +104,6 @@ Supported native reanalysis/observational datasets The following native reanalysis/observational datasets are supported under the ``native6`` project. - - To use these datasets, put the files containing the data in the directory that you have :ref:`configured ` for the ``rootpath`` of the ``native6`` project, in a subdirectory called From a031ea9516ce1db269396d46e380fad89d1191bd Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Tue, 8 Oct 2024 09:46:59 +0200 Subject: [PATCH 55/64] Fix doc build --- doc/quickstart/find_data.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index abc23bb9c8..e96756957c 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -148,8 +148,9 @@ ERA5 (in GRIB format available on DKRZ's Levante) ERA5 data in monthly, daily, and hourly resolution is `available on Levante `__ in its native GRIB format. -To read these data with ESMValCore, use the root path ``/pool/data/ERA5`` with -DRS ``DKRZ-ERA5-GRIB`` in your :ref:`user configuration file`, for example: +To read these data with ESMValCore, use the :ref:`rootpath +` ``/pool/data/ERA5`` with :ref:`DRS +` ``DKRZ-ERA5-GRIB`` in your configuration, for example: .. code-block:: yaml @@ -722,6 +723,8 @@ first discuss the ``drs`` parameter: as we've seen in the previous section, the DRS as a standard is used for both file naming conventions and for directory structures. +.. _config_option_drs: + Explaining ``drs: CMIP5:`` or ``drs: CMIP6:`` --------------------------------------------- Whereas ESMValCore will by default use the CMOR standard for file naming (please From 44040c37027c245e4f42f7f451c132439519812f Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Tue, 8 Oct 2024 09:59:38 +0200 Subject: [PATCH 56/64] Remove unnecessary unit conversions --- esmvalcore/cmor/_fixes/native6/era5.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py index cf41c2d2e1..cafbe96b23 100644 --- a/esmvalcore/cmor/_fixes/native6/era5.py +++ b/esmvalcore/cmor/_fixes/native6/era5.py @@ -210,9 +210,9 @@ class O3(Fix): def fix_metadata(self, cubes): """Convert mass mixing ratios to mole fractions.""" for cube in cubes: - cube.units = "kg kg-1" - # Convert to molar mixing ratios, which is almost identical to mole - # fraction for small amounts of substances (which we have here) + # Original units are kg kg-1. Convert these to molar mixing ratios, + # which is almost identical to mole fraction for small amounts of + # substances (which we have here) cube.data = cube.core_data() * 28.9644 / 47.9982 cube.units = "mol mol-1" return cubes @@ -463,8 +463,11 @@ class Toz(Fix): def fix_metadata(self, cubes): """Convert 'kg m-2' to 'm'.""" for cube in cubes: - cube.units = "kg m-2" - # 1 DU = 1e-5 m = 2.1415e-5 kg m-2 --> 1m = 2.1415 kg m-2 + # Original units are kg m-2. Convert these to m here. + # 1 DU = 0.4462 mmol m-2 = 21.415 mg m-2 = 2.1415e-5 kg m-2 + # (assuming O3 molar mass of 48 g mol-1) + # Since 1 mm of pure O3 layer is defined as 100 DU + # --> 1m ~ 2.1415 kg m-2 cube.data = cube.core_data() / 2.1415 cube.units = "m" return cubes From e4d642a465417cf17fca7d329150389f37b0f616 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 14 Oct 2024 16:55:57 +0200 Subject: [PATCH 57/64] Add note about downloading GRIB data from the CDS --- doc/quickstart/find_data.rst | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index e96756957c..51992e9be5 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -142,12 +142,24 @@ DRS for ``native6``). .. _read_native_era5_grib: -ERA5 (in GRIB format available on DKRZ's Levante) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +ERA5 (in GRIB format available on DKRZ's Levante or downloaded from the CDS) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ERA5 data in monthly, daily, and hourly resolution is `available on Levante `__ in its native GRIB format. + +.. note:: + ERA5 data in its native GRIB format can also be downloaded from the + `Copernicus Climate Data Store (CDS) + `__. + For example, hourly data on pressure levels is available `here + `__. + Reading self-downloaded ERA5 data in GRIB format is experimental and likely + requires additional setup from the user like setting up the proper directory + structure for the input files and/or creating a custom :ref:`DRS + `. + To read these data with ESMValCore, use the :ref:`rootpath ` ``/pool/data/ERA5`` with :ref:`DRS ` ``DKRZ-ERA5-GRIB`` in your configuration, for example: From 086422fe526146e02d94471845ad3cd9ca69421a Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 14 Oct 2024 18:07:53 +0200 Subject: [PATCH 58/64] Fix units for lat and lon --- esmvalcore/cmor/_fixes/native6/era5.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py index cafbe96b23..a981ca70f8 100644 --- a/esmvalcore/cmor/_fixes/native6/era5.py +++ b/esmvalcore/cmor/_fixes/native6/era5.py @@ -507,7 +507,7 @@ def _fix_coordinates(self, cube): coord = cube.coord(axis=axis) if axis == "T": coord.convert_units("days since 1850-1-1 00:00:00.0") - if axis == "Z": + if axis in ("X", "Y", "Z"): coord.convert_units(coord_def.units) coord.standard_name = coord_def.standard_name coord.var_name = coord_def.out_name From 57c478f79b4ce25211f78126f0874b5b84877328 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 14 Oct 2024 18:08:23 +0200 Subject: [PATCH 59/64] Fix GRIB_PARAM attribute in resulting netcdf file --- esmvalcore/cmor/_fixes/native6/era5.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py index a981ca70f8..2311197d5a 100644 --- a/esmvalcore/cmor/_fixes/native6/era5.py +++ b/esmvalcore/cmor/_fixes/native6/era5.py @@ -577,6 +577,10 @@ def fix_metadata(self, cubes): "Contains modified Copernicus Climate Change " f"Service Information {year}" ) + if "GRIB_PARAM" in cube.attributes: + cube.attributes["GRIB_PARAM"] = str( + cube.attributes["GRIB_PARAM"] + ) fixed_cubes.append(cube) From e123244d16b99564c29a62e6d84f15568fcf6030 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Thu, 17 Oct 2024 14:44:41 +0200 Subject: [PATCH 60/64] Avoid double regridding for ERA5 GRIB data --- doc/quickstart/find_data.rst | 8 +- esmvalcore/_recipe/recipe.py | 25 ++++ esmvalcore/cmor/_fixes/native6/era5.py | 16 --- .../config/extra_facets/native6-era5.yml | 2 +- esmvalcore/preprocessor/_io.py | 4 +- .../cmor/_fixes/native6/test_era5.py | 38 +----- tests/integration/conftest.py | 36 +++--- tests/integration/recipe/test_recipe.py | 113 ++++++++++++++++++ 8 files changed, 166 insertions(+), 76 deletions(-) diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index 51992e9be5..1811fd7c68 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -205,16 +205,14 @@ By default, these data is regridded to a regular 0.25°x0.25° grid as `__ using bilinear interpolation. -To disable this, you can use the facet ``regrid: false`` in the recipe: +To disable this, you can use the facet ``automatic_regrid: false`` in the +recipe: .. code-block:: yaml datasets: - {project: native6, dataset: ERA5, timerange: '2000/2001', - short_name: tas, mip: Amon, regrid: false} - -It is recommended to disable the default regridding if regridding is setup in -the :ref:`preprocessor `. + short_name: tas, mip: Amon, automatic_regrid: false} - Supported variables: ``albsn``, ``cl``, ``cli``, ``clt``, ``clw``, ``hur``, ``hus``, ``o3``, ``prw``, ``ps``, ``psl``, ``rainmxrat27``, ``sftlf``, diff --git a/esmvalcore/_recipe/recipe.py b/esmvalcore/_recipe/recipe.py index 55e789d6f4..6173d1b60b 100644 --- a/esmvalcore/_recipe/recipe.py +++ b/esmvalcore/_recipe/recipe.py @@ -37,6 +37,7 @@ PreprocessorFile, ) from esmvalcore.preprocessor._area import _update_shapefile_path +from esmvalcore.preprocessor._io import GRIB_FORMATS from esmvalcore.preprocessor._multimodel import _get_stat_identifier from esmvalcore.preprocessor._regrid import ( _spec_to_latlonvals, @@ -227,6 +228,29 @@ def _get_default_settings(dataset): return settings +def _add_dataset_specific_settings(dataset: Dataset, settings: dict) -> None: + """Add dataset-specific settings.""" + project = dataset.facets["project"] + dataset_name = dataset.facets["dataset"] + file_suffixes = [Path(file.name).suffix for file in dataset.files] + + # Automatic regridding for native ERA5 data in GRIB format if regridding + # step is not already present (can be disabled with facet + # automatic_regrid=False) + if all( + [ + project == "native6", + dataset_name == "ERA5", + any(grib_format in file_suffixes for grib_format in GRIB_FORMATS), + "regrid" not in settings, + dataset.facets.get("automatic_regrid", True), + ] + ): + # Settings recommended by ECMWF + # (https://confluence.ecmwf.int/display/CKB/ERA5%3A+What+is+the+spatial+reference#heading-Interpolation) + settings["regrid"] = {"target_grid": "0.25x0.25", "scheme": "linear"} + + def _exclude_dataset(settings, facets, step): """Exclude dataset from specific preprocessor step if requested.""" exclude = { @@ -541,6 +565,7 @@ def _get_preprocessor_products( _apply_preprocessor_profile(settings, profile) _update_multi_dataset_settings(dataset.facets, settings) _update_preproc_functions(settings, dataset, datasets, missing_vars) + _add_dataset_specific_settings(dataset, settings) check.preprocessor_supplementaries(dataset, settings) input_datasets = _get_input_datasets(dataset) missing = _check_input_files(input_datasets) diff --git a/esmvalcore/cmor/_fixes/native6/era5.py b/esmvalcore/cmor/_fixes/native6/era5.py index 2311197d5a..18d34d0521 100644 --- a/esmvalcore/cmor/_fixes/native6/era5.py +++ b/esmvalcore/cmor/_fixes/native6/era5.py @@ -11,12 +11,9 @@ from esmvalcore.cmor._fixes.shared import add_scalar_height_coord from esmvalcore.cmor.table import CMOR_TABLES from esmvalcore.iris_helpers import date2num, has_unstructured_grid -from esmvalcore.preprocessor import regrid logger = logging.getLogger(__name__) -DEFAULT_ERA5_GRID = "0.25x0.25" - def get_frequency(cube): """Determine time frequency of input cube.""" @@ -585,16 +582,3 @@ def fix_metadata(self, cubes): fixed_cubes.append(cube) return fixed_cubes - - def fix_data(self, cube): - """Fix data.""" - regridding_enabled = self.extra_facets.get("regrid", True) - if regridding_enabled and has_unstructured_grid(cube): - logger.debug( - "Automatically regrid ERA5 data (variable %s) to %s° " - "grid using bilinear regridding", - self.vardef.short_name, - DEFAULT_ERA5_GRID, - ) - cube = regrid(cube, DEFAULT_ERA5_GRID, "linear") - return cube diff --git a/esmvalcore/config/extra_facets/native6-era5.yml b/esmvalcore/config/extra_facets/native6-era5.yml index fc514acb11..4ab1915da9 100644 --- a/esmvalcore/config/extra_facets/native6-era5.yml +++ b/esmvalcore/config/extra_facets/native6-era5.yml @@ -17,8 +17,8 @@ ERA5: # Settings for all variables of all MIPs '*': '*': + automatic_regrid: true family: E5 - regrid: true type: an typeid: '00' version: v1 diff --git a/esmvalcore/preprocessor/_io.py b/esmvalcore/preprocessor/_io.py index fff32f6cce..5038d680c9 100644 --- a/esmvalcore/preprocessor/_io.py +++ b/esmvalcore/preprocessor/_io.py @@ -36,6 +36,7 @@ "reference_dataset", "alternative_dataset", } +GRIB_FORMATS = (".grib2", ".grib", ".grb2", ".grb", ".gb2", ".gb") iris.FUTURE.save_split_attrs = True @@ -142,8 +143,7 @@ def load( # GRIB files need to be loaded with iris.load, otherwise we will # get separate (lat, lon) slices for each time step, pressure # level, etc. - grib_formats = (".grib2", ".grib", ".grb2", ".grb", ".gb2", ".gb") - if file.suffix in grib_formats: + if file.suffix in GRIB_FORMATS: raw_cubes = iris.load(file, callback=_load_callback) else: raw_cubes = iris.load_raw(file, callback=_load_callback) diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py index 0139ae50a1..26636979db 100644 --- a/tests/integration/cmor/_fixes/native6/test_era5.py +++ b/tests/integration/cmor/_fixes/native6/test_era5.py @@ -9,7 +9,6 @@ from iris.coords import AuxCoord, DimCoord from iris.cube import Cube, CubeList -import esmvalcore.cmor._fixes.native6.era5 from esmvalcore.cmor._fixes.fix import Fix, GenericFix from esmvalcore.cmor._fixes.native6.era5 import ( AllVars, @@ -18,7 +17,7 @@ fix_accumulated_units, get_frequency, ) -from esmvalcore.cmor.fix import fix_data, fix_metadata +from esmvalcore.cmor.fix import fix_metadata from esmvalcore.cmor.table import CMOR_TABLES, get_var_info from esmvalcore.preprocessor import cmor_check_metadata @@ -1551,38 +1550,3 @@ def test_unstructured_grid(unstructured_grid_cubes): lon = fixed_cube.coord("longitude") np.testing.assert_allclose(lon.points, [179, 180, 180, 179]) assert lon.bounds is None - - -@pytest.mark.parametrize("regrid", [None, False, True]) -def test_automatic_regridding_unstructured_cube( - regrid, unstructured_grid_cubes, monkeypatch -): - """Test automatic regridding.""" - monkeypatch.setattr( - esmvalcore.cmor._fixes.native6.era5, "DEFAULT_ERA5_GRID", "60x60" - ) - cube = unstructured_grid_cubes[0] - - fix_kwargs = {} - if regrid is not None: - fix_kwargs["regrid"] = regrid - fixed_cube = fix_data(cube, "tas", "native6", "era5", "Amon", **fix_kwargs) - - if regrid is None or regrid is True: - assert fixed_cube.shape == (2, 3, 6) - else: - assert fixed_cube.shape == (2, 4) - - -@pytest.mark.parametrize("regrid", [None, False, True]) -def test_automatic_regridding_regular_cube(regrid): - """Test automatic regridding.""" - cube = era5_2d("monthly")[0] - - fix_kwargs = {} - if regrid is not None: - fix_kwargs["regrid"] = regrid - fixed_cube = fix_data(cube, "tas", "native6", "era5", "Amon", **fix_kwargs) - - assert fixed_cube.shape == (3, 3, 3) - assert fixed_cube is cube diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index e32e3ca3fa..5b3a2678d3 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -100,21 +100,33 @@ def _get_files(root_path, facets, tracking_id): return files, globs -@pytest.fixture -def patched_datafinder(tmp_path, monkeypatch): - def tracking_ids(i=0): - while True: - yield i - i += 1 +def _tracking_ids(i=0): + while True: + yield i + i += 1 - tracking_id = tracking_ids() +def _get_find_files_func(path: Path, suffix: str = ".nc"): def find_files(*, debug: bool = False, **facets): - files, file_globs = _get_files(tmp_path, facets, tracking_id) + files, file_globs = _get_files(path, facets, _tracking_ids()) + files = [f.with_suffix(suffix) for f in files] + file_globs = [g.with_suffix(suffix) for g in file_globs] if debug: return files, file_globs return files + return find_files + + +@pytest.fixture +def patched_datafinder(tmp_path, monkeypatch): + find_files = _get_find_files_func(tmp_path) + monkeypatch.setattr(esmvalcore.local, "find_files", find_files) + + +@pytest.fixture +def patched_datafinder_grib(tmp_path, monkeypatch): + find_files = _get_find_files_func(tmp_path, suffix=".grib") monkeypatch.setattr(esmvalcore.local, "find_files", find_files) @@ -129,13 +141,7 @@ def patched_failing_datafinder(tmp_path, monkeypatch): Otherwise, return files just like `patched_datafinder`. """ - - def tracking_ids(i=0): - while True: - yield i - i += 1 - - tracking_id = tracking_ids() + tracking_id = _tracking_ids() def find_files(*, debug: bool = False, **facets): files, file_globs = _get_files(tmp_path, facets, tracking_id) diff --git a/tests/integration/recipe/test_recipe.py b/tests/integration/recipe/test_recipe.py index f486db1657..a3fefaccc6 100644 --- a/tests/integration/recipe/test_recipe.py +++ b/tests/integration/recipe/test_recipe.py @@ -3390,3 +3390,116 @@ def test_invalid_interpolate(tmp_path, patched_datafinder, session): get_recipe(tmp_path, content, session) assert str(exc.value) == INITIALIZATION_ERROR_MSG assert exc.value.failed_tasks[0].message == msg + + +def test_automatic_regrid_era5_nc(tmp_path, patched_datafinder, session): + content = dedent(""" + diagnostics: + diagnostic_name: + variables: + tas: + mip: Amon + timerange: '20000101/20001231' + additional_datasets: + - {project: native6, dataset: ERA5, tier: 3} + scripts: null + """) + recipe = get_recipe(tmp_path, content, session) + + assert len(recipe.tasks) == 1 + task = recipe.tasks.pop() + + assert len(task.products) == 1 + product = task.products.pop() + + assert "regrid" not in product.settings + + +def test_automatic_regrid_era5_grib( + tmp_path, patched_datafinder_grib, session +): + content = dedent(""" + diagnostics: + diagnostic_name: + variables: + tas: + mip: Amon + timerange: '20000101/20001231' + additional_datasets: + - {project: native6, dataset: ERA5, tier: 3} + scripts: null + """) + recipe = get_recipe(tmp_path, content, session) + + assert len(recipe.tasks) == 1 + task = recipe.tasks.pop() + + assert len(task.products) == 1 + product = task.products.pop() + + assert "regrid" in product.settings + assert product.settings["regrid"] == { + "target_grid": "0.25x0.25", + "scheme": "linear", + } + + +def test_automatic_no_regrid_era5_grib( + tmp_path, patched_datafinder_grib, session +): + content = dedent(""" + diagnostics: + diagnostic_name: + variables: + tas: + mip: Amon + timerange: '20000101/20001231' + additional_datasets: + - {project: native6, dataset: ERA5, tier: 3, automatic_regrid: false} + scripts: null + """) + recipe = get_recipe(tmp_path, content, session) + + assert len(recipe.tasks) == 1 + task = recipe.tasks.pop() + + assert len(task.products) == 1 + product = task.products.pop() + + assert "regrid" not in product.settings + + +def test_automatic_already_regrid_era5_grib( + tmp_path, patched_datafinder_grib, session +): + content = dedent(""" + preprocessors: + test_automatic_regrid_era5: + regrid: + target_grid: 1x1 + scheme: nearest + + diagnostics: + diagnostic_name: + variables: + tas: + preprocessor: test_automatic_regrid_era5 + mip: Amon + timerange: '20000101/20001231' + additional_datasets: + - {project: native6, dataset: ERA5, tier: 3} + scripts: null + """) + recipe = get_recipe(tmp_path, content, session) + + assert len(recipe.tasks) == 1 + task = recipe.tasks.pop() + + assert len(task.products) == 1 + product = task.products.pop() + + assert "regrid" in product.settings + assert product.settings["regrid"] == { + "target_grid": "1x1", + "scheme": "nearest", + } From e6389dbdda3f3823f0b081f9e20cf094a4f70597 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Thu, 17 Oct 2024 14:49:51 +0200 Subject: [PATCH 61/64] Add debug message --- esmvalcore/_recipe/recipe.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/esmvalcore/_recipe/recipe.py b/esmvalcore/_recipe/recipe.py index 6173d1b60b..5d71cdd022 100644 --- a/esmvalcore/_recipe/recipe.py +++ b/esmvalcore/_recipe/recipe.py @@ -249,6 +249,11 @@ def _add_dataset_specific_settings(dataset: Dataset, settings: dict) -> None: # Settings recommended by ECMWF # (https://confluence.ecmwf.int/display/CKB/ERA5%3A+What+is+the+spatial+reference#heading-Interpolation) settings["regrid"] = {"target_grid": "0.25x0.25", "scheme": "linear"} + logger.debug( + "Automatically regrid native6 ERA5 data in GRIB format with the " + "settings %s", + settings["regrid"], + ) def _exclude_dataset(settings, facets, step): From 66ecd845f10d1b8c21175d47c72da9dff6e4af89 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Thu, 17 Oct 2024 15:05:13 +0200 Subject: [PATCH 62/64] 100% coverage --- tests/integration/cmor/_fixes/native6/test_era5.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/cmor/_fixes/native6/test_era5.py b/tests/integration/cmor/_fixes/native6/test_era5.py index 26636979db..b65acabbff 100644 --- a/tests/integration/cmor/_fixes/native6/test_era5.py +++ b/tests/integration/cmor/_fixes/native6/test_era5.py @@ -1517,6 +1517,7 @@ def unstructured_grid_cubes(): units="K", dim_coords_and_dims=[(time, 0)], aux_coords_and_dims=[(lat, 1), (lon, 1)], + attributes={"GRIB_PARAM": (1, 1)}, ) return CubeList([cube]) @@ -1550,3 +1551,5 @@ def test_unstructured_grid(unstructured_grid_cubes): lon = fixed_cube.coord("longitude") np.testing.assert_allclose(lon.points, [179, 180, 180, 179]) assert lon.bounds is None + + assert fixed_cube.attributes["GRIB_PARAM"] == "(1, 1)" From 1845cdc7f92865b49fe421d21bc003f1adf769d4 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Thu, 17 Oct 2024 15:10:28 +0200 Subject: [PATCH 63/64] Fix test fixture --- tests/integration/conftest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 5b3a2678d3..f6251a8bb0 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -107,8 +107,10 @@ def _tracking_ids(i=0): def _get_find_files_func(path: Path, suffix: str = ".nc"): + tracking_id = _tracking_ids() + def find_files(*, debug: bool = False, **facets): - files, file_globs = _get_files(path, facets, _tracking_ids()) + files, file_globs = _get_files(path, facets, tracking_id) files = [f.with_suffix(suffix) for f in files] file_globs = [g.with_suffix(suffix) for g in file_globs] if debug: From 9af867a955594cd27cd251a21453eee6f8f474a9 Mon Sep 17 00:00:00 2001 From: Manuel Schlund <32543114+schlunma@users.noreply.github.com> Date: Fri, 6 Dec 2024 09:26:29 +0100 Subject: [PATCH 64/64] Apply suggestions from code review Co-authored-by: Bettina Gier --- doc/quickstart/find_data.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index 1811fd7c68..d93f114f21 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -107,7 +107,7 @@ The following native reanalysis/observational datasets are supported under the To use these datasets, put the files containing the data in the directory that you have :ref:`configured ` for the ``rootpath`` of the ``native6`` project, in a subdirectory called -``Tier{tier}/{dataset}/{version}/{frequency}/{short_name}`` (assuming your are +``Tier{tier}/{dataset}/{version}/{frequency}/{short_name}`` (assuming you are using the ``default`` DRS for ``native6``). Replace the items in curly braces by the values used in the variable/dataset definition in the :ref:`recipe `. @@ -121,7 +121,7 @@ ERA5 data can be downloaded from the Copernicus Climate Data Store (CDS) using the convenient tool `era5cli `__. For example for monthly data, place the files in the ``/Tier3/ERA5/version/mon/pr`` subdirectory of your ``rootpath`` that you have -configured for the ``native6`` project (assuming your are using the ``default`` +configured for the ``native6`` project (assuming you are using the ``default`` DRS for ``native6``). - Supported variables: ``cl``, ``clt``, ``evspsbl``, ``evspsblpot``, ``mrro``, @@ -200,7 +200,7 @@ Thus, example dataset entries could look like this: The native ERA5 output in GRIB format is stored on a `reduced Gaussian grid `__. -By default, these data is regridded to a regular 0.25°x0.25° grid as +By default, these data are regridded to a regular 0.25°x0.25° grid as `recommended by the ECMWF `__ using bilinear interpolation. @@ -230,7 +230,7 @@ MSWEP For example for monthly data, place the files in the ``/Tier3/MSWEP/version/mon/pr`` subdirectory of your ``rootpath`` that you have -configured for the ``native6`` project (assuming your are using the ``default`` +configured for the ``native6`` project (assuming you are using the ``default`` DRS for ``native6``). .. note::