diff --git a/.cirrus.yml b/.cirrus.yml index e55b69dc69..e6bd0fc424 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -38,7 +38,7 @@ env: # Conda packages to be installed. CONDA_CACHE_PACKAGES: "nox pip" # Git commit hash for iris test data. - IRIS_TEST_DATA_VERSION: "2.2" + IRIS_TEST_DATA_VERSION: "2.4" # Base directory for the iris-test-data. IRIS_TEST_DATA_DIR: ${HOME}/iris-test-data @@ -193,4 +193,4 @@ task: - mkdir -p ${MPL_RC_DIR} - echo "backend : agg" > ${MPL_RC_FILE} - echo "image.cmap : viridis" >> ${MPL_RC_FILE} - - nox --session linkcheck -- --verbose \ No newline at end of file + - nox --session linkcheck -- --verbose diff --git a/lib/iris/experimental/ugrid/__init__.py b/lib/iris/experimental/ugrid/__init__.py index 144c3e4da6..bfc570fcfd 100644 --- a/lib/iris/experimental/ugrid/__init__.py +++ b/lib/iris/experimental/ugrid/__init__.py @@ -15,7 +15,9 @@ from collections.abc import Iterable from contextlib import contextmanager from functools import wraps +from itertools import groupby import logging +from pathlib import Path import re import threading @@ -39,12 +41,15 @@ from ...exceptions import ConnectivityNotFoundError, CoordinateNotFoundError from ...fileformats import cf, netcdf from ...fileformats._nc_load_rules.helpers import get_attr_units, get_names +from ...io import decode_uri, expand_filespecs from ...util import guess_coord_axis __all__ = [ "CFUGridReader", "Connectivity", "ConnectivityMetadata", + "load_mesh", + "load_meshes", "Mesh", "Mesh1DConnectivities", "Mesh1DCoords", @@ -200,9 +205,9 @@ def __init__( for Fortran and legacy NetCDF files). * src_dim (int): Either ``0`` or ``1``. Default is ``0``. Denotes which dimension - of :attr:`indices` varies over the :attr:`src_location`'s (the + of :attr:`indices` varies over the :attr:`src_location`\\ s (the alternate dimension therefore varying within individual - :attr:`src_location`'s). (This parameter allows support for fastest varying index being + :attr:`src_location`\\ s). (This parameter allows support for fastest varying index being either first or last). E.g. for ``face_node_connectivity``, for 10 faces: ``indices.shape[src_dim] = 10``. @@ -353,7 +358,7 @@ def start_index(self): def src_dim(self): """ The dimension of the connectivity's :attr:`indices` array that varies - over the connectivity's :attr:`src_location`'s. Either ``0`` or ``1``. + over the connectivity's :attr:`src_location`\\ s. Either ``0`` or ``1``. **Read-only** - validity of :attr:`indices` is dependent on :attr:`src_dim`. Use :meth:`transpose` to create a new, transposed :class:`Connectivity` if a different :attr:`src_dim` is needed. @@ -367,7 +372,7 @@ def tgt_dim(self): Derived as the alternate value of :attr:`src_dim` - each must equal either ``0`` or ``1``. The dimension of the connectivity's :attr:`indices` array that varies - within the connectivity's individual :attr:`src_location`'s. + within the connectivity's individual :attr:`src_location`\\ s. """ return self._tgt_dim @@ -491,7 +496,7 @@ def validate_indices(self): """ Perform a thorough validity check of this connectivity's :attr:`indices`. Includes checking the sizes of individual - :attr:`src_location`'s (specified using masks on the + :attr:`src_location`\\ s (specified using masks on the :attr:`indices` array) against the :attr:`cf_role`. Raises a ``ValueError`` if any problems are encountered, otherwise @@ -1003,6 +1008,11 @@ def __eq__(self, other): # TBD: this is a minimalist implementation and requires to be revisited return id(self) == id(other) + def __hash__(self): + # Allow use in sets and as dictionary keys, as is done for :class:`iris.cube.Cube`. + # See https://github.com/SciTools/iris/pull/1772 + return hash(id(self)) + def __getstate__(self): return ( self._metadata_manager, @@ -1928,7 +1938,7 @@ def to_MeshCoord(self, location, axis): def to_MeshCoords(self, location): """ - Generate a tuple of :class:`MeshCoord`'s, each referencing the current + Generate a tuple of :class:`MeshCoord`\\ s, each referencing the current :class:`Mesh`, one for each :attr:`AXES` value, passing through the ``location`` argument. @@ -1942,7 +1952,7 @@ def to_MeshCoords(self, location): The ``location`` argument for :class:`MeshCoord` instantiation. Returns: - tuple of :class:`MeshCoord`'s referencing the current :class:`Mesh`. + tuple of :class:`MeshCoord`\\ s referencing the current :class:`Mesh`. One for each value in :attr:`AXES`, using the value for the ``axis`` argument. @@ -3274,6 +3284,135 @@ def context(self): PARSE_UGRID_ON_LOAD = ParseUGridOnLoad() +def _meshes_from_cf(cf_reader): + """ + Common behaviour for extracting meshes from a CFReader. + + Simple now, but expected to increase in complexity as Mesh sharing develops. + + """ + # Mesh instances are shared between file phenomena. + # TODO: more sophisticated Mesh sharing between files. + # TODO: access external Mesh cache? + mesh_vars = cf_reader.cf_group.meshes + meshes = { + name: _build_mesh(cf_reader, var, cf_reader.filename) + for name, var in mesh_vars.items() + } + return meshes + + +def load_mesh(uris, var_name=None): + """ + Load a single :class:`Mesh` object from one or more NetCDF files. + + Raises an error if more/less than one :class:`Mesh` is found. + + Parameters + ---------- + uris : str or iterable of str + One or more filenames/URI's. Filenames can include wildcards. Any URI's + must support OpenDAP. + var_name : str, optional + Only return a :class:`Mesh` if its var_name matches this value. + + Returns + ------- + :class:`Mesh` + + """ + meshes_result = load_meshes(uris, var_name) + result = set([mesh for file in meshes_result.values() for mesh in file]) + mesh_count = len(result) + if mesh_count != 1: + message = ( + f"Expecting 1 mesh, but input file(s) produced: {mesh_count} ." + ) + raise ValueError(message) + return result.pop() # Return the single element + + +def load_meshes(uris, var_name=None): + """ + Load :class:`Mesh` objects from one or more NetCDF files. + + Parameters + ---------- + uris : str or iterable of str + One or more filenames/URI's. Filenames can include wildcards. Any URI's + must support OpenDAP. + var_name : str, optional + Only return :class:`Mesh`\\ es that have var_names matching this value. + + Returns + ------- + dict + A dictionary mapping each mesh-containing file path/URL in the input + ``uris`` to a list of the :class:`Mesh`\\ es returned from each. + + """ + # TODO: rationalise UGRID/mesh handling once experimental.ugrid is folded + # into standard behaviour. + # No constraints or callbacks supported - these assume they are operating + # on a Cube. + + from iris.fileformats import FORMAT_AGENT + + if not PARSE_UGRID_ON_LOAD: + # Explicit behaviour, consistent with netcdf.load_cubes(), rather than + # an invisible assumption. + message = ( + f"PARSE_UGRID_ON_LOAD is {bool(PARSE_UGRID_ON_LOAD)}. Must be " + f"True to enable mesh loading." + ) + raise ValueError(message) + + if isinstance(uris, str): + uris = [uris] + + # Group collections of uris by their iris handler + # Create list of tuples relating schemes to part names. + uri_tuples = sorted(decode_uri(uri) for uri in uris) + + valid_sources = [] + for scheme, groups in groupby(uri_tuples, key=lambda x: x[0]): + # Call each scheme handler with the appropriate URIs + if scheme == "file": + filenames = [x[1] for x in groups] + sources = expand_filespecs(filenames) + elif scheme in ["http", "https"]: + sources = [":".join(x) for x in groups] + else: + message = f"Iris cannot handle the URI scheme: {scheme}" + raise ValueError(message) + + for source in sources: + if scheme == "file": + with open(source, "rb") as fh: + handling_format_spec = FORMAT_AGENT.get_spec( + Path(source).name, fh + ) + else: + handling_format_spec = FORMAT_AGENT.get_spec(source, None) + + if handling_format_spec.handler == netcdf.load_cubes: + valid_sources.append(source) + else: + message = f"Ignoring non-NetCDF file: {source}" + logger.info(msg=message, extra=dict(cls=None)) + + result = {} + for source in valid_sources: + meshes_dict = _meshes_from_cf(CFUGridReader(source)) + meshes = list(meshes_dict.values()) + if var_name is not None: + meshes = list(filter(lambda m: m.var_name == var_name, meshes)) + if meshes: + result[source] = meshes + + return result + + ############ # CF Overrides. # These are not included in __all__ since they are not [currently] needed @@ -3469,7 +3608,17 @@ def identify(cls, variables, ignore=None, target=None, warn=True): log_level = logging.WARNING if warn else logging.DEBUG # Identify all CF-UGRID mesh variables. + all_vars = target == variables for nc_var_name, nc_var in target.items(): + if all_vars: + # SPECIAL BEHAVIOUR FOR MESH VARIABLES. + # We are looking for all mesh variables. Check if THIS variable + # is a mesh using its own attributes. + if getattr(nc_var, "cf_role", "") == "mesh_topology": + result[nc_var_name] = CFUGridMeshVariable( + nc_var_name, nc_var + ) + # Check for mesh variable references. nc_var_att = getattr(nc_var, cls.cf_identity, None) @@ -3679,6 +3828,21 @@ def _build_mesh(cf, mesh_var, file_path): attributes = {} attr_units = get_attr_units(mesh_var, attributes) + cf_role_message = None + if not hasattr(mesh_var, "cf_role"): + cf_role_message = f"{mesh_var.cf_name} has no cf_role attribute." + cf_role = "mesh_topology" + else: + cf_role = getattr(mesh_var, "cf_role") + if cf_role != "mesh_topology": + cf_role_message = ( + f"{mesh_var.cf_name} has an inappropriate cf_role: {cf_role}." + ) + if cf_role_message: + cf_role_message += " Correcting to 'mesh_topology'." + # TODO: reconsider logging level when we have consistent practice. + logger.warning(cf_role_message, extra=dict(cls=None)) + if hasattr(mesh_var, "volume_node_connectivity"): topology_dimension = 3 elif hasattr(mesh_var, "face_node_connectivity"): @@ -3782,7 +3946,6 @@ def _build_mesh(cf, mesh_var, file_path): edge_dimension=edge_dimension, face_dimension=face_dimension, ) - assert mesh.cf_role == mesh_var.cf_role mesh_elements = ( list(mesh.all_coords) + list(mesh.all_connectivities) + [mesh] diff --git a/lib/iris/fileformats/cf.py b/lib/iris/fileformats/cf.py index 1429e4f65e..b22fbd3b51 100644 --- a/lib/iris/fileformats/cf.py +++ b/lib/iris/fileformats/cf.py @@ -1067,6 +1067,11 @@ def __init__(self, filename, warn=False, monotonic=False): self._build_cf_groups() self._reset() + @property + def filename(self): + """The file that the CFReader is reading.""" + return self._filename + def __repr__(self): return "%s(%r)" % (self.__class__.__name__, self._filename) diff --git a/lib/iris/fileformats/netcdf.py b/lib/iris/fileformats/netcdf.py index a4727ea624..7bb90665b6 100644 --- a/lib/iris/fileformats/netcdf.py +++ b/lib/iris/fileformats/netcdf.py @@ -792,8 +792,8 @@ def load_cubes(filenames, callback=None): from iris.experimental.ugrid import ( PARSE_UGRID_ON_LOAD, CFUGridReader, - _build_mesh, _build_mesh_coords, + _meshes_from_cf, ) from iris.io import run_callback @@ -808,15 +808,7 @@ def load_cubes(filenames, callback=None): meshes = {} if PARSE_UGRID_ON_LOAD: cf = CFUGridReader(filename) - - # Mesh instances are shared between file phenomena. - # TODO: more sophisticated Mesh sharing between files. - # TODO: access external Mesh cache? - mesh_vars = cf.cf_group.meshes - meshes = { - name: _build_mesh(cf, var, filename) - for name, var in mesh_vars.items() - } + meshes = _meshes_from_cf(cf) else: cf = iris.fileformats.cf.CFReader(filename) diff --git a/lib/iris/tests/integration/experimental/test_ugrid_load.py b/lib/iris/tests/integration/experimental/test_ugrid_load.py index 1503225d6f..5d40b1e69b 100644 --- a/lib/iris/tests/integration/experimental/test_ugrid_load.py +++ b/lib/iris/tests/integration/experimental/test_ugrid_load.py @@ -10,15 +10,20 @@ standard behaviour. """ +# Import iris.tests first so that some things can be initialised before +# importing anything else. +import iris.tests as tests # isort:skip from collections.abc import Iterable from iris import Constraint, load -from iris.experimental.ugrid import PARSE_UGRID_ON_LOAD, logger - -# Import iris.tests first so that some things can be initialised before -# importing anything else. -import iris.tests as tests +from iris.experimental.ugrid import ( + PARSE_UGRID_ON_LOAD, + Mesh, + load_mesh, + load_meshes, + logger, +) from iris.tests.stock.netcdf import ( _file_from_cdl_template as create_file_from_cdl_template, ) @@ -109,6 +114,15 @@ def test_3D_veg_pseudo_levels(self): "3D_veg_pseudo_levels.cml", ) + def test_no_mesh(self): + with PARSE_UGRID_ON_LOAD.context(): + cube_list = load( + tests.get_data_path( + ["NetCDF", "unstructured_grid", "theta_nodal_not_ugrid.nc"] + ) + ) + self.assertTrue(all([cube.mesh is None for cube in cube_list])) + @tests.skip_data class TestMultiplePhenomena(tests.IrisTest): @@ -178,6 +192,56 @@ def test_mesh_no_topology_dimension(self): # Check that the result has the correct topology-dimension value. self.assertEqual(cube.mesh.topology_dimension, 2) + def test_mesh_bad_cf_role(self): + # Check that the load generates a suitable warning. + log_regex = r"inappropriate cf_role" + with self.assertLogs(logger, level="WARNING", msg_regex=log_regex): + template = "minimal_bad_mesh_cf_role" + dim_line = 'mesh_var:cf_role = "foo" ;' + _ = self.create_synthetic_test_cube( + template=template, subs=dict(CF_ROLE_DEFINITION=dim_line) + ) + + def test_mesh_no_cf_role(self): + # Check that the load generates a suitable warning. + log_regex = r"no cf_role attribute" + with self.assertLogs(logger, level="WARNING", msg_regex=log_regex): + template = "minimal_bad_mesh_cf_role" + dim_line = "" + _ = self.create_synthetic_test_cube( + template=template, subs=dict(CF_ROLE_DEFINITION=dim_line) + ) + + +@tests.skip_data +class Test_load_mesh(tests.IrisTest): + def common_test(self, file_name, mesh_var_name): + with PARSE_UGRID_ON_LOAD.context(): + mesh = load_mesh( + tests.get_data_path(["NetCDF", "unstructured_grid", file_name]) + ) + # NOTE: cannot use CML tests as this isn't supported for non-Cubes. + self.assertIsInstance(mesh, Mesh) + self.assertEqual(mesh.var_name, mesh_var_name) + + def test_full_file(self): + self.common_test( + "lfric_ngvat_2D_1t_face_half_levels_main_conv_rain.nc", + "Mesh2d_half_levels", + ) + + def test_mesh_file(self): + self.common_test("mesh_C12.nc", "dynamics") + + def test_no_mesh(self): + with PARSE_UGRID_ON_LOAD.context(): + meshes = load_meshes( + tests.get_data_path( + ["NetCDF", "unstructured_grid", "theta_nodal_not_ugrid.nc"] + ) + ) + self.assertDictEqual({}, meshes) + if __name__ == "__main__": tests.main() diff --git a/lib/iris/tests/stock/file_headers/minimal_bad_mesh_cf_role.cdl b/lib/iris/tests/stock/file_headers/minimal_bad_mesh_cf_role.cdl new file mode 100644 index 0000000000..6fb90fd8c6 --- /dev/null +++ b/lib/iris/tests/stock/file_headers/minimal_bad_mesh_cf_role.cdl @@ -0,0 +1,38 @@ +// Tolerant loading test example : the mesh has the wrong 'mesh_topology' +// NOTE: *not* truly minimal, as we cannot (yet) handle data with no face coords. +netcdf ${DATASET_NAME} { +dimensions: + NODES = ${NUM_NODES} ; + FACES = ${NUM_FACES} ; + FACE_CORNERS = 4 ; +variables: + int mesh_var ; + ${CF_ROLE_DEFINITION} + mesh_var:topology_dimension = 2 ; + mesh_var:node_coordinates = "mesh_node_x mesh_node_y" ; + mesh_var:face_node_connectivity = "mesh_face_nodes" ; + mesh_var:face_coordinates = "mesh_face_x mesh_face_y" ; + float mesh_node_x(NODES) ; + mesh_node_x:standard_name = "longitude" ; + mesh_node_x:long_name = "Longitude of mesh nodes." ; + mesh_node_x:units = "degrees_east" ; + float mesh_node_y(NODES) ; + mesh_node_y:standard_name = "latitude" ; + mesh_node_y:long_name = "Latitude of mesh nodes." ; + mesh_node_y:units = "degrees_north" ; + float mesh_face_x(FACES) ; + mesh_face_x:standard_name = "longitude" ; + mesh_face_x:long_name = "Longitude of mesh nodes." ; + mesh_face_x:units = "degrees_east" ; + float mesh_face_y(FACES) ; + mesh_face_y:standard_name = "latitude" ; + mesh_face_y:long_name = "Latitude of mesh nodes." ; + mesh_face_y:units = "degrees_north" ; + int mesh_face_nodes(FACES, FACE_CORNERS) ; + mesh_face_nodes:cf_role = "face_node_connectivity" ; + mesh_face_nodes:long_name = "Maps every face to its corner nodes." ; + mesh_face_nodes:start_index = 0 ; + float data_var(FACES) ; + data_var:mesh = "mesh_var" ; + data_var:location = "face" ; +} diff --git a/lib/iris/tests/unit/experimental/ugrid/test_CFUGridMeshVariable.py b/lib/iris/tests/unit/experimental/ugrid/test_CFUGridMeshVariable.py index e08dbc769e..8736a82fc1 100644 --- a/lib/iris/tests/unit/experimental/ugrid/test_CFUGridMeshVariable.py +++ b/lib/iris/tests/unit/experimental/ugrid/test_CFUGridMeshVariable.py @@ -10,13 +10,13 @@ standard behaviour. """ +# Import iris.tests first so that some things can be initialised before +# importing anything else. +import iris.tests as tests # isort:skip + import numpy as np from iris.experimental.ugrid import CFUGridMeshVariable, logger - -# Import iris.tests first so that some things can be initialised before -# importing anything else. -import iris.tests as tests from iris.tests.unit.experimental.ugrid.test_CFUGridReader import ( netcdf_ugrid_variable, ) @@ -31,7 +31,26 @@ class TestIdentify(tests.IrisTest): def setUp(self): self.cf_identity = "mesh" + def test_cf_role(self): + # Test that mesh variables can be identified by having `cf_role="mesh_topology"`. + match_name = "match" + match = named_variable(match_name) + setattr(match, "cf_role", "mesh_topology") + + not_match_name = f"not_{match_name}" + not_match = named_variable(not_match_name) + setattr(not_match, "cf_role", "foo") + + vars_all = {match_name: match, not_match_name: not_match} + + # ONLY expecting match, excluding not_match. + expected = {match_name: CFUGridMeshVariable(match_name, match)} + result = CFUGridMeshVariable.identify(vars_all) + self.assertDictEqual(expected, result) + def test_cf_identity(self): + # Test that mesh variables can be identified by being another variable's + # `mesh` attribute. subject_name = "ref_subject" ref_subject = named_variable(subject_name) ref_source = named_variable("ref_source") @@ -49,6 +68,37 @@ def test_cf_identity(self): result = CFUGridMeshVariable.identify(vars_all) self.assertDictEqual(expected, result) + def test_cf_role_and_identity(self): + # Test that identification can successfully handle a combination of + # mesh variables having `cf_role="mesh_topology"` AND being referenced as + # another variable's `mesh` attribute. + role_match_name = "match" + role_match = named_variable(role_match_name) + setattr(role_match, "cf_role", "mesh_topology") + ref_source_1 = named_variable("ref_source_1") + setattr(ref_source_1, self.cf_identity, role_match_name) + + subject_name = "ref_subject" + ref_subject = named_variable(subject_name) + ref_source_2 = named_variable("ref_source_2") + setattr(ref_source_2, self.cf_identity, subject_name) + + vars_all = { + role_match_name: role_match, + subject_name: ref_subject, + "ref_not_subject": named_variable("ref_not_subject"), + "ref_source_1": ref_source_1, + "ref_source_2": ref_source_2, + } + + # Expecting role_match and ref_subject but excluding other variables. + expected = { + role_match_name: CFUGridMeshVariable(role_match_name, role_match), + subject_name: CFUGridMeshVariable(subject_name, ref_subject), + } + result = CFUGridMeshVariable.identify(vars_all) + self.assertDictEqual(expected, result) + def test_duplicate_refs(self): subject_name = "ref_subject" ref_subject = named_variable(subject_name) diff --git a/lib/iris/tests/unit/experimental/ugrid/test_load_mesh.py b/lib/iris/tests/unit/experimental/ugrid/test_load_mesh.py new file mode 100644 index 0000000000..8bb78929f8 --- /dev/null +++ b/lib/iris/tests/unit/experimental/ugrid/test_load_mesh.py @@ -0,0 +1,51 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the LGPL license. +# See COPYING and COPYING.LESSER in the root of the repository for full +# licensing details. +""" +Unit tests for the :func:`iris.experimental.ugrid.load_mesh` function. + +""" + +# Import iris.tests first so that some things can be initialised before +# importing anything else. +import iris.tests as tests # isort:skip + +from iris.experimental.ugrid import PARSE_UGRID_ON_LOAD, load_mesh + + +class Tests(tests.IrisTest): + # All 'real' tests have been done for load_meshes(). Here we just check + # that load_mesh() works with load_meshes() correctly, using mocking. + def setUp(self): + self.load_meshes_mock = self.patch( + "iris.experimental.ugrid.load_meshes" + ) + # The expected return from load_meshes - a dict of files, each with + # a list of meshes. + self.load_meshes_mock.return_value = {"file": ["mesh"]} + + def test_calls_load_meshes(self): + args = [("file_1", "file_2"), "my_var_name"] + with PARSE_UGRID_ON_LOAD.context(): + _ = load_mesh(args) + self.assertTrue(self.load_meshes_mock.called_with(args)) + + def test_returns_mesh(self): + with PARSE_UGRID_ON_LOAD.context(): + mesh = load_mesh([]) + self.assertEqual(mesh, "mesh") + + def test_single_mesh(self): + # Override the load_meshes_mock return values to provoke errors. + def common(ret_val): + self.load_meshes_mock.return_value = ret_val + with self.assertRaisesRegex(ValueError, "Expecting 1 mesh.*"): + with PARSE_UGRID_ON_LOAD.context(): + _ = load_mesh([]) + + # Too many. + common({"file": ["mesh1", "mesh2"]}) + # Too few. + common({"file": []}) diff --git a/lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py b/lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py new file mode 100644 index 0000000000..8ff0d1ac63 --- /dev/null +++ b/lib/iris/tests/unit/experimental/ugrid/test_load_meshes.py @@ -0,0 +1,232 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the LGPL license. +# See COPYING and COPYING.LESSER in the root of the repository for full +# licensing details. +""" +Unit tests for the :func:`iris.experimental.ugrid.load_meshes` function. + +""" + +# Import iris.tests first so that some things can be initialised before +# importing anything else. +import iris.tests as tests # isort:skip + +from pathlib import Path +from shutil import rmtree +from subprocess import check_call +import tempfile +from uuid import uuid4 + +from iris.experimental.ugrid import PARSE_UGRID_ON_LOAD, load_meshes, logger + + +def setUpModule(): + global TMP_DIR + TMP_DIR = Path(tempfile.mkdtemp()) + + +def tearDownModule(): + if TMP_DIR is not None: + rmtree(TMP_DIR) + + +def cdl_to_nc(cdl): + cdl_path = TMP_DIR / "tst.cdl" + nc_path = TMP_DIR / f"{uuid4()}.nc" + # Write CDL string into a temporary CDL file. + with open(cdl_path, "w") as f_out: + f_out.write(cdl) + # Use ncgen to convert this into an actual (temporary) netCDF file. + command = "ncgen -o {} {}".format(nc_path, cdl_path) + check_call(command, shell=True) + return str(nc_path) + + +class TestsBasic(tests.IrisTest): + def setUp(self): + self.ref_cdl = """ + netcdf mesh_test { + dimensions: + node = 3 ; + face = 1 ; + vertex = 3 ; + levels = 2 ; + variables: + int mesh ; + mesh:cf_role = "mesh_topology" ; + mesh:topology_dimension = 2 ; + mesh:node_coordinates = "node_x node_y" ; + mesh:face_node_connectivity = "face_nodes" ; + float node_x(node) ; + node_x:standard_name = "longitude" ; + float node_y(node) ; + node_y:standard_name = "latitude" ; + int face_nodes(face, vertex) ; + face_nodes:cf_role = "face_node_connectivity" ; + face_nodes:start_index = 0 ; + int levels(levels) ; + float node_data(levels, node) ; + node_data:coordinates = "node_x node_y" ; + node_data:location = "node" ; + node_data:mesh = "mesh" ; + data: + mesh = 0; + node_x = 0., 2., 1.; + node_y = 0., 0., 1.; + face_nodes = 0, 1, 2; + levels = 1, 2; + node_data = 0., 0., 0.; + } + """ + self.nc_path = cdl_to_nc(self.ref_cdl) + + def add_second_mesh(self): + second_name = "mesh2" + cdl_extra = f""" + int {second_name} ; + {second_name}:cf_role = "mesh_topology" ; + {second_name}:topology_dimension = 2 ; + {second_name}:node_coordinates = "node_x node_y" ; + {second_name}:face_coordinates = "face_x face_y" ; + {second_name}:face_node_connectivity = "face_nodes" ; + """ + vars_string = "variables:" + vars_start = self.ref_cdl.index(vars_string) + len(vars_string) + new_cdl = ( + self.ref_cdl[:vars_start] + cdl_extra + self.ref_cdl[vars_start:] + ) + return new_cdl, second_name + + def test_with_data(self): + nc_path = cdl_to_nc(self.ref_cdl) + with PARSE_UGRID_ON_LOAD.context(): + meshes = load_meshes(nc_path) + + files = list(meshes.keys()) + self.assertEqual(1, len(files)) + file_meshes = meshes[files[0]] + self.assertEqual(1, len(file_meshes)) + mesh = file_meshes[0] + self.assertEqual("mesh", mesh.var_name) + + def test_no_data(self): + cdl_lines = self.ref_cdl.split("\n") + cdl_lines = filter( + lambda line: ':mesh = "mesh"' not in line, cdl_lines + ) + ref_cdl = "\n".join(cdl_lines) + + nc_path = cdl_to_nc(ref_cdl) + with PARSE_UGRID_ON_LOAD.context(): + meshes = load_meshes(nc_path) + + files = list(meshes.keys()) + self.assertEqual(1, len(files)) + file_meshes = meshes[files[0]] + self.assertEqual(1, len(file_meshes)) + mesh = file_meshes[0] + self.assertEqual("mesh", mesh.var_name) + + def test_no_mesh(self): + cdl_lines = self.ref_cdl.split("\n") + cdl_lines = filter( + lambda line: all( + [s not in line for s in (':mesh = "mesh"', "mesh_topology")] + ), + cdl_lines, + ) + ref_cdl = "\n".join(cdl_lines) + + nc_path = cdl_to_nc(ref_cdl) + with PARSE_UGRID_ON_LOAD.context(): + meshes = load_meshes(nc_path) + + self.assertDictEqual({}, meshes) + + def test_multi_files(self): + files_count = 3 + nc_paths = [cdl_to_nc(self.ref_cdl) for _ in range(files_count)] + with PARSE_UGRID_ON_LOAD.context(): + meshes = load_meshes(nc_paths) + self.assertEqual(files_count, len(meshes)) + + def test_multi_meshes(self): + ref_cdl, second_name = self.add_second_mesh() + nc_path = cdl_to_nc(ref_cdl) + with PARSE_UGRID_ON_LOAD.context(): + meshes = load_meshes(nc_path) + + files = list(meshes.keys()) + self.assertEqual(1, len(files)) + file_meshes = meshes[files[0]] + self.assertEqual(2, len(file_meshes)) + mesh_names = [mesh.var_name for mesh in file_meshes] + self.assertIn("mesh", mesh_names) + self.assertIn(second_name, mesh_names) + + def test_var_name(self): + second_cdl, second_name = self.add_second_mesh() + cdls = [self.ref_cdl, second_cdl] + nc_paths = [cdl_to_nc(cdl) for cdl in cdls] + with PARSE_UGRID_ON_LOAD.context(): + meshes = load_meshes(nc_paths, second_name) + + files = list(meshes.keys()) + self.assertEqual(1, len(files)) + file_meshes = meshes[files[0]] + self.assertEqual(1, len(file_meshes)) + self.assertEqual(second_name, file_meshes[0].var_name) + + def test_no_parsing(self): + nc_path = cdl_to_nc(self.ref_cdl) + with self.assertRaisesRegex( + ValueError, ".*Must be True to enable mesh loading." + ): + _ = load_meshes(nc_path) + + def test_invalid_scheme(self): + with self.assertRaisesRegex( + ValueError, "Iris cannot handle the URI scheme:.*" + ): + with PARSE_UGRID_ON_LOAD.context(): + _ = load_meshes("foo://bar") + + @tests.skip_data + def test_non_nc(self): + log_regex = r"Ignoring non-NetCDF file:.*" + with self.assertLogs(logger, level="INFO", msg_regex=log_regex): + with PARSE_UGRID_ON_LOAD.context(): + meshes = load_meshes( + tests.get_data_path(["PP", "simple_pp", "global.pp"]) + ) + self.assertDictEqual({}, meshes) + + +class TestsHttp(tests.IrisTest): + # Tests of HTTP (OpenDAP) loading need mocking since we can't have tests + # that rely on 3rd party servers. + def setUp(self): + self.format_agent_mock = self.patch( + "iris.fileformats.FORMAT_AGENT.get_spec" + ) + + def test_http(self): + url = "http://foo" + with PARSE_UGRID_ON_LOAD.context(): + _ = load_meshes(url) + self.format_agent_mock.assert_called_with(url, None) + + def test_mixed_sources(self): + url = "http://foo" + file = TMP_DIR / f"{uuid4()}.nc" + file.touch() + glob = f"{TMP_DIR}/*.nc" + + with PARSE_UGRID_ON_LOAD.context(): + _ = load_meshes([url, glob]) + file_uris = [ + call[0][0] for call in self.format_agent_mock.call_args_list + ] + for source in (url, Path(file).name): + self.assertIn(source, file_uris)