From 0162faf1ab38b5c43f9b6b8b0fab6920ebd9f679 Mon Sep 17 00:00:00 2001 From: keltonhalbert Date: Mon, 21 Sep 2020 18:44:38 +0000 Subject: [PATCH 01/50] patched back in our API additions after pulling from master --- yt/frontends/api.py | 1 + yt/frontends/nc4_cm1/__init__.py | 14 ++ yt/frontends/nc4_cm1/api.py | 25 +++ yt/frontends/nc4_cm1/data_structures.py | 215 ++++++++++++++++++++++++ yt/frontends/nc4_cm1/definitions.py | 1 + yt/frontends/nc4_cm1/fields.py | 79 +++++++++ yt/frontends/nc4_cm1/io.py | 72 ++++++++ yt/frontends/nc4_cm1/misc.py | 0 8 files changed, 407 insertions(+) create mode 100644 yt/frontends/nc4_cm1/__init__.py create mode 100644 yt/frontends/nc4_cm1/api.py create mode 100644 yt/frontends/nc4_cm1/data_structures.py create mode 100644 yt/frontends/nc4_cm1/definitions.py create mode 100644 yt/frontends/nc4_cm1/fields.py create mode 100644 yt/frontends/nc4_cm1/io.py create mode 100644 yt/frontends/nc4_cm1/misc.py diff --git a/yt/frontends/api.py b/yt/frontends/api.py index 982d664742a..ed86a8d369e 100644 --- a/yt/frontends/api.py +++ b/yt/frontends/api.py @@ -40,6 +40,7 @@ "swift", "tipsy", "ytdata", + "nc4_cm1", ] diff --git a/yt/frontends/nc4_cm1/__init__.py b/yt/frontends/nc4_cm1/__init__.py new file mode 100644 index 00000000000..94beb392d20 --- /dev/null +++ b/yt/frontends/nc4_cm1/__init__.py @@ -0,0 +1,14 @@ +""" +API for yt.frontends.nc4_cm1 + + + +""" + +#----------------------------------------------------------------------------- +# Copyright (c) 2013, yt Development Team. +# +# Distributed under the terms of the Modified BSD License. +# +# The full license is in the file COPYING.txt, distributed with this software. +#----------------------------------------------------------------------------- diff --git a/yt/frontends/nc4_cm1/api.py b/yt/frontends/nc4_cm1/api.py new file mode 100644 index 00000000000..1c15c6b9112 --- /dev/null +++ b/yt/frontends/nc4_cm1/api.py @@ -0,0 +1,25 @@ +""" +API for yt.frontends.nc4_cm1 + + + +""" + +#----------------------------------------------------------------------------- +# Copyright (c) 2013, yt Development Team. +# +# Distributed under the terms of the Modified BSD License. +# +# The full license is in the file COPYING.txt, distributed with this software. +#----------------------------------------------------------------------------- + +from .data_structures import \ + CM1Grid, \ + CM1Hierarchy, \ + CM1Dataset + +from .fields import \ + CM1FieldInfo + +from .io import \ + CM1IOHandler diff --git a/yt/frontends/nc4_cm1/data_structures.py b/yt/frontends/nc4_cm1/data_structures.py new file mode 100644 index 00000000000..9b91302de83 --- /dev/null +++ b/yt/frontends/nc4_cm1/data_structures.py @@ -0,0 +1,215 @@ +""" +Skeleton data structures + + + +""" + +#----------------------------------------------------------------------------- +# Copyright (c) 2013, yt Development Team. +# +# Distributed under the terms of the Modified BSD License. +# +# The full license is in the file COPYING.txt, distributed with this software. +#----------------------------------------------------------------------------- + +## Written by Kelton Halbert and Leigh Orf at the 2019 yt developers workshop @ NCSA +## for the purpose of reading in George Bryan's Cloud Model 1 output for plotting in yt. + +import os +import stat +import numpy as np +import weakref +import xarray + +from yt.data_objects.grid_patch import \ + AMRGridPatch +from yt.geometry.grid_geometry_handler import \ + GridIndex +from yt.data_objects.static_output import \ + Dataset +from .fields import CM1FieldInfo +from yt.geometry.coordinates.cartesian_coordinates import CartesianCoordinateHandler + + +class CM1Grid(AMRGridPatch): + _id_offset = 0 + + def __init__(self, id, index, level, dimensions): + super(CM1Grid, self).__init__( + id, filename=index.index_filename, index=index) + self.Parent = None + self.Children = [] + self.Level = level + self.ActiveDimensions = dimensions + + def __repr__(self): + return "CM1Grid_%04i (%s)" % (self.id, self.ActiveDimensions) + + +class CM1Hierarchy(GridIndex): + grid = CM1Grid + + def __init__(self, ds, dataset_type='cm1'): + self.dataset_type = dataset_type + self.dataset = weakref.proxy(ds) + # for now, the index file is the dataset! + self.index_filename = self.dataset.parameter_filename + self.directory = os.path.dirname(self.index_filename) + # float type for the simulation edges and must be float64 now + self.float_type = np.float64 + super(CM1Hierarchy, self).__init__(ds, dataset_type) + + def _initialize_state_variables(self): + super(CM1Hierarchy, self)._initialize_state_variables() + self.num_grids = 1 + + def _detect_output_fields(self): + # This needs to set a self.field_list that contains all the available, + # on-disk fields. No derived fields should be defined here. + # NOTE: Each should be a tuple, where the first element is the on-disk + # fluid type or particle type. Convention suggests that the on-disk + # fluid type is usually the dataset_type and the on-disk particle type + # (for a single population of particles) is "io". + self.field_list = [] + + ## loop over the variable names in the netCDF file + for key in self.ds._handle.variables.keys(): + if all(x in self.ds._handle[key].dims for x in ['time', 'zh', 'yh', 'xh']) is True: + field_tup = ('cm1', key) + self.field_list.append(field_tup) + + def _count_grids(self): + # This needs to set self.num_grids + self.num_grids = 1 + + def _parse_index(self): + # This needs to fill the following arrays, where N is self.num_grids: + # self.grid_left_edge (N, 3) <= float64 + # self.grid_right_edge (N, 3) <= float64 + # self.grid_dimensions (N, 3) <= int + # self.grid_particle_count (N, 1) <= int + # self.grid_levels (N, 1) <= int + # self.grids (N, 1) <= grid objects + # self.max_level = self.grid_levels.max() + self.grid_left_edge[0][:] = self.ds.domain_left_edge[:] + self.grid_right_edge[0][:] = self.ds.domain_right_edge[:] + self.grid_dimensions[0][:] = self.ds.domain_dimensions[:] + self.grid_particle_count[0][0] = 0 + self.grid_levels[0][0] = 1 + self.max_level = 1 + + + def _populate_grid_objects(self): + # For each grid g, this must call: + # g._prepare_grid() + # g._setup_dx() + # This must also set: + # g.Children <= list of child grids + # g.Parent <= parent grid + # This is handled by the frontend because often the children must be + # identified. + self.grids = np.empty(self.num_grids, dtype='object') + for i in range(self.num_grids): + g = self.grid(i, self, self.grid_levels.flat[i], self.grid_dimensions[i]) + g._prepare_grid() + g._setup_dx() + self.grids[i] = g + + +class CM1Dataset(Dataset): + _index_class = CM1Hierarchy + _field_info_class = CM1FieldInfo + + def __init__(self, filename, dataset_type='cm1', + storage_filename=None, + units_override=None): + self.fluid_types += ('cm1',) + self._handle = xarray.open_mfdataset(filename) + # refinement factor between a grid and its subgrid + self.refine_by = 2 + super(CM1Dataset, self).__init__(filename, dataset_type, + units_override=units_override) + self.storage_filename = storage_filename + + def _set_code_unit_attributes(self): + # This is where quantities are created that represent the various + # on-disk units. These are the currently available quantities which + # should be set, along with examples of how to set them to standard + # values. + # + # self.length_unit = self.quan(1.0, "cm") + # self.mass_unit = self.quan(1.0, "g") + # self.time_unit = self.quan(1.0, "s") + # self.time_unit = self.quan(1.0, "s") + # + # These can also be set: + # self.velocity_unit = self.quan(1.0, "cm/s") + # self.magnetic_unit = self.quan(1.0, "gauss") + length_unit = self._handle.variables['xh'].attrs['units'] + self.length_unit = self.quan(1.0, length_unit) + self.mass_unit = self.quan(1.0, "kg") + self.time_unit = self.quan(1.0, "s") + self.velocity_unit = self.quan(1.0, "m/s") + + def _parse_parameter_file(self): + # This needs to set up the following items. Note that these are all + # assumed to be in code units; domain_left_edge and domain_right_edge + # will be converted to YTArray automatically at a later time. + # This includes the cosmological parameters. + # + # self.unique_identifier <= unique identifier for the dataset + # being read (e.g., UUID or ST_CTIME) + self.unique_identifier = int(os.stat(self.parameter_filename)[stat.ST_CTIME]) + # self.parameters <= full of code-specific items of use + self.parameters = {} + coords = self._handle.coords + # TO DO: Possibly figure out a way to generalize this to be coordiante variable name + # agnostic in order to make useful for WRF or climate data. For now, we're hard coding + # for CM1 specifically and have named the classes appropriately, but generalizing is good. + xh, yh, zh = [coords[i] for i in ["xh", "yh", "zh"]] + # self.domain_left_edge <= array of float64 + self.domain_left_edge = np.array([xh.min(), yh.min(), zh.min()], dtype='float64') + # self.domain_right_edge <= array of float64 + self.domain_right_edge = np.array([xh.max(), yh.max(), zh.max()], dtype='float64') + # self.dimensionality <= int + self.dimensionality = 3 + # self.domain_dimensions <= array of int64 + dims = [self._handle.dims[i] for i in ["xh", "yh", "zh"]] + self.domain_dimensions = np.array(dims, dtype='int64') + # self.periodicity <= three-element tuple of booleans + self.periodicity = (False, False, False) + # self.current_time <= simulation time in code units + self.current_time = self._handle.time.values + # We also set up cosmological information. Set these to zero if + # non-cosmological. + self.cosmological_simulation = 0.0 + self.current_redshift = 0.0 + self.omega_lambda = 0.0 + self.omega_matter = 0.0 + self.hubble_constant = 0.0 + + + @classmethod + def _is_valid(self, *args, **kwargs): + # This accepts a filename or a set of arguments and returns True or + # False depending on if the file is of the type requested. + try: + import xarray + except ImportError: + return False + + try: + ds = xarray.open_dataset(args[0]) + except (FileNotFoundError, OSError): + return False + + try: + variables = ds.variables.keys() + except KeyError: + return False + + if 'xh' in variables: + return True + + return False diff --git a/yt/frontends/nc4_cm1/definitions.py b/yt/frontends/nc4_cm1/definitions.py new file mode 100644 index 00000000000..94175d4544d --- /dev/null +++ b/yt/frontends/nc4_cm1/definitions.py @@ -0,0 +1 @@ +# This file is often empty. It can hold definitions related to a frontend. diff --git a/yt/frontends/nc4_cm1/fields.py b/yt/frontends/nc4_cm1/fields.py new file mode 100644 index 00000000000..a099fcb5636 --- /dev/null +++ b/yt/frontends/nc4_cm1/fields.py @@ -0,0 +1,79 @@ +""" +Skeleton-specific fields + + + +""" + +#----------------------------------------------------------------------------- +# Copyright (c) 2013, yt Development Team. +# +# Distributed under the terms of the Modified BSD License. +# +# The full license is in the file COPYING.txt, distributed with this software. +#----------------------------------------------------------------------------- + +from yt.fields.field_info_container import \ + FieldInfoContainer + +# We need to specify which fields we might have in our dataset. The field info +# container subclass here will define which fields it knows about. There are +# optionally methods on it that get called which can be subclassed. + +class CM1FieldInfo(FieldInfoContainer): + known_other_fields = ( + # Each entry here is of the form + # ( "name", ("units", ["fields", "to", "alias"], # "display_name")), + ("uinterp", ("m/s", ["velocity_x"], None)), + ("vinterp", ("m/s", ["velocity_y"], None)), + ("winterp", ("m/s", ["velocity_z"], None)), + ("hwin_sr", ("m/s", ["storm_relative_horizontal_wind_speed"], None)), + ("windmag_sr", ("m/s", ["storm_relative_3D_wind_speed"], None)), + ("hwin_gr", ("m/s", ["ground_relative_horizontal_wind_speed"], None)), + ("thpert", ("K", ["potential_temperature_perturbation"], None)), + ("thrhopert", ("K", ["density_potential_temperature_perturbation"], None)), + ("prespert", ("hPa", ["presure_perturbation"], None)), + ("rhopert", ("kg/m**3", ["density_perturbation"], None)), + ("dbz", ("dBZ", ["simulated_reflectivity"], None)), + ("qvpert", ("g/kg", ["water_vapor_mixing_ratio_perturbation"], None)), + ("qc", ("g/kg", ["cloud_liquid_water_mixing_ratio"], None)), + ("qr", ("g/kg", ["rain_mixing_ratio"], None)), + ("qi", ("g/kg", ["cloud_ice_mixing_ratio"], None)), + ("qs", ("g/kg", ["snow_mixing_ratio"], None)), + ("qg", ("g/kg", ["graupel_or_hail_mixing_ratio"], None)), + ("qcloud", ("g/kg", ["sum_of_cloud_water_and_cloud_ice_mixing_ratios"], None)), + ("qprecip", ("g/kg", ["sum_of_rain_graupel_snow_mixing_ratios"], None)), + ("nci", ("1/cm**3", ["number_concerntration_of_cloud_ice"], None)), + ("ncr", ("1/cm**3", ["number_concentration_of_rain"], None)), + ("ncs", ("1/cm**3", ["number_concentration_of_snow"], None)), + ("ncg", ("1/cm**3", ["number_concentration_of_graupel_or_hail"], None)), + ("xvort", ("1/s", ["vorticity_x"], None)), + ("yvort", ("1/s", ["vorticity_y"], None)), + ("zvort", ("1/s", ["vorticity_z"], None)), + ("hvort", ("1/s", ["horizontal_vorticity_magnitude"], None)), + ("vortmag", ("1/s", ["vorticity_magnitude"], None)), + ("streamvort", ("1/s",["streamwise_vorticity"], None)), + ("khh", ("m**2/s", ["khh"], None)), + ("khv", ("m**2/s", ["khv"], None)), + ("kmh", ("m**2/s", ["kmh"], None)), + ("kmv", ("m**2/s", ["kmv"], None)) + ) + + known_particle_fields = ( + # Identical form to above + # ( "name", ("units", ["fields", "to", "alias"], # "display_name")), + ) + + def __init__(self, ds, field_list): + super(CM1FieldInfo, self).__init__(ds, field_list) + # If you want, you can check self.field_list + + def setup_fluid_fields(self): + # Here we do anything that might need info about the dataset. + # You can use self.alias, self.add_output_field (for on-disk fields) + # and self.add_field (for derived fields). + pass + + def setup_particle_fields(self, ptype): + super(CM1FieldInfo, self).setup_particle_fields(ptype) + # This will get called for every particle type. diff --git a/yt/frontends/nc4_cm1/io.py b/yt/frontends/nc4_cm1/io.py new file mode 100644 index 00000000000..abfaa2bf2ac --- /dev/null +++ b/yt/frontends/nc4_cm1/io.py @@ -0,0 +1,72 @@ +""" +Skeleton-specific IO functions + + + +""" + +#----------------------------------------------------------------------------- +# Copyright (c) 2013, yt Development Team. +# +# Distributed under the terms of the Modified BSD License. +# +# The full license is in the file COPYING.txt, distributed with this software. +#----------------------------------------------------------------------------- + +from yt.utilities.io_handler import \ + BaseIOHandler +import numpy as np + + + +class CM1IOHandler(BaseIOHandler): + _particle_reader = False + _dataset_type = 'cm1' + + def _read_particle_coords(self, chunks, ptf): + # This needs to *yield* a series of tuples of (ptype, (x, y, z)). + # chunks is a list of chunks, and ptf is a dict where the keys are + # ptypes and the values are lists of fields. + pass + + def _read_particle_fields(self, chunks, ptf, selector): + # This gets called after the arrays have been allocated. It needs to + # yield ((ptype, field), data) where data is the masked results of + # reading ptype, field and applying the selector to the data read in. + # Selector objects have a .select_points(x,y,z) that returns a mask, so + # you need to do your masking here. + pass + + def _read_fluid_selection(self, chunks, selector, fields, size): + # This needs to allocate a set of arrays inside a dictionary, where the + # keys are the (ftype, fname) tuples and the values are arrays that + # have been masked using whatever selector method is appropriate. The + # dict gets returned at the end and it should be flat, with selected + # data. Note that if you're reading grid data, you might need to + # special-case a grid selector object. + # Also note that "chunks" is a generator for multiple chunks, each of + # which contains a list of grids. The returned numpy arrays should be + # in 64-bit float and contiguous along the z direction. Therefore, for + # a C-like input array with the dimension [x][y][z] or a + # Fortran-like input array with the dimension (z,y,x), a matrix + # transpose is required (e.g., using np_array.transpose() or + # np_array.swapaxes(0,2)). + + data = {} + offset = 0 + for field in fields: + data[field] = np.empty(size, dtype='float64') + for chunk in chunks: + for grid in chunk.objs: + ds = self.ds._handle + variable = ds.variables[field[1]] + values = np.squeeze(variable.values[0].T) + offset += grid.select(selector, values, data[field], offset) + return data + + def _read_chunk_data(self, chunk, fields): + # This reads the data from a single chunk without doing any selection, + # and is only used for caching data that might be used by multiple + # different selectors later. For instance, this can speed up ghost zone + # computation. + pass diff --git a/yt/frontends/nc4_cm1/misc.py b/yt/frontends/nc4_cm1/misc.py new file mode 100644 index 00000000000..e69de29bb2d From 58126a4e7c2fb4bfda8185741155c384d15de2ed Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Mon, 21 Sep 2020 14:30:31 -0500 Subject: [PATCH 02/50] fix grid_geometry_handler import, style fixes --- yt/frontends/nc4_cm1/data_structures.py | 70 +++++++++++++------------ 1 file changed, 37 insertions(+), 33 deletions(-) diff --git a/yt/frontends/nc4_cm1/data_structures.py b/yt/frontends/nc4_cm1/data_structures.py index 9b91302de83..b6c33546e32 100644 --- a/yt/frontends/nc4_cm1/data_structures.py +++ b/yt/frontends/nc4_cm1/data_structures.py @@ -5,39 +5,36 @@ """ -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- # Copyright (c) 2013, yt Development Team. # # Distributed under the terms of the Modified BSD License. # # The full license is in the file COPYING.txt, distributed with this software. -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- ## Written by Kelton Halbert and Leigh Orf at the 2019 yt developers workshop @ NCSA ## for the purpose of reading in George Bryan's Cloud Model 1 output for plotting in yt. import os import stat -import numpy as np import weakref + +import numpy as np import xarray -from yt.data_objects.grid_patch import \ - AMRGridPatch -from yt.geometry.grid_geometry_handler import \ - GridIndex -from yt.data_objects.static_output import \ - Dataset +from yt.data_objects.index_subobjects.grid_patch import AMRGridPatch +from yt.data_objects.static_output import Dataset +from yt.geometry.grid_geometry_handler import GridIndex + from .fields import CM1FieldInfo -from yt.geometry.coordinates.cartesian_coordinates import CartesianCoordinateHandler class CM1Grid(AMRGridPatch): _id_offset = 0 def __init__(self, id, index, level, dimensions): - super(CM1Grid, self).__init__( - id, filename=index.index_filename, index=index) + super(CM1Grid, self).__init__(id, filename=index.index_filename, index=index) self.Parent = None self.Children = [] self.Level = level @@ -50,7 +47,7 @@ def __repr__(self): class CM1Hierarchy(GridIndex): grid = CM1Grid - def __init__(self, ds, dataset_type='cm1'): + def __init__(self, ds, dataset_type="cm1"): self.dataset_type = dataset_type self.dataset = weakref.proxy(ds) # for now, the index file is the dataset! @@ -72,11 +69,14 @@ def _detect_output_fields(self): # fluid type is usually the dataset_type and the on-disk particle type # (for a single population of particles) is "io". self.field_list = [] - + ## loop over the variable names in the netCDF file for key in self.ds._handle.variables.keys(): - if all(x in self.ds._handle[key].dims for x in ['time', 'zh', 'yh', 'xh']) is True: - field_tup = ('cm1', key) + if ( + all(x in self.ds._handle[key].dims for x in ["time", "zh", "yh", "xh"]) + is True + ): + field_tup = ("cm1", key) self.field_list.append(field_tup) def _count_grids(self): @@ -99,7 +99,6 @@ def _parse_index(self): self.grid_levels[0][0] = 1 self.max_level = 1 - def _populate_grid_objects(self): # For each grid g, this must call: # g._prepare_grid() @@ -109,7 +108,7 @@ def _populate_grid_objects(self): # g.Parent <= parent grid # This is handled by the frontend because often the children must be # identified. - self.grids = np.empty(self.num_grids, dtype='object') + self.grids = np.empty(self.num_grids, dtype="object") for i in range(self.num_grids): g = self.grid(i, self, self.grid_levels.flat[i], self.grid_dimensions[i]) g._prepare_grid() @@ -121,15 +120,16 @@ class CM1Dataset(Dataset): _index_class = CM1Hierarchy _field_info_class = CM1FieldInfo - def __init__(self, filename, dataset_type='cm1', - storage_filename=None, - units_override=None): - self.fluid_types += ('cm1',) + def __init__( + self, filename, dataset_type="cm1", storage_filename=None, units_override=None + ): + self.fluid_types += ("cm1",) self._handle = xarray.open_mfdataset(filename) # refinement factor between a grid and its subgrid self.refine_by = 2 - super(CM1Dataset, self).__init__(filename, dataset_type, - units_override=units_override) + super(CM1Dataset, self).__init__( + filename, dataset_type, units_override=units_override + ) self.storage_filename = storage_filename def _set_code_unit_attributes(self): @@ -146,7 +146,7 @@ def _set_code_unit_attributes(self): # These can also be set: # self.velocity_unit = self.quan(1.0, "cm/s") # self.magnetic_unit = self.quan(1.0, "gauss") - length_unit = self._handle.variables['xh'].attrs['units'] + length_unit = self._handle.variables["xh"].attrs["units"] self.length_unit = self.quan(1.0, length_unit) self.mass_unit = self.quan(1.0, "kg") self.time_unit = self.quan(1.0, "s") @@ -164,19 +164,24 @@ def _parse_parameter_file(self): # self.parameters <= full of code-specific items of use self.parameters = {} coords = self._handle.coords - # TO DO: Possibly figure out a way to generalize this to be coordiante variable name - # agnostic in order to make useful for WRF or climate data. For now, we're hard coding - # for CM1 specifically and have named the classes appropriately, but generalizing is good. + # TO DO: Possibly figure out a way to generalize this to be coordiante variable + # name agnostic in order to make useful for WRF or climate data. For now, we're + # hard coding for CM1 specifically and have named the classes appropriately, but + # generalizing is good. xh, yh, zh = [coords[i] for i in ["xh", "yh", "zh"]] # self.domain_left_edge <= array of float64 - self.domain_left_edge = np.array([xh.min(), yh.min(), zh.min()], dtype='float64') + self.domain_left_edge = np.array( + [xh.min(), yh.min(), zh.min()], dtype="float64" + ) # self.domain_right_edge <= array of float64 - self.domain_right_edge = np.array([xh.max(), yh.max(), zh.max()], dtype='float64') + self.domain_right_edge = np.array( + [xh.max(), yh.max(), zh.max()], dtype="float64" + ) # self.dimensionality <= int self.dimensionality = 3 # self.domain_dimensions <= array of int64 dims = [self._handle.dims[i] for i in ["xh", "yh", "zh"]] - self.domain_dimensions = np.array(dims, dtype='int64') + self.domain_dimensions = np.array(dims, dtype="int64") # self.periodicity <= three-element tuple of booleans self.periodicity = (False, False, False) # self.current_time <= simulation time in code units @@ -189,7 +194,6 @@ def _parse_parameter_file(self): self.omega_matter = 0.0 self.hubble_constant = 0.0 - @classmethod def _is_valid(self, *args, **kwargs): # This accepts a filename or a set of arguments and returns True or @@ -209,7 +213,7 @@ def _is_valid(self, *args, **kwargs): except KeyError: return False - if 'xh' in variables: + if "xh" in variables: return True return False From 922a4e60b4fa75d65d219737b0e660ca5e7dc36a Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Mon, 21 Sep 2020 14:56:08 -0500 Subject: [PATCH 03/50] more style fixes --- yt/frontends/nc4_cm1/__init__.py | 4 +- yt/frontends/nc4_cm1/api.py | 17 ++---- yt/frontends/nc4_cm1/data_structures.py | 2 +- yt/frontends/nc4_cm1/fields.py | 74 +++++++++++-------------- yt/frontends/nc4_cm1/io.py | 17 ++---- 5 files changed, 45 insertions(+), 69 deletions(-) diff --git a/yt/frontends/nc4_cm1/__init__.py b/yt/frontends/nc4_cm1/__init__.py index 94beb392d20..9f6ddfa37b3 100644 --- a/yt/frontends/nc4_cm1/__init__.py +++ b/yt/frontends/nc4_cm1/__init__.py @@ -5,10 +5,10 @@ """ -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- # Copyright (c) 2013, yt Development Team. # # Distributed under the terms of the Modified BSD License. # # The full license is in the file COPYING.txt, distributed with this software. -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- diff --git a/yt/frontends/nc4_cm1/api.py b/yt/frontends/nc4_cm1/api.py index 1c15c6b9112..388d77508e3 100644 --- a/yt/frontends/nc4_cm1/api.py +++ b/yt/frontends/nc4_cm1/api.py @@ -5,21 +5,14 @@ """ -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- # Copyright (c) 2013, yt Development Team. # # Distributed under the terms of the Modified BSD License. # # The full license is in the file COPYING.txt, distributed with this software. -#----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- -from .data_structures import \ - CM1Grid, \ - CM1Hierarchy, \ - CM1Dataset - -from .fields import \ - CM1FieldInfo - -from .io import \ - CM1IOHandler +from .data_structures import CM1Dataset, CM1Grid, CM1Hierarchy +from .fields import CM1FieldInfo +from .io import CM1IOHandler diff --git a/yt/frontends/nc4_cm1/data_structures.py b/yt/frontends/nc4_cm1/data_structures.py index b6c33546e32..aa301e54aa7 100644 --- a/yt/frontends/nc4_cm1/data_structures.py +++ b/yt/frontends/nc4_cm1/data_structures.py @@ -205,7 +205,7 @@ def _is_valid(self, *args, **kwargs): try: ds = xarray.open_dataset(args[0]) - except (FileNotFoundError, OSError): + except OSError: return False try: diff --git a/yt/frontends/nc4_cm1/fields.py b/yt/frontends/nc4_cm1/fields.py index a099fcb5636..5cba2b7282c 100644 --- a/yt/frontends/nc4_cm1/fields.py +++ b/yt/frontends/nc4_cm1/fields.py @@ -5,58 +5,50 @@ """ -#----------------------------------------------------------------------------- -# Copyright (c) 2013, yt Development Team. -# -# Distributed under the terms of the Modified BSD License. -# -# The full license is in the file COPYING.txt, distributed with this software. -#----------------------------------------------------------------------------- - -from yt.fields.field_info_container import \ - FieldInfoContainer +from yt.fields.field_info_container import FieldInfoContainer # We need to specify which fields we might have in our dataset. The field info # container subclass here will define which fields it knows about. There are # optionally methods on it that get called which can be subclassed. + class CM1FieldInfo(FieldInfoContainer): known_other_fields = ( # Each entry here is of the form # ( "name", ("units", ["fields", "to", "alias"], # "display_name")), - ("uinterp", ("m/s", ["velocity_x"], None)), - ("vinterp", ("m/s", ["velocity_y"], None)), - ("winterp", ("m/s", ["velocity_z"], None)), - ("hwin_sr", ("m/s", ["storm_relative_horizontal_wind_speed"], None)), + ("uinterp", ("m/s", ["velocity_x"], None)), + ("vinterp", ("m/s", ["velocity_y"], None)), + ("winterp", ("m/s", ["velocity_z"], None)), + ("hwin_sr", ("m/s", ["storm_relative_horizontal_wind_speed"], None)), ("windmag_sr", ("m/s", ["storm_relative_3D_wind_speed"], None)), - ("hwin_gr", ("m/s", ["ground_relative_horizontal_wind_speed"], None)), - ("thpert", ("K", ["potential_temperature_perturbation"], None)), - ("thrhopert", ("K", ["density_potential_temperature_perturbation"], None)), - ("prespert", ("hPa", ["presure_perturbation"], None)), + ("hwin_gr", ("m/s", ["ground_relative_horizontal_wind_speed"], None)), + ("thpert", ("K", ["potential_temperature_perturbation"], None)), + ("thrhopert", ("K", ["density_potential_temperature_perturbation"], None)), + ("prespert", ("hPa", ["presure_perturbation"], None)), ("rhopert", ("kg/m**3", ["density_perturbation"], None)), - ("dbz", ("dBZ", ["simulated_reflectivity"], None)), - ("qvpert", ("g/kg", ["water_vapor_mixing_ratio_perturbation"], None)), - ("qc", ("g/kg", ["cloud_liquid_water_mixing_ratio"], None)), - ("qr", ("g/kg", ["rain_mixing_ratio"], None)), - ("qi", ("g/kg", ["cloud_ice_mixing_ratio"], None)), - ("qs", ("g/kg", ["snow_mixing_ratio"], None)), - ("qg", ("g/kg", ["graupel_or_hail_mixing_ratio"], None)), - ("qcloud", ("g/kg", ["sum_of_cloud_water_and_cloud_ice_mixing_ratios"], None)), - ("qprecip", ("g/kg", ["sum_of_rain_graupel_snow_mixing_ratios"], None)), - ("nci", ("1/cm**3", ["number_concerntration_of_cloud_ice"], None)), - ("ncr", ("1/cm**3", ["number_concentration_of_rain"], None)), - ("ncs", ("1/cm**3", ["number_concentration_of_snow"], None)), - ("ncg", ("1/cm**3", ["number_concentration_of_graupel_or_hail"], None)), - ("xvort", ("1/s", ["vorticity_x"], None)), - ("yvort", ("1/s", ["vorticity_y"], None)), - ("zvort", ("1/s", ["vorticity_z"], None)), - ("hvort", ("1/s", ["horizontal_vorticity_magnitude"], None)), - ("vortmag", ("1/s", ["vorticity_magnitude"], None)), - ("streamvort", ("1/s",["streamwise_vorticity"], None)), - ("khh", ("m**2/s", ["khh"], None)), - ("khv", ("m**2/s", ["khv"], None)), - ("kmh", ("m**2/s", ["kmh"], None)), - ("kmv", ("m**2/s", ["kmv"], None)) + ("dbz", ("dBZ", ["simulated_reflectivity"], None)), + ("qvpert", ("g/kg", ["water_vapor_mixing_ratio_perturbation"], None)), + ("qc", ("g/kg", ["cloud_liquid_water_mixing_ratio"], None)), + ("qr", ("g/kg", ["rain_mixing_ratio"], None)), + ("qi", ("g/kg", ["cloud_ice_mixing_ratio"], None)), + ("qs", ("g/kg", ["snow_mixing_ratio"], None)), + ("qg", ("g/kg", ["graupel_or_hail_mixing_ratio"], None)), + ("qcloud", ("g/kg", ["sum_of_cloud_water_and_cloud_ice_mixing_ratios"], None)), + ("qprecip", ("g/kg", ["sum_of_rain_graupel_snow_mixing_ratios"], None)), + ("nci", ("1/cm**3", ["number_concerntration_of_cloud_ice"], None)), + ("ncr", ("1/cm**3", ["number_concentration_of_rain"], None)), + ("ncs", ("1/cm**3", ["number_concentration_of_snow"], None)), + ("ncg", ("1/cm**3", ["number_concentration_of_graupel_or_hail"], None)), + ("xvort", ("1/s", ["vorticity_x"], None)), + ("yvort", ("1/s", ["vorticity_y"], None)), + ("zvort", ("1/s", ["vorticity_z"], None)), + ("hvort", ("1/s", ["horizontal_vorticity_magnitude"], None)), + ("vortmag", ("1/s", ["vorticity_magnitude"], None)), + ("streamvort", ("1/s", ["streamwise_vorticity"], None)), + ("khh", ("m**2/s", ["khh"], None)), + ("khv", ("m**2/s", ["khv"], None)), + ("kmh", ("m**2/s", ["kmh"], None)), + ("kmv", ("m**2/s", ["kmv"], None)), ) known_particle_fields = ( diff --git a/yt/frontends/nc4_cm1/io.py b/yt/frontends/nc4_cm1/io.py index abfaa2bf2ac..5be73bc92c4 100644 --- a/yt/frontends/nc4_cm1/io.py +++ b/yt/frontends/nc4_cm1/io.py @@ -5,23 +5,14 @@ """ -#----------------------------------------------------------------------------- -# Copyright (c) 2013, yt Development Team. -# -# Distributed under the terms of the Modified BSD License. -# -# The full license is in the file COPYING.txt, distributed with this software. -#----------------------------------------------------------------------------- - -from yt.utilities.io_handler import \ - BaseIOHandler import numpy as np +from yt.utilities.io_handler import BaseIOHandler class CM1IOHandler(BaseIOHandler): _particle_reader = False - _dataset_type = 'cm1' + _dataset_type = "cm1" def _read_particle_coords(self, chunks, ptf): # This needs to *yield* a series of tuples of (ptype, (x, y, z)). @@ -55,7 +46,7 @@ def _read_fluid_selection(self, chunks, selector, fields, size): data = {} offset = 0 for field in fields: - data[field] = np.empty(size, dtype='float64') + data[field] = np.empty(size, dtype="float64") for chunk in chunks: for grid in chunk.objs: ds = self.ds._handle @@ -63,7 +54,7 @@ def _read_fluid_selection(self, chunks, selector, fields, size): values = np.squeeze(variable.values[0].T) offset += grid.select(selector, values, data[field], offset) return data - + def _read_chunk_data(self, chunk, fields): # This reads the data from a single chunk without doing any selection, # and is only used for caching data that might be used by multiple From de1b58a394f7cf4a5422a793dffaa38a045d0acf Mon Sep 17 00:00:00 2001 From: keltonhalbert Date: Mon, 21 Sep 2020 20:07:31 +0000 Subject: [PATCH 04/50] fixed the units for simulated reflectivity --- yt/frontends/nc4_cm1/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/frontends/nc4_cm1/fields.py b/yt/frontends/nc4_cm1/fields.py index a099fcb5636..7612b67813f 100644 --- a/yt/frontends/nc4_cm1/fields.py +++ b/yt/frontends/nc4_cm1/fields.py @@ -34,7 +34,7 @@ class CM1FieldInfo(FieldInfoContainer): ("thrhopert", ("K", ["density_potential_temperature_perturbation"], None)), ("prespert", ("hPa", ["presure_perturbation"], None)), ("rhopert", ("kg/m**3", ["density_perturbation"], None)), - ("dbz", ("dBZ", ["simulated_reflectivity"], None)), + ("dbz", ("dB", ["simulated_reflectivity"], None)), ("qvpert", ("g/kg", ["water_vapor_mixing_ratio_perturbation"], None)), ("qc", ("g/kg", ["cloud_liquid_water_mixing_ratio"], None)), ("qr", ("g/kg", ["rain_mixing_ratio"], None)), From 30e6253372655911bfbf67d0c0019ba0d4c40b1d Mon Sep 17 00:00:00 2001 From: keltonhalbert Date: Mon, 21 Sep 2020 20:24:16 +0000 Subject: [PATCH 05/50] empty commit for tests --- yt/frontends/nc4_cm1/tests/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 yt/frontends/nc4_cm1/tests/__init__.py diff --git a/yt/frontends/nc4_cm1/tests/__init__.py b/yt/frontends/nc4_cm1/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From 5847686041acfd7b188a2b1725a8fe71c35aedd4 Mon Sep 17 00:00:00 2001 From: keltonhalbert Date: Mon, 21 Sep 2020 20:24:49 +0000 Subject: [PATCH 06/50] empty commit for tests --- yt/frontends/nc4_cm1/tests/test_outputs.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 yt/frontends/nc4_cm1/tests/test_outputs.py diff --git a/yt/frontends/nc4_cm1/tests/test_outputs.py b/yt/frontends/nc4_cm1/tests/test_outputs.py new file mode 100644 index 00000000000..e69de29bb2d From c9d48ddb9baef4895465396a27724694f2cec640 Mon Sep 17 00:00:00 2001 From: Madicken Munk Date: Mon, 21 Sep 2020 15:28:39 -0500 Subject: [PATCH 07/50] remove copyright from nc4 frontend files --- yt/frontends/nc4_cm1/__init__.py | 7 ------- yt/frontends/nc4_cm1/api.py | 8 -------- yt/frontends/nc4_cm1/data_structures.py | 8 -------- 3 files changed, 23 deletions(-) diff --git a/yt/frontends/nc4_cm1/__init__.py b/yt/frontends/nc4_cm1/__init__.py index 9f6ddfa37b3..fcd921acc9d 100644 --- a/yt/frontends/nc4_cm1/__init__.py +++ b/yt/frontends/nc4_cm1/__init__.py @@ -5,10 +5,3 @@ """ -# ----------------------------------------------------------------------------- -# Copyright (c) 2013, yt Development Team. -# -# Distributed under the terms of the Modified BSD License. -# -# The full license is in the file COPYING.txt, distributed with this software. -# ----------------------------------------------------------------------------- diff --git a/yt/frontends/nc4_cm1/api.py b/yt/frontends/nc4_cm1/api.py index 388d77508e3..f7acb342075 100644 --- a/yt/frontends/nc4_cm1/api.py +++ b/yt/frontends/nc4_cm1/api.py @@ -5,14 +5,6 @@ """ -# ----------------------------------------------------------------------------- -# Copyright (c) 2013, yt Development Team. -# -# Distributed under the terms of the Modified BSD License. -# -# The full license is in the file COPYING.txt, distributed with this software. -# ----------------------------------------------------------------------------- - from .data_structures import CM1Dataset, CM1Grid, CM1Hierarchy from .fields import CM1FieldInfo from .io import CM1IOHandler diff --git a/yt/frontends/nc4_cm1/data_structures.py b/yt/frontends/nc4_cm1/data_structures.py index aa301e54aa7..1d9bafcefa6 100644 --- a/yt/frontends/nc4_cm1/data_structures.py +++ b/yt/frontends/nc4_cm1/data_structures.py @@ -5,14 +5,6 @@ """ -# ----------------------------------------------------------------------------- -# Copyright (c) 2013, yt Development Team. -# -# Distributed under the terms of the Modified BSD License. -# -# The full license is in the file COPYING.txt, distributed with this software. -# ----------------------------------------------------------------------------- - ## Written by Kelton Halbert and Leigh Orf at the 2019 yt developers workshop @ NCSA ## for the purpose of reading in George Bryan's Cloud Model 1 output for plotting in yt. From 66b4e03c4c93dc7d6a242eea1c83c4e1133ae18b Mon Sep 17 00:00:00 2001 From: keltonhalbert Date: Mon, 21 Sep 2020 20:38:26 +0000 Subject: [PATCH 08/50] added the staggered velocity variables --- yt/frontends/nc4_cm1/fields.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/yt/frontends/nc4_cm1/fields.py b/yt/frontends/nc4_cm1/fields.py index af0ef898233..337192a440d 100644 --- a/yt/frontends/nc4_cm1/fields.py +++ b/yt/frontends/nc4_cm1/fields.py @@ -19,6 +19,9 @@ class CM1FieldInfo(FieldInfoContainer): ("uinterp", ("m/s", ["velocity_x"], None)), ("vinterp", ("m/s", ["velocity_y"], None)), ("winterp", ("m/s", ["velocity_z"], None)), + ("u", ("m/s", ["velocity_x"], None)), + ("v", ("m/s", ["velocity_y"], None)), + ("w", ("m/s", ["velocity_z"], None)), ("hwin_sr", ("m/s", ["storm_relative_horizontal_wind_speed"], None)), ("windmag_sr", ("m/s", ["storm_relative_3D_wind_speed"], None)), ("hwin_gr", ("m/s", ["ground_relative_horizontal_wind_speed"], None)), From 42b6abf8c53892edd8990042f953ae0b224c9e8d Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Mon, 21 Sep 2020 15:43:34 -0500 Subject: [PATCH 09/50] moved xarray to on_demand_imports --- yt/frontends/nc4_cm1/data_structures.py | 8 ++------ yt/utilities/on_demand_imports.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/yt/frontends/nc4_cm1/data_structures.py b/yt/frontends/nc4_cm1/data_structures.py index 1d9bafcefa6..bfd4533f496 100644 --- a/yt/frontends/nc4_cm1/data_structures.py +++ b/yt/frontends/nc4_cm1/data_structures.py @@ -13,11 +13,11 @@ import weakref import numpy as np -import xarray from yt.data_objects.index_subobjects.grid_patch import AMRGridPatch from yt.data_objects.static_output import Dataset from yt.geometry.grid_geometry_handler import GridIndex +from yt.utilities.on_demand_imports import _xarray as xarray from .fields import CM1FieldInfo @@ -116,7 +116,7 @@ def __init__( self, filename, dataset_type="cm1", storage_filename=None, units_override=None ): self.fluid_types += ("cm1",) - self._handle = xarray.open_mfdataset(filename) + self._handle = xarray.open_dataset(filename) # refinement factor between a grid and its subgrid self.refine_by = 2 super(CM1Dataset, self).__init__( @@ -190,10 +190,6 @@ def _parse_parameter_file(self): def _is_valid(self, *args, **kwargs): # This accepts a filename or a set of arguments and returns True or # False depending on if the file is of the type requested. - try: - import xarray - except ImportError: - return False try: ds = xarray.open_dataset(args[0]) diff --git a/yt/utilities/on_demand_imports.py b/yt/utilities/on_demand_imports.py index 059b8be78a9..ce6cecaabf5 100644 --- a/yt/utilities/on_demand_imports.py +++ b/yt/utilities/on_demand_imports.py @@ -617,3 +617,21 @@ def __getattr__(self, attr): _f90nml = f90nml_imports() + + +class xarray_imports: + _name = "xarray" + _open_dataset = None + + @property + def open_dataset(self): + if self._open_dataset is None: + try: + from xarray import open_dataset + except ImportError: + open_dataset = NotAModule(self._name) + self._open_dataset = open_dataset + return self._open_dataset + + +_xarray = xarray_imports() From 15ac1bfba84bf7fd578ba29beb5daf57817676d7 Mon Sep 17 00:00:00 2001 From: keltonhalbert Date: Mon, 21 Sep 2020 20:52:24 +0000 Subject: [PATCH 10/50] updated the is_valid method to have a more meaningful check. This checks all of the variables and their coordinates and compares them to the global dataset coordinates to make sure all coordinates are present and accounted for --- yt/frontends/nc4_cm1/data_structures.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/yt/frontends/nc4_cm1/data_structures.py b/yt/frontends/nc4_cm1/data_structures.py index 1d9bafcefa6..8161ba5c844 100644 --- a/yt/frontends/nc4_cm1/data_structures.py +++ b/yt/frontends/nc4_cm1/data_structures.py @@ -116,7 +116,7 @@ def __init__( self, filename, dataset_type="cm1", storage_filename=None, units_override=None ): self.fluid_types += ("cm1",) - self._handle = xarray.open_mfdataset(filename) + self._handle = xarray.open_dataset(filename, engine="netcdf4") # refinement factor between a grid and its subgrid self.refine_by = 2 super(CM1Dataset, self).__init__( @@ -205,7 +205,16 @@ def _is_valid(self, *args, **kwargs): except KeyError: return False - if "xh" in variables: - return True - - return False + ## check each variable array in the dataset + ## and make sure that the coordinates in the + ## variable arrays matches the coordinates + ## of the dataset + coords = ds.coords ## get the dataset wide coordinates + nvars = len(ds.variables.keys()) ## number of variables in dataset + passed = 0 ## number of variables passing the tet + for var in ds.variables.keys(): ## iterate over the variables + vcoords = ds[var].coords ## get the coordinates for the variable + for vc in vcoords: ## iterate over the coordinates for the variable + if vc in coords: passed += 1 ## check that the coordinate exists in global dataset + if (passed == nvars): return True ## if all vars pass return True + else: reurn False From 96306e8ee913d1cac22ef82d4b57b13692fabe09 Mon Sep 17 00:00:00 2001 From: keltonhalbert Date: Mon, 21 Sep 2020 20:55:53 +0000 Subject: [PATCH 11/50] stupid typos... --- yt/frontends/nc4_cm1/data_structures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/frontends/nc4_cm1/data_structures.py b/yt/frontends/nc4_cm1/data_structures.py index c6609aea951..cb83ad3d4bd 100644 --- a/yt/frontends/nc4_cm1/data_structures.py +++ b/yt/frontends/nc4_cm1/data_structures.py @@ -213,4 +213,4 @@ def _is_valid(self, *args, **kwargs): for vc in vcoords: ## iterate over the coordinates for the variable if vc in coords: passed += 1 ## check that the coordinate exists in global dataset if (passed == nvars): return True ## if all vars pass return True - else: reurn False + else: return False From dc181b2b5669b7279054954dae2edad72274a6a6 Mon Sep 17 00:00:00 2001 From: keltonhalbert Date: Mon, 21 Sep 2020 21:05:42 +0000 Subject: [PATCH 12/50] tested and working implementation of _is_valid now properly checks coordinates for each variable --- yt/frontends/nc4_cm1/data_structures.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/yt/frontends/nc4_cm1/data_structures.py b/yt/frontends/nc4_cm1/data_structures.py index cb83ad3d4bd..8ec47e14d4c 100644 --- a/yt/frontends/nc4_cm1/data_structures.py +++ b/yt/frontends/nc4_cm1/data_structures.py @@ -207,10 +207,15 @@ def _is_valid(self, *args, **kwargs): ## of the dataset coords = ds.coords ## get the dataset wide coordinates nvars = len(ds.variables.keys()) ## number of variables in dataset - passed = 0 ## number of variables passing the tet + varspassed = 0 ## number of variables passing the tet for var in ds.variables.keys(): ## iterate over the variables vcoords = ds[var].coords ## get the coordinates for the variable + ncoords = len(vcoords) ## number of coordinates in variable + coordspassed = 0 ## number of coordinates that pass for a variable for vc in vcoords: ## iterate over the coordinates for the variable - if vc in coords: passed += 1 ## check that the coordinate exists in global dataset - if (passed == nvars): return True ## if all vars pass return True + if vc in coords: coordspassed += 1 ## variable coordinate and global coordinate are same + else: return False + if (coordspassed == ncoords): varspassed += 1 ## if all coordinates in a variable pass, the variable passes + else: return False + if (varspassed == nvars): return True ## if all vars pass return True else: return False From df98c1f41f4b2d862e3ce494adc9500cfbafa6d9 Mon Sep 17 00:00:00 2001 From: keltonhalbert Date: Mon, 21 Sep 2020 21:40:59 +0000 Subject: [PATCH 13/50] some dummy tests copied from the FLASH frontend --- yt/frontends/nc4_cm1/tests/test_outputs.py | 62 ++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/yt/frontends/nc4_cm1/tests/test_outputs.py b/yt/frontends/nc4_cm1/tests/test_outputs.py index e69de29bb2d..e2f16c47ba6 100644 --- a/yt/frontends/nc4_cm1/tests/test_outputs.py +++ b/yt/frontends/nc4_cm1/tests/test_outputs.py @@ -0,0 +1,62 @@ +from collections import OrderedDict + +import os +import stat +import weakref +import numpy as np + +from yt.utilities.on_demand_imports import _xarray as xarray +from yt.frontends.nc4_cm1.api import CM1Dataset +from yt.testing import ( + assert_equal, + requires_file, + units_override_check, +) +from yt.utilities.answer_testing.framework import ( + data_dir_load, + requires_ds, + small_patch_amr, +) + + +_fields = ("dbz", "thrhopert", "zvort") + +## To-Do - Need to know the actual location of test +## data for these tests! Probably need to give a small, +## static NetCDF file near the tornado. +cm1sim = "CM1Tornado/test_dataset.nc" + + +@requires_ds(cm1sim, big_data=True) +def test_tornado(): + ds = data_dir_load(sloshing) + assert_equal(str(ds), "test_dataset.nc") + for test in small_patch_amr(ds, _fields): + test_tprmadp.__name__ = test.description + yield test + + +@requires_file(cm1sim) +def test_CM1Dataset(): + assert isinstance(data_dir_load(cm1sim), CM1Dataset) + + +@requires_file(cm1sim) +def test_units_override(): + units_override_check(cm1sim) + + +@requires_file(cm1sim) +def test_tornado_dataset(): + ds = data_dir_load(cm1sim) + ## To-Do: Static tests for the specific + ## NetCDF file given above! + assert_equal(ds.parameters["time"], 751000000000.0) + assert_equal(ds.domain_dimensions, np.array([8, 8, 8])) + assert_equal(ds.domain_left_edge, ds.arr([-2e18, -2e18, -2e18], "code_length")) + + assert_equal(ds.index.num_grids, 73) + dd = ds.all_data() + dd["density"] + + From 11b8c9a7b44ed0ee0f30201b5ef0b8c839c09fcc Mon Sep 17 00:00:00 2001 From: Madicken Munk Date: Mon, 21 Sep 2020 16:48:46 -0500 Subject: [PATCH 14/50] add xarray as optional dependency --- appveyor.yml | 2 +- doc/install_script.sh | 1 + tests/test_install_script.py | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index e800f3d40df..284d1838fc3 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -31,7 +31,7 @@ install: # Install specified version of numpy and dependencies - "conda install --yes -c conda-forge numpy scipy nose pytest setuptools ipython git unyt Cython sympy fastcache h5py matplotlib=3.1.3 mock pandas cartopy conda-build pooch pyyaml - nose-timer pykdtree" + nose-timer pykdtree xarray" # install yt - "pip install -e ." diff --git a/doc/install_script.sh b/doc/install_script.sh index 62d165a3635..93cce57cf34 100644 --- a/doc/install_script.sh +++ b/doc/install_script.sh @@ -36,6 +36,7 @@ INST_CARTOPY=0 # Install cartopy? INST_NOSE=1 # Install nose? INST_NETCDF4=1 # Install netcdf4 and its python bindings? INST_POOCH=1 # Install pooch? +INST_XARRAY=0 # Install xarray? # This is the branch we will install from for INST_YT_SOURCE=1 BRANCH="master" diff --git a/tests/test_install_script.py b/tests/test_install_script.py index 1c9a3b75c22..cf9d5cae62a 100644 --- a/tests/test_install_script.py +++ b/tests/test_install_script.py @@ -28,6 +28,7 @@ "astropy", "cartopy", "pooch", + "xarray", ] # dependencies that are only installable when yt is built from source From e0bcc5d20d850a89ba7d30fce711210433c68d2a Mon Sep 17 00:00:00 2001 From: keltonhalbert Date: Mon, 21 Sep 2020 21:59:42 +0000 Subject: [PATCH 15/50] sorta maybe adding tests? --- yt/frontends/nc4_cm1/api.py | 1 + yt/frontends/nc4_cm1/tests/test_outputs.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/yt/frontends/nc4_cm1/api.py b/yt/frontends/nc4_cm1/api.py index f7acb342075..074cd098e7f 100644 --- a/yt/frontends/nc4_cm1/api.py +++ b/yt/frontends/nc4_cm1/api.py @@ -5,6 +5,7 @@ """ +from . import tests from .data_structures import CM1Dataset, CM1Grid, CM1Hierarchy from .fields import CM1FieldInfo from .io import CM1IOHandler diff --git a/yt/frontends/nc4_cm1/tests/test_outputs.py b/yt/frontends/nc4_cm1/tests/test_outputs.py index e2f16c47ba6..74e8e9edf54 100644 --- a/yt/frontends/nc4_cm1/tests/test_outputs.py +++ b/yt/frontends/nc4_cm1/tests/test_outputs.py @@ -29,7 +29,8 @@ @requires_ds(cm1sim, big_data=True) def test_tornado(): - ds = data_dir_load(sloshing) + ds = data_dir_load(cm1sim) + print(ds) assert_equal(str(ds), "test_dataset.nc") for test in small_patch_amr(ds, _fields): test_tprmadp.__name__ = test.description @@ -51,12 +52,11 @@ def test_tornado_dataset(): ds = data_dir_load(cm1sim) ## To-Do: Static tests for the specific ## NetCDF file given above! - assert_equal(ds.parameters["time"], 751000000000.0) - assert_equal(ds.domain_dimensions, np.array([8, 8, 8])) - assert_equal(ds.domain_left_edge, ds.arr([-2e18, -2e18, -2e18], "code_length")) - - assert_equal(ds.index.num_grids, 73) - dd = ds.all_data() - dd["density"] + #assert_equal(ds.parameters["time"], 751000000000.0) + #assert_equal(ds.domain_dimensions, np.array([8, 8, 8])) + #assert_equal(ds.domain_left_edge, ds.arr([-2e18, -2e18, -2e18], "code_length")) + #assert_equal(ds.index.num_grids, 73) + #dd = ds.all_data() + #dd["density"] From 306cf092253849c9cc4def187548c9dbf87905cc Mon Sep 17 00:00:00 2001 From: keltonhalbert Date: Mon, 21 Sep 2020 22:22:11 +0000 Subject: [PATCH 16/50] uncommitted changes to data structures --- yt/frontends/nc4_cm1/data_structures.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/yt/frontends/nc4_cm1/data_structures.py b/yt/frontends/nc4_cm1/data_structures.py index 8ec47e14d4c..b010b0ecb03 100644 --- a/yt/frontends/nc4_cm1/data_structures.py +++ b/yt/frontends/nc4_cm1/data_structures.py @@ -217,5 +217,8 @@ def _is_valid(self, *args, **kwargs): else: return False if (coordspassed == ncoords): varspassed += 1 ## if all coordinates in a variable pass, the variable passes else: return False + ## TO-DO - add specific CM1 check to make sure this + ## frontend doesn't step on the toes of anyone running + ## xarray in the future! if (varspassed == nvars): return True ## if all vars pass return True else: return False From 58149e1b1a7a299d9ef1d45b197c405bb6ff3cdd Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Tue, 22 Sep 2020 13:53:10 -0500 Subject: [PATCH 17/50] pulling in appveyor update --- appveyor.yml | 45 ++++++--------------------------------------- 1 file changed, 6 insertions(+), 39 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 284d1838fc3..98782fca2b2 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,48 +1,15 @@ -# AppVeyor.com is a Continuous Integration service to build and run tests under -# Windows - -environment: - - global: - PYTHON: "C:\\Miniconda36-x64" - - matrix: - - PYTHON_VERSION: "3.8" +# This is a dummy configuration file that *does not run any test*. +# The file should be deleted, and Appveyor deactivated as soon as all +# opened PRs have this configuration merged in. +# SEE PR#2908 (https://github.com/yt-project/yt/pull/2908/files) platform: - x64 install: - - "if not exist \"%userprofile%\\.config\\yt\" mkdir %userprofile%\\.config\\yt" - - "echo [yt] > %userprofile%\\.config\\yt\\ytrc" - - "echo suppressStreamLogging = True >> %userprofile%\\.config\\yt\\ytrc" - - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - - "copy tests\\matplotlibrc ." - - # Install the build and runtime dependencies of the project. - # Create a conda environment - - "conda update -q --yes conda" - - "conda create -q --yes -n test python=%PYTHON_VERSION%" - - "activate test" + - "echo 'Not installing anything.'" - # Check that we have the expected version of Python - - "python --version" - - # Install specified version of numpy and dependencies - - "conda install --yes -c conda-forge numpy scipy nose pytest setuptools ipython git unyt - Cython sympy fastcache h5py matplotlib=3.1.3 mock pandas cartopy conda-build pooch pyyaml - nose-timer pykdtree xarray" - # install yt - - "pip install -e ." - -# Not a .NET project build: false test_script: - - "nosetests --with-timer --timer-top-n=20 --nologcapture --with-xunit -sv --traverse-namespace yt" - -# Enable this to be able to login to the build worker. You can use the -# `remmina` program in Ubuntu, use the login information that the line below -# prints into the log. -#on_finish: -#- ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) + - "echo 'Not testing anything. This is done on Travis'" \ No newline at end of file From 557621b3fb1bd76b6db0259174d052b4947f60fe Mon Sep 17 00:00:00 2001 From: keltonhalbert Date: Tue, 22 Sep 2020 19:30:43 +0000 Subject: [PATCH 18/50] Updated some of the data_structures file to have new metadata info in the parameters dict. Removed some unecessary example code in the comments. In test_outputs, we now have some working frontend tests based on my local dataset. This will need to be modified once we decide on a single test file later and upload it to the data hub. May also need an image test at some point. --- yt/frontends/nc4_cm1/data_structures.py | 24 +++++---------- yt/frontends/nc4_cm1/tests/test_outputs.py | 36 +++++++++++++--------- 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/yt/frontends/nc4_cm1/data_structures.py b/yt/frontends/nc4_cm1/data_structures.py index b010b0ecb03..ff31e22e3ee 100644 --- a/yt/frontends/nc4_cm1/data_structures.py +++ b/yt/frontends/nc4_cm1/data_structures.py @@ -118,7 +118,7 @@ def __init__( self.fluid_types += ("cm1",) self._handle = xarray.open_dataset(filename, engine="netcdf4") # refinement factor between a grid and its subgrid - self.refine_by = 2 + self.refine_by = 1 super(CM1Dataset, self).__init__( filename, dataset_type, units_override=units_override ) @@ -129,20 +129,12 @@ def _set_code_unit_attributes(self): # on-disk units. These are the currently available quantities which # should be set, along with examples of how to set them to standard # values. - # - # self.length_unit = self.quan(1.0, "cm") - # self.mass_unit = self.quan(1.0, "g") - # self.time_unit = self.quan(1.0, "s") - # self.time_unit = self.quan(1.0, "s") - # - # These can also be set: - # self.velocity_unit = self.quan(1.0, "cm/s") - # self.magnetic_unit = self.quan(1.0, "gauss") length_unit = self._handle.variables["xh"].attrs["units"] self.length_unit = self.quan(1.0, length_unit) self.mass_unit = self.quan(1.0, "kg") self.time_unit = self.quan(1.0, "s") self.velocity_unit = self.quan(1.0, "m/s") + self.time_unit = self.quan(1.0, "s") def _parse_parameter_file(self): # This needs to set up the following items. Note that these are all @@ -156,28 +148,26 @@ def _parse_parameter_file(self): # self.parameters <= full of code-specific items of use self.parameters = {} coords = self._handle.coords + self.parameters["coords"] = coords + # TO DO: Possibly figure out a way to generalize this to be coordiante variable # name agnostic in order to make useful for WRF or climate data. For now, we're # hard coding for CM1 specifically and have named the classes appropriately, but # generalizing is good. xh, yh, zh = [coords[i] for i in ["xh", "yh", "zh"]] - # self.domain_left_edge <= array of float64 self.domain_left_edge = np.array( [xh.min(), yh.min(), zh.min()], dtype="float64" ) - # self.domain_right_edge <= array of float64 self.domain_right_edge = np.array( [xh.max(), yh.max(), zh.max()], dtype="float64" ) - # self.dimensionality <= int self.dimensionality = 3 - # self.domain_dimensions <= array of int64 dims = [self._handle.dims[i] for i in ["xh", "yh", "zh"]] self.domain_dimensions = np.array(dims, dtype="int64") - # self.periodicity <= three-element tuple of booleans self.periodicity = (False, False, False) - # self.current_time <= simulation time in code units - self.current_time = self._handle.time.values + self.current_time = self._handle.time.values[0] + self.parameters["time"] = self.current_time + # We also set up cosmological information. Set these to zero if # non-cosmological. self.cosmological_simulation = 0.0 diff --git a/yt/frontends/nc4_cm1/tests/test_outputs.py b/yt/frontends/nc4_cm1/tests/test_outputs.py index 74e8e9edf54..ad78bb1f635 100644 --- a/yt/frontends/nc4_cm1/tests/test_outputs.py +++ b/yt/frontends/nc4_cm1/tests/test_outputs.py @@ -21,17 +21,14 @@ _fields = ("dbz", "thrhopert", "zvort") -## To-Do - Need to know the actual location of test -## data for these tests! Probably need to give a small, -## static NetCDF file near the tornado. -cm1sim = "CM1Tornado/test_dataset.nc" +cm1sim = "budget-test.04400.000000.nc" @requires_ds(cm1sim, big_data=True) -def test_tornado(): +def test_mesh(): ds = data_dir_load(cm1sim) print(ds) - assert_equal(str(ds), "test_dataset.nc") + assert_equal(str(ds), "budget-test.04400.000000.nc") for test in small_patch_amr(ds, _fields): test_tprmadp.__name__ = test.description yield test @@ -48,15 +45,24 @@ def test_units_override(): @requires_file(cm1sim) -def test_tornado_dataset(): +def test_dims_and_meta(): ds = data_dir_load(cm1sim) - ## To-Do: Static tests for the specific - ## NetCDF file given above! - #assert_equal(ds.parameters["time"], 751000000000.0) - #assert_equal(ds.domain_dimensions, np.array([8, 8, 8])) - #assert_equal(ds.domain_left_edge, ds.arr([-2e18, -2e18, -2e18], "code_length")) - #assert_equal(ds.index.num_grids, 73) - #dd = ds.all_data() - #dd["density"] + known_dims = ["time", "zf", "zh", "yf", "yh", "xf", "xh"] + dims = ds.parameters["coords"].dims.keys() + ## check the file for 2 grids and a time dimension - + ## (time, xf, xh, yf, yh, zf, zh). The dimesions ending in + ## f are the staggered velocity grid components and the + ## dimensions ending in h are the scalar grid components + assert_equal(len(dims), 7) + for kdim in known_dims: + assert kdim in dims + + ## check the simulation time + assert_equal(ds.parameters["time"], 4400.) + +## TO DO: perhaps an image test of 1km AGL reflectivity? +@requires_ds(cm1sim, bid_data=True) +def test_reflectivity_plot(): + return From b58ead2b0ad6c54893e8e487c0d30a6712feb488 Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Tue, 22 Sep 2020 15:01:50 -0500 Subject: [PATCH 19/50] adjusting _is_valid --- yt/frontends/nc4_cm1/data_structures.py | 54 ++++++++++++++++--------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/yt/frontends/nc4_cm1/data_structures.py b/yt/frontends/nc4_cm1/data_structures.py index ff31e22e3ee..64017477c52 100644 --- a/yt/frontends/nc4_cm1/data_structures.py +++ b/yt/frontends/nc4_cm1/data_structures.py @@ -166,7 +166,7 @@ def _parse_parameter_file(self): self.domain_dimensions = np.array(dims, dtype="int64") self.periodicity = (False, False, False) self.current_time = self._handle.time.values[0] - self.parameters["time"] = self.current_time + self.parameters["time"] = self.current_time # We also set up cosmological information. Set these to zero if # non-cosmological. @@ -185,6 +185,14 @@ def _is_valid(self, *args, **kwargs): ds = xarray.open_dataset(args[0]) except OSError: return False + except ImportError: + # xarray not installed. If we can tell the file is a nc4_cm1 file without + # xarray, would be good to warn the user here. + return False + + # TO-DO check for global attribute here. e.g., + # if 'convention' not in ds.attrs.keys(): + # return False try: variables = ds.variables.keys() @@ -192,23 +200,31 @@ def _is_valid(self, *args, **kwargs): return False ## check each variable array in the dataset - ## and make sure that the coordinates in the + ## and make sure that the coordinates in the ## variable arrays matches the coordinates ## of the dataset - coords = ds.coords ## get the dataset wide coordinates - nvars = len(ds.variables.keys()) ## number of variables in dataset - varspassed = 0 ## number of variables passing the tet - for var in ds.variables.keys(): ## iterate over the variables - vcoords = ds[var].coords ## get the coordinates for the variable - ncoords = len(vcoords) ## number of coordinates in variable - coordspassed = 0 ## number of coordinates that pass for a variable - for vc in vcoords: ## iterate over the coordinates for the variable - if vc in coords: coordspassed += 1 ## variable coordinate and global coordinate are same - else: return False - if (coordspassed == ncoords): varspassed += 1 ## if all coordinates in a variable pass, the variable passes - else: return False - ## TO-DO - add specific CM1 check to make sure this - ## frontend doesn't step on the toes of anyone running - ## xarray in the future! - if (varspassed == nvars): return True ## if all vars pass return True - else: return False + coords = ds.coords # get the dataset wide coordinates + nvars = len(variables) # number of variables in dataset + varspassed = 0 # number of variables passing the tet + for var in variables: # iterate over the variables + vcoords = ds[var].coords # get the coordinates for the variable + ncoords = len(vcoords) # number of coordinates in variable + coordspassed = 0 # number of coordinates that pass for a variable + for vc in vcoords: # iterate over the coordinates for the variable + if vc in coords: + coordspassed += ( + 1 # variable coordinate and global coordinate are same + ) + else: + return False + if coordspassed == ncoords: + varspassed += ( + 1 # if all coordinates in a variable pass, the variable passes + ) + else: + return False + + if varspassed == nvars: + return True # if all vars pass return True + else: + return False From 0fc8367e7c593a89a4bb7572c66b66b54be3f5f6 Mon Sep 17 00:00:00 2001 From: keltonhalbert Date: Tue, 22 Sep 2020 20:44:54 +0000 Subject: [PATCH 20/50] removed unnecessary imports and image test --- yt/frontends/nc4_cm1/tests/test_outputs.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/yt/frontends/nc4_cm1/tests/test_outputs.py b/yt/frontends/nc4_cm1/tests/test_outputs.py index ad78bb1f635..2b86e62da0c 100644 --- a/yt/frontends/nc4_cm1/tests/test_outputs.py +++ b/yt/frontends/nc4_cm1/tests/test_outputs.py @@ -1,10 +1,3 @@ -from collections import OrderedDict - -import os -import stat -import weakref -import numpy as np - from yt.utilities.on_demand_imports import _xarray as xarray from yt.frontends.nc4_cm1.api import CM1Dataset from yt.testing import ( @@ -20,7 +13,6 @@ _fields = ("dbz", "thrhopert", "zvort") - cm1sim = "budget-test.04400.000000.nc" @@ -62,7 +54,3 @@ def test_dims_and_meta(): ## check the simulation time assert_equal(ds.parameters["time"], 4400.) -## TO DO: perhaps an image test of 1km AGL reflectivity? -@requires_ds(cm1sim, bid_data=True) -def test_reflectivity_plot(): - return From bd5abb57b05721670a1e37446727e2264c6dbdd8 Mon Sep 17 00:00:00 2001 From: keltonhalbert Date: Tue, 22 Sep 2020 20:49:51 +0000 Subject: [PATCH 21/50] updated the file to the testfile provided by Leigh --- yt/frontends/nc4_cm1/tests/test_outputs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yt/frontends/nc4_cm1/tests/test_outputs.py b/yt/frontends/nc4_cm1/tests/test_outputs.py index 2b86e62da0c..75f747c153b 100644 --- a/yt/frontends/nc4_cm1/tests/test_outputs.py +++ b/yt/frontends/nc4_cm1/tests/test_outputs.py @@ -13,14 +13,14 @@ _fields = ("dbz", "thrhopert", "zvort") -cm1sim = "budget-test.04400.000000.nc" +cm1sim = "testyt.05500.000000.nc" @requires_ds(cm1sim, big_data=True) def test_mesh(): ds = data_dir_load(cm1sim) print(ds) - assert_equal(str(ds), "budget-test.04400.000000.nc") + assert_equal(str(ds), "testyt.05500.000000.nc") for test in small_patch_amr(ds, _fields): test_tprmadp.__name__ = test.description yield test @@ -52,5 +52,5 @@ def test_dims_and_meta(): assert kdim in dims ## check the simulation time - assert_equal(ds.parameters["time"], 4400.) + assert_equal(ds.parameters["time"], 5500.) From f3dff05809b44ceb5e823774211795204bbf1a16 Mon Sep 17 00:00:00 2001 From: keltonhalbert Date: Tue, 22 Sep 2020 20:55:13 +0000 Subject: [PATCH 22/50] added storage of global metadata in parameters dict --- yt/frontends/nc4_cm1/data_structures.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/yt/frontends/nc4_cm1/data_structures.py b/yt/frontends/nc4_cm1/data_structures.py index 64017477c52..dda613a64a5 100644 --- a/yt/frontends/nc4_cm1/data_structures.py +++ b/yt/frontends/nc4_cm1/data_structures.py @@ -149,6 +149,8 @@ def _parse_parameter_file(self): self.parameters = {} coords = self._handle.coords self.parameters["coords"] = coords + self.parameters["lofs_version"] = ds.attrs["cm1_lofs_version"] + self.parameters["is_uniform"] = ds.attrs["uniform_mesh"] # TO DO: Possibly figure out a way to generalize this to be coordiante variable # name agnostic in order to make useful for WRF or climate data. For now, we're From e1914b3d05f9b84ea8f6a0d162ac2dd0285b10d3 Mon Sep 17 00:00:00 2001 From: keltonhalbert Date: Tue, 22 Sep 2020 20:59:04 +0000 Subject: [PATCH 23/50] added very simple test of cm1 output version --- yt/frontends/nc4_cm1/tests/test_outputs.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/yt/frontends/nc4_cm1/tests/test_outputs.py b/yt/frontends/nc4_cm1/tests/test_outputs.py index 75f747c153b..0ea68a4a91f 100644 --- a/yt/frontends/nc4_cm1/tests/test_outputs.py +++ b/yt/frontends/nc4_cm1/tests/test_outputs.py @@ -54,3 +54,7 @@ def test_dims_and_meta(): ## check the simulation time assert_equal(ds.parameters["time"], 5500.) +@requires_file(cm1sim) +def test_if_cm1(): + ds = data_dir_load(cm1sim) + assert float(ds.parameters["lofs_version"]) >= 1.0 From 1dc86946e448fba07316f6a33d8585f877622d25 Mon Sep 17 00:00:00 2001 From: keltonhalbert Date: Tue, 22 Sep 2020 21:00:41 +0000 Subject: [PATCH 24/50] I promise I know how the handlers work --- yt/frontends/nc4_cm1/data_structures.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yt/frontends/nc4_cm1/data_structures.py b/yt/frontends/nc4_cm1/data_structures.py index dda613a64a5..b3f68f76bb3 100644 --- a/yt/frontends/nc4_cm1/data_structures.py +++ b/yt/frontends/nc4_cm1/data_structures.py @@ -149,8 +149,8 @@ def _parse_parameter_file(self): self.parameters = {} coords = self._handle.coords self.parameters["coords"] = coords - self.parameters["lofs_version"] = ds.attrs["cm1_lofs_version"] - self.parameters["is_uniform"] = ds.attrs["uniform_mesh"] + self.parameters["lofs_version"] = self._handle.attrs["cm1_lofs_version"] + self.parameters["is_uniform"] = self._handle.attrs["uniform_mesh"] # TO DO: Possibly figure out a way to generalize this to be coordiante variable # name agnostic in order to make useful for WRF or climate data. For now, we're From f13bf04922fc8b3d1d9cf917669691f64569fbea Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Tue, 22 Sep 2020 16:05:09 -0500 Subject: [PATCH 25/50] is_valid update --- yt/frontends/nc4_cm1/data_structures.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/yt/frontends/nc4_cm1/data_structures.py b/yt/frontends/nc4_cm1/data_structures.py index b3f68f76bb3..52c3bf448ca 100644 --- a/yt/frontends/nc4_cm1/data_structures.py +++ b/yt/frontends/nc4_cm1/data_structures.py @@ -17,7 +17,8 @@ from yt.data_objects.index_subobjects.grid_patch import AMRGridPatch from yt.data_objects.static_output import Dataset from yt.geometry.grid_geometry_handler import GridIndex -from yt.utilities.on_demand_imports import _xarray as xarray +from yt.utilities.file_handler import warn_netcdf +from yt.utilities.on_demand_imports import _netCDF4 as netCDF4, _xarray as xarray from .fields import CM1FieldInfo @@ -183,18 +184,22 @@ def _is_valid(self, *args, **kwargs): # This accepts a filename or a set of arguments and returns True or # False depending on if the file is of the type requested. + # first use standard netcdf4 to check for our identifying attribute + # following exodus_ii frontend _is_valid + warn_netcdf(args[0]) + try: + with netCDF4.Dataset(args[0], keepweakref=True) as f: + is_cm1 = hasattr(f, "cm1_lofs_version") + if is_cm1 is False: + return False + except Exception: + return False + + # try to open the dataset -- will raise xarray import error if not installed. try: ds = xarray.open_dataset(args[0]) except OSError: return False - except ImportError: - # xarray not installed. If we can tell the file is a nc4_cm1 file without - # xarray, would be good to warn the user here. - return False - - # TO-DO check for global attribute here. e.g., - # if 'convention' not in ds.attrs.keys(): - # return False try: variables = ds.variables.keys() From fa1ee324431f8c7b9ed92130f003e54a7358d928 Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Tue, 22 Sep 2020 17:33:09 -0500 Subject: [PATCH 26/50] style fixes --- yt/frontends/nc4_cm1/__init__.py | 1 - yt/frontends/nc4_cm1/fields.py | 46 +++++++++++----------- yt/frontends/nc4_cm1/tests/test_outputs.py | 17 +++----- 3 files changed, 29 insertions(+), 35 deletions(-) diff --git a/yt/frontends/nc4_cm1/__init__.py b/yt/frontends/nc4_cm1/__init__.py index fcd921acc9d..7a4dbb7a31b 100644 --- a/yt/frontends/nc4_cm1/__init__.py +++ b/yt/frontends/nc4_cm1/__init__.py @@ -4,4 +4,3 @@ """ - diff --git a/yt/frontends/nc4_cm1/fields.py b/yt/frontends/nc4_cm1/fields.py index 337192a440d..db27ef6557e 100644 --- a/yt/frontends/nc4_cm1/fields.py +++ b/yt/frontends/nc4_cm1/fields.py @@ -29,29 +29,29 @@ class CM1FieldInfo(FieldInfoContainer): ("thrhopert", ("K", ["density_potential_temperature_perturbation"], None)), ("prespert", ("hPa", ["presure_perturbation"], None)), ("rhopert", ("kg/m**3", ["density_perturbation"], None)), - ("dbz", ("dB", ["simulated_reflectivity"], None)), - ("qvpert", ("g/kg", ["water_vapor_mixing_ratio_perturbation"], None)), - ("qc", ("g/kg", ["cloud_liquid_water_mixing_ratio"], None)), - ("qr", ("g/kg", ["rain_mixing_ratio"], None)), - ("qi", ("g/kg", ["cloud_ice_mixing_ratio"], None)), - ("qs", ("g/kg", ["snow_mixing_ratio"], None)), - ("qg", ("g/kg", ["graupel_or_hail_mixing_ratio"], None)), - ("qcloud", ("g/kg", ["sum_of_cloud_water_and_cloud_ice_mixing_ratios"], None)), - ("qprecip", ("g/kg", ["sum_of_rain_graupel_snow_mixing_ratios"], None)), - ("nci", ("1/cm**3", ["number_concerntration_of_cloud_ice"], None)), - ("ncr", ("1/cm**3", ["number_concentration_of_rain"], None)), - ("ncs", ("1/cm**3", ["number_concentration_of_snow"], None)), - ("ncg", ("1/cm**3", ["number_concentration_of_graupel_or_hail"], None)), - ("xvort", ("1/s", ["vorticity_x"], None)), - ("yvort", ("1/s", ["vorticity_y"], None)), - ("zvort", ("1/s", ["vorticity_z"], None)), - ("hvort", ("1/s", ["horizontal_vorticity_magnitude"], None)), - ("vortmag", ("1/s", ["vorticity_magnitude"], None)), - ("streamvort", ("1/s",["streamwise_vorticity"], None)), - ("khh", ("m**2/s", ["khh"], None)), - ("khv", ("m**2/s", ["khv"], None)), - ("kmh", ("m**2/s", ["kmh"], None)), - ("kmv", ("m**2/s", ["kmv"], None)) + ("dbz", ("dB", ["simulated_reflectivity"], None)), + ("qvpert", ("g/kg", ["water_vapor_mixing_ratio_perturbation"], None)), + ("qc", ("g/kg", ["cloud_liquid_water_mixing_ratio"], None)), + ("qr", ("g/kg", ["rain_mixing_ratio"], None)), + ("qi", ("g/kg", ["cloud_ice_mixing_ratio"], None)), + ("qs", ("g/kg", ["snow_mixing_ratio"], None)), + ("qg", ("g/kg", ["graupel_or_hail_mixing_ratio"], None)), + ("qcloud", ("g/kg", ["sum_of_cloud_water_and_cloud_ice_mixing_ratios"], None)), + ("qprecip", ("g/kg", ["sum_of_rain_graupel_snow_mixing_ratios"], None)), + ("nci", ("1/cm**3", ["number_concerntration_of_cloud_ice"], None)), + ("ncr", ("1/cm**3", ["number_concentration_of_rain"], None)), + ("ncs", ("1/cm**3", ["number_concentration_of_snow"], None)), + ("ncg", ("1/cm**3", ["number_concentration_of_graupel_or_hail"], None)), + ("xvort", ("1/s", ["vorticity_x"], None)), + ("yvort", ("1/s", ["vorticity_y"], None)), + ("zvort", ("1/s", ["vorticity_z"], None)), + ("hvort", ("1/s", ["horizontal_vorticity_magnitude"], None)), + ("vortmag", ("1/s", ["vorticity_magnitude"], None)), + ("streamvort", ("1/s", ["streamwise_vorticity"], None)), + ("khh", ("m**2/s", ["khh"], None)), + ("khv", ("m**2/s", ["khv"], None)), + ("kmh", ("m**2/s", ["kmh"], None)), + ("kmv", ("m**2/s", ["kmv"], None)), ) known_particle_fields = ( diff --git a/yt/frontends/nc4_cm1/tests/test_outputs.py b/yt/frontends/nc4_cm1/tests/test_outputs.py index 0ea68a4a91f..e6f7799446a 100644 --- a/yt/frontends/nc4_cm1/tests/test_outputs.py +++ b/yt/frontends/nc4_cm1/tests/test_outputs.py @@ -1,17 +1,11 @@ -from yt.utilities.on_demand_imports import _xarray as xarray from yt.frontends.nc4_cm1.api import CM1Dataset -from yt.testing import ( - assert_equal, - requires_file, - units_override_check, -) +from yt.testing import assert_equal, requires_file, units_override_check from yt.utilities.answer_testing.framework import ( data_dir_load, requires_ds, small_patch_amr, ) - _fields = ("dbz", "thrhopert", "zvort") cm1sim = "testyt.05500.000000.nc" @@ -22,7 +16,7 @@ def test_mesh(): print(ds) assert_equal(str(ds), "testyt.05500.000000.nc") for test in small_patch_amr(ds, _fields): - test_tprmadp.__name__ = test.description + test_mesh.__name__ = test.description yield test @@ -43,7 +37,7 @@ def test_dims_and_meta(): known_dims = ["time", "zf", "zh", "yf", "yh", "xf", "xh"] dims = ds.parameters["coords"].dims.keys() - ## check the file for 2 grids and a time dimension - + ## check the file for 2 grids and a time dimension - ## (time, xf, xh, yf, yh, zf, zh). The dimesions ending in ## f are the staggered velocity grid components and the ## dimensions ending in h are the scalar grid components @@ -51,8 +45,9 @@ def test_dims_and_meta(): for kdim in known_dims: assert kdim in dims - ## check the simulation time - assert_equal(ds.parameters["time"], 5500.) + ## check the simulation time + assert_equal(ds.parameters["time"], 5500.0) + @requires_file(cm1sim) def test_if_cm1(): From 214de7f4abb2b45b76304a40f76c9336e73cf0f7 Mon Sep 17 00:00:00 2001 From: keltonhalbert Date: Wed, 23 Sep 2020 18:25:16 +0000 Subject: [PATCH 27/50] I had untracked changes apparently and need to pull --- yt/frontends/nc4_cm1/data_structures.py | 1 + 1 file changed, 1 insertion(+) diff --git a/yt/frontends/nc4_cm1/data_structures.py b/yt/frontends/nc4_cm1/data_structures.py index b3f68f76bb3..6c5500d52f9 100644 --- a/yt/frontends/nc4_cm1/data_structures.py +++ b/yt/frontends/nc4_cm1/data_structures.py @@ -166,6 +166,7 @@ def _parse_parameter_file(self): self.dimensionality = 3 dims = [self._handle.dims[i] for i in ["xh", "yh", "zh"]] self.domain_dimensions = np.array(dims, dtype="int64") + print(dims, self.domain_dimensions) self.periodicity = (False, False, False) self.current_time = self._handle.time.values[0] self.parameters["time"] = self.current_time From 48aed38f85c5e004500988f05ee28eda45549f96 Mon Sep 17 00:00:00 2001 From: keltonhalbert Date: Wed, 23 Sep 2020 19:08:10 +0000 Subject: [PATCH 28/50] added default MKS units system --- yt/frontends/nc4_cm1/data_structures.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/yt/frontends/nc4_cm1/data_structures.py b/yt/frontends/nc4_cm1/data_structures.py index 2963a04e217..770907e43ad 100644 --- a/yt/frontends/nc4_cm1/data_structures.py +++ b/yt/frontends/nc4_cm1/data_structures.py @@ -114,7 +114,7 @@ class CM1Dataset(Dataset): _field_info_class = CM1FieldInfo def __init__( - self, filename, dataset_type="cm1", storage_filename=None, units_override=None + self, filename, dataset_type="cm1", units_override=units_override, unit_system="mks" ): self.fluid_types += ("cm1",) self._handle = xarray.open_dataset(filename, engine="netcdf4") @@ -167,7 +167,6 @@ def _parse_parameter_file(self): self.dimensionality = 3 dims = [self._handle.dims[i] for i in ["xh", "yh", "zh"]] self.domain_dimensions = np.array(dims, dtype="int64") - print(dims, self.domain_dimensions) self.periodicity = (False, False, False) self.current_time = self._handle.time.values[0] self.parameters["time"] = self.current_time From 3f49b4c44495afdd30ecc990f7fa28f51a701226 Mon Sep 17 00:00:00 2001 From: keltonhalbert Date: Wed, 23 Sep 2020 19:35:27 +0000 Subject: [PATCH 29/50] Made recommended changes from Matt that helps fix the issues with coordinate axes and slices in YT. Z is now respected as up for both XZ and YZ slices. Additionally, fixes the axis labeling. --- yt/frontends/nc4_cm1/data_structures.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/yt/frontends/nc4_cm1/data_structures.py b/yt/frontends/nc4_cm1/data_structures.py index 770907e43ad..2765b1c1c28 100644 --- a/yt/frontends/nc4_cm1/data_structures.py +++ b/yt/frontends/nc4_cm1/data_structures.py @@ -114,17 +114,22 @@ class CM1Dataset(Dataset): _field_info_class = CM1FieldInfo def __init__( - self, filename, dataset_type="cm1", units_override=units_override, unit_system="mks" + self, filename, dataset_type="cm1", storage_filename=None, units_override=None ): self.fluid_types += ("cm1",) self._handle = xarray.open_dataset(filename, engine="netcdf4") # refinement factor between a grid and its subgrid self.refine_by = 1 super(CM1Dataset, self).__init__( - filename, dataset_type, units_override=units_override + filename, dataset_type, units_override=units_override, unit_system="mks" ) self.storage_filename = storage_filename + def _setup_coordinate_handler(self): + super(CM1Dataset, self)._setup_coordinate_handler() + self.coordinates._x_pairs = (("x", "y"), ("y", "x"), ("z", "x")) + self.coordinates._y_pairs = (("x", "z"), ("y", "z"), ("z", "y")) + def _set_code_unit_attributes(self): # This is where quantities are created that represent the various # on-disk units. These are the currently available quantities which From 2f3225392453c7ea65f58488ea37a0fa7353602a Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Wed, 23 Sep 2020 14:41:55 -0500 Subject: [PATCH 30/50] working answer test (locally) --- yt/frontends/nc4_cm1/tests/test_outputs.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/yt/frontends/nc4_cm1/tests/test_outputs.py b/yt/frontends/nc4_cm1/tests/test_outputs.py index e6f7799446a..2965bc9ab49 100644 --- a/yt/frontends/nc4_cm1/tests/test_outputs.py +++ b/yt/frontends/nc4_cm1/tests/test_outputs.py @@ -1,24 +1,39 @@ from yt.frontends.nc4_cm1.api import CM1Dataset from yt.testing import assert_equal, requires_file, units_override_check from yt.utilities.answer_testing.framework import ( + FieldValuesTest, + GridValuesTest, + can_run_ds, data_dir_load, requires_ds, small_patch_amr, ) -_fields = ("dbz", "thrhopert", "zvort") +_fields = ("thrhopert", "zvort") cm1sim = "testyt.05500.000000.nc" @requires_ds(cm1sim, big_data=True) def test_mesh(): ds = data_dir_load(cm1sim) - print(ds) assert_equal(str(ds), "testyt.05500.000000.nc") - for test in small_patch_amr(ds, _fields): + + # run the small_patch_amr tests on safe fields + ic = ds.domain_center + for test in small_patch_amr(ds, _fields, input_center=ic, input_weight=None): test_mesh.__name__ = test.description yield test + # manually run the Grid and Field Values tests on dbz (do not want to run the + # ProjectionValuesTest for this field) + if can_run_ds(ds): + dso = [None, ("sphere", (ic, (0.1, "unitary")))] + for field in ["dbz"]: + yield GridValuesTest(ds, field) + for axis in [0, 1, 2]: + for dobj_name in dso: + yield FieldValuesTest(ds, field, dobj_name) + @requires_file(cm1sim) def test_CM1Dataset(): From f20ab6b32954f43d957c4dbda3791da8a2d8f733 Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Wed, 23 Sep 2020 14:55:35 -0500 Subject: [PATCH 31/50] answer test update --- yt/frontends/nc4_cm1/tests/test_outputs.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/yt/frontends/nc4_cm1/tests/test_outputs.py b/yt/frontends/nc4_cm1/tests/test_outputs.py index 2965bc9ab49..e88a6be1820 100644 --- a/yt/frontends/nc4_cm1/tests/test_outputs.py +++ b/yt/frontends/nc4_cm1/tests/test_outputs.py @@ -13,7 +13,7 @@ cm1sim = "testyt.05500.000000.nc" -@requires_ds(cm1sim, big_data=True) +@requires_ds(cm1sim) def test_mesh(): ds = data_dir_load(cm1sim) assert_equal(str(ds), "testyt.05500.000000.nc") @@ -30,9 +30,8 @@ def test_mesh(): dso = [None, ("sphere", (ic, (0.1, "unitary")))] for field in ["dbz"]: yield GridValuesTest(ds, field) - for axis in [0, 1, 2]: - for dobj_name in dso: - yield FieldValuesTest(ds, field, dobj_name) + for dobj_name in dso: + yield FieldValuesTest(ds, field, dobj_name) @requires_file(cm1sim) From 2ea2cdd5ff939b37d00ad2e649bf375ca947ad54 Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Wed, 23 Sep 2020 15:03:58 -0500 Subject: [PATCH 32/50] test filename update --- yt/frontends/nc4_cm1/tests/test_outputs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yt/frontends/nc4_cm1/tests/test_outputs.py b/yt/frontends/nc4_cm1/tests/test_outputs.py index e88a6be1820..f7d942fde8d 100644 --- a/yt/frontends/nc4_cm1/tests/test_outputs.py +++ b/yt/frontends/nc4_cm1/tests/test_outputs.py @@ -10,13 +10,13 @@ ) _fields = ("thrhopert", "zvort") -cm1sim = "testyt.05500.000000.nc" +cm1sim = "nc4_cm1_lofs_tornado_test.nc" @requires_ds(cm1sim) def test_mesh(): ds = data_dir_load(cm1sim) - assert_equal(str(ds), "testyt.05500.000000.nc") + assert_equal(str(ds), cm1sim) # run the small_patch_amr tests on safe fields ic = ds.domain_center From cc4f4608ab74d69314a9130da0d4b5aadd43ecc7 Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Wed, 23 Sep 2020 15:40:19 -0500 Subject: [PATCH 33/50] add check and message for cm1 vs cm1_lofs --- yt/frontends/nc4_cm1/data_structures.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/yt/frontends/nc4_cm1/data_structures.py b/yt/frontends/nc4_cm1/data_structures.py index 2765b1c1c28..3239b1388f3 100644 --- a/yt/frontends/nc4_cm1/data_structures.py +++ b/yt/frontends/nc4_cm1/data_structures.py @@ -18,6 +18,7 @@ from yt.data_objects.static_output import Dataset from yt.geometry.grid_geometry_handler import GridIndex from yt.utilities.file_handler import warn_netcdf +from yt.utilities.logger import ytLogger as mylog from yt.utilities.on_demand_imports import _netCDF4 as netCDF4, _xarray as xarray from .fields import CM1FieldInfo @@ -194,8 +195,19 @@ def _is_valid(self, *args, **kwargs): warn_netcdf(args[0]) try: with netCDF4.Dataset(args[0], keepweakref=True) as f: - is_cm1 = hasattr(f, "cm1_lofs_version") - if is_cm1 is False: + is_cm1_lofs = hasattr(f, "cm1_lofs_version") + is_cm1 = hasattr(f, "cm1 version") + + if is_cm1_lofs is False: + if is_cm1: + mylog.error( + ( + "It looks like you are trying to load a cm1 netcdf file, " + "but at present yt only supports cm1_lofs output. Until" + " support is added, you can likely use" + " yt.load_uniform_grid() to load your cm1 file manually." + ) + ) return False except Exception: return False From fe74465a42fbe6a7a23d6f861808cd28cebc8b46 Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Wed, 23 Sep 2020 15:48:50 -0500 Subject: [PATCH 34/50] answer test update, tests.yaml bump --- tests/tests.yaml | 3 +++ yt/frontends/nc4_cm1/tests/test_outputs.py | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/tests.yaml b/tests/tests.yaml index 35e5720a633..a33bb38eee7 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -167,6 +167,9 @@ answer_tests: #local_particle_trajectory_001: # - yt/data_objects/tests/test_particle_trajectories.py + + local_nc4_cm1_000: # PR 2176 + - yt/frontends/nc4_cm1/tests/test_outputs.py:test_cm1_mesh_fields other_tests: unittests: diff --git a/yt/frontends/nc4_cm1/tests/test_outputs.py b/yt/frontends/nc4_cm1/tests/test_outputs.py index f7d942fde8d..51d0e927b86 100644 --- a/yt/frontends/nc4_cm1/tests/test_outputs.py +++ b/yt/frontends/nc4_cm1/tests/test_outputs.py @@ -10,18 +10,18 @@ ) _fields = ("thrhopert", "zvort") -cm1sim = "nc4_cm1_lofs_tornado_test.nc" +cm1sim = "cm1_tornado_lofs/nc4_cm1_lofs_tornado_test.nc" @requires_ds(cm1sim) -def test_mesh(): +def test_cm1_mesh_fields(): ds = data_dir_load(cm1sim) - assert_equal(str(ds), cm1sim) + assert_equal(str(ds), "nc4_cm1_lofs_tornado_test.nc") # run the small_patch_amr tests on safe fields ic = ds.domain_center for test in small_patch_amr(ds, _fields, input_center=ic, input_weight=None): - test_mesh.__name__ = test.description + test_cm1_mesh_fields.__name__ = test.description yield test # manually run the Grid and Field Values tests on dbz (do not want to run the From 4ed2d23f201a0cb0984b601d47416594934613b7 Mon Sep 17 00:00:00 2001 From: Madicken Munk Date: Wed, 23 Sep 2020 16:58:13 -0500 Subject: [PATCH 35/50] open with netcdf and xarray safely --- yt/frontends/nc4_cm1/data_structures.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/yt/frontends/nc4_cm1/data_structures.py b/yt/frontends/nc4_cm1/data_structures.py index 3239b1388f3..eec3b9e885c 100644 --- a/yt/frontends/nc4_cm1/data_structures.py +++ b/yt/frontends/nc4_cm1/data_structures.py @@ -118,7 +118,8 @@ def __init__( self, filename, dataset_type="cm1", storage_filename=None, units_override=None ): self.fluid_types += ("cm1",) - self._handle = xarray.open_dataset(filename, engine="netcdf4") + self._handle = xarray.open_dataset(filename, engine="netcdf4", + backend_kwargs={"clobber":False}) # refinement factor between a grid and its subgrid self.refine_by = 1 super(CM1Dataset, self).__init__( @@ -194,7 +195,7 @@ def _is_valid(self, *args, **kwargs): # following exodus_ii frontend _is_valid warn_netcdf(args[0]) try: - with netCDF4.Dataset(args[0], keepweakref=True) as f: + with netCDF4.Dataset(args[0], "r", keepweakref=True) as f: is_cm1_lofs = hasattr(f, "cm1_lofs_version") is_cm1 = hasattr(f, "cm1 version") @@ -214,7 +215,8 @@ def _is_valid(self, *args, **kwargs): # try to open the dataset -- will raise xarray import error if not installed. try: - ds = xarray.open_dataset(args[0]) + ds = xarray.open_dataset(args[0], engine="netcdf4", + backend_kwargs={"clobber":False}) except OSError: return False From 991497a5cdfb65aeb94fc89bc61d628242834c36 Mon Sep 17 00:00:00 2001 From: Madicken Munk Date: Wed, 23 Sep 2020 17:07:53 -0500 Subject: [PATCH 36/50] style fixes --- yt/frontends/nc4_cm1/data_structures.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/yt/frontends/nc4_cm1/data_structures.py b/yt/frontends/nc4_cm1/data_structures.py index eec3b9e885c..e84726c0cda 100644 --- a/yt/frontends/nc4_cm1/data_structures.py +++ b/yt/frontends/nc4_cm1/data_structures.py @@ -118,8 +118,8 @@ def __init__( self, filename, dataset_type="cm1", storage_filename=None, units_override=None ): self.fluid_types += ("cm1",) - self._handle = xarray.open_dataset(filename, engine="netcdf4", - backend_kwargs={"clobber":False}) + self._handle = xarray.open_dataset( + filename, engine="netcdf4", backend_kwargs={"clobber":False}) # refinement factor between a grid and its subgrid self.refine_by = 1 super(CM1Dataset, self).__init__( @@ -215,8 +215,9 @@ def _is_valid(self, *args, **kwargs): # try to open the dataset -- will raise xarray import error if not installed. try: - ds = xarray.open_dataset(args[0], engine="netcdf4", - backend_kwargs={"clobber":False}) + ds = xarray.open_dataset( + args[0], engine="netcdf4", backend_kwargs={"clobber":False} + ) except OSError: return False From d90a8b2ad39d3fce2f0834eaafd039839d3d4d41 Mon Sep 17 00:00:00 2001 From: Chris Havlin Date: Fri, 25 Sep 2020 10:16:03 -0500 Subject: [PATCH 37/50] switch to netcdf4 loader --- yt/frontends/nc4_cm1/data_structures.py | 171 +++++++++------------ yt/frontends/nc4_cm1/fields.py | 7 - yt/frontends/nc4_cm1/io.py | 29 ++-- yt/frontends/nc4_cm1/tests/test_outputs.py | 4 +- 4 files changed, 89 insertions(+), 122 deletions(-) diff --git a/yt/frontends/nc4_cm1/data_structures.py b/yt/frontends/nc4_cm1/data_structures.py index e84726c0cda..ba57d7c4117 100644 --- a/yt/frontends/nc4_cm1/data_structures.py +++ b/yt/frontends/nc4_cm1/data_structures.py @@ -1,25 +1,16 @@ -""" -Skeleton data structures - - - -""" - -## Written by Kelton Halbert and Leigh Orf at the 2019 yt developers workshop @ NCSA -## for the purpose of reading in George Bryan's Cloud Model 1 output for plotting in yt. - import os import stat import weakref +from collections import OrderedDict import numpy as np from yt.data_objects.index_subobjects.grid_patch import AMRGridPatch from yt.data_objects.static_output import Dataset from yt.geometry.grid_geometry_handler import GridIndex -from yt.utilities.file_handler import warn_netcdf +from yt.utilities.file_handler import NetCDF4FileHandler, warn_netcdf from yt.utilities.logger import ytLogger as mylog -from yt.utilities.on_demand_imports import _netCDF4 as netCDF4, _xarray as xarray +from yt.utilities.on_demand_imports import _netCDF4 as netCDF4 from .fields import CM1FieldInfo @@ -56,22 +47,10 @@ def _initialize_state_variables(self): self.num_grids = 1 def _detect_output_fields(self): - # This needs to set a self.field_list that contains all the available, - # on-disk fields. No derived fields should be defined here. - # NOTE: Each should be a tuple, where the first element is the on-disk - # fluid type or particle type. Convention suggests that the on-disk - # fluid type is usually the dataset_type and the on-disk particle type - # (for a single population of particles) is "io". + # build list of on-disk fields for dataset_type 'cm1' self.field_list = [] - - ## loop over the variable names in the netCDF file - for key in self.ds._handle.variables.keys(): - if ( - all(x in self.ds._handle[key].dims for x in ["time", "zh", "yh", "xh"]) - is True - ): - field_tup = ("cm1", key) - self.field_list.append(field_tup) + for vname in self.dataset.parameters["variable_names"]: + self.field_list.append(("cm1", vname)) def _count_grids(self): # This needs to set self.num_grids @@ -118,14 +97,14 @@ def __init__( self, filename, dataset_type="cm1", storage_filename=None, units_override=None ): self.fluid_types += ("cm1",) - self._handle = xarray.open_dataset( - filename, engine="netcdf4", backend_kwargs={"clobber":False}) + self._handle = NetCDF4FileHandler(filename) # refinement factor between a grid and its subgrid self.refine_by = 1 super(CM1Dataset, self).__init__( filename, dataset_type, units_override=units_override, unit_system="mks" ) self.storage_filename = storage_filename + self.filename = filename def _setup_coordinate_handler(self): super(CM1Dataset, self)._setup_coordinate_handler() @@ -137,7 +116,8 @@ def _set_code_unit_attributes(self): # on-disk units. These are the currently available quantities which # should be set, along with examples of how to set them to standard # values. - length_unit = self._handle.variables["xh"].attrs["units"] + with self._handle.open_ds() as ds: + length_unit = ds.variables["xh"].units self.length_unit = self.quan(1.0, length_unit) self.mass_unit = self.quan(1.0, "kg") self.time_unit = self.quan(1.0, "s") @@ -153,33 +133,48 @@ def _parse_parameter_file(self): # self.unique_identifier <= unique identifier for the dataset # being read (e.g., UUID or ST_CTIME) self.unique_identifier = int(os.stat(self.parameter_filename)[stat.ST_CTIME]) - # self.parameters <= full of code-specific items of use - self.parameters = {} - coords = self._handle.coords + self.parameters = {} # code-specific items + with self._handle.open_ds() as ds: + dims = [ds.dimensions[i].size for i in ["xh", "yh", "zh"]] + coords = {i: ds.variables[i][:] for i in ["xh", "yh", "zh"]} + + # TO DO: eneralize this to be coordiante variable name agnostic in order to + # make useful for WRF or climate data. For now, we're hard coding for CM1 + # specifically and have named the classes appropriately + xh, yh, zh = [ds.variables[i][:] for i in ["xh", "yh", "zh"]] + self.domain_left_edge = np.array( + [xh.min(), yh.min(), zh.min()], dtype="float64" + ) + self.domain_right_edge = np.array( + [xh.max(), yh.max(), zh.max()], dtype="float64" + ) + + # loop over the variable names in the netCDF file, record only those on the + # "zh","yh","xh" grid. + varnames = [] + for key in ds.variables.keys(): + vardims = ds.variables[key].dimensions + if all(x in vardims for x in ["time", "zh", "yh", "xh"]) is True: + varnames.append(key) + self.parameters["variable_names"] = varnames + self.parameters["lofs_version"] = ds.cm1_lofs_version + self.parameters["is_uniform"] = ds.uniform_mesh + self.current_time = ds.variables["time"][:][0] + + # record the dimension metadata + dim_info = OrderedDict() + for dim, meta in ds.dimensions.items(): + dim_info[dim] = meta.size + self.parameters["dimensions"] = dim_info + self.parameters["coords"] = coords - self.parameters["lofs_version"] = self._handle.attrs["cm1_lofs_version"] - self.parameters["is_uniform"] = self._handle.attrs["uniform_mesh"] - - # TO DO: Possibly figure out a way to generalize this to be coordiante variable - # name agnostic in order to make useful for WRF or climate data. For now, we're - # hard coding for CM1 specifically and have named the classes appropriately, but - # generalizing is good. - xh, yh, zh = [coords[i] for i in ["xh", "yh", "zh"]] - self.domain_left_edge = np.array( - [xh.min(), yh.min(), zh.min()], dtype="float64" - ) - self.domain_right_edge = np.array( - [xh.max(), yh.max(), zh.max()], dtype="float64" - ) + self.dimensionality = 3 - dims = [self._handle.dims[i] for i in ["xh", "yh", "zh"]] self.domain_dimensions = np.array(dims, dtype="int64") self.periodicity = (False, False, False) - self.current_time = self._handle.time.values[0] self.parameters["time"] = self.current_time - # We also set up cosmological information. Set these to zero if - # non-cosmological. + # Set cosmological information to zero for non-cosmological. self.cosmological_simulation = 0.0 self.current_redshift = 0.0 self.omega_lambda = 0.0 @@ -191,13 +186,34 @@ def _is_valid(self, *args, **kwargs): # This accepts a filename or a set of arguments and returns True or # False depending on if the file is of the type requested. - # first use standard netcdf4 to check for our identifying attribute - # following exodus_ii frontend _is_valid warn_netcdf(args[0]) try: - with netCDF4.Dataset(args[0], "r", keepweakref=True) as f: - is_cm1_lofs = hasattr(f, "cm1_lofs_version") - is_cm1 = hasattr(f, "cm1 version") + with netCDF4.Dataset(args[0], "r", keepweakref=True) as ds: + is_cm1_lofs = hasattr(ds, "cm1_lofs_version") + is_cm1 = hasattr(ds, "cm1 version") + # ensure coordinates of each variable array exists in the dataset + coords = ds.dimensions # get the dataset wide coordinates + failed_vars = [] # list of failed variables + for var in ds.variables: # iterate over the variables + vcoords = ds[var].dimensions # get the dimensions for the variable + ncoords = len(vcoords) # number of coordinates in variable + coordspassed = 0 # number of coordinates that pass for a variable + for vc in vcoords: # iterate over the coordinates for the variable + if vc in coords: + # variable coordinate and global coordinate are same + coordspassed += 1 + if coordspassed != ncoords: + failed_vars.append(var) + + if len(failed_vars) > 0: + mylog.error( + ( + "Trying to load a cm1_lofs netcdf file but the coordinates " + "of the following fields do not match the coordinates of " + f"the dataset: {failed_vars}" + ) + ) + return False if is_cm1_lofs is False: if is_cm1: @@ -213,45 +229,4 @@ def _is_valid(self, *args, **kwargs): except Exception: return False - # try to open the dataset -- will raise xarray import error if not installed. - try: - ds = xarray.open_dataset( - args[0], engine="netcdf4", backend_kwargs={"clobber":False} - ) - except OSError: - return False - - try: - variables = ds.variables.keys() - except KeyError: - return False - - ## check each variable array in the dataset - ## and make sure that the coordinates in the - ## variable arrays matches the coordinates - ## of the dataset - coords = ds.coords # get the dataset wide coordinates - nvars = len(variables) # number of variables in dataset - varspassed = 0 # number of variables passing the tet - for var in variables: # iterate over the variables - vcoords = ds[var].coords # get the coordinates for the variable - ncoords = len(vcoords) # number of coordinates in variable - coordspassed = 0 # number of coordinates that pass for a variable - for vc in vcoords: # iterate over the coordinates for the variable - if vc in coords: - coordspassed += ( - 1 # variable coordinate and global coordinate are same - ) - else: - return False - if coordspassed == ncoords: - varspassed += ( - 1 # if all coordinates in a variable pass, the variable passes - ) - else: - return False - - if varspassed == nvars: - return True # if all vars pass return True - else: - return False + return True diff --git a/yt/frontends/nc4_cm1/fields.py b/yt/frontends/nc4_cm1/fields.py index db27ef6557e..df759da1f15 100644 --- a/yt/frontends/nc4_cm1/fields.py +++ b/yt/frontends/nc4_cm1/fields.py @@ -1,10 +1,3 @@ -""" -Skeleton-specific fields - - - -""" - from yt.fields.field_info_container import FieldInfoContainer # We need to specify which fields we might have in our dataset. The field info diff --git a/yt/frontends/nc4_cm1/io.py b/yt/frontends/nc4_cm1/io.py index 5be73bc92c4..22c282715fe 100644 --- a/yt/frontends/nc4_cm1/io.py +++ b/yt/frontends/nc4_cm1/io.py @@ -1,12 +1,6 @@ -""" -Skeleton-specific IO functions - - - -""" - import numpy as np +from yt.utilities.file_handler import NetCDF4FileHandler from yt.utilities.io_handler import BaseIOHandler @@ -14,6 +8,11 @@ class CM1IOHandler(BaseIOHandler): _particle_reader = False _dataset_type = "cm1" + def __init__(self, ds): + self.filename = ds.filename + self._handle = NetCDF4FileHandler(self.filename) + super(CM1IOHandler, self).__init__(ds) + def _read_particle_coords(self, chunks, ptf): # This needs to *yield* a series of tuples of (ptype, (x, y, z)). # chunks is a list of chunks, and ptf is a dict where the keys are @@ -45,14 +44,14 @@ def _read_fluid_selection(self, chunks, selector, fields, size): data = {} offset = 0 - for field in fields: - data[field] = np.empty(size, dtype="float64") - for chunk in chunks: - for grid in chunk.objs: - ds = self.ds._handle - variable = ds.variables[field[1]] - values = np.squeeze(variable.values[0].T) - offset += grid.select(selector, values, data[field], offset) + with self._handle.open_ds() as ds: + for field in fields: + data[field] = np.empty(size, dtype="float64") + for chunk in chunks: + for grid in chunk.objs: + variable = ds.variables[field[1]][:][0] + values = np.squeeze(variable.T) + offset += grid.select(selector, values, data[field], offset) return data def _read_chunk_data(self, chunk, fields): diff --git a/yt/frontends/nc4_cm1/tests/test_outputs.py b/yt/frontends/nc4_cm1/tests/test_outputs.py index 51d0e927b86..cf9b005db2c 100644 --- a/yt/frontends/nc4_cm1/tests/test_outputs.py +++ b/yt/frontends/nc4_cm1/tests/test_outputs.py @@ -49,13 +49,13 @@ def test_dims_and_meta(): ds = data_dir_load(cm1sim) known_dims = ["time", "zf", "zh", "yf", "yh", "xf", "xh"] - dims = ds.parameters["coords"].dims.keys() + dims = ds.parameters["dimensions"] ## check the file for 2 grids and a time dimension - ## (time, xf, xh, yf, yh, zf, zh). The dimesions ending in ## f are the staggered velocity grid components and the ## dimensions ending in h are the scalar grid components - assert_equal(len(dims), 7) + assert_equal(len(dims), len(known_dims)) for kdim in known_dims: assert kdim in dims From dfea4127f28e17b7239a882db57d3a9db4a4df52 Mon Sep 17 00:00:00 2001 From: Chris Havlin Date: Fri, 25 Sep 2020 15:03:21 -0500 Subject: [PATCH 38/50] set NetCDF$FileHandler file mode to r always --- yt/utilities/file_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/utilities/file_handler.py b/yt/utilities/file_handler.py index 58eb99bc296..9de487cfdce 100644 --- a/yt/utilities/file_handler.py +++ b/yt/utilities/file_handler.py @@ -111,6 +111,6 @@ def __init__(self, filename): def open_ds(self): from yt.utilities.on_demand_imports import _netCDF4 as netCDF4 - ds = netCDF4.Dataset(self.filename) + ds = netCDF4.Dataset(self.filename, "r") yield ds ds.close() From 80c6a20621c8a36e521ce50b5e1312168e23b208 Mon Sep 17 00:00:00 2001 From: Madicken Munk Date: Fri, 25 Sep 2020 15:17:02 -0500 Subject: [PATCH 39/50] remove xarray from on demand imports --- doc/install_script.sh | 1 - tests/test_install_script.py | 1 - yt/utilities/on_demand_imports.py | 18 ------------------ 3 files changed, 20 deletions(-) diff --git a/doc/install_script.sh b/doc/install_script.sh index 93cce57cf34..62d165a3635 100644 --- a/doc/install_script.sh +++ b/doc/install_script.sh @@ -36,7 +36,6 @@ INST_CARTOPY=0 # Install cartopy? INST_NOSE=1 # Install nose? INST_NETCDF4=1 # Install netcdf4 and its python bindings? INST_POOCH=1 # Install pooch? -INST_XARRAY=0 # Install xarray? # This is the branch we will install from for INST_YT_SOURCE=1 BRANCH="master" diff --git a/tests/test_install_script.py b/tests/test_install_script.py index cf9d5cae62a..1c9a3b75c22 100644 --- a/tests/test_install_script.py +++ b/tests/test_install_script.py @@ -28,7 +28,6 @@ "astropy", "cartopy", "pooch", - "xarray", ] # dependencies that are only installable when yt is built from source diff --git a/yt/utilities/on_demand_imports.py b/yt/utilities/on_demand_imports.py index ce6cecaabf5..059b8be78a9 100644 --- a/yt/utilities/on_demand_imports.py +++ b/yt/utilities/on_demand_imports.py @@ -617,21 +617,3 @@ def __getattr__(self, attr): _f90nml = f90nml_imports() - - -class xarray_imports: - _name = "xarray" - _open_dataset = None - - @property - def open_dataset(self): - if self._open_dataset is None: - try: - from xarray import open_dataset - except ImportError: - open_dataset = NotAModule(self._name) - self._open_dataset = open_dataset - return self._open_dataset - - -_xarray = xarray_imports() From eef55ac87dd916c504d21c9e85ce218f5db187df Mon Sep 17 00:00:00 2001 From: Chris Havlin Date: Fri, 25 Sep 2020 15:31:47 -0500 Subject: [PATCH 40/50] move the netcdf4 import to isvalid try block --- yt/frontends/nc4_cm1/data_structures.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/yt/frontends/nc4_cm1/data_structures.py b/yt/frontends/nc4_cm1/data_structures.py index ba57d7c4117..8fe3fa2a41d 100644 --- a/yt/frontends/nc4_cm1/data_structures.py +++ b/yt/frontends/nc4_cm1/data_structures.py @@ -10,7 +10,6 @@ from yt.geometry.grid_geometry_handler import GridIndex from yt.utilities.file_handler import NetCDF4FileHandler, warn_netcdf from yt.utilities.logger import ytLogger as mylog -from yt.utilities.on_demand_imports import _netCDF4 as netCDF4 from .fields import CM1FieldInfo @@ -188,7 +187,9 @@ def _is_valid(self, *args, **kwargs): warn_netcdf(args[0]) try: - with netCDF4.Dataset(args[0], "r", keepweakref=True) as ds: + from netCDF4 import Dataset + + with Dataset(args[0], "r", keepweakref=True) as ds: is_cm1_lofs = hasattr(ds, "cm1_lofs_version") is_cm1 = hasattr(ds, "cm1 version") # ensure coordinates of each variable array exists in the dataset From dec3f0b454340e986f2af6717cf11d14e01c2887 Mon Sep 17 00:00:00 2001 From: Chris Havlin Date: Fri, 25 Sep 2020 16:01:11 -0500 Subject: [PATCH 41/50] ensure netcd4 import before h5py --- yt/utilities/on_demand_imports.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/yt/utilities/on_demand_imports.py b/yt/utilities/on_demand_imports.py index 059b8be78a9..5f62aee96b3 100644 --- a/yt/utilities/on_demand_imports.py +++ b/yt/utilities/on_demand_imports.py @@ -56,6 +56,15 @@ class netCDF4_imports: _name = "netCDF4" _Dataset = None + def __init__(self): + try: + import netCDF4 + + netCDF4.__version__ + except ImportError: + pass + super(netCDF4_imports, self).__init__() + @property def Dataset(self): if self._Dataset is None: @@ -67,7 +76,7 @@ def Dataset(self): return self._Dataset -_netCDF4 = netCDF4_imports() +_netCDF4 = netCDF4_imports() # Always do that before initializing h5py_imports() !!! class astropy_imports: From 860fa19b16a07a055a99f384ee0221e25ec079fc Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Wed, 30 Sep 2020 13:35:11 -0500 Subject: [PATCH 42/50] Apply quick suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Robert Co-authored-by: Corentin Cadiou --- yt/frontends/nc4_cm1/api.py | 1 - yt/frontends/nc4_cm1/data_structures.py | 38 ++++++------------------- yt/frontends/nc4_cm1/io.py | 4 +-- 3 files changed, 11 insertions(+), 32 deletions(-) diff --git a/yt/frontends/nc4_cm1/api.py b/yt/frontends/nc4_cm1/api.py index 074cd098e7f..f7acb342075 100644 --- a/yt/frontends/nc4_cm1/api.py +++ b/yt/frontends/nc4_cm1/api.py @@ -5,7 +5,6 @@ """ -from . import tests from .data_structures import CM1Dataset, CM1Grid, CM1Hierarchy from .fields import CM1FieldInfo from .io import CM1IOHandler diff --git a/yt/frontends/nc4_cm1/data_structures.py b/yt/frontends/nc4_cm1/data_structures.py index 8fe3fa2a41d..0eacec6e5e8 100644 --- a/yt/frontends/nc4_cm1/data_structures.py +++ b/yt/frontends/nc4_cm1/data_structures.py @@ -25,7 +25,7 @@ def __init__(self, id, index, level, dimensions): self.ActiveDimensions = dimensions def __repr__(self): - return "CM1Grid_%04i (%s)" % (self.id, self.ActiveDimensions) + return f"CM1Grid_{self.id:d} ({self.ActiveDimensions})" class CM1Hierarchy(GridIndex): @@ -56,14 +56,6 @@ def _count_grids(self): self.num_grids = 1 def _parse_index(self): - # This needs to fill the following arrays, where N is self.num_grids: - # self.grid_left_edge (N, 3) <= float64 - # self.grid_right_edge (N, 3) <= float64 - # self.grid_dimensions (N, 3) <= int - # self.grid_particle_count (N, 1) <= int - # self.grid_levels (N, 1) <= int - # self.grids (N, 1) <= grid objects - # self.max_level = self.grid_levels.max() self.grid_left_edge[0][:] = self.ds.domain_left_edge[:] self.grid_right_edge[0][:] = self.ds.domain_right_edge[:] self.grid_dimensions[0][:] = self.ds.domain_dimensions[:] @@ -72,14 +64,6 @@ def _parse_index(self): self.max_level = 1 def _populate_grid_objects(self): - # For each grid g, this must call: - # g._prepare_grid() - # g._setup_dx() - # This must also set: - # g.Children <= list of child grids - # g.Parent <= parent grid - # This is handled by the frontend because often the children must be - # identified. self.grids = np.empty(self.num_grids, dtype="object") for i in range(self.num_grids): g = self.grid(i, self, self.grid_levels.flat[i], self.grid_dimensions[i]) @@ -151,9 +135,8 @@ def _parse_parameter_file(self): # loop over the variable names in the netCDF file, record only those on the # "zh","yh","xh" grid. varnames = [] - for key in ds.variables.keys(): - vardims = ds.variables[key].dimensions - if all(x in vardims for x in ["time", "zh", "yh", "xh"]) is True: + for key, var in ds.variables.items(): + if all(x in var.dimensions for x in ["time", "zh", "yh", "xh"]): varnames.append(key) self.parameters["variable_names"] = varnames self.parameters["lofs_version"] = ds.cm1_lofs_version @@ -174,14 +157,14 @@ def _parse_parameter_file(self): self.parameters["time"] = self.current_time # Set cosmological information to zero for non-cosmological. - self.cosmological_simulation = 0.0 + self.cosmological_simulation = 0 self.current_redshift = 0.0 self.omega_lambda = 0.0 self.omega_matter = 0.0 self.hubble_constant = 0.0 @classmethod - def _is_valid(self, *args, **kwargs): + def _is_valid(cls, *args, **kwargs): # This accepts a filename or a set of arguments and returns True or # False depending on if the file is of the type requested. @@ -198,15 +181,12 @@ def _is_valid(self, *args, **kwargs): for var in ds.variables: # iterate over the variables vcoords = ds[var].dimensions # get the dimensions for the variable ncoords = len(vcoords) # number of coordinates in variable - coordspassed = 0 # number of coordinates that pass for a variable - for vc in vcoords: # iterate over the coordinates for the variable - if vc in coords: - # variable coordinate and global coordinate are same - coordspassed += 1 + # number of coordinates that pass for a variable + coordspassed = sum(vc in coords for vc in vcoords) if coordspassed != ncoords: failed_vars.append(var) - if len(failed_vars) > 0: + if failed_vars: mylog.error( ( "Trying to load a cm1_lofs netcdf file but the coordinates " @@ -216,7 +196,7 @@ def _is_valid(self, *args, **kwargs): ) return False - if is_cm1_lofs is False: + if not is_cm1_lofs: if is_cm1: mylog.error( ( diff --git a/yt/frontends/nc4_cm1/io.py b/yt/frontends/nc4_cm1/io.py index 22c282715fe..fb1a8ff673e 100644 --- a/yt/frontends/nc4_cm1/io.py +++ b/yt/frontends/nc4_cm1/io.py @@ -17,7 +17,7 @@ def _read_particle_coords(self, chunks, ptf): # This needs to *yield* a series of tuples of (ptype, (x, y, z)). # chunks is a list of chunks, and ptf is a dict where the keys are # ptypes and the values are lists of fields. - pass + raise NotImplementedError def _read_particle_fields(self, chunks, ptf, selector): # This gets called after the arrays have been allocated. It needs to @@ -25,7 +25,7 @@ def _read_particle_fields(self, chunks, ptf, selector): # reading ptype, field and applying the selector to the data read in. # Selector objects have a .select_points(x,y,z) that returns a mask, so # you need to do your masking here. - pass + raise NotImplementedError def _read_fluid_selection(self, chunks, selector, fields, size): # This needs to allocate a set of arrays inside a dictionary, where the From 5c423ba590865d5f7f657bc0a89b0880187ad434 Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Wed, 30 Sep 2020 13:50:50 -0500 Subject: [PATCH 43/50] incorporating more suggested changes --- yt/frontends/api.py | 2 +- yt/frontends/nc4_cm1/data_structures.py | 16 +++++++++++++--- yt/utilities/on_demand_imports.py | 10 ++++++---- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/yt/frontends/api.py b/yt/frontends/api.py index ed86a8d369e..7f163fe7491 100644 --- a/yt/frontends/api.py +++ b/yt/frontends/api.py @@ -30,6 +30,7 @@ "halo_catalog", "http_stream", "moab", + "nc4_cm1", "open_pmd", "owls", "owls_subfind", @@ -40,7 +41,6 @@ "swift", "tipsy", "ytdata", - "nc4_cm1", ] diff --git a/yt/frontends/nc4_cm1/data_structures.py b/yt/frontends/nc4_cm1/data_structures.py index 0eacec6e5e8..62a09975d0b 100644 --- a/yt/frontends/nc4_cm1/data_structures.py +++ b/yt/frontends/nc4_cm1/data_structures.py @@ -77,19 +77,29 @@ class CM1Dataset(Dataset): _field_info_class = CM1FieldInfo def __init__( - self, filename, dataset_type="cm1", storage_filename=None, units_override=None + self, + filename, + dataset_type="cm1", + storage_filename=None, + units_override=None, + unit_system="mks", ): self.fluid_types += ("cm1",) self._handle = NetCDF4FileHandler(filename) # refinement factor between a grid and its subgrid self.refine_by = 1 super(CM1Dataset, self).__init__( - filename, dataset_type, units_override=units_override, unit_system="mks" + filename, + dataset_type, + units_override=units_override, + unit_system=unit_system, ) self.storage_filename = storage_filename self.filename = filename def _setup_coordinate_handler(self): + # ensure correct ordering of axes so plots aren't rotated (z should always be + # on the vertical axis). super(CM1Dataset, self)._setup_coordinate_handler() self.coordinates._x_pairs = (("x", "y"), ("y", "x"), ("z", "x")) self.coordinates._y_pairs = (("x", "z"), ("y", "z"), ("z", "y")) @@ -174,7 +184,7 @@ def _is_valid(cls, *args, **kwargs): with Dataset(args[0], "r", keepweakref=True) as ds: is_cm1_lofs = hasattr(ds, "cm1_lofs_version") - is_cm1 = hasattr(ds, "cm1 version") + is_cm1 = hasattr(ds, "cm1 version") # note: not a typo, it is a space # ensure coordinates of each variable array exists in the dataset coords = ds.dimensions # get the dataset wide coordinates failed_vars = [] # list of failed variables diff --git a/yt/utilities/on_demand_imports.py b/yt/utilities/on_demand_imports.py index 5f62aee96b3..c7811f5025e 100644 --- a/yt/utilities/on_demand_imports.py +++ b/yt/utilities/on_demand_imports.py @@ -57,10 +57,12 @@ class netCDF4_imports: _Dataset = None def __init__(self): + # this ensures the import ordering between netcdf4 and h5py. If hy5py is + # imported first, can get file lock errors on some systems (including travis-ci) + # so we need to do this before initializing h5py_imports()! + # similar to this issue https://github.com/pydata/xarray/issues/2560 try: - import netCDF4 - - netCDF4.__version__ + import netCDF4 # noqa F401 except ImportError: pass super(netCDF4_imports, self).__init__() @@ -76,7 +78,7 @@ def Dataset(self): return self._Dataset -_netCDF4 = netCDF4_imports() # Always do that before initializing h5py_imports() !!! +_netCDF4 = netCDF4_imports() class astropy_imports: From 093208852565fbe80bc70e559e6d6ad2c5455280 Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Wed, 30 Sep 2020 14:08:22 -0500 Subject: [PATCH 44/50] switched netcdf handle to _handle instead of ds --- yt/frontends/nc4_cm1/data_structures.py | 23 ++++++++++------------ yt/frontends/nc4_cm1/tests/test_outputs.py | 2 +- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/yt/frontends/nc4_cm1/data_structures.py b/yt/frontends/nc4_cm1/data_structures.py index 62a09975d0b..314f861926a 100644 --- a/yt/frontends/nc4_cm1/data_structures.py +++ b/yt/frontends/nc4_cm1/data_structures.py @@ -127,14 +127,14 @@ def _parse_parameter_file(self): # being read (e.g., UUID or ST_CTIME) self.unique_identifier = int(os.stat(self.parameter_filename)[stat.ST_CTIME]) self.parameters = {} # code-specific items - with self._handle.open_ds() as ds: - dims = [ds.dimensions[i].size for i in ["xh", "yh", "zh"]] - coords = {i: ds.variables[i][:] for i in ["xh", "yh", "zh"]} + with self._handle.open_ds() as _handle: + # _handle here is a netcdf Dataset object + dims = [_handle.dimensions[i].size for i in ["xh", "yh", "zh"]] - # TO DO: eneralize this to be coordiante variable name agnostic in order to + # TO DO: generalize this to be coordiante variable name agnostic in order to # make useful for WRF or climate data. For now, we're hard coding for CM1 # specifically and have named the classes appropriately - xh, yh, zh = [ds.variables[i][:] for i in ["xh", "yh", "zh"]] + xh, yh, zh = [_handle.variables[i][:] for i in ["xh", "yh", "zh"]] self.domain_left_edge = np.array( [xh.min(), yh.min(), zh.min()], dtype="float64" ) @@ -145,26 +145,23 @@ def _parse_parameter_file(self): # loop over the variable names in the netCDF file, record only those on the # "zh","yh","xh" grid. varnames = [] - for key, var in ds.variables.items(): + for key, var in _handle.variables.items(): if all(x in var.dimensions for x in ["time", "zh", "yh", "xh"]): varnames.append(key) self.parameters["variable_names"] = varnames - self.parameters["lofs_version"] = ds.cm1_lofs_version - self.parameters["is_uniform"] = ds.uniform_mesh - self.current_time = ds.variables["time"][:][0] + self.parameters["lofs_version"] = _handle.cm1_lofs_version + self.parameters["is_uniform"] = _handle.uniform_mesh + self.current_time = _handle.variables["time"][:][0] # record the dimension metadata dim_info = OrderedDict() - for dim, meta in ds.dimensions.items(): + for dim, meta in _handle.dimensions.items(): dim_info[dim] = meta.size self.parameters["dimensions"] = dim_info - self.parameters["coords"] = coords - self.dimensionality = 3 self.domain_dimensions = np.array(dims, dtype="int64") self.periodicity = (False, False, False) - self.parameters["time"] = self.current_time # Set cosmological information to zero for non-cosmological. self.cosmological_simulation = 0 diff --git a/yt/frontends/nc4_cm1/tests/test_outputs.py b/yt/frontends/nc4_cm1/tests/test_outputs.py index cf9b005db2c..fff9f8a7698 100644 --- a/yt/frontends/nc4_cm1/tests/test_outputs.py +++ b/yt/frontends/nc4_cm1/tests/test_outputs.py @@ -60,7 +60,7 @@ def test_dims_and_meta(): assert kdim in dims ## check the simulation time - assert_equal(ds.parameters["time"], 5500.0) + assert_equal(ds.current_time, 5500.0) @requires_file(cm1sim) From 1aada12f7b7fad087fa37a805c3fa8fd14d8ad5a Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Wed, 30 Sep 2020 14:50:03 -0500 Subject: [PATCH 45/50] add kwargs to nc4 file handler, commenting --- yt/frontends/nc4_cm1/data_structures.py | 29 +++++++++++++------------ yt/utilities/file_handler.py | 4 ++-- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/yt/frontends/nc4_cm1/data_structures.py b/yt/frontends/nc4_cm1/data_structures.py index 314f861926a..f87b2aab495 100644 --- a/yt/frontends/nc4_cm1/data_structures.py +++ b/yt/frontends/nc4_cm1/data_structures.py @@ -109,8 +109,8 @@ def _set_code_unit_attributes(self): # on-disk units. These are the currently available quantities which # should be set, along with examples of how to set them to standard # values. - with self._handle.open_ds() as ds: - length_unit = ds.variables["xh"].units + with self._handle.open_ds() as _handle: + length_unit = _handle.variables["xh"].units self.length_unit = self.quan(1.0, length_unit) self.mass_unit = self.quan(1.0, "kg") self.time_unit = self.quan(1.0, "s") @@ -128,7 +128,8 @@ def _parse_parameter_file(self): self.unique_identifier = int(os.stat(self.parameter_filename)[stat.ST_CTIME]) self.parameters = {} # code-specific items with self._handle.open_ds() as _handle: - # _handle here is a netcdf Dataset object + # _handle here is a netcdf Dataset object, we need to parse some metadata + # for constructing our yt ds. dims = [_handle.dimensions[i].size for i in ["xh", "yh", "zh"]] # TO DO: generalize this to be coordiante variable name agnostic in order to @@ -153,7 +154,8 @@ def _parse_parameter_file(self): self.parameters["is_uniform"] = _handle.uniform_mesh self.current_time = _handle.variables["time"][:][0] - # record the dimension metadata + # record the dimension metadata: __handle.dimensions contains netcdf + # objects so we need to manually copy over attributes. dim_info = OrderedDict() for dim, meta in _handle.dimensions.items(): dim_info[dim] = meta.size @@ -177,16 +179,15 @@ def _is_valid(cls, *args, **kwargs): warn_netcdf(args[0]) try: - from netCDF4 import Dataset - - with Dataset(args[0], "r", keepweakref=True) as ds: - is_cm1_lofs = hasattr(ds, "cm1_lofs_version") - is_cm1 = hasattr(ds, "cm1 version") # note: not a typo, it is a space + nc4_file = NetCDF4FileHandler(args[0]) + with nc4_file.open_ds(keepweakref=True) as _handle: + is_cm1_lofs = hasattr(_handle, "cm1_lofs_version") + is_cm1 = hasattr(_handle, "cm1 version") # not a typo, it is a space... # ensure coordinates of each variable array exists in the dataset - coords = ds.dimensions # get the dataset wide coordinates + coords = _handle.dimensions # get the dataset wide coordinates failed_vars = [] # list of failed variables - for var in ds.variables: # iterate over the variables - vcoords = ds[var].dimensions # get the dimensions for the variable + for var in _handle.variables: # iterate over the variables + vcoords = _handle[var].dimensions # get the dims for the variable ncoords = len(vcoords) # number of coordinates in variable # number of coordinates that pass for a variable coordspassed = sum(vc in coords for vc in vcoords) @@ -194,7 +195,7 @@ def _is_valid(cls, *args, **kwargs): failed_vars.append(var) if failed_vars: - mylog.error( + mylog.warning( ( "Trying to load a cm1_lofs netcdf file but the coordinates " "of the following fields do not match the coordinates of " @@ -205,7 +206,7 @@ def _is_valid(cls, *args, **kwargs): if not is_cm1_lofs: if is_cm1: - mylog.error( + mylog.warning( ( "It looks like you are trying to load a cm1 netcdf file, " "but at present yt only supports cm1_lofs output. Until" diff --git a/yt/utilities/file_handler.py b/yt/utilities/file_handler.py index 9de487cfdce..4ee2fa880b2 100644 --- a/yt/utilities/file_handler.py +++ b/yt/utilities/file_handler.py @@ -108,9 +108,9 @@ def __init__(self, filename): self.filename = filename @contextmanager - def open_ds(self): + def open_ds(self, **kwargs): from yt.utilities.on_demand_imports import _netCDF4 as netCDF4 - ds = netCDF4.Dataset(self.filename, "r") + ds = netCDF4.Dataset(self.filename, mode="r", **kwargs) yield ds ds.close() From 11116a22e800f3d3b3b48f101f453cb9a4ab99ac Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Fri, 2 Oct 2020 16:47:40 -0500 Subject: [PATCH 46/50] remaining suggestions --- yt/frontends/nc4_cm1/data_structures.py | 31 ++++++++++--------------- yt/frontends/nc4_cm1/fields.py | 4 ---- yt/utilities/on_demand_imports.py | 1 - 3 files changed, 12 insertions(+), 24 deletions(-) diff --git a/yt/frontends/nc4_cm1/data_structures.py b/yt/frontends/nc4_cm1/data_structures.py index f87b2aab495..8d7fd5df9e1 100644 --- a/yt/frontends/nc4_cm1/data_structures.py +++ b/yt/frontends/nc4_cm1/data_structures.py @@ -41,10 +41,6 @@ def __init__(self, ds, dataset_type="cm1"): self.float_type = np.float64 super(CM1Hierarchy, self).__init__(ds, dataset_type) - def _initialize_state_variables(self): - super(CM1Hierarchy, self)._initialize_state_variables() - self.num_grids = 1 - def _detect_output_fields(self): # build list of on-disk fields for dataset_type 'cm1' self.field_list = [] @@ -173,16 +169,17 @@ def _parse_parameter_file(self): self.hubble_constant = 0.0 @classmethod - def _is_valid(cls, *args, **kwargs): + def _is_valid(cls, filename, *args, **kwargs): # This accepts a filename or a set of arguments and returns True or # False depending on if the file is of the type requested. - warn_netcdf(args[0]) + warn_netcdf(filename) try: - nc4_file = NetCDF4FileHandler(args[0]) + nc4_file = NetCDF4FileHandler(filename) with nc4_file.open_ds(keepweakref=True) as _handle: is_cm1_lofs = hasattr(_handle, "cm1_lofs_version") is_cm1 = hasattr(_handle, "cm1 version") # not a typo, it is a space... + # ensure coordinates of each variable array exists in the dataset coords = _handle.dimensions # get the dataset wide coordinates failed_vars = [] # list of failed variables @@ -196,26 +193,22 @@ def _is_valid(cls, *args, **kwargs): if failed_vars: mylog.warning( - ( - "Trying to load a cm1_lofs netcdf file but the coordinates " - "of the following fields do not match the coordinates of " - f"the dataset: {failed_vars}" - ) + "Trying to load a cm1_lofs netcdf file but the " + "coordinates of the following fields do not match the " + f"coordinates of the dataset: {failed_vars}" ) return False if not is_cm1_lofs: if is_cm1: mylog.warning( - ( - "It looks like you are trying to load a cm1 netcdf file, " - "but at present yt only supports cm1_lofs output. Until" - " support is added, you can likely use" - " yt.load_uniform_grid() to load your cm1 file manually." - ) + "It looks like you are trying to load a cm1 netcdf file, " + "but at present yt only supports cm1_lofs output. Until" + " support is added, you can likely use" + " yt.load_uniform_grid() to load your cm1 file manually." ) return False - except Exception: + except (OSError, AttributeError): return False return True diff --git a/yt/frontends/nc4_cm1/fields.py b/yt/frontends/nc4_cm1/fields.py index df759da1f15..6b9c299f193 100644 --- a/yt/frontends/nc4_cm1/fields.py +++ b/yt/frontends/nc4_cm1/fields.py @@ -52,10 +52,6 @@ class CM1FieldInfo(FieldInfoContainer): # ( "name", ("units", ["fields", "to", "alias"], # "display_name")), ) - def __init__(self, ds, field_list): - super(CM1FieldInfo, self).__init__(ds, field_list) - # If you want, you can check self.field_list - def setup_fluid_fields(self): # Here we do anything that might need info about the dataset. # You can use self.alias, self.add_output_field (for on-disk fields) diff --git a/yt/utilities/on_demand_imports.py b/yt/utilities/on_demand_imports.py index c7811f5025e..bec116438f0 100644 --- a/yt/utilities/on_demand_imports.py +++ b/yt/utilities/on_demand_imports.py @@ -65,7 +65,6 @@ def __init__(self): import netCDF4 # noqa F401 except ImportError: pass - super(netCDF4_imports, self).__init__() @property def Dataset(self): From 58bca157b9294e3645316ae485d385175d281b51 Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Fri, 2 Oct 2020 16:52:40 -0500 Subject: [PATCH 47/50] inline comment noting xh,yh,zh definition --- yt/frontends/nc4_cm1/data_structures.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/yt/frontends/nc4_cm1/data_structures.py b/yt/frontends/nc4_cm1/data_structures.py index 8d7fd5df9e1..581f0c2d462 100644 --- a/yt/frontends/nc4_cm1/data_structures.py +++ b/yt/frontends/nc4_cm1/data_structures.py @@ -126,11 +126,13 @@ def _parse_parameter_file(self): with self._handle.open_ds() as _handle: # _handle here is a netcdf Dataset object, we need to parse some metadata # for constructing our yt ds. - dims = [_handle.dimensions[i].size for i in ["xh", "yh", "zh"]] # TO DO: generalize this to be coordiante variable name agnostic in order to # make useful for WRF or climate data. For now, we're hard coding for CM1 - # specifically and have named the classes appropriately + # specifically and have named the classes appropriately. Additionaly, we + # are only handling the cell-centered grid ("xh","yh","zh") at present. + # The cell-centered grid contains scalar fields and interpolated velocities. + dims = [_handle.dimensions[i].size for i in ["xh", "yh", "zh"]] xh, yh, zh = [_handle.variables[i][:] for i in ["xh", "yh", "zh"]] self.domain_left_edge = np.array( [xh.min(), yh.min(), zh.min()], dtype="float64" From 53e6162fa64ca8725317911d51a64d2c67ec9d6f Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Fri, 2 Oct 2020 16:58:00 -0500 Subject: [PATCH 48/50] removing empty files --- yt/frontends/nc4_cm1/definitions.py | 1 - yt/frontends/nc4_cm1/misc.py | 0 2 files changed, 1 deletion(-) delete mode 100644 yt/frontends/nc4_cm1/definitions.py delete mode 100644 yt/frontends/nc4_cm1/misc.py diff --git a/yt/frontends/nc4_cm1/definitions.py b/yt/frontends/nc4_cm1/definitions.py deleted file mode 100644 index 94175d4544d..00000000000 --- a/yt/frontends/nc4_cm1/definitions.py +++ /dev/null @@ -1 +0,0 @@ -# This file is often empty. It can hold definitions related to a frontend. diff --git a/yt/frontends/nc4_cm1/misc.py b/yt/frontends/nc4_cm1/misc.py deleted file mode 100644 index e69de29bb2d..00000000000 From 689cdf6c14b10d28948745578bcec2bf4baeb362 Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Mon, 5 Oct 2020 10:34:47 -0500 Subject: [PATCH 49/50] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Robert --- yt/frontends/nc4_cm1/data_structures.py | 5 ++--- yt/utilities/on_demand_imports.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/yt/frontends/nc4_cm1/data_structures.py b/yt/frontends/nc4_cm1/data_structures.py index 581f0c2d462..d71d4adb20b 100644 --- a/yt/frontends/nc4_cm1/data_structures.py +++ b/yt/frontends/nc4_cm1/data_structures.py @@ -43,9 +43,8 @@ def __init__(self, ds, dataset_type="cm1"): def _detect_output_fields(self): # build list of on-disk fields for dataset_type 'cm1' - self.field_list = [] - for vname in self.dataset.parameters["variable_names"]: - self.field_list.append(("cm1", vname)) + vnames = self.dataset.parameters["variable_names"] + self.field_list = [("cm1", vname) for vname in vnames] def _count_grids(self): # This needs to set self.num_grids diff --git a/yt/utilities/on_demand_imports.py b/yt/utilities/on_demand_imports.py index bec116438f0..ea68f879e7f 100644 --- a/yt/utilities/on_demand_imports.py +++ b/yt/utilities/on_demand_imports.py @@ -57,7 +57,7 @@ class netCDF4_imports: _Dataset = None def __init__(self): - # this ensures the import ordering between netcdf4 and h5py. If hy5py is + # this ensures the import ordering between netcdf4 and h5py. If h5py is # imported first, can get file lock errors on some systems (including travis-ci) # so we need to do this before initializing h5py_imports()! # similar to this issue https://github.com/pydata/xarray/issues/2560 From 12166dad71d5f4ac6c075430de9b467d6fe2c81c Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Mon, 5 Oct 2020 11:39:18 -0500 Subject: [PATCH 50/50] need to also catch ImportError in is_valid --- yt/frontends/nc4_cm1/data_structures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/frontends/nc4_cm1/data_structures.py b/yt/frontends/nc4_cm1/data_structures.py index d71d4adb20b..af800b5b313 100644 --- a/yt/frontends/nc4_cm1/data_structures.py +++ b/yt/frontends/nc4_cm1/data_structures.py @@ -209,7 +209,7 @@ def _is_valid(cls, filename, *args, **kwargs): " yt.load_uniform_grid() to load your cm1 file manually." ) return False - except (OSError, AttributeError): + except (OSError, AttributeError, ImportError): return False return True