From 06b5fa18dc683f3d630841e4509dac333acfceec Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 27 Oct 2022 16:29:48 +0100 Subject: [PATCH 01/10] dev --- cfdm/read_write/netcdf/netcdfwrite.py | 61 +++++++++++++++++++-------- cfdm/read_write/write.py | 2 + 2 files changed, 46 insertions(+), 17 deletions(-) diff --git a/cfdm/read_write/netcdf/netcdfwrite.py b/cfdm/read_write/netcdf/netcdfwrite.py index d902671b76..534daf7456 100644 --- a/cfdm/read_write/netcdf/netcdfwrite.py +++ b/cfdm/read_write/netcdf/netcdfwrite.py @@ -229,6 +229,7 @@ def _write_attributes(self, parent, ncvar, extra=None, omit=()): # Make sure that _FillValue and missing data have the same # data type as the data + dtype = None for attr in ("_FillValue", "missing_value"): if attr not in netcdf_attrs: continue @@ -263,7 +264,13 @@ def _write_attributes(self, parent, ncvar, extra=None, omit=()): del netcdf_attrs["_FillValue"] if not g["dry_run"]: + if dtype is None: + _FillValue = netcdf_attrs.pop("_FillValue ", None) + g["nc"][ncvar].setncatts(netcdf_attrs) + if dtype is None: + netcdf_attrs["_FillValue"] = _FillValue + if skip_set_fill_value: # Re-add as known attribute since this FV is already set @@ -705,7 +712,8 @@ def _write_dimension_coordinate(self, f, key, coord, ncdim, coordinates): # Create a new dimension coordinate variable self._write_netcdf_variable( - ncvar, ncdimensions, coord, extra=extra + ncvar, ncdimensions, coord, extra=extra, + omit_data=coord.construct_type in g['omit_data'] ) else: ncvar = seen[id(coord)]["ncvar"] @@ -758,7 +766,7 @@ def _write_count_variable( extra = {"sample_dimension": sample_ncdim} - # Create a new list variable + # Create a new count variable self._write_netcdf_variable( ncvar, (ncdim,), count_variable, extra=extra ) @@ -1333,7 +1341,10 @@ def _write_bounds( omit.append(prop) # Create the bounds netCDF variable - self._write_netcdf_variable(ncvar, ncdimensions, bounds, omit=omit) + self._write_netcdf_variable( + ncvar, ncdimensions, bounds, omit=omit, + omit_data=coord.construct_type in g['omit_data'] + ) extra["bounds"] = ncvar axes = self.implementation.get_construct_data_axes(f, coord_key) @@ -1493,7 +1504,7 @@ def _write_node_coordinates(self, coord, coord_ncvar, coord_ncdimensions): ncvar = self._netcdf_name(ncvar) # Create the netCDF node coordinates variable - self._write_netcdf_variable(ncvar, (ncdim,), nodes) + self._write_netcdf_variable(ncvar, (ncdim,), nodes) # TODO encodings = {} @@ -2041,7 +2052,8 @@ def _write_scalar_coordinate( # Create a new scalar coordinate variable self._write_netcdf_variable( - ncvar, (), scalar_coord, extra=bounds_extra + ncvar, (), scalar_coord, extra=bounds_extra, + omit_data=scalar_coord.construct_type in g['omit_data'] ) else: @@ -2134,7 +2146,8 @@ def _write_auxiliary_coordinate(self, f, key, coord, coordinates): # Create a new auxiliary coordinate variable, if it has data if self.implementation.get_data(coord, None) is not None: self._write_netcdf_variable( - ncvar, ncdimensions, coord, extra=extra + ncvar, ncdimensions, coord, extra=extra, + omit_data=coord.construct_type in g['omit_data'] ) g["key_to_ncvar"][key] = ncvar @@ -2214,7 +2227,8 @@ def _write_domain_ancillary(self, f, key, anc): self._write_bounds(f, anc, key, ncdimensions, ncvar) # Create a new domain ancillary variable - self._write_netcdf_variable(ncvar, ncdimensions, anc) + self._write_netcdf_variable(ncvar, ncdimensions, anc, + omit_data=anc.construct_type in g['omit_data']) g["key_to_ncvar"][key] = ncvar g["key_to_ncdims"][key] = ncdimensions @@ -2259,7 +2273,8 @@ def _write_field_ancillary(self, f, key, anc): ) # Create a new field ancillary variable - self._write_netcdf_variable(ncvar, ncdimensions, anc) + self._write_netcdf_variable(ncvar, ncdimensions, anc, + omit_data=anc.construct_type in g['omit_data']) g["key_to_ncvar"][key] = ncvar g["key_to_ncdims"][key] = ncdimensions @@ -2325,7 +2340,8 @@ def _write_cell_measure(self, field, key, cell_measure): ) # Create a new cell measure variable - self._write_netcdf_variable(ncvar, ncdimensions, cell_measure) + self._write_netcdf_variable(ncvar, ncdimensions, cell_measure, + omit_data= cell_measure.construct_type in g['omit_data']) g["key_to_ncvar"][key] = ncvar g["key_to_ncdims"][key] = ncdimensions @@ -2502,7 +2518,8 @@ def _write_netcdf_variable( extra=None, fill=False, data_variable=False, - domain_variable=False, + domain_variable=False, + omit_data=False, ): """Creates a new netCDF variable for a construct. @@ -2590,12 +2607,12 @@ def _write_netcdf_variable( # filled before any data is written. if the fill value is # False then the variable is not pre-filled. # ------------------------------------------------------------ - if fill or g["post_dry_run"]: # or append mode's appending iteration + if fill: # or g["post_dry_run"]: # or append mode's appending iteration fill_value = self.implementation.get_property( cfvar, "_FillValue", None ) else: - fill_value = None + fill_value = False if data_variable: lsd = g["least_significant_digit"] @@ -2645,9 +2662,10 @@ def _write_netcdf_variable( "endian": g["endian"], "chunksizes": chunksizes, "least_significant_digit": lsd, + "fill_value": fill_value, } - if fill_value is not None: - kwargs["fill_value"] = fill_value +# if fill_value is not None: +# kwargs["fill_value"] = fill_value # Add compression parameters (but not for vlen strings). if kwargs["datatype"] != str: @@ -2661,6 +2679,7 @@ def _write_netcdf_variable( f" to netCDF variable: {ncvar}({', '.join(ncdimensions)})" ) # pragma: no cover + print ('fill_value=', fill_value, datatype, repr(cfvar)) try: self._createVariable(**kwargs) except RuntimeError as error: @@ -2706,7 +2725,7 @@ def _write_netcdf_variable( # scale_factor or add_offset are set as properties on the # variable. # ------------------------------------------------------------ - if data is not None: + if data is not None and not omit_data: # Find the missing data values, if any. _FillValue = self.implementation.get_property( cfvar, "_FillValue", None @@ -3895,6 +3914,7 @@ def _write_field_or_domain( extra=extra, data_variable=field, domain_variable=domain, + omit_data=f.construct_type in g['omit_data'] # TODO: impl ) # Update the 'seen' dictionary, if required @@ -4321,6 +4341,9 @@ def file_open(self, filename, mode, fmt, fields): except RuntimeError as error: raise RuntimeError(f"{error}: {filename}") + # No need to pre-fill with fill values + nc.set_fill_off() + return nc @_manage_log_level_via_verbosity @@ -4349,6 +4372,7 @@ def write( warn_valid=True, group=True, coordinates=False, + omit_data=(), ): """Write field and domain constructs to a netCDF file. @@ -4654,12 +4678,15 @@ def write( # Dry run: populate 'seen' dict without actually writing to file. "dry_run": False, # To indicate if the previous iteration was a dry run: - "post_dry_run": False, + # # Note: need write_vars keys to specify dry runs (iterations) # and subsequent runs despite them being implied by the mode ('r' # and 'a' for dry_run and post_dry_run respectively) so that the # mode does not need to be passed to various methods, where a # pair of such keys seem clearer than one "effective mode" key. + "post_dry_run": False, + # + "omit_data": omit_data, } if mode not in ("w", "a", "r+"): @@ -4736,7 +4763,7 @@ def write( "Can't append fields with an incompatible 'featureType' " "global attribute to the original file." ) - + self._file_io_iteration( mode=effective_mode, overwrite=overwrite, diff --git a/cfdm/read_write/write.py b/cfdm/read_write/write.py index b54aacb7d2..9ec5ef6057 100644 --- a/cfdm/read_write/write.py +++ b/cfdm/read_write/write.py @@ -26,6 +26,7 @@ def write( warn_valid=True, group=True, coordinates=False, + omit_data=(), _implementation=_implementation, ): """Write field and domain constructs to a netCDF file. @@ -541,4 +542,5 @@ def write( group=group, coordinates=coordinates, extra_write_vars=None, + omit_data=omit_data, ) From 4f74a58c75ad5a2fea2388a3ecaedde4f78a0cd4 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 28 Oct 2022 10:17:05 +0100 Subject: [PATCH 02/10] dev --- Changelog.rst | 2 + cfdm/cfdmimplementation.py | 19 ++++++ cfdm/read_write/netcdf/netcdfwrite.py | 90 ++++++++++++++++++--------- cfdm/read_write/write.py | 38 ++++++++++- cfdm/test/test_read_write.py | 30 +++++++++ 5 files changed, 148 insertions(+), 31 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 4c24fc3aa1..8d509289bd 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -10,6 +10,8 @@ Version 1.10.0.1 (https://github.com/NCAS-CMS/cfdm/issues/215) * New method: `cfdm.Field.get_original_filenames` * New method: `cfdm.Data.get_original_filenames` +* New keyword parameter to `cfdm.write`: ``omit_data`` + (https://github.com/NCAS-CMS/cfdm/issues/221) * Fixed bug that caused a failure when printing date-time data with the first element masked (https://github.com/NCAS-CMS/cfdm/issues/211) diff --git a/cfdm/cfdmimplementation.py b/cfdm/cfdmimplementation.py index e14be1d415..0195b069d9 100644 --- a/cfdm/cfdmimplementation.py +++ b/cfdm/cfdmimplementation.py @@ -760,6 +760,25 @@ def get_construct_data_axes(self, field, key): except KeyError: return None + def get_construct_type(self, variable): + """Return the construct type of a variable. + + .. versionadded:: (cfdm) 1.10.0.1 + + :Parameters: + + variable: object + The object to get a construct type from. + + :Returns: + + `str` or `None` + The construct type, or `None` if the variable is not a + construct. + + """ + return getattr(variable, "construct_type", None) + def get_constructs(self, field, axes=(), data=False): """Return constructs that span particular axes. diff --git a/cfdm/read_write/netcdf/netcdfwrite.py b/cfdm/read_write/netcdf/netcdfwrite.py index 534daf7456..1be6acfe89 100644 --- a/cfdm/read_write/netcdf/netcdfwrite.py +++ b/cfdm/read_write/netcdf/netcdfwrite.py @@ -271,7 +271,6 @@ def _write_attributes(self, parent, ncvar, extra=None, omit=()): if dtype is None: netcdf_attrs["_FillValue"] = _FillValue - if skip_set_fill_value: # Re-add as known attribute since this FV is already set netcdf_attrs["_FillValue"] = self.implementation.get_data( @@ -712,8 +711,10 @@ def _write_dimension_coordinate(self, f, key, coord, ncdim, coordinates): # Create a new dimension coordinate variable self._write_netcdf_variable( - ncvar, ncdimensions, coord, extra=extra, - omit_data=coord.construct_type in g['omit_data'] + ncvar, + ncdimensions, + coord, + extra=extra, ) else: ncvar = seen[id(coord)]["ncvar"] @@ -1342,8 +1343,11 @@ def _write_bounds( # Create the bounds netCDF variable self._write_netcdf_variable( - ncvar, ncdimensions, bounds, omit=omit, - omit_data=coord.construct_type in g['omit_data'] + ncvar, + ncdimensions, + bounds, + omit=omit, + omit_data=coord.construct_type in g["omit_data"], ) extra["bounds"] = ncvar @@ -1504,7 +1508,7 @@ def _write_node_coordinates(self, coord, coord_ncvar, coord_ncdimensions): ncvar = self._netcdf_name(ncvar) # Create the netCDF node coordinates variable - self._write_netcdf_variable(ncvar, (ncdim,), nodes) # TODO + self._write_netcdf_variable(ncvar, (ncdim,), nodes) # TODO encodings = {} @@ -2052,8 +2056,10 @@ def _write_scalar_coordinate( # Create a new scalar coordinate variable self._write_netcdf_variable( - ncvar, (), scalar_coord, extra=bounds_extra, - omit_data=scalar_coord.construct_type in g['omit_data'] + ncvar, + (), + scalar_coord, + extra=bounds_extra, ) else: @@ -2146,8 +2152,10 @@ def _write_auxiliary_coordinate(self, f, key, coord, coordinates): # Create a new auxiliary coordinate variable, if it has data if self.implementation.get_data(coord, None) is not None: self._write_netcdf_variable( - ncvar, ncdimensions, coord, extra=extra, - omit_data=coord.construct_type in g['omit_data'] + ncvar, + ncdimensions, + coord, + extra=extra, ) g["key_to_ncvar"][key] = ncvar @@ -2227,8 +2235,7 @@ def _write_domain_ancillary(self, f, key, anc): self._write_bounds(f, anc, key, ncdimensions, ncvar) # Create a new domain ancillary variable - self._write_netcdf_variable(ncvar, ncdimensions, anc, - omit_data=anc.construct_type in g['omit_data']) + self._write_netcdf_variable(ncvar, ncdimensions, anc) g["key_to_ncvar"][key] = ncvar g["key_to_ncdims"][key] = ncdimensions @@ -2273,8 +2280,7 @@ def _write_field_ancillary(self, f, key, anc): ) # Create a new field ancillary variable - self._write_netcdf_variable(ncvar, ncdimensions, anc, - omit_data=anc.construct_type in g['omit_data']) + self._write_netcdf_variable(ncvar, ncdimensions, anc) g["key_to_ncvar"][key] = ncvar g["key_to_ncdims"][key] = ncdimensions @@ -2340,8 +2346,7 @@ def _write_cell_measure(self, field, key, cell_measure): ) # Create a new cell measure variable - self._write_netcdf_variable(ncvar, ncdimensions, cell_measure, - omit_data= cell_measure.construct_type in g['omit_data']) + self._write_netcdf_variable(ncvar, ncdimensions, cell_measure) g["key_to_ncvar"][key] = ncvar g["key_to_ncdims"][key] = ncdimensions @@ -2518,8 +2523,8 @@ def _write_netcdf_variable( extra=None, fill=False, data_variable=False, - domain_variable=False, - omit_data=False, + domain_variable=False, + omit_data=None, ): """Creates a new netCDF variable for a construct. @@ -2602,17 +2607,26 @@ def _write_netcdf_variable( logger.info(f" Writing {cfvar!r}") # pragma: no cover + # Set 'omit_data' + if omit_data is None: + omit_data = ( + self.implementation.get_construct_type(cfvar) in g["omit_data"] + ) + # ------------------------------------------------------------ # Find the fill value - the value that the variable's data get # filled before any data is written. if the fill value is # False then the variable is not pre-filled. # ------------------------------------------------------------ - if fill: # or g["post_dry_run"]: # or append mode's appending iteration + if fill or g["post_dry_run"]: # or append mode's appending iteration fill_value = self.implementation.get_property( cfvar, "_FillValue", None ) else: - fill_value = False + fill_value = None + + if omit_data: + fill_value = None if data_variable: lsd = g["least_significant_digit"] @@ -2664,8 +2678,6 @@ def _write_netcdf_variable( "least_significant_digit": lsd, "fill_value": fill_value, } -# if fill_value is not None: -# kwargs["fill_value"] = fill_value # Add compression parameters (but not for vlen strings). if kwargs["datatype"] != str: @@ -2679,7 +2691,6 @@ def _write_netcdf_variable( f" to netCDF variable: {ncvar}({', '.join(ncdimensions)})" ) # pragma: no cover - print ('fill_value=', fill_value, datatype, repr(cfvar)) try: self._createVariable(**kwargs) except RuntimeError as error: @@ -3914,7 +3925,6 @@ def _write_field_or_domain( extra=extra, data_variable=field, domain_variable=domain, - omit_data=f.construct_type in g['omit_data'] # TODO: impl ) # Update the 'seen' dictionary, if required @@ -4341,9 +4351,6 @@ def file_open(self, filename, mode, fmt, fields): except RuntimeError as error: raise RuntimeError(f"{error}: {filename}") - # No need to pre-fill with fill values - nc.set_fill_off() - return nc @_manage_log_level_via_verbosity @@ -4372,7 +4379,7 @@ def write( warn_valid=True, group=True, coordinates=False, - omit_data=(), + omit_data=None, ): """Write field and domain constructs to a netCDF file. @@ -4592,6 +4599,13 @@ def write( .. versionadded:: (cfdm) 1.8.7.0 + omit_data: (sequence of) `str`, optional + Do not write the data of the named construct types. + + See `cfdm.write` for details. + + .. versionadded:: (cfdm) 1.10.0.1 + :Returns: `None` @@ -4606,6 +4620,22 @@ def write( # Expand file name filename = os.path.expanduser(os.path.expandvars(filename)) + # Parse the omit_data parameter + if omit_data is None: + omit_data = () + elif isinstance(omit_data, str): + omit_data = (omit_data,) + + if "all" in omit_data: + omit_data = ( + "field", + "field_ancillary", + "domain_ancillary", + "auxiliary_coordinate", + "cell_measure", + "dimension_coordinate", + ) + # ------------------------------------------------------------ # Initialise netCDF write parameters # ------------------------------------------------------------ @@ -4685,7 +4715,7 @@ def write( # mode does not need to be passed to various methods, where a # pair of such keys seem clearer than one "effective mode" key. "post_dry_run": False, - # + # TODO "omit_data": omit_data, } @@ -4763,7 +4793,7 @@ def write( "Can't append fields with an incompatible 'featureType' " "global attribute to the original file." ) - + self._file_io_iteration( mode=effective_mode, overwrite=overwrite, diff --git a/cfdm/read_write/write.py b/cfdm/read_write/write.py index 9ec5ef6057..c4bf7650da 100644 --- a/cfdm/read_write/write.py +++ b/cfdm/read_write/write.py @@ -26,7 +26,7 @@ def write( warn_valid=True, group=True, coordinates=False, - omit_data=(), + omit_data=None, _implementation=_implementation, ): """Write field and domain constructs to a netCDF file. @@ -494,6 +494,42 @@ def write( .. versionadded:: (cfdm) 1.8.7.0 + omit_data: (sequence of) `str`, optional + Do not write the data of the named construct types. + + This does not affect the amount of netCDF variables and + dimensions that are written to the file, nor the netCDF + variables' attributes, but does not create data on disk + for the requested variables. The resulting file will be + smaller than it otherwise would have been, and when the + new file is read the data of these variables will be + represented by an array of all missing data. + + The *omit_data* parameter may be one, or a sequence, of: + + ========================== =============================== + *omit_data* Constructs + ========================== =============================== + ``'field'`` Field constructs + ``'field_ancillary'`` Field ancillary constructs + ``'domain_ancillary'`` Domain ancillary constructs + ``'dimension_coordinate'`` Dimension coordinate constructs + ``'auxiliary_coordinate'`` Auxiliary coordinate constructs + ``'cell_measure'`` Cell measure constructs + ``'all'`` All of the above constructs + ========================== =============================== + + *Parameter example:* + To omit the data from only field constructs: + ``omit_data='field'`` or ``omit_data=['field']``. + + *Parameter example:* + To omit the data from domain ancillary and cell measure + constucts: ``omit_data=['domain_ancillary', + 'cell_measure']``. + + .. versionadded:: (cfdm) 1.10.0.1 + _implementation: (subclass of) `CFDMImplementation`, optional Define the CF data model implementation that defines field and metadata constructs and their components. diff --git a/cfdm/test/test_read_write.py b/cfdm/test/test_read_write.py index d137067220..d389732e07 100644 --- a/cfdm/test/test_read_write.py +++ b/cfdm/test/test_read_write.py @@ -905,6 +905,36 @@ def test_read_original_filenames(self): set((cfdm.abspath(parent_file), cfdm.abspath(external_file))), ) + def test_write_omit_data(self): + """Test the `omit_data` parameter to `write`.""" + f = cfdm.example_field(1) + cfdm.write(f, tmpfile) + + cfdm.write(f, tmpfile, omit_data="all") + g = cfdm.read(tmpfile) + self.assertEqual(len(g), 1) + g = g[0] + + # Check that the data are missing + self.assertFalse(g.array.count()) + self.assertFalse(g.construct("grid_latitude").array.count()) + + cfdm.write(f, tmpfile, omit_data=("field", "dimension_coordinate")) + g = cfdm.read(tmpfile)[0] + + # Check that only the field and dimension coordinate data are + # missing + self.assertFalse(g.array.count()) + self.assertFalse(g.construct("grid_latitude").array.count()) + self.assertTrue(g.construct("latitude").array.count()) + + cfdm.write(f, tmpfile, omit_data="field") + g = cfdm.read(tmpfile)[0] + + # Check that only the field data are missing + self.assertFalse(g.array.count()) + self.assertTrue(g.construct("grid_latitude").array.count()) + if __name__ == "__main__": print("Run date:", datetime.datetime.now()) From 749c478ec46b04822c9af1b7f741d0554949255d Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 28 Oct 2022 10:24:42 +0100 Subject: [PATCH 03/10] remove not needed dev. code --- cfdm/read_write/netcdf/netcdfwrite.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/cfdm/read_write/netcdf/netcdfwrite.py b/cfdm/read_write/netcdf/netcdfwrite.py index 1be6acfe89..90253b1b45 100644 --- a/cfdm/read_write/netcdf/netcdfwrite.py +++ b/cfdm/read_write/netcdf/netcdfwrite.py @@ -229,7 +229,6 @@ def _write_attributes(self, parent, ncvar, extra=None, omit=()): # Make sure that _FillValue and missing data have the same # data type as the data - dtype = None for attr in ("_FillValue", "missing_value"): if attr not in netcdf_attrs: continue @@ -264,12 +263,7 @@ def _write_attributes(self, parent, ncvar, extra=None, omit=()): del netcdf_attrs["_FillValue"] if not g["dry_run"]: - if dtype is None: - _FillValue = netcdf_attrs.pop("_FillValue ", None) - g["nc"][ncvar].setncatts(netcdf_attrs) - if dtype is None: - netcdf_attrs["_FillValue"] = _FillValue if skip_set_fill_value: # Re-add as known attribute since this FV is already set @@ -1508,7 +1502,7 @@ def _write_node_coordinates(self, coord, coord_ncvar, coord_ncdimensions): ncvar = self._netcdf_name(ncvar) # Create the netCDF node coordinates variable - self._write_netcdf_variable(ncvar, (ncdim,), nodes) # TODO + self._write_netcdf_variable(ncvar, (ncdim,), nodes) encodings = {} @@ -4715,7 +4709,7 @@ def write( # mode does not need to be passed to various methods, where a # pair of such keys seem clearer than one "effective mode" key. "post_dry_run": False, - # TODO + # Do not write the data of the named construct types. "omit_data": omit_data, } From dfd408ceec749d15f023ede586be4a6d2fd20414 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 28 Oct 2022 10:28:29 +0100 Subject: [PATCH 04/10] docs --- cfdm/read_write/write.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cfdm/read_write/write.py b/cfdm/read_write/write.py index c4bf7650da..2b5a251ac6 100644 --- a/cfdm/read_write/write.py +++ b/cfdm/read_write/write.py @@ -502,7 +502,7 @@ def write( variables' attributes, but does not create data on disk for the requested variables. The resulting file will be smaller than it otherwise would have been, and when the - new file is read the data of these variables will be + new file is read then the data of these variables will be represented by an array of all missing data. The *omit_data* parameter may be one, or a sequence, of: From 7e8340efd7d6b27ec39b9049b02f44272d0729b6 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 28 Oct 2022 10:37:03 +0100 Subject: [PATCH 05/10] use implementation method --- cfdm/read_write/netcdf/netcdfwrite.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cfdm/read_write/netcdf/netcdfwrite.py b/cfdm/read_write/netcdf/netcdfwrite.py index 90253b1b45..398354fbdd 100644 --- a/cfdm/read_write/netcdf/netcdfwrite.py +++ b/cfdm/read_write/netcdf/netcdfwrite.py @@ -1341,7 +1341,8 @@ def _write_bounds( ncdimensions, bounds, omit=omit, - omit_data=coord.construct_type in g["omit_data"], + omit_data=self.implementation.get_construct_type(coord) + in g["omit_data"], ) extra["bounds"] = ncvar From e0d61cb40e13c05cba91e701455b45b830c903d9 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 28 Oct 2022 17:04:08 +0100 Subject: [PATCH 06/10] passed omit_data to node coords and interior ring --- cfdm/read_write/netcdf/netcdfwrite.py | 18 +++++++++++++++--- cfdm/read_write/write.py | 2 +- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/cfdm/read_write/netcdf/netcdfwrite.py b/cfdm/read_write/netcdf/netcdfwrite.py index 398354fbdd..5e1bc94d87 100644 --- a/cfdm/read_write/netcdf/netcdfwrite.py +++ b/cfdm/read_write/netcdf/netcdfwrite.py @@ -822,7 +822,7 @@ def _write_index_variable( size=self.implementation.get_data_size(index_variable), ) - # Create a new list variable + # Create a new index variable extra = {"instance_dimension": instance_dimension} self._write_netcdf_variable( ncvar, (ncdim,), index_variable, extra=extra @@ -1503,7 +1503,13 @@ def _write_node_coordinates(self, coord, coord_ncvar, coord_ncdimensions): ncvar = self._netcdf_name(ncvar) # Create the netCDF node coordinates variable - self._write_netcdf_variable(ncvar, (ncdim,), nodes) + self._write_netcdf_variable( + ncvar, + (ncdim,), + nodes, + omit_data=self.implementation.get_construct_type(coord) + in g["omit_data"], + ) encodings = {} @@ -1994,7 +2000,13 @@ def _write_interior_ring(self, coord, bounds, encodings): ncvar = self._netcdf_name(ncvar) # Create the netCDF interior ring variable - self._write_netcdf_variable(ncvar, (ncdim,), interior_ring) + self._write_netcdf_variable( + ncvar, + (ncdim,), + interior_ring, + omit_data=self.implementation.get_construct_type(coord) + in g["omit_data"], + ) g["part_ncdim"] = ncdim diff --git a/cfdm/read_write/write.py b/cfdm/read_write/write.py index 2b5a251ac6..a354ef0ebe 100644 --- a/cfdm/read_write/write.py +++ b/cfdm/read_write/write.py @@ -508,7 +508,7 @@ def write( The *omit_data* parameter may be one, or a sequence, of: ========================== =============================== - *omit_data* Constructs + *omit_data* Construct types ========================== =============================== ``'field'`` Field constructs ``'field_ancillary'`` Field ancillary constructs From 3ae74fe0916df87256843ae1ea2017f9b2b54fd5 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 28 Oct 2022 18:01:05 +0100 Subject: [PATCH 07/10] remove ncvar from Z coord in example 6 --- cfdm/examplefield.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cfdm/examplefield.py b/cfdm/examplefield.py index c5bb0d3794..7d9acfb68c 100644 --- a/cfdm/examplefield.py +++ b/cfdm/examplefield.py @@ -4677,7 +4677,6 @@ def example_field(n, _implementation=_implementation): # auxiliary_coordinate: Z c = AuxiliaryCoordinate() - c.nc_set_variable("z") c.set_geometry("polygon") b = Bounds() b.set_properties( From ff61272db9104250998f43995c0c904ced3b5858 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 28 Oct 2022 18:10:16 +0100 Subject: [PATCH 08/10] add a dump call to test_write_omit_data --- cfdm/test/test_read_write.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cfdm/test/test_read_write.py b/cfdm/test/test_read_write.py index d389732e07..2cb19d9f53 100644 --- a/cfdm/test/test_read_write.py +++ b/cfdm/test/test_read_write.py @@ -919,6 +919,9 @@ def test_write_omit_data(self): self.assertFalse(g.array.count()) self.assertFalse(g.construct("grid_latitude").array.count()) + # Check that a dump works + f.dump(display=False) + cfdm.write(f, tmpfile, omit_data=("field", "dimension_coordinate")) g = cfdm.read(tmpfile)[0] From afebbd6ee918d7ad602d702eea7aa3d4f72b578c Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 28 Oct 2022 18:13:03 +0100 Subject: [PATCH 09/10] add a dump call to test_write_omit_data --- cfdm/test/test_read_write.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cfdm/test/test_read_write.py b/cfdm/test/test_read_write.py index 2cb19d9f53..1a2009ff44 100644 --- a/cfdm/test/test_read_write.py +++ b/cfdm/test/test_read_write.py @@ -920,7 +920,7 @@ def test_write_omit_data(self): self.assertFalse(g.construct("grid_latitude").array.count()) # Check that a dump works - f.dump(display=False) + g.dump(display=False) cfdm.write(f, tmpfile, omit_data=("field", "dimension_coordinate")) g = cfdm.read(tmpfile)[0] From f580ea50888dbe1e4bdc0c32275582cd40e5beee Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 28 Oct 2022 18:13:06 +0100 Subject: [PATCH 10/10] tidy --- cfdm/read_write/netcdf/netcdfwrite.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cfdm/read_write/netcdf/netcdfwrite.py b/cfdm/read_write/netcdf/netcdfwrite.py index 5e1bc94d87..67c0eff816 100644 --- a/cfdm/read_write/netcdf/netcdfwrite.py +++ b/cfdm/read_write/netcdf/netcdfwrite.py @@ -2625,16 +2625,15 @@ def _write_netcdf_variable( # filled before any data is written. if the fill value is # False then the variable is not pre-filled. # ------------------------------------------------------------ - if fill or g["post_dry_run"]: # or append mode's appending iteration + if ( + omit_data or fill or g["post_dry_run"] + ): # or append mode's appending iteration fill_value = self.implementation.get_property( cfvar, "_FillValue", None ) else: fill_value = None - if omit_data: - fill_value = None - if data_variable: lsd = g["least_significant_digit"] else: