Skip to content

Commit

Permalink
Tweak netcdf files written by preprocessor
Browse files Browse the repository at this point in the history
Change output format from NETCDF4 to NETCDF4_CLASSIC (for backwards
compatibility; *believe* but haven't tested that this still permits
writing >2Gb output files).

If variable has a time axis, make it unlimited. Set units on the time
bounds variable, if present, to address issue noted in
pydata/xarray#2921 (comment)
  • Loading branch information
tsjackson-noaa committed Jan 18, 2021
1 parent 606dc25 commit 41b6886
Showing 1 changed file with 14 additions and 4 deletions.
18 changes: 14 additions & 4 deletions src/preprocessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -470,7 +470,7 @@ def edit_request(self, data_mgr, pod):
# arguments passed to xr.to_netcdf
save_dataset_kwargs = {
"engine": "netcdf4",
"format": "NETCDF4" # required by this choice of engine (?)
"format": "NETCDF4_CLASSIC" # NETCDF3* not supported by this engine (?)
}

def read_one_file(self, var, path_list):
Expand Down Expand Up @@ -520,12 +520,22 @@ def write_dataset(self, var, ds):
os.makedirs(os.path.dirname(var.dest_path), exist_ok=True)
_log.debug("xr.Dataset.to_netcdf on %s", var.dest_path)
ds = self.clean_encoding(ds)
if var.is_static:
unlimited_dims = []
else:
t_coord = var.T
ds_T = ds[t_coord.name]
unlimited_dims = [t_coord.name]
if 'units' in ds_T.attrs and 'units' not in ds_T.encoding:
ds_T.encoding['units'] = ds_T.attrs['units']
if t_coord.has_bounds:
ds[t_coord.bounds].encoding['units'] = ds_T.encoding['units']

ds.to_netcdf(
path=var.dest_path,
mode='w',
**self.save_dataset_kwargs
# Currently don't make time unlimited, since data might be static
# and we analyze a fixed date range
**self.save_dataset_kwargs,
unlimited_dims=unlimited_dims
)
ds.close()

Expand Down

0 comments on commit 41b6886

Please sign in to comment.