Skip to content

Commit

Permalink
ENH: Fill Value
Browse files Browse the repository at this point in the history
  • Loading branch information
@jklymak
jklymak committed Apr 29, 2024
1 parent 88e673c commit f49bd2c
Show file tree
Hide file tree
Showing 4 changed files with 64 additions and 60 deletions.
7 changes: 4 additions & 3 deletions pyglider/ncprocess.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,16 +114,17 @@ def extract_timeseries_profiles(inname, outdir, deploymentyaml):
'salinity', 'density', 'lon', 'lat', 'depth']
for name in to_fill:
dss[name].attrs['ancillary_variables'] = name + '_qc'

# outname = outdir + '/' + utils.get_file_id(dss) + '.nc'
_log.info('Writing %s', outname)
timeunits = 'nanoseconds since 1970-01-01T00:00:00Z'
timecalendar = 'gregorian'
dss.to_netcdf(outname, encoding={'time': {'units': timeunits,
'calendar': timecalendar},
'calendar': timecalendar,
'dtype': 'float64'},
'profile_time':
{'units': timeunits,
'_FillValue': -99999}}
'_FillValue': -99999.0,
'dtype': 'float64'}}
)

# add traj_strlen using bare ntcdf to make IOOS happy
Expand Down
4 changes: 3 additions & 1 deletion pyglider/slocum.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -944,7 +944,9 @@ def binary_to_timeseries(indir, cachedir, outdir, deploymentyaml, *,
outname = (outdir + '/' + ds.attrs['deployment_name'] + fnamesuffix + '.nc')
_log.info('writing %s', outname)
ds.to_netcdf(outname, 'w',
encoding={'time': {'units': 'seconds since 1970-01-01T00:00:00Z', '_FillValue': -999999}})
encoding={'time': {'units': 'seconds since 1970-01-01T00:00:00Z',
'_FillValue': -999999,
'dtype': 'int64'}})

return outname

Expand Down
47 changes: 24 additions & 23 deletions pyglider/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,9 +69,10 @@ def get_glider_depth(ds):
'comment': 'from science pressure and interpolated',
'instrument': 'instrument_ctd',
'observation_type': 'calulated',
'accuracy': '1', 'precision': '2', 'resolution': '0.02',
'accuracy': 1.0,
'precision': 2.0, 'resolution': 0.02,
'platform': 'platform',
'valid_min': '0', 'valid_max': '2000',
'valid_min': 0.0, 'valid_max': 2000.0,
'reference_datum': 'surface', 'positive': 'down'}
ds['depth'].attrs = attr
return ds
Expand Down Expand Up @@ -300,11 +301,11 @@ def get_derived_eos_raw(ds):
('method', 'get_derived_eos_raw'),
('observation_type', 'calulated'),
('instrument', 'instrument_ctd'),
('valid_max', '40.0'),
('valid_min', '0.0'),
('accuracy', '0.01'),
('precision', '0.01'),
('resolution', '0.001')])
('valid_max', 40.0),
('valid_min', 0.0),
('accuracy', 0.01),
('precision', 0.01),
('resolution', 0.001)])
attrs = fill_required_attrs(attrs)
ds['salinity'].attrs = attrs
sa = gsw.SA_from_SP(ds['salinity'], ds['pressure'], ds['longitude'],
Expand All @@ -320,9 +321,9 @@ def get_derived_eos_raw(ds):
('method', 'get_derived_eos_raw'),
('observation_type', 'calulated'),
('instrument', 'instrument_ctd'),
('accuracy', '0.01'),
('precision', '0.01'),
('resolution', '0.001')
('accuracy', 0.01),
('precision', 0.01),
('resolution', 0.001)
])
attrs = fill_required_attrs(attrs)
ds['potential_density'].attrs = attrs
Expand All @@ -338,11 +339,11 @@ def get_derived_eos_raw(ds):
('sources', 'salinity temperature pressure'),
('instrument', 'instrument_ctd'),
('method', 'get_derived_eos_raw'),
('valid_min', '1000.0'),
('valid_max', '1040.0'),
('accuracy', '0.01'),
('precision', '0.01'),
('resolution', '0.001')
('valid_min', 990.0),
('valid_max', 1040.0),
('accuracy', 0.01),
('precision', 0.01),
('resolution', 0.001)
])
attrs = fill_required_attrs(attrs)
ds['density'].attrs = attrs
Expand All @@ -357,9 +358,9 @@ def get_derived_eos_raw(ds):
('observation_type', 'calulated'),
('method', 'get_derived_eos_raw'),
('instrument', 'instrument_ctd'),
('accuracy', '0.002'),
('precision', '0.001'),
('resolution', '0.0001')
('accuracy', 0.002),
('precision', 0.001),
('resolution', 0.0001)
])
attrs = fill_required_attrs(attrs)
ds['potential_temperature'].attrs = attrs
Expand Down Expand Up @@ -576,16 +577,16 @@ def find_gaps(sample_time, timebase, maxgap):
# figure out which sample each time in time base belongs to:
time_index = np.searchsorted(sample_time, timebase, side='right')
time_index = np.clip(time_index, 0, len(sample_time)-1)

# figure out the space between sample pairs
dt = np.concatenate(([0], np.diff(sample_time)))
# get the gap size for each timebase data point:
ddt = dt[time_index]
# get the indices of timebase that are too large and account for the
# degenerate case when a timebase point falls directly on a sample time.

# get the indices of timebase that are too large and account for the
# degenerate case when a timebase point falls directly on a sample time.
index = ~np.logical_or((ddt <= maxgap),(np.isin(timebase,sample_time)))

return index

def _parse_gliderxml_pos(fname):
Expand Down
66 changes: 33 additions & 33 deletions tests/example-data/example-slocum/deploymentRealtime.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,8 +101,8 @@ netcdf_variables:
observation_type: measured
platform: platform
reference: WGS84
valid_max: "90.0"
valid_min: "-90.0"
valid_max: 90.0
valid_min: -90.0
coordinate_reference_frame: urn:ogc:crs:EPSG::4326

longitude:
Expand All @@ -116,8 +116,8 @@ netcdf_variables:
observation_type: measured
platform: platform
reference: WGS84
valid_max: "180.0"
valid_min: "-180.0"
valid_max: 180.0
valid_min: -180.0
coordinate_reference_frame: urn:ogc:crs:EPSG::4326

heading:
Expand Down Expand Up @@ -164,12 +164,12 @@ netcdf_variables:
units: S m-1
coordinates: time depth latitude longitude
instrument: instrument_ctd
valid_min: "0."
valid_max: "10."
valid_min: 0.
valid_max: 10.
observation_type: "measured"
accuracy: "0.0003"
precision: "0.0001"
resolution: "0.00002"
accuracy: 0.0003
precision: 0.0001
resolution: 0.00002

temperature:
source: sci_water_temp
Expand All @@ -178,12 +178,12 @@ netcdf_variables:
units: Celsius
coordinates: time depth latitude longitude
instrument: instrument_ctd
valid_min: "-5"
valid_max: "50"
valid_min: -5.0
valid_max: 50.0
observation_type: "measured"
accuracy: "0.002"
precision: "0.001"
resolution: "0.0002"
accuracy: 0.002
precision: 0.001
resolution: 0.0002

pressure:
source: sci_water_pressure
Expand All @@ -192,15 +192,15 @@ netcdf_variables:
units: dbar
coordinates: time depth latitude longitude
conversion: bar2dbar
valid_min: "0"
valid_max: "2000"
valid_min: 0.0
valid_max: 2000.0
positive: "down"
reference_datum: "sea-surface"
instrument: "instrument_ctd"
observation_type: "measured"
accuracy: "1"
precision: "2"
resolution: "0.02"
accuracy: 1.0
precision: 2.0
resolution: 0.02
comment: "ctd pressure sensor"


Expand Down Expand Up @@ -252,8 +252,8 @@ profile_variables:
profile_id:
comment: Sequential profile number within the trajectory. This value is unique in each file that is part of a single trajectory/deployment.
long_name: 'Profile ID'
valid_max: '2147483647'
valid_min: '1'
valid_max: 2147483647.0
valid_min: 1.0

profile_time:
comment: Timestamp corresponding to the mid-point of the profile
Expand Down Expand Up @@ -283,8 +283,8 @@ profile_variables:
platform: platform
standard_name: latitude
units: degrees_north
valid_max: "90.0"
valid_min: "-90.0"
valid_max: 90.0
valid_min: -90.0

profile_lon:
comment: Value is interpolated to provide an estimate of the latitude at the mid-point of the profile
Expand All @@ -293,8 +293,8 @@ profile_variables:
platform: platform
standard_name: longitude
units: degrees_east
valid_max: "180.0"
valid_min: "-180.0"
valid_max: 180.0
valid_min: -180.0

u:
comment: The depth-averaged current is an estimate of the net current measured while the glider is underwater. The value is calculated over the entire underwater segment, which may consist of 1 or more dives.
Expand All @@ -303,8 +303,8 @@ profile_variables:
platform: platform
standard_name: eastward_sea_water_velocity
units: m s-1
valid_max: "10.0"
valid_min: "-10.0"
valid_max: 10.0
valid_min: -10.0

v:
comment: The depth-averaged current is an estimate of the net current measured while the glider is underwater. The value is calculated over the entire underwater segment, which may consist of 1 or more dives.
Expand All @@ -313,8 +313,8 @@ profile_variables:
platform: platform
standard_name: northward_sea_water_velocity
units: m s-1
valid_max: "10.0"
valid_min: "-10.0"
valid_max: 10.0
valid_min: -10.0

lon_uv:
comment: Not computed
Expand All @@ -323,8 +323,8 @@ profile_variables:
platform: platform
standard_name: longitude
units: degrees_east
valid_max: "180.0"
valid_min: "-180.0"
valid_max: 180.0
valid_min: -180.0

lat_uv:
comment: Not computed
Expand All @@ -333,8 +333,8 @@ profile_variables:
platform: platform
standard_name: latitude
units: degrees_north
valid_max: "90.0"
valid_min: "-90.0"
valid_max: 90.0
valid_min: -90.0

time_uv:
comment: Not computed
Expand Down

0 comments on commit f49bd2c

Please sign in to comment.