diff --git a/docs/examples/minimal_example.py b/docs/examples/minimal_example.py index b3469f0..133e121 100644 --- a/docs/examples/minimal_example.py +++ b/docs/examples/minimal_example.py @@ -10,14 +10,17 @@ #select a set of parameters # Filter the locations dataframe with the desired parameters and stations. -bool_stations = locations.index.isin(['IJMDBTHVN', 'DANTZGZD','HOEKVHLD']) -# measured (WATHTE) versus computed/astro +bool_stations = locations.index.isin(['ijmuiden.buitenhaven', 'dantziggat.zuid', 'hoekvanholland', 'ameland.nes']) +# meting/astronomisch/verwachting +bool_procestype = locations['ProcesType'].isin(['meting']) +# waterlevel/waterhoogte (WATHTE) bool_grootheid = locations['Grootheid.Code'].isin(['WATHTE']) # timeseries (NVT) versus extremes -bool_groepering = locations['Groepering.Code'].isin(['NVT']) +bool_groepering = locations['Groepering.Code'].isin(['']) # vertical reference (NAP/MSL) bool_hoedanigheid = locations['Hoedanigheid.Code'].isin(['NAP']) -selected = locations.loc[bool_stations & bool_grootheid & +selected = locations.loc[bool_procestype & + bool_stations & bool_grootheid & bool_groepering & bool_hoedanigheid] start_date = dt.datetime(2023, 1, 1) diff --git a/docs/examples/retrieve_parallel_to_netcdf.py b/docs/examples/retrieve_parallel_to_netcdf.py index 7ce5c35..fced612 100644 --- a/docs/examples/retrieve_parallel_to_netcdf.py +++ b/docs/examples/retrieve_parallel_to_netcdf.py @@ -13,7 +13,6 @@ def get_data(location, start_date, end_date, dir_output, overwrite=True): - station_id = location.name station_messageid = location["Locatie_MessageID"] filename = os.path.join(dir_output, f"{station_id}-{station_messageid}.nc") @@ -44,19 +43,18 @@ def get_data(location, start_date, end_date, dir_output, overwrite=True): ds.to_netcdf(filename, format="NETCDF4_CLASSIC") -if ( __name__ == "__main__" ): - +if __name__ == "__main__": dir_output = './ddl_retrieved_data' os.makedirs(dir_output, exist_ok=True) # get locations locations = ddlpy.locations() - bool_stations = locations.index.isin(['IJMDBTHVN', 'DANTZGZD', 'HOEKVHLD', 'VLISSGN', 'HOEK', 'VLIS', "OLST"]) - bool_grootheid = locations['Grootheid.Code'].isin(['WATHTE']) # measured (WATHTE) versus computed/astro - bool_groepering = locations['Groepering.Code'].isin(['NVT']) # timeseries (NVT) versus extremes + bool_stations = locations.index.isin(['ijmuiden.buitenhaven', 'dantziggat.zuid', 'hoekvanholland', 'ameland.nes', 'vlissingen', 'olst']) + bool_procestype = locations['ProcesType'].isin(['meting']) # meting/astronomisch/verwachting + bool_grootheid = locations['Grootheid.Code'].isin(['WATHTE']) # waterlevel (WATHTE) + bool_groepering = locations['Groepering.Code'].isin(['']) # timeseries (NVT) versus extremes bool_hoedanigheid = locations['Hoedanigheid.Code'].isin(['NAP']) # vertical reference (NAP/MSL) - selected = locations.loc[bool_stations & bool_grootheid & bool_groepering & bool_hoedanigheid] - + selected = locations.loc[bool_stations & bool_procestype & bool_grootheid & bool_groepering & bool_hoedanigheid] start_date = dt.datetime(2022, 1, 1) end_date = dt.datetime(2022, 3, 1) @@ -70,7 +68,6 @@ def get_data(location, start_date, end_date, dir_output, overwrite=True): for station_code, location in selected.iterrows(): executor.submit(get_data, location, start_date, end_date, dir_output) - file_list = glob.glob(os.path.join(dir_output, "*.nc")) fig, ax = plt.subplots() for file_nc in file_list: diff --git a/docs/notebooks/measurements.ipynb b/docs/notebooks/measurements.ipynb index 91905d0..012dbae 100644 --- a/docs/notebooks/measurements.ipynb +++ b/docs/notebooks/measurements.ipynb @@ -39,7 +39,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "DEBUG:ddlpy.ddlpy:Requesting at https://waterwebservices.rijkswaterstaat.nl/METADATASERVICES_DBO/OphalenCatalogus with request: {\"CatalogusFilter\": {\"Eenheden\": true, \"Grootheden\": true, \"Hoedanigheden\": true, \"Groeperingen\": true, \"Parameters\": true, \"Compartimenten\": true}}\n" + "DEBUG:ddlpy.ddlpy:Requesting at https://waterwebservices.beta.rijkswaterstaat.nl/test/METADATASERVICES/OphalenCatalogus with request: {\"CatalogusFilter\": {\"Eenheden\": true, \"Grootheden\": true, \"Hoedanigheden\": true, \"Groeperingen\": true, \"Parameters\": true, \"ProcesTypes\": true, \"Compartimenten\": true}}\n" ] } ], @@ -83,17 +83,19 @@ "
5 rows × 21 columns
3 rows × 21 columns
2 rows × 21 columns
39739 rows × 53 columns
39745 rows × 48 columns
<xarray.Dataset> Size: 2MB\n", - "Dimensions: (time: 39739)\n", + "<xarray.Dataset> Size: 3MB\n", + "Dimensions: (time: 39745)\n", "Coordinates:\n", - " * time (time) datetime64[ns] 318kB ...\n", + " * time (time) datetime64[ns] 318kB 2019...\n", "Data variables:\n", - " WaarnemingMetadata.StatuswaardeLijst (time) object 318kB 'Gecont...\n", - " WaarnemingMetadata.KwaliteitswaardecodeLijst (time) object 318kB '00' .....\n", - " MeetApparaat.Code (time) object 318kB '127' ....\n", - " WaardeBepalingsmethode.Code (time) object 318kB 'other:...\n", - " Meetwaarde.Waarde_Numeriek (time) float64 318kB 44.0 ....\n", - "Attributes: (12/27)\n", - " BemonsteringsApparaat.Code: NVT\n", - " BemonsteringsMethode.Code: NVT\n", - " BioTaxon.Code: NVT\n", - " BioTaxon_Compartiment.Code: NVT\n", - " MonsterBewerkingsMethode.Code: NVT\n", - " Orgaan.Code: NVT\n", - " ... ...\n", - " Hoedanigheid.Code: NAP\n", - " Code: SCHEVNGN\n", - " Naam: Scheveningen\n", - " Coordinatenstelsel: 25831\n", - " X: 586550.994420996\n", - " Y: 5772806.43069697xarray.DatasetDimensions:time: 39739Coordinates: (1)time(time)datetime64[ns]2019-05-01T01:00:00 ... 2020-02-01array(['2019-05-01T01:00:00.000000000', '2019-05-01T01:10:00.000000000',\n", - " '2019-05-01T01:20:00.000000000', ..., '2020-01-31T23:40:00.000000000',\n", + " WaarnemingMetadata.Statuswaarde (time) object 318kB 'Ongecontrol...\n", + " WaarnemingMetadata.Kwaliteitswaardecode (time) object 318kB '00' ... '00'\n", + " Groepering.Code (time) object 318kB '' '' ... '' ''\n", + " BemonsteringsApparaat.Code (time) object 318kB '' '' ... '' ''\n", + " BemonsteringsMethode.Code (time) object 318kB '' '' ... '' ''\n", + " MeetApparaat.Code (time) object 318kB '8000' ... '...\n", + " WaardeBepalingsMethode.Code (time) object 318kB 'other:F012'...\n", + " WaardeBepalingsTechniek.Code (time) object 318kB '' '' ... '' ''\n", + " ProcesType (time) object 318kB 'astronomisc...\n", + " Meetwaarde.Waarde_Numeriek (time) float64 318kB -103.0 ... ...\n", + "Attributes: (12/19)\n", + " Parameter.Code: NVT\n", + " BioTaxon.Code: NVT\n", + " Orgaan.Code: NVT\n", + " Typering.Code: NVT\n", + " WaardeBewerkingsMethode.Code: NVT\n", + " WaarnemingMetadata.OpdrachtgevendeInstantie: RIKZMON_WAT\n", + " ... ...\n", + " Hoedanigheid.Code: NAP\n", + " Code: ameland.nes\n", + " Naam: Ameland, Nes\n", + " Coordinatenstelsel: ETRS89\n", + " Lon: 5.759448\n", + " Lat: 53.429766xarray.DatasetDimensions:time: 39745Coordinates: (1)time(time)datetime64[ns]2019-05-01 ... 2020-02-01array(['2019-05-01T00:00:00.000000000', '2019-05-01T00:10:00.000000000',\n", + " '2019-05-01T00:20:00.000000000', ..., '2020-01-31T23:40:00.000000000',\n", " '2020-01-31T23:50:00.000000000', '2020-02-01T00:00:00.000000000'],\n", - " dtype='datetime64[ns]')Data variables: (5)WaarnemingMetadata.StatuswaardeLijst(time)object'Gecontroleerd' ... 'Gecontroleerd'array(['Gecontroleerd', 'Gecontroleerd', 'Gecontroleerd', ...,\n", - " 'Gecontroleerd', 'Gecontroleerd', 'Gecontroleerd'], dtype=object)WaarnemingMetadata.KwaliteitswaardecodeLijst(time)object'00' '00' '00' ... '00' '00' '00'array(['00', '00', '00', ..., '00', '00', '00'], dtype=object)MeetApparaat.Code(time)object'127' '127' '127' ... '127' '127'127 :Vlotterarray(['127', '127', '127', ..., '127', '127', '127'], dtype=object)WaardeBepalingsmethode.Code(time)object'other:F007' ... 'other:F007'other:F007 :Rekenkundig gemiddelde waarde over vorige 5 en volgende 5 minutenarray(['other:F007', 'other:F007', 'other:F007', ..., 'other:F007',\n", - " 'other:F007', 'other:F007'], dtype=object)Meetwaarde.Waarde_Numeriek(time)float6444.0 41.0 36.0 ... -22.0 -26.0array([ 44., 41., 36., ..., -23., -22., -26.])Indexes: (1)timePandasIndexPandasIndex(DatetimeIndex(['2019-05-01 01:00:00', '2019-05-01 01:10:00',\n", + " dtype='datetime64[ns]')Data variables: (10)WaarnemingMetadata.Statuswaarde(time)object'Ongecontroleerd' ... 'Ongecontr...array(['Ongecontroleerd', 'Ongecontroleerd', 'Ongecontroleerd', ...,\n", + " 'Ongecontroleerd', 'Ongecontroleerd', 'Ongecontroleerd'],\n", + " dtype=object)WaarnemingMetadata.Kwaliteitswaardecode(time)object'00' '00' '00' ... '00' '00' '00'array(['00', '00', '00', ..., '00', '00', '00'], dtype=object)Groepering.Code(time)object'' '' '' '' '' ... '' '' '' '' '' :array(['', '', '', ..., '', '', ''], dtype=object)BemonsteringsApparaat.Code(time)object'' '' '' '' '' ... '' '' '' '' '' :array(['', '', '', ..., '', '', ''], dtype=object)BemonsteringsMethode.Code(time)object'' '' '' '' '' ... '' '' '' '' '' :array(['', '', '', ..., '', '', ''], dtype=object)MeetApparaat.Code(time)object'8000' '8000' ... '8000' '8000'8000 :NVTarray(['8000', '8000', '8000', ..., '8000', '8000', '8000'], dtype=object)WaardeBepalingsMethode.Code(time)object'other:F012' ... 'other:F012'other:F012 :Astronomische waterhoogte mbv harmonische analysearray(['other:F012', 'other:F012', 'other:F012', ..., 'other:F012',\n", + " 'other:F012', 'other:F012'], dtype=object)WaardeBepalingsTechniek.Code(time)object'' '' '' '' '' ... '' '' '' '' '' :array(['', '', '', ..., '', '', ''], dtype=object)ProcesType(time)object'astronomisch' ... 'astronomisch'array(['astronomisch', 'astronomisch', 'astronomisch', ...,\n", + " 'astronomisch', 'astronomisch', 'astronomisch'], dtype=object)Meetwaarde.Waarde_Numeriek(time)float64-103.0 -106.0 -108.0 ... 93.0 96.0array([-103., -106., -108., ..., 89., 93., 96.])Indexes: (1)timePandasIndexPandasIndex(DatetimeIndex(['2019-05-01 00:00:00', '2019-05-01 00:10:00',\n", + " '2019-05-01 00:20:00', '2019-05-01 00:30:00',\n", + " '2019-05-01 00:40:00', '2019-05-01 00:50:00',\n", + " '2019-05-01 01:00:00', '2019-05-01 01:10:00',\n", " '2019-05-01 01:20:00', '2019-05-01 01:30:00',\n", - " '2019-05-01 01:40:00', '2019-05-01 01:50:00',\n", - " '2019-05-01 02:00:00', '2019-05-01 02:10:00',\n", - " '2019-05-01 02:20:00', '2019-05-01 02:30:00',\n", " ...\n", " '2020-01-31 22:30:00', '2020-01-31 22:40:00',\n", " '2020-01-31 22:50:00', '2020-01-31 23:00:00',\n", " '2020-01-31 23:10:00', '2020-01-31 23:20:00',\n", " '2020-01-31 23:30:00', '2020-01-31 23:40:00',\n", " '2020-01-31 23:50:00', '2020-02-01 00:00:00'],\n", - " dtype='datetime64[ns]', name='time', length=39739, freq=None))Attributes: (27)BemonsteringsApparaat.Code :NVTBemonsteringsMethode.Code :NVTBioTaxon.Code :NVTBioTaxon_Compartiment.Code :NVTMonsterBewerkingsMethode.Code :NVTOrgaan.Code :NVTParameter.Code :NVTPlaatsBepalingsApparaat.Code :NVTTypering.Code :NVTGroepering.Code :NVTWaardeBepalingstechniek.Code :NVTWaardeBewerkingsmethode.Code :NVTWaarnemingMetadata.OpdrachtgevendeInstantieLijst :RIKZMON_WATWaarnemingMetadata.BemonsteringshoogteLijst :-999999999WaarnemingMetadata.ReferentievlakLijst :NVTAquoMetadata_MessageID :2573BioTaxonType :NVTBemonsteringsSoort.Code :SBCompartiment.Code :OWEenheid.Code :cmGrootheid.Code :WATHTEHoedanigheid.Code :NAPCode :SCHEVNGNNaam :ScheveningenCoordinatenstelsel :25831X :586550.994420996Y :5772806.43069697" + " dtype='datetime64[ns]', name='time', length=39745, freq=None))
<xarray.Dataset> Size: 3MB\n", + "Dimensions: (time: 39745)\n", "Coordinates:\n", - " * time (time) datetime64[ns] 318kB ...\n", + " * time (time) datetime64[ns] 318kB 2019...\n", "Data variables:\n", - " WaarnemingMetadata.StatuswaardeLijst (time) object 318kB 'Gecont...\n", - " WaarnemingMetadata.KwaliteitswaardecodeLijst (time) object 318kB '00' .....\n", - " MeetApparaat.Code (time) object 318kB '127' ....\n", - " WaardeBepalingsmethode.Code (time) object 318kB 'other:...\n", - " Meetwaarde.Waarde_Numeriek (time) float64 318kB 44.0 ....\n", - "Attributes: (12/27)\n", - " BemonsteringsApparaat.Code: NVT\n", - " BemonsteringsMethode.Code: NVT\n", - " BioTaxon.Code: NVT\n", - " BioTaxon_Compartiment.Code: NVT\n", - " MonsterBewerkingsMethode.Code: NVT\n", - " Orgaan.Code: NVT\n", - " ... ...\n", - " Hoedanigheid.Code: NAP\n", - " Code: SCHEVNGN\n", - " Naam: Scheveningen\n", - " Coordinatenstelsel: 25831\n", - " X: 586550.994420996\n", - " Y: 5772806.43069697
array(['2019-05-01T01:00:00.000000000', '2019-05-01T01:10:00.000000000',\n", - " '2019-05-01T01:20:00.000000000', ..., '2020-01-31T23:40:00.000000000',\n", + " WaarnemingMetadata.Statuswaarde (time) object 318kB 'Ongecontrol...\n", + " WaarnemingMetadata.Kwaliteitswaardecode (time) object 318kB '00' ... '00'\n", + " Groepering.Code (time) object 318kB '' '' ... '' ''\n", + " BemonsteringsApparaat.Code (time) object 318kB '' '' ... '' ''\n", + " BemonsteringsMethode.Code (time) object 318kB '' '' ... '' ''\n", + " MeetApparaat.Code (time) object 318kB '8000' ... '...\n", + " WaardeBepalingsMethode.Code (time) object 318kB 'other:F012'...\n", + " WaardeBepalingsTechniek.Code (time) object 318kB '' '' ... '' ''\n", + " ProcesType (time) object 318kB 'astronomisc...\n", + " Meetwaarde.Waarde_Numeriek (time) float64 318kB -103.0 ... ...\n", + "Attributes: (12/19)\n", + " Parameter.Code: NVT\n", + " BioTaxon.Code: NVT\n", + " Orgaan.Code: NVT\n", + " Typering.Code: NVT\n", + " WaardeBewerkingsMethode.Code: NVT\n", + " WaarnemingMetadata.OpdrachtgevendeInstantie: RIKZMON_WAT\n", + " ... ...\n", + " Hoedanigheid.Code: NAP\n", + " Code: ameland.nes\n", + " Naam: Ameland, Nes\n", + " Coordinatenstelsel: ETRS89\n", + " Lon: 5.759448\n", + " Lat: 53.429766
array(['2019-05-01T00:00:00.000000000', '2019-05-01T00:10:00.000000000',\n", + " '2019-05-01T00:20:00.000000000', ..., '2020-01-31T23:40:00.000000000',\n", " '2020-01-31T23:50:00.000000000', '2020-02-01T00:00:00.000000000'],\n", - " dtype='datetime64[ns]')
array(['Gecontroleerd', 'Gecontroleerd', 'Gecontroleerd', ...,\n", - " 'Gecontroleerd', 'Gecontroleerd', 'Gecontroleerd'], dtype=object)
array(['00', '00', '00', ..., '00', '00', '00'], dtype=object)
array(['127', '127', '127', ..., '127', '127', '127'], dtype=object)
array(['other:F007', 'other:F007', 'other:F007', ..., 'other:F007',\n", - " 'other:F007', 'other:F007'], dtype=object)
array([ 44., 41., 36., ..., -23., -22., -26.])
PandasIndex(DatetimeIndex(['2019-05-01 01:00:00', '2019-05-01 01:10:00',\n", + " dtype='datetime64[ns]')
array(['Ongecontroleerd', 'Ongecontroleerd', 'Ongecontroleerd', ...,\n", + " 'Ongecontroleerd', 'Ongecontroleerd', 'Ongecontroleerd'],\n", + " dtype=object)
array(['', '', '', ..., '', '', ''], dtype=object)
array(['8000', '8000', '8000', ..., '8000', '8000', '8000'], dtype=object)
array(['other:F012', 'other:F012', 'other:F012', ..., 'other:F012',\n", + " 'other:F012', 'other:F012'], dtype=object)
array(['astronomisch', 'astronomisch', 'astronomisch', ...,\n", + " 'astronomisch', 'astronomisch', 'astronomisch'], dtype=object)
array([-103., -106., -108., ..., 89., 93., 96.])
PandasIndex(DatetimeIndex(['2019-05-01 00:00:00', '2019-05-01 00:10:00',\n", + " '2019-05-01 00:20:00', '2019-05-01 00:30:00',\n", + " '2019-05-01 00:40:00', '2019-05-01 00:50:00',\n", + " '2019-05-01 01:00:00', '2019-05-01 01:10:00',\n", " '2019-05-01 01:20:00', '2019-05-01 01:30:00',\n", - " '2019-05-01 01:40:00', '2019-05-01 01:50:00',\n", - " '2019-05-01 02:00:00', '2019-05-01 02:10:00',\n", - " '2019-05-01 02:20:00', '2019-05-01 02:30:00',\n", " ...\n", " '2020-01-31 22:30:00', '2020-01-31 22:40:00',\n", " '2020-01-31 22:50:00', '2020-01-31 23:00:00',\n", " '2020-01-31 23:10:00', '2020-01-31 23:20:00',\n", " '2020-01-31 23:30:00', '2020-01-31 23:40:00',\n", " '2020-01-31 23:50:00', '2020-02-01 00:00:00'],\n", - " dtype='datetime64[ns]', name='time', length=39739, freq=None))