Skip to content

Commit

Permalink
Merge pull request #392 from GuillaumeVandekerckhove/all-wfs-layers
Browse files Browse the repository at this point in the history 
Combine PFAS data of the same matrix
  • Loading branch information
@Roel
Roel authored Aug 29, 2023
2 parents 24960cf + e022a7a commit 23d97d1
Show file tree
Hide file tree
Showing 3 changed files with 832 additions and 311 deletions.
232 changes: 229 additions & 3 deletions contrib/PFAS_concentrations/PFAS_concentrations.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ class RequestPFASdata:
def __init__(self):
"""Initialize the class.
Create a metadata file that contains the date and necessary package versions.
Create a metadata file that contains the date, necessary package versions and the datapoints count.
"""

def json_serial(obj):
Expand All @@ -44,6 +44,7 @@ def json_serial(obj):
package_versions = (f'pandas: {version("pandas")}', f'pydov: {version("pydov")}')

self.dictionary = {"date": date, "versions": package_versions, "nb_datapoints": [{}]}
self.combined_datasets = {}

def wfs_request(self, layer, location, max_features, query=None, sort_by=None):
"""Download the available PFAS-data through a wfs request.
Expand Down Expand Up @@ -249,6 +250,9 @@ def groundwater(self, location, max_features):
data_wfs_Lantis_gw = data_wfs_Lantis_gw.drop_duplicates(
subset=data_wfs_Lantis_gw.columns)

combined_groundwater = {'Groundwater': [data_pydov_VMM_gw, data_wfs_OVAM, data_wfs_Lantis_gw]}
self.combined_datasets.update(combined_groundwater)

data_pydov_VMM_gw_len = len(data_pydov_VMM_gw)
data_wfs_OVAM_len = len(data_wfs_OVAM)
data_wfs_Lantis_gw_len = len(data_wfs_Lantis_gw)
Expand Down Expand Up @@ -436,6 +440,9 @@ def soil(self, location, max_features):
data_wfs_Lantis_soil = data_wfs_Lantis_soil.drop_duplicates(
subset=data_wfs_Lantis_soil.columns)

combined_soil = {'Soil': [data_wfs_OVAM, data_wfs_Lantis_soil]}
self.combined_datasets.update(combined_soil)

data_wfs_OVAM_len = len(data_wfs_OVAM)
data_wfs_Lantis_soil_len = len(data_wfs_Lantis_soil)

Expand Down Expand Up @@ -489,6 +496,9 @@ def soil_water(self, location, max_features):
data_wfs_OVAM_sediment = data_wfs_OVAM[data_wfs_OVAM['medium'] == 'Waterbodem - sediment']
data_wfs_OVAM_fixed = data_wfs_OVAM[data_wfs_OVAM['medium'] == 'Waterbodem - vaste deel van waterbodem']

combined_soil_water = {'Soil_water': [data_wfs_VMM_ws, data_wfs_OVAM_sediment, data_wfs_OVAM_fixed]}
self.combined_datasets.update(combined_soil_water)

data_wfs_VMM_ws_len = len(data_wfs_VMM_ws)
data_wfs_OVAM_sediment_len = len(data_wfs_OVAM_sediment)
data_wfs_OVAM_fixed_len = len(data_wfs_OVAM_fixed)
Expand Down Expand Up @@ -541,10 +551,12 @@ def surface_water(self, location, max_features):
data_wfs_OVAM = data_wfs_OVAM.drop_duplicates(
subset=data_wfs_OVAM.columns)

combined_surface_water = {'Surface_water': [data_wfs_VMM_sw, data_wfs_OVAM]}
self.combined_datasets.update(combined_surface_water)

data_wfs_VMM_sw_len = len(data_wfs_VMM_sw)
data_wfs_OVAM_len = len(data_wfs_OVAM)


nb_datapoints = {"Surface_water_VMM" : data_wfs_VMM_sw_len}
self.dictionary["nb_datapoints"][0].update(nb_datapoints)
nb_datapoints = {"Surface_water_OVAM": data_wfs_OVAM_len}
Expand Down Expand Up @@ -593,6 +605,180 @@ def waste_water(self, location, max_features, query=None, sort_by=None):

return data_wfs_VMM_ww

def combined_groundwater(self, location, max_features):
"""
Download the groundwater data and combine it in one dataframe.
Parameters
----------
location:
Query on location.
(https://pydov.readthedocs.io/en/stable/query_location.html)
max_features: int
Limit the number of WFS features you want to be returned.
(https://pydov.readthedocs.io/en/stable/sort_limit.html)
Returns
-------
The downloaded and combined groundwater data.
"""
if 'Groundwater' in self.combined_datasets:
gw_datasets = self.combined_datasets['Groundwater']
gw_VMM = gw_datasets[0]
gw_OVAM = gw_datasets[1]
gw_Lantis = gw_datasets[2]
else:
gw_VMM, gw_OVAM, gw_Lantis = RequestPFASdata().groundwater(location, max_features)

gw_VMM['top_m_mv'] = gw_VMM['diepte_onderkant_filter']-gw_VMM['lengte_filter']

gw_VMM = gw_VMM.rename(columns={'grondwatermonsternummer': 'id', 'datum_monstername': 'datum', 'x': 'x_m_L72', 'y': 'y_m_L72', 'detectie': 'detectieconditie', 'waarde': 'meetwaarde', 'eenheid': 'meeteenheid', 'diepte_onderkant_filter': 'basis_m_mv'})
gw_OVAM = gw_OVAM.rename(columns={'top_in_m': 'top_m_mv', 'basis_in_m': 'basis_m_mv', 'x_ml72': 'x_m_L72', 'y_ml72': 'y_m_L72'})
gw_Lantis = gw_Lantis.rename(columns={'filter_van_m': 'top_m_mv', 'filter_tot_m': 'basis_m_mv', 'analysemonster': 'id', 'datum_bemonstering': 'datum', 'waarde': 'meetwaarde', 'eenheid': 'meeteenheid', 'x_ml72': 'x_m_L72', 'y_ml72': 'y_m_L72'})

gw_VMM['bron'] = 'VMM'
gw_OVAM['bron'] = 'OVAM'
gw_Lantis['bron'] = 'Lantis'

gw_VMM = gw_VMM[['id', 'datum', 'x_m_L72', 'y_m_L72', 'top_m_mv', 'basis_m_mv', 'parameter', 'detectieconditie', 'meetwaarde', 'meeteenheid', 'bron']]
gw_OVAM = gw_OVAM[['id', 'datum', 'x_m_L72', 'y_m_L72', 'top_m_mv', 'basis_m_mv', 'parameter', 'detectieconditie', 'meetwaarde', 'meeteenheid', 'bron']]
gw_Lantis = gw_Lantis[['id', 'datum', 'x_m_L72', 'y_m_L72', 'top_m_mv', 'basis_m_mv', 'parameter', 'detectieconditie', 'meetwaarde', 'meeteenheid', 'bron']]

groundwater = pd.concat([gw_VMM, gw_OVAM, gw_Lantis])

data_groundwater_len = len(groundwater)
nb_datapoints = {"Combined_groundwater": data_groundwater_len}
self.dictionary["nb_datapoints"][0].update(nb_datapoints)

return groundwater

def combined_soil(self, location, max_features):
"""
Download the soil data and combine it in one dataframe.
Parameters
----------
location:
Query on location.
(https://pydov.readthedocs.io/en/stable/query_location.html)
max_features: int
Limit the number of WFS features you want to be returned.
(https://pydov.readthedocs.io/en/stable/sort_limit.html)
Returns
-------
The downloaded and combined soil data.
"""
if 'Soil' in self.combined_datasets:
soil_datasets = self.combined_datasets['Soil']
soil_OVAM = soil_datasets[0]
soil_Lantis = soil_datasets[1]
else:
soil_OVAM, soil_Lantis = RequestPFASdata().soil(location, max_features)

soil_OVAM = soil_OVAM.rename(columns={'top_in_m': 'top_m_mv', 'basis_in_m': 'basis_m_mv', 'x_ml72': 'x_m_L72', 'y_ml72': 'y_m_L72'})
soil_Lantis = soil_Lantis.rename(columns={'analysemonster': 'id', 'diepte_van_m': 'top_m_mv', 'diepte_tot_m': 'basis_m_mv', 'datum_bemonstering': 'datum', 'waarde': 'meetwaarde', 'eenheid': 'meeteenheid', 'x_ml72': 'x_m_L72', 'y_ml72': 'y_m_L72'})

soil_OVAM['bron'] = 'OVAM'
soil_Lantis['bron'] = 'Lantis'

soil_OVAM = soil_OVAM[['id', 'datum', 'x_m_L72', 'y_m_L72', 'top_m_mv', 'basis_m_mv', 'parameter', 'detectieconditie', 'meetwaarde', 'meeteenheid', 'bron']]
soil_Lantis = soil_Lantis[['id', 'datum', 'x_m_L72', 'y_m_L72', 'top_m_mv', 'basis_m_mv', 'parameter', 'detectieconditie', 'meetwaarde', 'meeteenheid', 'bron']]

soil = pd.concat([soil_OVAM, soil_Lantis])

data_soil_len = len(soil)
nb_datapoints = {"Combined_soil": data_soil_len}
self.dictionary["nb_datapoints"][0].update(nb_datapoints)

return soil

def combined_soil_water(self, location, max_features):
"""
Download the soilwater data and combine it in one dataframe.
Parameters
----------
location:
Query on location.
(https://pydov.readthedocs.io/en/stable/query_location.html)
max_features: int
Limit the number of WFS features you want to be returned.
(https://pydov.readthedocs.io/en/stable/sort_limit.html)
Returns
-------
The downloaded and combined soilwater data.
"""
if 'Soil_water' in self.combined_datasets:
soilwater_datasets = self.combined_datasets['Soil_water']
soilwater_VMM = soilwater_datasets[0]
soilwater_OVAM_sediment = soilwater_datasets[1]
soilwater_OVAM_fixed = soilwater_datasets[2]
else:
soilwater_VMM, soilwater_OVAM_sediment, soilwater_OVAM_fixed = RequestPFASdata().soil_water(location, max_features)

soilwater_VMM = soilwater_VMM.rename(columns={'VHA_code': 'id', 'X': 'x_m_L72', 'Y': 'y_m_L72'})
soilwater_OVAM_sediment = soilwater_OVAM_sediment.rename(columns={'x_ml72': 'x_m_L72', 'y_ml72': 'y_m_L72'})
soilwater_OVAM_fixed = soilwater_OVAM_fixed.rename(columns={'x_ml72': 'x_m_L72', 'y_ml72': 'y_m_L72'})

soilwater_VMM['bron'] = 'VMM'
soilwater_OVAM_sediment['bron'] = 'OVAM_sediment'
soilwater_OVAM_fixed['bron'] = 'OVAM_fixed'

soilwater_VMM = soilwater_VMM[['id', 'datum', 'x_m_L72', 'y_m_L72', 'parameter', 'detectieconditie', 'meetwaarde', 'meeteenheid', 'bron']]
soilwater_OVAM_sediment = soilwater_OVAM_sediment[['id', 'datum', 'x_m_L72', 'y_m_L72', 'parameter', 'detectieconditie', 'meetwaarde', 'meeteenheid', 'bron']]
soilwater_OVAM_fixed = soilwater_OVAM_fixed[['id', 'datum', 'x_m_L72', 'y_m_L72', 'parameter', 'detectieconditie', 'meetwaarde', 'meeteenheid', 'bron']]

soilwater = pd.concat([soilwater_VMM, soilwater_OVAM_sediment, soilwater_OVAM_fixed])

data_soilwater_len = len(soilwater)
nb_datapoints = {"Combined_soil_water": data_soilwater_len}
self.dictionary["nb_datapoints"][0].update(nb_datapoints)

return soilwater

def combined_surface_water(self, location, max_features):
"""
Download the surface water data and combine it in one dataframe.
Parameters
----------
location:
Query on location.
(https://pydov.readthedocs.io/en/stable/query_location.html)
max_features: int
Limit the number of WFS features you want to be returned.
(https://pydov.readthedocs.io/en/stable/sort_limit.html)
Returns
-------
The downloaded and combined surface water data.
"""
if 'Surface_water' in self.combined_datasets:
surfacewater_datasets = self.combined_datasets['Surface_water']
surfacewater_VMM = surfacewater_datasets[0]
surfacewater_OVAM = surfacewater_datasets[1]
else:
surfacewater_VMM, surfacewater_OVAM = RequestPFASdata().surface_water(location, max_features)

surfacewater_VMM = surfacewater_VMM.rename(columns={'ogc_fid': 'id', 'x_mL72': 'x_m_L72', 'y_mL72': 'y_m_L72'})
surfacewater_OVAM = surfacewater_OVAM.rename(columns={'x_ml72': 'x_m_L72', 'y_ml72': 'y_m_L72'})

surfacewater_VMM['bron'] = 'VMM'
surfacewater_OVAM['bron'] = 'OVAM'

surfacewater_VMM = surfacewater_VMM[['id', 'datum', 'x_m_L72', 'y_m_L72', 'parameter', 'detectieconditie', 'meetwaarde', 'meeteenheid', 'bron']]
surfacewater_OVAM = surfacewater_OVAM[['id', 'datum', 'x_m_L72', 'y_m_L72', 'parameter', 'detectieconditie', 'meetwaarde', 'meeteenheid', 'bron']]

surfacewater = pd.concat([surfacewater_VMM, surfacewater_OVAM])

data_surfacewater_len = len(surfacewater)
nb_datapoints = {"Combined_surface_water": data_surfacewater_len}
self.dictionary["nb_datapoints"][0].update(nb_datapoints)

return surfacewater

def main(self, medium, location=None, max_features=None, save=False):

"""
Expand All @@ -615,6 +801,10 @@ def main(self, medium, location=None, max_features=None, save=False):
- 'soil water'
- 'surface water'
- 'waste water'
- 'combined_groundwater'
- 'combined_soil'
- 'combined_soil_water'
- 'combined_surface_water'
location:
Query on location.
(https://pydov.readthedocs.io/en/stable/query_location.html)
Expand Down Expand Up @@ -644,10 +834,14 @@ def main(self, medium, location=None, max_features=None, save=False):
data_wfs_VMM_ws, data_wfs_OVAM_ws_sediment, data_wfs_OVAM_ws_fixed = self.soil_water(location, max_features)
data_wfs_VMM_sw, data_wfs_OVAM_sw = self.surface_water(location, max_features)
data_wfs_VMM_ww = self.waste_water(location, max_features)
data_groundwater = self.combined_groundwater(location, max_features)
data_soil = self.combined_soil(location, max_features)
data_soil_water = self.combined_soil_water(location, max_features)
data_surface_water = self.combined_surface_water(location, max_features)
return_list.extend([data_wfs_VMM_biota, data_wfs_OVAM_effluent, data_pydov_VMM_gw, data_wfs_OVAM_gw,
data_wfs_Lantis_gw, data_wfs_OVAM_migration, data_wfs_OVAM_pp, data_wfs_OVAM_rainwater,
data_wfs_OVAM_soil, data_wfs_Lantis_soil, data_wfs_VMM_ws, data_wfs_OVAM_ws_sediment, data_wfs_OVAM_ws_fixed, data_wfs_VMM_sw,
data_wfs_OVAM_sw, data_wfs_VMM_ww])
data_wfs_OVAM_sw, data_wfs_VMM_ww, data_groundwater, data_soil, data_soil_water, data_surface_water])
elif i == 'biota':
data_wfs_VMM_biota = self.biota(location, max_features)
return_list.extend([data_wfs_VMM_biota])
Expand Down Expand Up @@ -678,6 +872,18 @@ def main(self, medium, location=None, max_features=None, save=False):
elif i == 'waste water':
data_wfs_VMM_ww = self.waste_water(location, max_features)
return_list.extend([data_wfs_VMM_ww])
elif i == 'combined_groundwater':
data_groundwater = self.combined_groundwater(location, max_features)
return_list.extend([data_groundwater])
elif i == 'combined_soil':
data_soil = self.combined_soil(location, max_features)
return_list.extend([data_soil])
elif i == 'combined_soil_water':
data_soil_water = self.combined_soil_water(location, max_features)
return_list.extend([data_soil_water])
elif i == 'combined_surface_water':
data_surface_water = self.combined_surface_water(location, max_features)
return_list.extend([data_surface_water])

metadata = json.dumps(self.dictionary, indent=3)

Expand Down Expand Up @@ -729,6 +935,14 @@ def main(self, medium, location=None, max_features=None, save=False):
pbar.update(metadata['nb_datapoints'][0]['Surface_water_OVAM'])
data_wfs_VMM_ww.to_excel(writer, sheet_name='Waste_water_VMM')
pbar.update(metadata['nb_datapoints'][0]['Waste_water_VMM'])
data_groundwater.to_excel(writer, sheet_name='Combined_groundwater')
pbar.update(metadata['nb_datapoints'][0]['Combined_groundwater'])
data_soil.to_excel(writer, sheet_name='Combined_soil')
pbar.update(metadata['nb_datapoints'][0]['Combined_soil'])
data_soil_water.to_excel(writer, sheet_name='Combined_soil_water')
pbar.update(metadata['nb_datapoints'][0]['Combined_soil_water'])
data_surface_water.to_excel(writer, sheet_name='Combined_surface_water')
pbar.update(metadata['nb_datapoints'][0]['Combined_surface_water'])
elif i == 'biota':
data_wfs_VMM_biota.to_excel(writer, sheet_name='Biota_VMM')
pbar.update(metadata['nb_datapoints'][0]['Biota_VMM'])
Expand Down Expand Up @@ -771,6 +985,18 @@ def main(self, medium, location=None, max_features=None, save=False):
elif i == 'waste water':
data_wfs_VMM_ww.to_excel(writer, sheet_name='Waste_water_VMM')
pbar.update(metadata['nb_datapoints'][0]['Waste_water_VMM'])
elif i == 'combined_groundwater':
data_groundwater.to_excel(writer, sheet_name='Combined_groundwater')
pbar.update(metadata['nb_datapoints'][0]['Combined_groundwater'])
elif i == 'combined_soil':
data_soil.to_excel(writer, sheet_name='Combined_soil')
pbar.update(metadata['nb_datapoints'][0]['Combined_soil'])
elif i == 'combined_soil_water':
data_soil_water.to_excel(writer, sheet_name='Combined_soil_water')
pbar.update(metadata['nb_datapoints'][0]['Combined_soil_water'])
elif i == 'combined_surface_water':
data_surface_water.to_excel(writer, sheet_name='Combined_surface_water')
pbar.update(metadata['nb_datapoints'][0]['Combined_surface_water'])
pbar.close()


Expand Down
Loading

0 comments on commit 23d97d1

Please sign in to comment.