From 4f635226d67c98d52dda2353b1ceeec47e93d5f4 Mon Sep 17 00:00:00 2001 From: Roel Huybrechts Date: Tue, 26 Sep 2023 17:00:06 +0200 Subject: [PATCH] Add custom XML field to parse mv_mtaw fields. --- pydov/search/abstract.py | 20 ++++++++++-- pydov/types/abstract.py | 11 +++++-- pydov/types/boring.py | 8 ++--- pydov/types/fields.py | 53 ++++++++++++++++++++++++++++++-- pydov/types/grondwaterfilter.py | 9 ++---- pydov/types/interpretaties.py | 4 +-- pydov/types/ligging.py | 54 +++++++++++++++++++++++++++++++++ pydov/types/sondering.py | 9 ++---- tests/abstract.py | 5 +-- 9 files changed, 143 insertions(+), 30 deletions(-) create mode 100644 pydov/types/ligging.py diff --git a/pydov/search/abstract.py b/pydov/search/abstract.py index 38e2de3d..1faffea7 100644 --- a/pydov/search/abstract.py +++ b/pydov/search/abstract.py @@ -557,7 +557,8 @@ def _build_fields(self, wfs_schema, feature_catalogue, xsd_schemas): fields[field['name']] = field - for custom_field in self._type.get_fields(source=['custom']).values(): + for custom_field in self._type.get_fields( + source=['custom_wfs']).values(): field = { 'name': custom_field['name'], 'type': custom_field['type'], @@ -568,6 +569,18 @@ def _build_fields(self, wfs_schema, feature_catalogue, xsd_schemas): } fields[field['name']] = field + for custom_field in self._type.get_fields( + source=['custom_xml']).values(): + field = { + 'name': custom_field['name'], + 'type': custom_field['type'], + 'definition': custom_field['definition'], + 'notnull': custom_field['notnull'], + 'query': False, + 'cost': 10 + } + fields[field['name']] = field + return fields def _pre_search_validation(self, location, query, sort_by, @@ -847,8 +860,9 @@ def _search(self, location=None, query=None, return_fields=None, geom_return_crs = None extra_custom_fields = set() - for custom_field in self._type.get_fields(source=('custom',)).values(): - extra_custom_fields.update(custom_field.requires_fields()) + for custom_field in self._type.get_fields( + source=('custom_wfs',)).values(): + extra_custom_fields.update(custom_field.requires_wfs_fields()) wfs_property_names.extend(extra_wfs_fields) wfs_property_names.extend(list(extra_custom_fields)) diff --git a/pydov/types/abstract.py b/pydov/types/abstract.py index a45335e7..6020e6d6 100644 --- a/pydov/types/abstract.py +++ b/pydov/types/abstract.py @@ -348,6 +348,11 @@ def _parse_xml_data(self, session=None): returntype=field.get('type', None) ) + for field in self.get_fields(source=('custom_xml',), + include_subtypes=False).values(): + self.data[field['name']] = field.calculate( + self.__class__, tree) or np.nan + self._parse_subtypes(xml) return True except XmlParseError: @@ -399,8 +404,8 @@ def from_wfs_element(cls, feature, namespace): returntype=field.get('type', str) ) - for field in cls.get_fields(source=('custom',)).values(): - for required_field in field.requires_fields(): + for field in cls.get_fields(source=('custom_wfs',)).values(): + for required_field in field.requires_wfs_fields(): instance.data[required_field] = cls._parse( func=feature.findtext, xpath=required_field, @@ -408,7 +413,7 @@ def from_wfs_element(cls, feature, namespace): returntype=field.get('type', str) ) - for field in cls.get_fields(source=('custom',)).values(): + for field in cls.get_fields(source=('custom_wfs',)).values(): instance.data[field['name']] = field.calculate(instance) or np.nan return instance diff --git a/pydov/types/boring.py b/pydov/types/boring.py index 54f2db5e..37c26812 100644 --- a/pydov/types/boring.py +++ b/pydov/types/boring.py @@ -2,6 +2,7 @@ """Module containing the DOV data type for boreholes (Boring), including subtypes.""" from pydov.types.fields import WfsField, XmlField +from pydov.types.ligging import MvMtawField from .abstract import AbstractDovSubType, AbstractDovType @@ -39,11 +40,8 @@ class Boring(AbstractDovType): datatype='string'), WfsField(name='x', source_field='X_mL72', datatype='float'), WfsField(name='y', source_field='Y_mL72', datatype='float'), - XmlField(name='mv_mtaw', - source_xpath='/boring/oorspronkelijk_maaiveld/waarde', - definition='Maaiveldhoogte in mTAW op dag dat de boring ' - 'uitgevoerd werd.', - datatype='float'), + MvMtawField('Maaiveldhoogte in mTAW op dag dat de boring ' + 'uitgevoerd werd.'), WfsField(name='start_boring_mtaw', source_field='Z_mTAW', datatype='float'), WfsField(name='gemeente', source_field='gemeente', datatype='string'), diff --git a/pydov/types/fields.py b/pydov/types/fields.py index 4bad0b73..9e457b61 100644 --- a/pydov/types/fields.py +++ b/pydov/types/fields.py @@ -32,7 +32,7 @@ def __init__(self, name, source, datatype, **kwargs): ---------- name : str Name of this field in the return dataframe. - source : one of 'wfs', 'xml', 'custom' + source : one of 'wfs', 'xml', 'custom_wfs', 'custom_xml' Source of this field. datatype : one of 'string', 'integer', 'float', 'date', 'datetime' \ or 'boolean' @@ -145,11 +145,11 @@ def __init__(self, name, datatype, definition='', notnull=False): True if this field is always present (mandatory), False otherwise. """ - super(_CustomWfsField, self).__init__(name, 'custom', datatype) + super(_CustomWfsField, self).__init__(name, 'custom_wfs', datatype) self.__setitem__('definition', definition) self.__setitem__('notnull', notnull) - def requires_fields(self): + def requires_wfs_fields(self): """Get a list of WFS fields that are required by (the calculation of) this custom field. @@ -187,6 +187,53 @@ def calculate(self, instance): raise NotImplementedError +class _CustomXmlField(AbstractField): + """Class for a custom field, created explicitly in pydov from other XML + fields.""" + + def __init__(self, name, datatype, definition='', notnull=False): + """Initialise a custom field. + + Parameters + ---------- + name : str + Name of this field in the return dataframe. + datatype : one of 'string', 'integer', 'float', 'date', 'datetime' \ + or 'boolean' + Datatype of the values of this field in the return dataframe. + definition : str, optional + Definition of this field. + notnull : bool, optional, defaults to False + True if this field is always present (mandatory), False otherwise. + + """ + super(_CustomXmlField, self).__init__(name, 'custom_xml', datatype) + self.__setitem__('definition', definition) + self.__setitem__('notnull', notnull) + + def calculate(self, cls, tree): + """Calculate the value of this custom field from the given XML tree. + + Parameters + ---------- + cls : AbstractDovType + Class of the type this field belongs to. + tree : etree.ElementTree + ElementTree of the DOV XML for this instance. + + Returns + ------- + Value to be used for this custom field for this instance. Its datatype + should match the one set in the initialisation of the custom field. + + Raises + ------ + NotImplementedError + Implement this in a subclass. + """ + raise NotImplementedError + + class ReturnFieldList(list): """List of return fields used in search methods. """ diff --git a/pydov/types/grondwaterfilter.py b/pydov/types/grondwaterfilter.py index e24502b7..db0deb84 100644 --- a/pydov/types/grondwaterfilter.py +++ b/pydov/types/grondwaterfilter.py @@ -2,6 +2,7 @@ """Module containing the DOV data type for screens (Filter), including subtypes.""" from pydov.types.fields import WfsField, XmlField, XsdType +from pydov.types.ligging import MvMtawField from pydov.util.dovutil import build_dov_url from .abstract import AbstractDovSubType, AbstractDovType @@ -78,12 +79,8 @@ class GrondwaterFilter(AbstractDovType): WfsField(name='y', source_field='Y_mL72', datatype='float'), WfsField(name='start_grondwaterlocatie_mtaw', source_field='Z_mTAW', datatype='float'), - XmlField(name='mv_mtaw', - source_xpath='/grondwaterlocatie/puntligging/' - 'oorspronkelijk_maaiveld/waarde', - definition='Maaiveldhoogte in mTAW op dag ' - 'dat de put/boring uitgevoerd werd', - datatype='float'), + MvMtawField('Maaiveldhoogte in mTAW op dag ' + 'dat de put/boring uitgevoerd werd'), WfsField(name='gemeente', source_field='gemeente', datatype='string'), XmlField(name='meetnet_code', source_xpath='/filter/meetnet', diff --git a/pydov/types/interpretaties.py b/pydov/types/interpretaties.py index 8b0217d0..4e051f5b 100644 --- a/pydov/types/interpretaties.py +++ b/pydov/types/interpretaties.py @@ -20,7 +20,7 @@ def __init__(self): 'boring).', datatype='string') - def requires_fields(self): + def requires_wfs_fields(self): return ['Type_proef', 'Proeffiche'] def calculate(self, instance): @@ -41,7 +41,7 @@ def __init__(self): 'aan een sondering).', datatype='string') - def requires_fields(self): + def requires_wfs_fields(self): return ['Type_proef', 'Proeffiche'] def calculate(self, instance): diff --git a/pydov/types/ligging.py b/pydov/types/ligging.py new file mode 100644 index 00000000..083540a9 --- /dev/null +++ b/pydov/types/ligging.py @@ -0,0 +1,54 @@ +import numpy as np + +from pydov.types.fields import _CustomXmlField + + +class MvMtawField(_CustomXmlField): + """Field for retrieving the mv_mtaw value from the height of the point in + relavant cases.""" + + def __init__(self, definition): + """Initialise a MvMtawField (mv_mtaw) with given definition. + + Parameters + ---------- + definition : string + Type-specific definition of the mv_mtaw field. + """ + super().__init__( + name='mv_mtaw', + definition=definition, + datatype='float', + notnull=False + ) + + def calculate(self, cls, tree): + # Support the old format too + oorspronkelijk_maaiveld = cls._parse( + func=tree.findtext, + xpath='.//oorspronkelijk_maaiveld/waarde', + namespace=None, + returntype='float' + ) + if oorspronkelijk_maaiveld is not np.nan: + return oorspronkelijk_maaiveld + + # Check if referentiepunt is Maaiveld + referentiepunt = cls._parse( + func=tree.findtext, + xpath='.//ligging/metadata_hoogtebepaling/referentiepunt_type', + namespace=None, + returntype='string' + ) + if referentiepunt != 'Maaiveld': + # If referentiepunt is not Maaiveld, we don't know the height of + # maaiveld + return np.nan + + # If referentiepunt is Maaiveld, height of the ligging is Maaiveld + point = tree.findtext( + './/ligging/{http://www.opengis.net/gml/3.2}Point/' + '{http://www.opengis.net/gml/3.2}pos' + ) + hoogte = float(point.split(' ')[-1]) + return hoogte diff --git a/pydov/types/sondering.py b/pydov/types/sondering.py index 851289ef..d86bbb98 100644 --- a/pydov/types/sondering.py +++ b/pydov/types/sondering.py @@ -3,6 +3,7 @@ including subtypes.""" from pydov.types.abstract import AbstractDovSubType, AbstractDovType from pydov.types.fields import WfsField, XmlField +from pydov.types.ligging import MvMtawField class Meetdata(AbstractDovSubType): @@ -62,12 +63,8 @@ class Sondering(AbstractDovType): datatype='string'), WfsField(name='x', source_field='X_mL72', datatype='float'), WfsField(name='y', source_field='Y_mL72', datatype='float'), - XmlField(name='mv_mtaw', - source_xpath='/sondering/sondeerpositie/' - 'oorspronkelijk_maaiveld/waarde', - definition='Maaiveldhoogte in mTAW op dag dat de sondering ' - 'uitgevoerd werd.', - datatype='float'), + MvMtawField('Maaiveldhoogte in mTAW op dag dat de sondering ' + 'uitgevoerd werd.'), WfsField(name='start_sondering_mtaw', source_field='Z_mTAW', datatype='float'), WfsField(name='diepte_sondering_van', source_field='diepte_van_m', diff --git a/tests/abstract.py b/tests/abstract.py index 1c18e5b2..7223f219 100644 --- a/tests/abstract.py +++ b/tests/abstract.py @@ -877,7 +877,7 @@ def test_get_df_array(self, wfs_feature, mp_dov_xml): wfs_feature, self.namespace) fields = [f for f in self.datatype_class.get_fields( - source=('wfs', 'xml', 'custom')).values() if not + source=('wfs', 'xml', 'custom_wfs', 'custom_xml')).values() if not f.get('wfs_injected', False)] df_array = feature.get_df_array() @@ -919,7 +919,8 @@ def test_get_df_array_wrongreturnfields(self, wfs_feature): wfs_feature, self.namespace) with pytest.raises(InvalidFieldError): - feature.get_df_array(return_fields=ReturnFieldList.from_field_names(self.inexistent_field)) + feature.get_df_array( + return_fields=ReturnFieldList.from_field_names(self.inexistent_field)) def test_from_wfs_str(self, wfs_getfeature): """Test the from_wfs method to construct objects from a WFS response,