Add custom XML field to parse mv_mtaw fields.

pydov/search/abstract.py

@@ -557,7 +557,8 @@ def _build_fields(self, wfs_schema, feature_catalogue, xsd_schemas):
 
             fields[field['name']] = field
 
-        for custom_field in self._type.get_fields(source=['custom']).values():
+        for custom_field in self._type.get_fields(
+                source=['custom_wfs']).values():
             field = {
                 'name': custom_field['name'],
                 'type': custom_field['type'],
@@ -568,6 +569,18 @@ def _build_fields(self, wfs_schema, feature_catalogue, xsd_schemas):
             }
             fields[field['name']] = field
 
+        for custom_field in self._type.get_fields(
+                source=['custom_xml']).values():
+            field = {
+                'name': custom_field['name'],
+                'type': custom_field['type'],
+                'definition': custom_field['definition'],
+                'notnull': custom_field['notnull'],
+                'query': False,
+                'cost': 10
+            }
+            fields[field['name']] = field
+
         return fields
 
     def _pre_search_validation(self, location, query, sort_by,
@@ -847,8 +860,9 @@ def _search(self, location=None, query=None, return_fields=None,
                 geom_return_crs = None
 
         extra_custom_fields = set()
-        for custom_field in self._type.get_fields(source=('custom',)).values():
-            extra_custom_fields.update(custom_field.requires_fields())
+        for custom_field in self._type.get_fields(
+                source=('custom_wfs',)).values():
+            extra_custom_fields.update(custom_field.requires_wfs_fields())
 
         wfs_property_names.extend(extra_wfs_fields)
         wfs_property_names.extend(list(extra_custom_fields))

pydov/types/abstract.py

@@ -348,6 +348,11 @@ def _parse_xml_data(self, session=None):
                     returntype=field.get('type', None)
                 )
 
+            for field in self.get_fields(source=('custom_xml',),
+                                         include_subtypes=False).values():
+                self.data[field['name']] = field.calculate(
+                    self.__class__, tree) or np.nan
+
             self._parse_subtypes(xml)
             return True
         except XmlParseError:
@@ -399,16 +404,16 @@ def from_wfs_element(cls, feature, namespace):
                     returntype=field.get('type', str)
                 )
 
-        for field in cls.get_fields(source=('custom',)).values():
-            for required_field in field.requires_fields():
+        for field in cls.get_fields(source=('custom_wfs',)).values():
+            for required_field in field.requires_wfs_fields():
                 instance.data[required_field] = cls._parse(
                     func=feature.findtext,
                     xpath=required_field,
                     namespace=namespace,
                     returntype=field.get('type', str)
                 )
 
-        for field in cls.get_fields(source=('custom',)).values():
+        for field in cls.get_fields(source=('custom_wfs',)).values():
             instance.data[field['name']] = field.calculate(instance) or np.nan
 
         return instance

pydov/types/boring.py

@@ -2,6 +2,7 @@
 """Module containing the DOV data type for boreholes (Boring), including
 subtypes."""
 from pydov.types.fields import WfsField, XmlField
+from pydov.types.ligging import MvMtawField
 
 from .abstract import AbstractDovSubType, AbstractDovType
 
@@ -39,11 +40,8 @@ class Boring(AbstractDovType):
                  datatype='string'),
         WfsField(name='x', source_field='X_mL72', datatype='float'),
         WfsField(name='y', source_field='Y_mL72', datatype='float'),
-        XmlField(name='mv_mtaw',
-                 source_xpath='/boring/oorspronkelijk_maaiveld/waarde',
-                 definition='Maaiveldhoogte in mTAW op dag dat de boring '
-                            'uitgevoerd werd.',
-                 datatype='float'),
+        MvMtawField('Maaiveldhoogte in mTAW op dag dat de boring '
+                    'uitgevoerd werd.'),
         WfsField(name='start_boring_mtaw', source_field='Z_mTAW',
                  datatype='float'),
         WfsField(name='gemeente', source_field='gemeente', datatype='string'),

pydov/types/fields.py

@@ -32,7 +32,7 @@ def __init__(self, name, source, datatype, **kwargs):
         ----------
         name : str
             Name of this field in the return dataframe.
-        source : one of 'wfs', 'xml', 'custom'
+        source : one of 'wfs', 'xml', 'custom_wfs', 'custom_xml'
             Source of this field.
         datatype : one of 'string', 'integer', 'float', 'date', 'datetime' \
                    or 'boolean'
@@ -145,11 +145,11 @@ def __init__(self, name, datatype, definition='', notnull=False):
             True if this field is always present (mandatory), False otherwise.
 
         """
-        super(_CustomWfsField, self).__init__(name, 'custom', datatype)
+        super(_CustomWfsField, self).__init__(name, 'custom_wfs', datatype)
         self.__setitem__('definition', definition)
         self.__setitem__('notnull', notnull)
 
-    def requires_fields(self):
+    def requires_wfs_fields(self):
         """Get a list of WFS fields that are required by (the calculation of)
         this custom field.
 
@@ -187,6 +187,53 @@ def calculate(self, instance):
         raise NotImplementedError
 
 
+class _CustomXmlField(AbstractField):
+    """Class for a custom field, created explicitly in pydov from other XML
+    fields."""
+
+    def __init__(self, name, datatype, definition='', notnull=False):
+        """Initialise a custom field.
+
+        Parameters
+        ----------
+        name : str
+            Name of this field in the return dataframe.
+        datatype : one of 'string', 'integer', 'float', 'date', 'datetime' \
+                   or 'boolean'
+            Datatype of the values of this field in the return dataframe.
+        definition : str, optional
+            Definition of this field.
+        notnull : bool, optional, defaults to False
+            True if this field is always present (mandatory), False otherwise.
+
+        """
+        super(_CustomXmlField, self).__init__(name, 'custom_xml', datatype)
+        self.__setitem__('definition', definition)
+        self.__setitem__('notnull', notnull)
+
+    def calculate(self, cls, tree):
+        """Calculate the value of this custom field from the given XML tree.
+
+        Parameters
+        ----------
+        cls : AbstractDovType
+            Class of the type this field belongs to.
+        tree : etree.ElementTree
+            ElementTree of the DOV XML for this instance.
+
+        Returns
+        -------
+        Value to be used for this custom field for this instance. Its datatype
+        should match the one set in the initialisation of the custom field.
+
+        Raises
+        ------
+        NotImplementedError
+            Implement this in a subclass.
+        """
+        raise NotImplementedError
+
+
 class ReturnFieldList(list):
     """List of return fields used in search methods. """
 

pydov/types/grondwaterfilter.py

@@ -2,6 +2,7 @@
 """Module containing the DOV data type for screens (Filter), including
 subtypes."""
 from pydov.types.fields import WfsField, XmlField, XsdType
+from pydov.types.ligging import MvMtawField
 from pydov.util.dovutil import build_dov_url
 
 from .abstract import AbstractDovSubType, AbstractDovType
@@ -78,12 +79,8 @@ class GrondwaterFilter(AbstractDovType):
         WfsField(name='y', source_field='Y_mL72', datatype='float'),
         WfsField(name='start_grondwaterlocatie_mtaw', source_field='Z_mTAW',
                  datatype='float'),
-        XmlField(name='mv_mtaw',
-                 source_xpath='/grondwaterlocatie/puntligging/'
-                              'oorspronkelijk_maaiveld/waarde',
-                 definition='Maaiveldhoogte in mTAW op dag '
-                            'dat de put/boring uitgevoerd werd',
-                 datatype='float'),
+        MvMtawField('Maaiveldhoogte in mTAW op dag '
+                    'dat de put/boring uitgevoerd werd'),
         WfsField(name='gemeente', source_field='gemeente', datatype='string'),
         XmlField(name='meetnet_code',
                  source_xpath='/filter/meetnet',

pydov/types/interpretaties.py

@@ -20,7 +20,7 @@ def __init__(self):
                          'boring).',
                          datatype='string')
 
-    def requires_fields(self):
+    def requires_wfs_fields(self):
         return ['Type_proef', 'Proeffiche']
 
     def calculate(self, instance):
@@ -41,7 +41,7 @@ def __init__(self):
                          'aan een sondering).',
                          datatype='string')
 
-    def requires_fields(self):
+    def requires_wfs_fields(self):
         return ['Type_proef', 'Proeffiche']
 
     def calculate(self, instance):

pydov/types/ligging.py

@@ -0,0 +1,54 @@
+import numpy as np
+
+from pydov.types.fields import _CustomXmlField
+
+
+class MvMtawField(_CustomXmlField):
+    """Field for retrieving the mv_mtaw value from the height of the point in
+    relavant cases."""
+
+    def __init__(self, definition):
+        """Initialise a MvMtawField (mv_mtaw) with given definition.
+
+        Parameters
+        ----------
+        definition : string
+            Type-specific definition of the mv_mtaw field.
+        """
+        super().__init__(
+            name='mv_mtaw',
+            definition=definition,
+            datatype='float',
+            notnull=False
+        )
+
+    def calculate(self, cls, tree):
+        # Support the old format too
+        oorspronkelijk_maaiveld = cls._parse(
+            func=tree.findtext,
+            xpath='.//oorspronkelijk_maaiveld/waarde',
+            namespace=None,
+            returntype='float'
+        )
+        if oorspronkelijk_maaiveld is not np.nan:
+            return oorspronkelijk_maaiveld
+
+        # Check if referentiepunt is Maaiveld
+        referentiepunt = cls._parse(
+            func=tree.findtext,
+            xpath='.//ligging/metadata_hoogtebepaling/referentiepunt_type',
+            namespace=None,
+            returntype='string'
+        )
+        if referentiepunt != 'Maaiveld':
+            # If referentiepunt is not Maaiveld, we don't know the height of
+            # maaiveld
+            return np.nan
+
+        # If referentiepunt is Maaiveld, height of the ligging is Maaiveld
+        point = tree.findtext(
+            './/ligging/{http://www.opengis.net/gml/3.2}Point/'
+            '{http://www.opengis.net/gml/3.2}pos'
+        )
+        hoogte = float(point.split(' ')[-1])
+        return hoogte

pydov/types/sondering.py

@@ -3,6 +3,7 @@
 including subtypes."""
 from pydov.types.abstract import AbstractDovSubType, AbstractDovType
 from pydov.types.fields import WfsField, XmlField
+from pydov.types.ligging import MvMtawField
 
 
 class Meetdata(AbstractDovSubType):
@@ -62,12 +63,8 @@ class Sondering(AbstractDovType):
                  datatype='string'),
         WfsField(name='x', source_field='X_mL72', datatype='float'),
         WfsField(name='y', source_field='Y_mL72', datatype='float'),
-        XmlField(name='mv_mtaw',
-                 source_xpath='/sondering/sondeerpositie/'
-                              'oorspronkelijk_maaiveld/waarde',
-                 definition='Maaiveldhoogte in mTAW op dag dat de sondering '
-                            'uitgevoerd werd.',
-                 datatype='float'),
+        MvMtawField('Maaiveldhoogte in mTAW op dag dat de sondering '
+                    'uitgevoerd werd.'),
         WfsField(name='start_sondering_mtaw', source_field='Z_mTAW',
                  datatype='float'),
         WfsField(name='diepte_sondering_van', source_field='diepte_van_m',

tests/abstract.py

@@ -877,7 +877,7 @@ def test_get_df_array(self, wfs_feature, mp_dov_xml):
             wfs_feature, self.namespace)
 
         fields = [f for f in self.datatype_class.get_fields(
-            source=('wfs', 'xml', 'custom')).values() if not
+            source=('wfs', 'xml', 'custom_wfs', 'custom_xml')).values() if not
             f.get('wfs_injected', False)]
 
         df_array = feature.get_df_array()
@@ -919,7 +919,8 @@ def test_get_df_array_wrongreturnfields(self, wfs_feature):
             wfs_feature, self.namespace)
 
         with pytest.raises(InvalidFieldError):
-            feature.get_df_array(return_fields=ReturnFieldList.from_field_names(self.inexistent_field))
+            feature.get_df_array(
+                return_fields=ReturnFieldList.from_field_names(self.inexistent_field))
 
     def test_from_wfs_str(self, wfs_getfeature):
         """Test the from_wfs method to construct objects from a WFS response,