Merge pull request #64 from Deltares/feature/raster

Grid model (raster)

.testdata/create_test_data.py

@@ -1,5 +1,6 @@
 from fiat.log import Log
 
+import copy
 import gc
 import math
 import os
@@ -212,6 +213,7 @@ def create_exposure_grid():
     for x, y in product(oneD, oneD):
         data[x, y] = 2000 + ((x + y) * 100)
     band.WriteArray(data)
+    band.SetMetadataItem("damage_function", "struct_1")
 
     band.FlushCache()
     src.FlushCache()
@@ -320,13 +322,11 @@ def create_settings():
             "elevation_reference": "DEM",
         },
         "exposure": {
-            "grid": {
-                "file": "exposure/spatial.nc",
-                "crs": "EPSG:4326",
+            "csv": {
+                "file": "exposure/spatial.csv",
             },
             "geom": {
                 "file1": "exposure/spatial.gpkg",
-                "csv": "exposure/spatial.csv",
                 "crs": "EPSG:4326",
             },
         },
@@ -345,20 +345,58 @@ def create_settings():
     with open(Path(p, "settings.toml"), "wb") as f:
         tomli_w.dump(doc, f)
 
-    doc["output"]["path"] = "output/event_2g"
-    doc["output"]["geom"]["name2"] = "spatial2.gpkg"
-    doc["exposure"]["geom"]["file2"] = "exposure/spatial2.gpkg"
+    doc2g = copy.deepcopy(doc)
+    doc2g["output"]["path"] = "output/event_2g"
+    doc2g["output"]["geom"]["name2"] = "spatial2.gpkg"
+    doc2g["exposure"]["geom"]["file2"] = "exposure/spatial2.gpkg"
 
     with open(Path(p, "settings_2g.toml"), "wb") as f:
-        tomli_w.dump(doc, f)
+        tomli_w.dump(doc2g, f)
 
-    doc["output"]["path"] = "output/event_missing"
-    del doc["output"]["geom"]["name2"]
-    del doc["exposure"]["geom"]["file2"]
-    doc["output"]["geom"]["name1"] = "spatial_missing.gpkg"
-    doc["exposure"]["geom"]["file1"] = "exposure/spatial_missing.gpkg"
+    doc_m = copy.deepcopy(doc)
+    doc_m["output"]["path"] = "output/event_missing"
+    doc_m["output"]["geom"]["name1"] = "spatial_missing.gpkg"
+    doc_m["exposure"]["geom"]["file1"] = "exposure/spatial_missing.gpkg"
 
     with open(Path(p, "settings_missing.toml"), "wb") as f:
+        tomli_w.dump(doc_m, f)
+
+
+def create_settings_grid():
+    doc = {
+        "global": {
+            "crs": "EPSG:4326",
+            "keep_temp_files": True,
+        },
+        "output": {
+            "path": "output/event",
+            "grid": {"name": "output.nc"},
+        },
+        "hazard": {
+            "file": "hazard/event_map.nc",
+            "crs": "EPSG:4326",
+            "risk": False,
+            "elevation_reference": "DEM",
+        },
+        "exposure": {
+            "grid": {
+                "file": "exposure/spatial.nc",
+                "crs": "EPSG:4326",
+            },
+        },
+        "vulnerability": {
+            "file": "vulnerability/vulnerability_curves.csv",
+            "step_size": 0.01,
+        },
+        "categorical_bins": {
+            "low": 0.25,
+            "medium-low": 0.5,
+            "medium-high": 0.75,
+            "high": 1,
+        },
+    }
+
+    with open(Path(p, "settings_grid.toml"), "wb") as f:
         tomli_w.dump(doc, f)
 
 
@@ -387,13 +425,11 @@ def create_settings_risk():
             },
         },
         "exposure": {
-            "grid": {
-                "file": "exposure/spatial.nc",
-                "crs": "EPSG:4326",
+            "csv": {
+                "file": "exposure/spatial.csv",
             },
             "geom": {
                 "file1": "exposure/spatial.gpkg",
-                "csv": "exposure/spatial.csv",
                 "crs": "EPSG:4326",
             },
         },
@@ -449,6 +485,7 @@ def log_base(b, x):
     create_hazard_map()
     create_risk_map()
     create_settings()
+    create_settings_grid()
     create_settings_risk()
     create_vulnerability()
     gc.collect()

src/fiat/cfg.py

@@ -1,4 +1,8 @@
-from fiat.check import check_config_entries
+from fiat.check import (
+    check_config_entries,
+    check_config_geom,
+    check_config_grid,
+)
 from fiat.util import (
     Path,
     create_hidden_folder,
@@ -9,13 +13,24 @@
 
 import os
 import tomli
+from osgeo import gdal
+from typing import Any
 
 
 class ConfigReader(dict):
     def __init__(
         self,
         file: str,
+        extra: dict = None,
     ):
+        """_summary_"""
+
+        # container for extra
+        self._build = True
+        self._extra = {}
+        if extra is not None:
+            self._extra.update(extra)
+
         # Set the root directory
         self.filepath = Path(file)
         self.path = self.filepath.parent
@@ -38,6 +53,13 @@ def __init__(
         # Create the hidden temporary folder
         self._create_temp_dir()
 
+        # Set the cache size per GDAL object
+        _cache_size = self.get("global.gdal_cache")
+        if _cache_size is not None:
+            gdal.SetCacheMax(_cache_size * 1024**2)
+        else:
+            gdal.SetCacheMax(50 * 1024**2)
+
         # Do some checking concerning the file paths in the settings file
         for key, item in self.items():
             if key.endswith(("file", "csv")) or key.rsplit(".", 1)[1].startswith(
@@ -52,9 +74,27 @@ def __init__(
                 if isinstance(item, str):
                     self[key] = item.lower()
 
+        self._build = False
+
+        # (Re)set the extra values
+        self.update(self._extra)
+
     def __repr__(self):
         return f"<ConfigReader object file='{self.filepath}'>"
 
+    def __reduce__(self):
+        """_summary_"""
+
+        return self.__class__, (
+            self.filepath,
+            self._extra,
+        )
+
+    def __setitem__(self, __key: Any, __value: Any):
+        if not self._build:
+            self._extra[__key] = __value
+        super().__setitem__(__key, __value)
+
     def _create_output_dir(
         self,
         path: Path | str,
@@ -81,10 +121,14 @@ def get_model_type(
     ):
         """_Summary_"""
 
-        if "exposure.geom_file" in self:
-            return 0
-        else:
-            return 1
+        _models = [False, False]
+
+        if check_config_geom(self):
+            _models[0] = True
+        if check_config_grid(self):
+            _models[1] = True
+
+        return _models
 
     def get_path(
         self,

src/fiat/check.py

@@ -1,6 +1,7 @@
 from fiat.log import spawn_logger, setup_default_log
 from fiat.util import NEWLINE_CHAR, deter_type, generic_path_check
 
+import fnmatch
 import sys
 from osgeo import gdal
 from osgeo import osr
@@ -38,6 +39,57 @@ def check_config_entries(
         sys.exit()
 
 
+def check_config_geom(
+    cfg: "ConfigReader",
+):
+    """_summary_"""
+
+    _req_fields = [
+        "exposure.csv.file",
+        "exposure.geom.crs",
+        "exposure.geom.file1",
+    ]
+    _all_geom = [
+        item for item in cfg if item.startswith(("exposure.geom", "exposure.csv"))
+    ]
+    if len(_all_geom) == 0:
+        return False
+
+    _check = [item in _all_geom for item in _req_fields]
+    if not all(_check):
+        _missing = [item for item, b in zip(_req_fields, _check) if not b]
+        logger.warning(
+            f"Info for the geometry model was found, but not all. {_missing} was/ were missing"
+        )
+        return False
+
+    return True
+
+
+def check_config_grid(
+    cfg: "ConfigReader",
+):
+    """_summary_"""
+
+    _req_fields = [
+        "exposure.grid.crs",
+        "exposure.grid.file",
+    ]
+    _all_grid = [item for item in cfg if item.startswith("exposure.grid")]
+    if len(_all_grid) == 0:
+        return False
+
+    _check = [item in _all_grid for item in _req_fields]
+    if not all(_check):
+        _missing = [item for item, b in zip(_req_fields, _check) if not b]
+        logger.warning(
+            f"Info for the grid (raster) model was found, but not all. {_missing} was/ were missing"
+        )
+        return False
+
+    return True
+
+
 def check_global_crs(
     srs: osr.SpatialReference,
     fname: str,
@@ -52,6 +104,31 @@ def check_global_crs(
 
 
 ## GIS
+def check_grid_exact(
+    haz,
+    exp,
+):
+    """_summary_"""
+
+    if not check_vs_srs(
+        haz.get_srs(),
+        exp.get_srs(),
+    ):
+        logger.error("")
+        sys.exit()
+
+    gtf1 = [round(_n, 2) for _n in haz.get_geotransform()]
+    gtf2 = [round(_n, 2) for _n in exp.get_geotransform()]
+
+    if gtf1 != gtf2:
+        logger.error("")
+        sys.exit()
+
+    if haz.shape != exp.shape:
+        logger.error("")
+        sys.exit()
+
+
 def check_internal_srs(
     source_srs: osr.SpatialReference,
     fname: str,
@@ -170,6 +247,65 @@ def check_hazard_subsets(
 
 
 ## Exposure
+def check_exp_columns(
+    columns: tuple | list,
+):
+    """_summary_"""
+
+    _man_columns = [
+        "Object ID",
+        "Ground Elevation",
+        "Ground Floor Height",
+    ]
+
+    _check = [item in columns for item in _man_columns]
+    if not all(_check):
+        _missing = [item for item, b in zip(_man_columns, _check) if not b]
+        logger.error(f"Missing mandatory exposure columns: {_missing}")
+        sys.exit()
+
+    dmg = fnmatch.filter(columns, "Damage Function: *")
+    dmg_suffix = [item.split(":")[1].strip() for item in dmg]
+    mpd = fnmatch.filter(columns, "Max Potential Damage: *")
+    mpd_suffix = [item.split(":")[1].strip() for item in mpd]
+
+    if not dmg:
+        logger.error("No damage function were given in ")
+        sys.exit()
+
+    if not mpd:
+        logger.error("No maximum potential damages were given in ")
+        sys.exit()
+
+    _check = [item in mpd_suffix for item in dmg_suffix]
+    if not any(_check):
+        logger.error(
+            "Damage function and maximum potential damage do not have a single match"
+        )
+        sys.exit()
+    if not all(_check):
+        _missing = [item for item, b in zip(dmg_suffix, _check) if not b]
+        logger.warning(
+            f"No every damage function has a corresponding maximum potential damage: {_missing}"
+        )
+
+
+def check_exp_grid_dmfs(
+    exp: object,
+    dmfs: tuple | list,
+):
+    """_summary_"""
+
+    _ef = [_i.get_metadata_item("damage_function") for _i in exp]
+    _i = None
+
+    _check = [item in dmfs for item in _ef]
+    if not all(_check):
+        _missing = [item for item, b in zip(_ef, _check) if not b]
+        logger.error(
+            f"Incorrect damage function identifier found in exposure grid: {_missing}",
+        )
+        sys.exit()
 
 
 ## Vulnerability

src/fiat/cli/main_old.py

@@ -88,7 +88,7 @@ def run(
         dst=cfg.get("output.path"),
     )
     sys.stdout.write(fiat_start_str)
-    logger.info(f"Delft-Fiat version: {__version__}")
+    logger.info(f"Delft-FIAT version: {__version__}")
     obj = FIAT(cfg)
     obj.run()