Merge pull request #116 from Serapieum-of-alex/big-data

Big data

.gitignore

@@ -128,7 +128,7 @@ venv.bak/
 
 HAPI_Nile.egg-info/
 *.pyc
-*.zip
+#*.zip
 *.drawio
 *.dbf
 *.prj
@@ -146,7 +146,7 @@ HAPI_Nile.egg-info/
 # conda smithy ci-skeleton start
 build_artifacts
 # conda smithy ci-skeleton end
-Hapi/hm/saintvenant.py
+#Hapi/hm/saintvenant.py
 Hapi/hm.py
 
 HAPI_Nile.egg-info/PKG-INFO
@@ -177,12 +177,12 @@ Hapi/java_functions.py
 Hapi/parameters/*.tif
 *.bin
 *.cue
-examples/Hydrological model/data/distributed_model/calibration/LB - tot.txt
+examples/hydrological-model/data/distributed_model/calibration/LB - tot.txt
 /examples/Hydrological model/data/meteo_data/meteodata_prepared/temp-rename-example/*.tif
 examples/Hydrodynamic models/test_case/results/q/right_results/*
 examples/Hydrodynamic models/test_case/results/h/right_results/*
 examples/Hydrodynamic models/test_case/results/USbnd/right_results/*
 examples/Hydrodynamic models/test_case/results/q/New folder/*
 examples/Hydrodynamic models/test_case/results/h/New folder/*
 examples/Hydrodynamic models/test_case/results/USbnd/New folder/*
-tests/mo/*
+tests/mo/*

.pre-commit-config.yaml

@@ -79,7 +79,7 @@ repos:
   hooks:
     - id: pytest-check
       name: pytest-check
-      entry: pytest
+      entry: pytest -vvv --cov=Hapi
       language: system
       pass_filenames: false
       always_run: true

HISTORY.rst

@@ -34,4 +34,10 @@ History
 * refactor code and change methods to camelcase
 * add hydrodynamic model 1d config file read function
 * simplify functions with too many parameters using decorator
-* add automatic pypi build and publish github actions
+* add automatic pypi build and publish github actions
+
+1.5.0 (2023-01-10)
+------------------
+* hydraulic model can read chunked big zip file
+* fix CI
+* fix missing module (saint venant script and module)

Hapi/hm/__init__.py

@@ -1,12 +0,0 @@
-"""Created on Sat Apr 24 20:57:45 2021.
-
-@author: mofarrag
-"""
-
-# import Hapi.hm.calibration as calibration
-# import Hapi.hm.crosssection as crosssection
-# import Hapi.hm.event as event
-# import Hapi.hm.inputs as inputs
-# import Hapi.hm.interface as interface
-# import Hapi.hm.river as river
-# import Hapi.hm.saintvenant as saintvenant

Hapi/hm/calibration.py

@@ -17,6 +17,7 @@
 
 from Hapi.hapi_warnings import SilenceNumpyWarning, SilenceShapelyWarning
 from Hapi.hm.river import River
+from Hapi.utils import class_attr_initialize
 
 datafn = lambda x: dt.datetime.strptime(x, "%Y-%m-%d")
 
@@ -30,14 +31,43 @@ class Calibration(River):
     Hydraulic model calibration class
     """
 
+    hm_gauges: DataFrame
+    rrm_gauges: DataFrame
+
+    calibration_attributes = dict(
+        q_hm=None,
+        WLHM=None,
+        q_rrm=None,
+        QRRM2=None,
+        rrm_gauges=None,
+        hm_gauges=None,
+        q_gauges=None,
+        WLGauges=None,
+        CalibrationQ=None,
+        CalibrationWL=None,
+        annual_max_obs_q=None,
+        annual_max_obs_wl=None,
+        annual_max_rrm=None,
+        annual_max_hm_q=None,
+        annual_max_hm_wl=None,
+        AnnualMaxDates=None,
+        MetricsHMvsRRM=None,
+        MetricsRRMvsObs=None,
+        MetricsHMWLvsObs=None,
+        MetricsHMQvsObs=None,
+        WLgaugesList=None,
+        QgaugesList=None,
+    )
+
+    @class_attr_initialize(calibration_attributes)
     def __init__(
         self,
         name: str,
         version: int = 3,
         start: Union[str, dt.datetime] = "1950-1-1",
         days: int = 36890,
         fmt: str = "%Y-%m-%d",
-        rrmstart: str = "",
+        rrmstart: str = None,
         rrmdays: int = 36890,
         novalue: int = -9,
         gauge_id_col: Any = "oid",
@@ -73,6 +103,7 @@ def __init__(
         -------
         None.
         """
+        # super().__init__()
         self.name = name
         self.version = version
         if isinstance(start, str):
@@ -86,49 +117,23 @@ def __init__(
         self.ReferenceIndex = pd.DataFrame(index=list(range(1, days + 1)))
         self.ReferenceIndex["date"] = Ref_ind[:-1]
 
-        if rrmstart == "":
+        if rrmstart is None:
             self.rrmstart = self.start
         else:
             try:
                 self.rrmstart = dt.datetime.strptime(rrmstart, fmt)
             except ValueError:
-                msg = (
-                    "plese check the fmt ({0}) you entered as it is different from the"
-                    " rrmstart data ({1})"
+                logger.debug(
+                    f"plese check the fmt ({fmt}) you entered as it is different from the"
+                    f" rrmstart data ({rrmstart})"
                 )
-                logger.debug(msg.format(fmt, rrmstart))
                 return
 
         self.rrmend = self.rrmstart + dt.timedelta(days=rrmdays)
         ref_ind = pd.date_range(self.rrmstart, self.rrmend, freq="D")
         self.rrmreferenceindex = pd.DataFrame(index=list(range(1, rrmdays + 1)))
         self.rrmreferenceindex["date"] = ref_ind[:-1]
 
-        self.q_hm = None  # ReadHMQ
-        self.WLHM = None  # ReadHMWL
-        self.q_rrm = None  # ReadRRM
-        self.QRRM2 = None  # ReadRRM
-        self.rrm_gauges = None  # ReadRRM
-
-        self.hm_gauges = None
-        self.q_gauges = None
-        self.WLGauges = None
-
-        self.CalibrationQ = None
-        self.CalibrationWL = None
-        self.annual_max_obs_q = None
-        self.annual_max_obs_wl = None
-        self.annual_max_rrm = None
-        self.annual_max_hm_q = None
-        self.annual_max_hm_wl = None
-        self.AnnualMaxDates = None
-        self.MetricsHMvsRRM = None
-        self.MetricsRRMvsObs = None
-        self.MetricsHMWLvsObs = None
-        self.MetricsHMQvsObs = None
-        self.WLgaugesList = None
-        self.QgaugesList = None
-
     def readGaugesTable(self, path: str):
         """ReadGaugesTable.
 
@@ -175,8 +180,8 @@ def readGaugesTable(self, path: str):
         # sort the gauges table based on the segment
         self.hm_gauges.sort_values(by="id", inplace=True, ignore_index=True)
 
-    def GetGauges(self, subid: int, gaugei: int = 0) -> DataFrame:
-        """GetGauges. Get_Gauge_ID get the id of the station for a given river segment.
+    def getGauges(self, subid: int, gaugei: int = 0) -> DataFrame:
+        """Get_Gauge_ID get the id of the station for a given river segment.
 
         parameters:
         ----------
@@ -691,7 +696,7 @@ def readHMQ(
 
         self.q_hm.index = pd.date_range(start, end, freq="D")
 
-    def ReadHMWL(
+    def readHMWL(
         self,
         path: str,
         fromday: Union[str, int] = "",
@@ -1089,7 +1094,7 @@ def getAnnualMax(
         else:
             self.annual_max_hm_wl = AnnualMax
 
-    def CalculateProfile(
+    def calculateProfile(
         self, Segmenti: int, BedlevelDS: float, Manning: float, BC_slope: float
     ):
         """CalculateProfile.
@@ -1154,7 +1159,7 @@ def CalculateProfile(
         except AttributeError:
             logger.debug(f"The Given river segment- {Segmenti} does not have a slope")
 
-    def SmoothBedLevel(self, segmenti):
+    def smoothBedLevel(self, segmenti):
         """SmoothXS.
 
         SmoothBedLevel method smoothes the bed level of a given segment ID by
@@ -1200,7 +1205,7 @@ def SmoothBedLevel(self, segmenti):
         # copy back the segment to the whole XS df
         self.crosssections.loc[self.crosssections["id"] == segmenti, :] = segment
 
-    def SmoothBankLevel(self, segmenti):
+    def smoothBankLevel(self, segmenti):
         """SmoothBankLevel.
 
         SmoothBankLevel method smoothes the bankfull depth for a given segment
@@ -1247,7 +1252,7 @@ def SmoothBankLevel(self, segmenti):
         # copy back the segment to the whole XS df
         self.crosssections.loc[self.crosssections["id"] == segmenti, :] = segment
 
-    def SmoothFloodplainHeight(self, segmenti):
+    def smoothFloodplainHeight(self, segmenti):
         """SmoothFloodplainHeight.
 
         SmoothFloodplainHeight method smoothes the Floodplain Height the
@@ -1315,7 +1320,7 @@ def SmoothFloodplainHeight(self, segmenti):
             self.crosssections["fpl"],
         )
 
-    def SmoothBedWidth(self, segmenti):
+    def smoothBedWidth(self, segmenti):
         """SmoothBedWidth.
 
         SmoothBedWidth method smoothes the Bed Width the in the cross section
@@ -1348,7 +1353,7 @@ def SmoothBedWidth(self, segmenti):
         # copy back the segment to the whole XS df
         self.crosssections.loc[self.crosssections["id"] == segmenti, :] = segment
 
-    def DownWardBedLevel(self, segmenti: int, height: Union[int, float]):
+    def downWardBedLevel(self, segmenti: int, height: Union[int, float]):
         """SmoothBedWidth.
 
         SmoothBedWidth method smoothes the Bed Width the in the cross section
@@ -1378,7 +1383,7 @@ def DownWardBedLevel(self, segmenti: int, height: Union[int, float]):
         # copy back the segment to the whole XS df
         self.crosssections.loc[self.crosssections["id"] == segmenti, :] = segment
 
-    def SmoothMaxSlope(self, segmenti, SlopePercentThreshold=1.5):
+    def smoothMaxSlope(self, segmenti, SlopePercentThreshold=1.5):
         """SmoothMaxSlope.
 
         SmoothMaxSlope method smoothes the bed level the in the cross section
@@ -1459,7 +1464,7 @@ def SmoothMaxSlope(self, segmenti, SlopePercentThreshold=1.5):
         # copy back the segment to the whole XS df
         self.crosssections.loc[self.crosssections["id"] == segmenti, :] = segment
 
-    def CheckFloodplain(self):
+    def checkFloodplain(self):
         """CheckFloodplain.
 
         CheckFloodplain method check if the dike levels is higher than the
@@ -1879,7 +1884,7 @@ def InspectGauge(
                 "please calculate first the MetricsHMvsRRM by the method HMvsRRM"
             )
 
-        gauge = self.GetGauges(subid, gaugei)
+        gauge = self.getGauges(subid, gaugei)
         gauge_id = gauge.loc[0, self.gauge_id_col]
         gaugename = str(gauge.loc[0, "name"])
 
@@ -1970,7 +1975,7 @@ def InspectGauge(
             return summary, fig, ax1
 
     @staticmethod
-    def PrepareToSave(df: DataFrame) -> DataFrame:
+    def prepareToSave(df: DataFrame) -> DataFrame:
         """PrepareToSave.
 
             PrepareToSave convert all the dates in the dataframe into string
@@ -2027,7 +2032,7 @@ def SaveMetices(self, path):
         if isinstance(self.MetricsHMvsRRM, GeoDataFrame) or isinstance(
             self.MetricsHMvsRRM, DataFrame
         ):
-            df = self.PrepareToSave(self.MetricsHMvsRRM.copy())
+            df = self.prepareToSave(self.MetricsHMvsRRM.copy())
             if isinstance(self.MetricsHMvsRRM, GeoDataFrame):
                 df.to_file(path + "MetricsHM_Q_RRM.geojson", driver="GeoJSON")
             if isinstance(self.MetricsHMvsRRM, DataFrame):
@@ -2036,7 +2041,7 @@ def SaveMetices(self, path):
         if isinstance(self.MetricsHMQvsObs, GeoDataFrame) or isinstance(
             self.MetricsHMQvsObs, DataFrame
         ):
-            df = self.PrepareToSave(self.MetricsHMQvsObs.copy())
+            df = self.prepareToSave(self.MetricsHMQvsObs.copy())
             if isinstance(self.MetricsHMQvsObs, GeoDataFrame):
                 df.to_file(path + "MetricsHM_Q_Obs.geojson", driver="GeoJSON")
             if isinstance(self.MetricsHMQvsObs, DataFrame):
@@ -2045,7 +2050,7 @@ def SaveMetices(self, path):
         if isinstance(self.MetricsRRMvsObs, GeoDataFrame) or isinstance(
             self.MetricsRRMvsObs, DataFrame
         ):
-            df = self.PrepareToSave(self.MetricsRRMvsObs.copy())
+            df = self.prepareToSave(self.MetricsRRMvsObs.copy())
             if isinstance(self.MetricsRRMvsObs, GeoDataFrame):
                 df.to_file(path + "MetricsRRM_Q_Obs.geojson", driver="GeoJSON")
             if isinstance(self.MetricsRRMvsObs, DataFrame):
@@ -2054,7 +2059,7 @@ def SaveMetices(self, path):
         if isinstance(self.MetricsHMWLvsObs, GeoDataFrame) or isinstance(
             self.MetricsHMWLvsObs, DataFrame
         ):
-            df = self.PrepareToSave(self.MetricsHMWLvsObs.copy())
+            df = self.prepareToSave(self.MetricsHMWLvsObs.copy())
             if isinstance(self.MetricsHMWLvsObs, GeoDataFrame):
                 df.to_file(path + "MetricsHM_WL_Obs.geojson", driver="GeoJSON")
             if isinstance(self.MetricsHMWLvsObs, DataFrame):