Release 0.6.0: interactive tool

.github/workflows/ci.yml

@@ -28,16 +28,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: mamba-org/setup-micromamba@v1
         with:
           activate-environment: test
           environment-file: environment.yml
           auto-activate-base: false
-      - name: conda check
-        shell: bash -l {0}
-        run: |
-          conda info
-          conda list
       - name: install hat package
         shell: bash -l {0}
         run: pip install .

.github/workflows/on-push.yml

@@ -32,16 +32,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: mamba-org/setup-micromamba@v1
         with:
           activate-environment: test
           environment-file: environment.yml
           auto-activate-base: false
-      - name: Conda check
-        shell: bash -l {0}
-        run: |
-          conda info
-          conda list
       - name: install hat package
         shell: bash -l {0}
         run: pip install .

.pre-commit-config.yaml

@@ -7,7 +7,7 @@ repos:
     rev: 23.3.0
     hooks:
       - id: black
-  - repo: https://github.com/PyCQA/flake8
-    rev: 6.0.0
-    hooks:
-      - id: flake8
+#  - repo: https://github.com/PyCQA/flake8
+#    rev: 6.0.0
+#    hooks:
+#      - id: flake8

README.md

@@ -14,7 +14,7 @@ Interfaces and functionality are likely to change, and the project itself may be
 
 Clone source code repository
 
-    $ git clone git@github.com:ecmwf-projects/hat.git
+    $ git clone https://github.com/ecmwf/hat.git
 
 Create conda python environment
 
@@ -63,4 +63,4 @@ does it submit to any jurisdiction.
 
 ### Citing
 
-In publications, please use a link to this repository (https://github.com/ecmwf/hat) and its documentation (https://hydro-analysis-toolkit.readthedocs.io)
+In publications, please use a link to this repository (https://github.com/ecmwf/hat) and its documentation (https://hydro-analysis-toolkit.readthedocs.io)

environment.yml

@@ -2,21 +2,21 @@ name: hat
 channels:
   - conda-forge
 dependencies:
-  - python=3.10
+  - python<=3.10
   - netCDF4
   - eccodes
   - cfgrib
-  - cftime
   - geopandas
   - xarray
   - plotly
   - matplotlib
-  - jupyterlab
+  - jupyter
   - tqdm
   - typer
   - humanize
-  - folium
   - typer
+  - ipyleaflet
+  - ipywidgets
   - pip
   # - pytest
   # - mkdocs

hat/data.py

@@ -37,6 +37,9 @@ def get_tmpdir():
     else:
         tmpdir = TemporaryDirectory().name
 
+    # Ensure the directory exists
+    os.makedirs(tmpdir, exist_ok=True)
+
     return tmpdir
 
 
@@ -257,8 +260,8 @@ def save_dataset_to_netcdf(ds: xr.Dataset, fpath: str):
     ds.to_netcdf(fpath)
 
 
-def find_main_var(ds):
-    variable_names = [k for k in ds.variables if len(ds.variables[k].dims) >= 3]
+def find_main_var(ds, min_dim=3):
+    variable_names = [k for k in ds.variables if len(ds.variables[k].dims) >= min_dim]
     if len(variable_names) > 1:
         raise Exception("More than one variable in dataset")
     elif len(variable_names) == 0:

hat/filters.py

@@ -144,7 +144,7 @@ def filter_dataframe(df, filters: str):
     return df
 
 
-def filter_timeseries(sims_ds: xr.Dataset, obs_ds: xr.Dataset, threshold=80):
+def filter_timeseries(sims_ds: xr.DataArray, obs_ds: xr.DataArray, threshold=80):
     """Clean the simulation and observation timeseries
 
     Only keep..
@@ -159,30 +159,36 @@ def filter_timeseries(sims_ds: xr.Dataset, obs_ds: xr.Dataset, threshold=80):
     matching_stations = sorted(
         set(sims_ds.station.values).intersection(obs_ds.station.values)
     )
+    print(len(matching_stations))
     sims_ds = sims_ds.sel(station=matching_stations)
     obs_ds = obs_ds.sel(station=matching_stations)
+    obs_ds = obs_ds.sel(time=sims_ds.time)
+
+    obs_ds = obs_ds.dropna(dim="station", how="all")
+    sims_ds = sims_ds.sel(station=obs_ds.station)
 
     # Only keep observations in the same time period as the simulations
-    obs_ds = obs_ds.where(sims_ds.time == obs_ds.time, drop=True)
+    # obs_ds = obs_ds.where(sims_ds.time == obs_ds.time, drop=True)
 
     # Only keep obsevations with enough valid data in this timeperiod
 
     # discharge data
-    dis = obs_ds.obsdis
+    print(sims_ds)
+    print(obs_ds)
 
     # Replace negative values with NaN
-    dis = dis.where(dis >= 0)
+    # dis = dis.where(dis >= 0)
 
-    # Percentage of valid discharge data at each point in time
-    valid_percent = dis.notnull().mean(dim="time") * 100
+    # # Percentage of valid discharge data at each point in time
+    # valid_percent = dis.notnull().mean(dim="time") * 100
 
-    # Boolean index of where there is enough valid data
-    enough_observation_data = valid_percent > threshold
+    # # Boolean index of where there is enough valid data
+    # enough_observation_data = valid_percent > threshold
 
-    # keep where there is enough observation data
-    obs_ds = obs_ds.where(enough_observation_data, drop=True)
+    # # keep where there is enough observation data
+    # obs_ds = obs_ds.where(enough_observation_data, drop=True)
 
-    # keep simulation that match remaining observations
-    sims_ds = sims_ds.where(enough_observation_data, drop=True)
+    # # keep simulation that match remaining observations
+    # sims_ds = sims_ds.where(enough_observation_data, drop=True)
 
     return (sims_ds, obs_ds)