Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

553 Adding files to Database when importing extrenal files #558

Closed
Show file tree
Hide file tree
Changes from 31 commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
b4c49bd
added dditional_files arg to all save() functions of object models.
LuukBlom Oct 11, 2024
618301b
implement saving additional files for all object_models. TODO add tes…
LuukBlom Oct 11, 2024
377fea3
WIP tests additional files. TODO finish event tests
LuukBlom Oct 14, 2024
15b0e8e
finish tests for events and measures
LuukBlom Oct 14, 2024
ba8f5d7
Merge branch 'main' into 553-new-development-area-shapefile-not-copie…
LuukBlom Oct 14, 2024
613e76a
go through PR and fix mistakes
LuukBlom Oct 15, 2024
50f2408
remove dditional_files argument and make default behaviour to always…
LuukBlom Oct 15, 2024
20f8fd7
Merge branch 'main' into 553-new-development-area-shapefile-not-copie…
LuukBlom Oct 15, 2024
1a03896
Whoops huge commit.
LuukBlom Oct 21, 2024
81396ad
small bug fixes + rename of util func
LuukBlom Oct 21, 2024
8ff7f38
moved code from pathbuilder to interface.database
LuukBlom Oct 22, 2024
287a269
revert pathbuilder change due to circular dependency
LuukBlom Oct 22, 2024
54e3dd0
finish implementation and use of IObject subclasses in other parts of…
LuukBlom Oct 22, 2024
5264ab6
temporary fix for until the hazard refactor to prevent trying to read…
LuukBlom Oct 22, 2024
0741ece
go through PR and fix obvious stuff
LuukBlom Oct 22, 2024
33b7498
corrected small bug when calling the resolve_filepath method
panosatha Oct 29, 2024
658a3e6
fix file handling for events
LuukBlom Oct 30, 2024
1fa9b8d
Merge branch '553-new-development-area-shapefile-not-copied-in-projec…
LuukBlom Oct 30, 2024
3c92acc
remove obsolete api func
LuukBlom Oct 30, 2024
e5f9eb1
Merge branch 'main' into 553-new-development-area-shapefile-not-copie…
LuukBlom Oct 30, 2024
ea23e77
remove duplicate api func
LuukBlom Oct 30, 2024
f0ce9f2
filter invalid datetime from index to stop errors in hydromt-sfincs
LuukBlom Oct 31, 2024
2b6e0a3
move validation of overlapping measures to the save function of dbs_s…
LuukBlom Nov 1, 2024
1b49e25
make sure that shapefiles can correctly be imported to the database
panosatha Nov 1, 2024
900f772
added overlapping measures check to dbs_strategy
LuukBlom Nov 1, 2024
5c00c24
api correction
panosatha Nov 1, 2024
a4c1736
Merge branch '553-new-development-area-shapefile-not-copied-in-projec…
LuukBlom Nov 1, 2024
ac15f72
instead of saving the name, also save the index in the cyc_db as ther…
LuukBlom Nov 1, 2024
a186cd4
hurricanes events expect either a .cyc file in the same dir as the to…
LuukBlom Nov 1, 2024
ca912ca
fixed broken test
LuukBlom Nov 1, 2024
e665c5f
fix copying of objects with additional files
LuukBlom Nov 5, 2024
3ce7dbe
fix exclude pattern
LuukBlom Nov 5, 2024
02f03f4
merge main into 553-new-development-area-shapefile-not-copied-in-proj…
LuukBlom Nov 27, 2024
fcbeaf5
merge main into 553-new-development-area-shapefile-not-copied-in-proj…
LuukBlom Nov 27, 2024
f26d949
Merge branch 'main' into 553-new-development-area-shapefile-not-copie…
LuukBlom Dec 2, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 15 additions & 14 deletions flood_adapt/api/events.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,13 @@

from flood_adapt.dbs_controller import Database
from flood_adapt.object_model.hazard.event.event import Event
from flood_adapt.object_model.hazard.event.event_factory import EventFactory
from flood_adapt.object_model.hazard.event.historical_nearshore import (
from flood_adapt.object_model.hazard.event.event_factory import (
EventFactory,
HistoricalHurricane,
HistoricalNearshore,
)
from flood_adapt.object_model.interface.events import (
HistoricalOffshore,
IEvent,
IHistoricalHurricane,
IHistoricalNearshore,
IHistoricalOffshore,
ISynthetic,
Synthetic,
)
from flood_adapt.object_model.io.unitfulvalue import UnitTypesLength

Expand All @@ -31,11 +28,11 @@ def get_event(name: str) -> IEvent:


def get_event_mode(name: str) -> str:
filename = Database().events.get_database_path() / f"{name}" / f"{name}.toml"
filename = Database().events.input_path / f"{name}" / f"{name}.toml"
return Event.get_mode(filename)


def create_synthetic_event(attrs: dict[str, Any]) -> ISynthetic:
def create_synthetic_event(attrs: dict[str, Any]) -> Synthetic:
panosatha marked this conversation as resolved.
Show resolved Hide resolved
"""Create a synthetic event object from a dictionary of attributes.

Parameters
Expand All @@ -51,7 +48,7 @@ def create_synthetic_event(attrs: dict[str, Any]) -> ISynthetic:
return EventFactory.get_event("Synthetic").load_dict(attrs)


def create_historical_nearshore_event(attrs: dict[str, Any]) -> IHistoricalNearshore:
def create_historical_nearshore_event(attrs: dict[str, Any]) -> HistoricalNearshore:
"""Create a historical nearshore event object from a dictionary of attributes.

Parameters
Expand All @@ -67,7 +64,7 @@ def create_historical_nearshore_event(attrs: dict[str, Any]) -> IHistoricalNears
return EventFactory.get_event("Historical_nearshore").load_dict(attrs)


def create_historical_offshore_event(attrs: dict[str, Any]) -> IHistoricalOffshore:
def create_historical_offshore_event(attrs: dict[str, Any]) -> HistoricalOffshore:
"""Create a historical offshore event object from a dictionary of attributes.

Parameters
Expand All @@ -83,7 +80,7 @@ def create_historical_offshore_event(attrs: dict[str, Any]) -> IHistoricalOffsho
return EventFactory.get_event("Historical_offshore").load_dict(attrs)


def create_historical_hurricane_event(attrs: dict[str, Any]) -> IHistoricalHurricane:
def create_historical_hurricane_event(attrs: dict[str, Any]) -> HistoricalHurricane:
"""Create a historical hurricane event object from a dictionary of attributes.

Parameters
Expand All @@ -99,7 +96,7 @@ def create_historical_hurricane_event(attrs: dict[str, Any]) -> IHistoricalHurri
return EventFactory.get_event("Historical_hurricane").load_dict(attrs)


def save_event_toml(event: IEvent) -> None:
def save_event(event: IEvent) -> None:
Database().events.save(event)


Expand Down Expand Up @@ -152,3 +149,7 @@ def plot_wind(event: IEvent, input_wind_df: pd.DataFrame = None) -> str:

def save_cyclone_track(event: IEvent, track: TropicalCyclone):
Database().write_cyc(event, track)


def get_cyclone_track_by_index(index: int) -> TropicalCyclone:
return Database().cyclone_track_database.get_track(index)
23 changes: 10 additions & 13 deletions flood_adapt/api/measures.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
from flood_adapt.object_model.interface.measures import (
IMeasure,
)
from flood_adapt.object_model.interface.site import ISite
from flood_adapt.object_model.interface.site import Site


def get_measures() -> dict[str, Any]:
Expand All @@ -35,29 +35,26 @@ def create_measure(attrs: dict[str, Any], type: str = None) -> IMeasure:
Dictionary of attributes for the measure.
type : str
Type of measure to create.
database : IDatabase, optional
Database to use for creating the measure, by default None

Returns
-------
IMeasure
Measure object.
"""
# If a database is provided, use it to set the input path for the measure. Otherwise, set it to None.
database_path = Database().input_path

if type == "elevate_properties":
return Elevate.load_dict(attrs, database_path)
return Elevate.load_dict(attrs)
elif type == "buyout_properties":
return Buyout.load_dict(attrs, database_path)
return Buyout.load_dict(attrs)
elif type == "floodproof_properties":
return FloodProof.load_dict(attrs, database_path)
return FloodProof.load_dict(attrs)
elif type in ["floodwall", "thin_dam", "levee"]:
return FloodWall.load_dict(attrs, database_path)
return FloodWall.load_dict(attrs)
elif type in ["pump", "culvert"]:
return Pump.load_dict(attrs, database_path)
return Pump.load_dict(attrs)
elif type in ["water_square", "total_storage", "greening"]:
return GreenInfrastructure.load_dict(attrs, database_path)
return GreenInfrastructure.load_dict(attrs)
else:
raise ValueError(f"Invalid measure type: {type}")


def save_measure(measure: IMeasure) -> None:
Expand All @@ -77,7 +74,7 @@ def copy_measure(old_name: str, new_name: str, new_description: str) -> None:


# Green infrastructure
def calculate_polygon_area(gdf: gpd.GeoDataFrame, site: ISite) -> float:
def calculate_polygon_area(gdf: gpd.GeoDataFrame, site: Site) -> float:
return GreenInfrastructure.calculate_polygon_area(gdf=gdf, site=site)


Expand Down
20 changes: 5 additions & 15 deletions flood_adapt/api/output.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,11 +66,7 @@ def get_obs_point_timeseries(name: str) -> gpd.GeoDataFrame:
f"Scenario {name} has not been run. Please run the scenario first."
)

output_path = (
Database()
.scenarios.get_database_path(get_input_path=False)
.joinpath(hazard.name)
)
output_path = Database().scenarios.output_path.joinpath(hazard.name)
gdf = Database().static.get_obs_points()
gdf["html"] = [
str(output_path.joinpath("Flooding", f"{station}_timeseries.html"))
Expand Down Expand Up @@ -106,9 +102,7 @@ def get_infographic(name: str) -> str:
)

config_path = database.static_path.joinpath("templates", "infographics")
output_path = database.scenarios.get_database_path(get_input_path=False).joinpath(
impact.name
)
output_path = database.scenarios.output_path.joinpath(impact.name)
metrics_outputs_path = output_path.joinpath(f"Infometrics_{impact.name}.csv")

infographic_path = InforgraphicFactory.create_infographic_file_writer(
Expand Down Expand Up @@ -142,13 +136,9 @@ def get_infometrics(name: str) -> pd.DataFrame:
If the metrics file does not exist.
"""
# Create the infographic path
metrics_path = (
Database()
.scenarios.get_database_path(get_input_path=False)
.joinpath(
name,
f"Infometrics_{name}.csv",
)
metrics_path = Database().scenarios.output_path.joinpath(
name,
f"Infometrics_{name}.csv",
)

# Check if the file exists
Expand Down
2 changes: 1 addition & 1 deletion flood_adapt/api/scenarios.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ def get_scenario(name: str) -> IScenario:


def create_scenario(attrs: dict[str, Any]) -> IScenario:
return Scenario.load_dict(attrs, Database().input_path)
return Scenario.load_dict(attrs)


def save_scenario(scenario: IScenario) -> tuple[bool, str]:
Expand Down
2 changes: 1 addition & 1 deletion flood_adapt/api/strategies.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ def get_strategy(name: str) -> IStrategy:


def create_strategy(attrs: dict[str, Any]) -> IStrategy:
return Strategy.load_dict(attrs, Database().input_path)
return Strategy.load_dict(attrs)


def save_strategy(strategy: IStrategy) -> None:
Expand Down
3 changes: 1 addition & 2 deletions flood_adapt/database_builder/create_database.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,8 @@
from flood_adapt.api.static import read_database
from flood_adapt.api.strategies import create_strategy, save_strategy
from flood_adapt.log import FloodAdaptLogging
from flood_adapt.object_model.interface.site import Obs_pointModel, SlrModel
from flood_adapt.object_model.interface.site import Obs_pointModel, Site, SlrModel
from flood_adapt.object_model.io.unitfulvalue import UnitfulDischarge, UnitfulLength
from flood_adapt.object_model.site import Site

config_path = None

Expand Down
15 changes: 3 additions & 12 deletions flood_adapt/dbs_classes/dbs_benefit.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,7 @@


class DbsBenefit(DbsTemplate):
_type = "benefit"
_folder_name = "benefits"
_object_model_class = Benefit
_object_class = Benefit

def save(self, benefit: IBenefit, overwrite: bool = False):
"""Save a benefit object in the database.
Expand Down Expand Up @@ -54,10 +52,7 @@ def delete(self, name: str, toml_only: bool = False):
super().delete(name, toml_only=toml_only)

# Delete output if edited
output_path = (
self._database.benefits.get_database_path(get_input_path=False) / name
)

output_path = self.output_path / name
if output_path.exists():
shutil.rmtree(output_path, ignore_errors=True)

Expand All @@ -78,10 +73,6 @@ def edit(self, benefit: IBenefit):
super().edit(benefit)

# Delete output if edited
output_path = (
self._database.benefits.get_database_path(get_input_path=False)
/ benefit.attrs.name
)

output_path = self.output_path / benefit.attrs.name
if output_path.exists():
shutil.rmtree(output_path, ignore_errors=True)
69 changes: 14 additions & 55 deletions flood_adapt/dbs_classes/dbs_event.py
Original file line number Diff line number Diff line change
@@ -1,20 +1,17 @@
import shutil
from pathlib import Path
from typing import Any

from flood_adapt.dbs_classes.dbs_template import DbsTemplate
from flood_adapt.object_model.hazard.event.event import Event
from flood_adapt.object_model.hazard.event.event_factory import EventFactory
from flood_adapt.object_model.hazard.event.eventset import EventSet
from flood_adapt.object_model.interface.events import IEvent, Mode
from flood_adapt.object_model.interface.events import Mode


class DbsEvent(DbsTemplate):
_type = "event"
_folder_name = "events"
_object_model_class = Event
_object_class = Event

def get(self, name: str) -> IEvent:
def get(self, name: str) -> Event:
"""Return an event object.

Parameters
Expand All @@ -28,11 +25,13 @@ def get(self, name: str) -> IEvent:
event object
"""
# Get event path
event_path = self._path / f"{name}" / f"{name}.toml"
event_path = self.input_path / f"{name}" / f"{name}.toml"

# Check if the object exists
if not Path(event_path).is_file():
raise ValueError(f"{self._type.capitalize()} '{name}' does not exist.")
raise ValueError(
f"{self._object_class.class_name} '{name}' does not exist."
)

# Load event
mode = Event.get_mode(event_path)
Expand All @@ -44,7 +43,7 @@ def get(self, name: str) -> IEvent:
elif mode == Mode.risk:
return EventSet.load_file(event_path)

def list_objects(self) -> dict[str, Any]:
def list_objects(self) -> dict[str, list[Any]]:
"""Return a dictionary with info on the events that currently exist in the database.

Returns
Expand All @@ -53,48 +52,11 @@ def list_objects(self) -> dict[str, Any]:
Includes 'name', 'description', 'path' and 'last_modification_date' info
"""
events = self._get_object_list()
objects = [self._database.events.get(name) for name in events["name"]]
objects = [self.get(name) for name in events["name"]]
events["description"] = [obj.attrs.description for obj in objects]
events["objects"] = objects
return events

def copy(self, old_name: str, new_name: str, new_description: str):
"""Copy (duplicate) an existing object, and give it a new name.

Parameters
----------
old_name : str
name of the existing measure
new_name : str
name of the new measure
new_description : str
description of the new measure
"""
# Check if the provided old_name is valid
if old_name not in self.list_objects()["name"]:
raise ValueError(f"'{old_name}' {self._type} does not exist.")

# First do a get and change the name and description
copy_object = self.get(old_name)
copy_object.attrs.name = new_name
copy_object.attrs.description = new_description

# After changing the name and description, receate the model to re-trigger the validators
copy_object.attrs = type(copy_object.attrs)(**copy_object.attrs.dict())

# Then a save. Checking whether the name is already in use is done in the save function
self.save(copy_object)

# Then save all the accompanied files
src = self._path / old_name
dest = self._path / new_name

EXCLUDE = [".spw", ".toml"]
for file in src.glob("*"):
if file.suffix in EXCLUDE:
continue
shutil.copy(file, dest / file.name)

def _check_standard_objects(self, name: str) -> bool:
"""Check if an event is a standard event.

Expand All @@ -109,10 +71,10 @@ def _check_standard_objects(self, name: str) -> bool:
True if the event is a standard event, False otherwise
"""
# Check if event is a standard event
if self._database.site.attrs.standard_objects.events:
if name in self._database.site.attrs.standard_objects.events:
return True

if self._database.site.attrs.standard_objects:
if self._database.site.attrs.standard_objects.events:
if name in self._database.site.attrs.standard_objects.events:
return True
return False

def check_higher_level_usage(self, name: str) -> list[str]:
Expand All @@ -129,10 +91,7 @@ def check_higher_level_usage(self, name: str) -> list[str]:
list of scenarios that use the event
"""
# Get all the scenarios
scenarios = [
self._database.scenarios.get(name)
for name in self._database.scenarios.list_objects()["name"]
]
scenarios = self._database.scenarios.list_objects()["objects"]

# Check if event is used in a scenario
used_in_scenario = [
Expand Down
Loading
Loading